@smoothbricks/lmao 0.2.2 → 0.2.3

package/src/lib/__tests__/vocabularyRegistry.test.ts

@@ -0,0 +1,313 @@
+import { describe, expect, it } from 'bun:test';
+import { createHash } from 'node:crypto';
+import {
+  getVocabularyGeneration,
+  registerVocabularyFragment,
+  type VocabularyFragment,
+  type VocabularyGeneration,
+  VocabularyRegistrationError,
+} from '../vocabularyRegistry.js';
+
+const encoder = new TextEncoder();
+const decoder = new TextDecoder();
+
+type RecordField = { name: string; column: string };
+type RecordInput = { text: string; fields?: readonly RecordField[] };
+
+function encodeRecord({ text, fields = [] }: RecordInput): Uint8Array {
+  const textBytes = encoder.encode(text);
+  const encodedFields = fields.map(({ name, column }) => ({
+    name: encoder.encode(name),
+    column: encoder.encode(column),
+  }));
+  let byteLength = 4 + textBytes.length + 2;
+  for (const field of encodedFields) byteLength += 2 + field.name.length + 2 + field.column.length;
+  const record = new Uint8Array(byteLength);
+  const view = new DataView(record.buffer);
+  let offset = 0;
+  view.setUint32(offset, textBytes.length, true);
+  offset += 4;
+  record.set(textBytes, offset);
+  offset += textBytes.length;
+  view.setUint16(offset, encodedFields.length, true);
+  offset += 2;
+  for (const field of encodedFields) {
+    view.setUint16(offset, field.name.length, true);
+    offset += 2;
+    record.set(field.name, offset);
+    offset += field.name.length;
+    view.setUint16(offset, field.column.length, true);
+    offset += 2;
+    record.set(field.column, offset);
+    offset += field.column.length;
+  }
+  return record;
+}
+
+function stableId(kindTag: number, record: Uint8Array): number {
+  const digest = createHash('sha256').update(Uint8Array.of(kindTag)).update(record).digest();
+  return (digest[0] << 16) | (digest[1] << 8) | digest[2];
+}
+
+function fragmentHash(fragment: Omit<VocabularyFragment, 'contentHash'>): string {
+  const algorithm = encoder.encode(fragment.idAlgorithm);
+  const byteLength =
+    1 +
+    2 +
+    algorithm.length +
+    4 +
+    fragment.ids.length * 4 +
+    4 +
+    fragment.kindTags.length +
+    4 +
+    fragment.utf8.length +
+    4 +
+    fragment.offsets.length * 4;
+  const bytes = new Uint8Array(byteLength);
+  const view = new DataView(bytes.buffer);
+  let offset = 0;
+  bytes[offset++] = fragment.schemaVersion;
+  view.setUint16(offset, algorithm.length, true);
+  offset += 2;
+  bytes.set(algorithm, offset);
+  offset += algorithm.length;
+  view.setUint32(offset, fragment.ids.length, true);
+  offset += 4;
+  for (const id of fragment.ids) {
+    view.setUint32(offset, id, true);
+    offset += 4;
+  }
+  view.setUint32(offset, fragment.kindTags.length, true);
+  offset += 4;
+  bytes.set(fragment.kindTags, offset);
+  offset += fragment.kindTags.length;
+  view.setUint32(offset, fragment.utf8.length, true);
+  offset += 4;
+  bytes.set(fragment.utf8, offset);
+  offset += fragment.utf8.length;
+  view.setUint32(offset, fragment.offsets.length, true);
+  offset += 4;
+  for (const boundary of fragment.offsets) {
+    view.setInt32(offset, boundary, true);
+    offset += 4;
+  }
+  return createHash('sha256').update(bytes).digest('hex');
+}
+
+function makeFragment(
+  inputs: readonly RecordInput[],
+  kindTags: readonly number[] = inputs.map(() => 1),
+): VocabularyFragment {
+  const records = inputs.map(encodeRecord);
+  const offsets = new Int32Array(records.length + 1);
+  let byteLength = 0;
+  for (let ordinal = 0; ordinal < records.length; ordinal++) {
+    byteLength += records[ordinal].length;
+    offsets[ordinal + 1] = byteLength;
+  }
+  const utf8 = new Uint8Array(byteLength);
+  let offset = 0;
+  for (const record of records) {
+    utf8.set(record, offset);
+    offset += record.length;
+  }
+  const withoutHash: Omit<VocabularyFragment, 'contentHash'> = {
+    schemaVersion: 1,
+    idAlgorithm: 'sha256-24-v1',
+    ids: Uint32Array.from(records, (record, ordinal) => stableId(kindTags[ordinal], record)),
+    kindTags: Uint8Array.from(kindTags),
+    utf8,
+    offsets,
+  };
+  return { ...withoutHash, contentHash: fragmentHash(withoutHash) };
+}
+
+function textAt(generation: VocabularyGeneration, denseIndex: number): string {
+  const record = generation.records.subarray(generation.offsets[denseIndex], generation.offsets[denseIndex + 1]);
+  const textLength = new DataView(record.buffer, record.byteOffset, record.byteLength).getUint32(0, true);
+  return decoder.decode(record.subarray(4, 4 + textLength));
+}
+
+function decodedBinding(binding: Uint32Array): string[] {
+  const generation = getVocabularyGeneration();
+  return Array.from(binding, (denseIndex) => textAt(generation, denseIndex));
+}
+
+function expectRegistrationError(fragment: unknown, code: string): void {
+  let thrown: unknown;
+  try {
+    Reflect.apply(registerVocabularyFragment, undefined, [fragment]);
+  } catch (error) {
+    thrown = error;
+  }
+  expect(thrown).toBeInstanceOf(VocabularyRegistrationError);
+  if (!(thrown instanceof VocabularyRegistrationError)) throw new Error(`expected ${code}`);
+  expect(thrown.message.startsWith(`${code}:`)).toBe(true);
+}
+
+describe('vocabulary fragment registration', () => {
+  it('binds fragment ordinals directly while duplicate and overlapping registration preserve append-only generations', () => {
+    const alpha = { text: 'registry overlap alpha', fields: [{ name: 'request.id', column: 'request_id' }] };
+    const beta = { text: 'registry overlap beta' };
+    const gamma = { text: 'registry overlap gamma', fields: [{ name: 'user', column: 'user_name' }] };
+    const firstFragment = makeFragment([alpha, beta], [1, 2]);
+    const overlappingFragment = makeFragment([beta, gamma], [2, 1]);
+
+    const before = getVocabularyGeneration();
+    const firstBinding = registerVocabularyFragment(firstFragment);
+    const firstGeneration = getVocabularyGeneration();
+    expect(firstGeneration.generation).toBe(before.generation + 1);
+    expect(decodedBinding(firstBinding)).toEqual([alpha.text, beta.text]);
+
+    const firstSnapshot = {
+      ids: Array.from(firstGeneration.ids),
+      kinds: Array.from(firstGeneration.kindTags),
+      records: Array.from(firstGeneration.records),
+      offsets: Array.from(firstGeneration.offsets),
+    };
+    const overlapBinding = registerVocabularyFragment(overlappingFragment);
+    const secondGeneration = getVocabularyGeneration();
+
+    expect(overlapBinding[0]).toBe(firstBinding[1]);
+    expect(decodedBinding(overlapBinding)).toEqual([beta.text, gamma.text]);
+    expect(secondGeneration.generation).toBe(firstGeneration.generation + 1);
+    expect(Array.from(secondGeneration.ids.subarray(0, firstGeneration.ids.length))).toEqual(firstSnapshot.ids);
+    expect(Array.from(secondGeneration.kindTags.subarray(0, firstGeneration.kindTags.length))).toEqual(
+      firstSnapshot.kinds,
+    );
+    expect(Array.from(secondGeneration.records.subarray(0, firstGeneration.records.length))).toEqual(
+      firstSnapshot.records,
+    );
+    expect(Array.from(secondGeneration.offsets.subarray(0, firstGeneration.offsets.length))).toEqual(
+      firstSnapshot.offsets,
+    );
+    expect(Array.from(firstGeneration.ids)).toEqual(firstSnapshot.ids);
+    expect(Array.from(firstGeneration.kindTags)).toEqual(firstSnapshot.kinds);
+    expect(Array.from(firstGeneration.records)).toEqual(firstSnapshot.records);
+    expect(Array.from(firstGeneration.offsets)).toEqual(firstSnapshot.offsets);
+
+    const duplicateBinding = registerVocabularyFragment(firstFragment);
+    expect(duplicateBinding).toBe(firstBinding);
+    expect(getVocabularyGeneration()).toBe(secondGeneration);
+  });
+
+  it('decodes each fragment in ordinal order regardless of the records registration order', () => {
+    const left = { text: 'registry ordinal left' };
+    const right = { text: 'registry ordinal right' };
+    const forward = makeFragment([left, right]);
+    const reverse = makeFragment([right, left]);
+
+    const forwardBinding = registerVocabularyFragment(forward);
+    const generationAfterForward = getVocabularyGeneration();
+    const reverseBinding = registerVocabularyFragment(reverse);
+
+    expect(decodedBinding(forwardBinding)).toEqual([left.text, right.text]);
+    expect(decodedBinding(reverseBinding)).toEqual([right.text, left.text]);
+    expect(Array.from(reverseBinding)).toEqual([forwardBinding[1], forwardBinding[0]]);
+    expect(getVocabularyGeneration()).toBe(generationAfterForward);
+  });
+
+  it('rejects malformed ABI, offsets, kind tags, record-derived IDs, UTF-8, and payload hashes before mutation', () => {
+    const valid = makeFragment([{ text: 'registry validation sentinel' }]);
+    const generation = getVocabularyGeneration();
+    const malformedCases: Array<{ name: string; fragment: unknown; code: string }> = [
+      {
+        name: 'schema version',
+        fragment: { ...valid, schemaVersion: 2 },
+        code: 'LMAO_VOCABULARY_ABI_UNAVAILABLE',
+      },
+      {
+        name: 'ID algorithm',
+        fragment: { ...valid, idAlgorithm: 'sha256-24-v2' },
+        code: 'LMAO_VOCABULARY_ABI_UNAVAILABLE',
+      },
+      {
+        name: 'offset coverage',
+        fragment: { ...valid, offsets: new Int32Array([1, valid.utf8.length]) },
+        code: 'LMAO_VOCABULARY_FRAGMENT_INVALID',
+      },
+      {
+        name: 'nonmonotonic offsets',
+        fragment: (() => {
+          const malformed = makeFragment([
+            { text: 'registry offset first' },
+            { text: 'registry offset second' },
+            { text: 'registry offset third' },
+          ]);
+          return {
+            ...malformed,
+            offsets: new Int32Array([0, malformed.offsets[2], malformed.offsets[1], malformed.utf8.length]),
+          };
+        })(),
+        code: 'LMAO_VOCABULARY_FRAGMENT_INVALID',
+      },
+      {
+        name: 'unknown kind',
+        fragment: makeFragment([{ text: 'registry unknown kind' }], [3]),
+        code: 'LMAO_VOCABULARY_FRAGMENT_INVALID',
+      },
+      {
+        name: 'record-derived ID mismatch',
+        fragment: (() => {
+          const mismatch = makeFragment([{ text: 'registry wrong stable ID' }]);
+          const withoutHash = { ...mismatch, ids: Uint32Array.of(mismatch.ids[0] ^ 1) };
+          return { ...withoutHash, contentHash: fragmentHash(withoutHash) };
+        })(),
+        code: 'LMAO_VOCABULARY_FRAGMENT_INVALID',
+      },
+      {
+        name: 'invalid UTF-8 text',
+        fragment: (() => {
+          const malformed = makeFragment([{ text: 'registry malformed UTF-8' }]);
+          const utf8 = malformed.utf8.slice();
+          utf8[4] = 0xff;
+          return { ...malformed, utf8 };
+        })(),
+        code: 'LMAO_VOCABULARY_FRAGMENT_INVALID',
+      },
+      {
+        name: 'payload hash mismatch',
+        fragment: { ...valid, contentHash: '0'.repeat(64) },
+        code: 'LMAO_VOCABULARY_CONTENT_HASH_MISMATCH',
+      },
+    ];
+
+    for (const testCase of malformedCases) {
+      try {
+        expectRegistrationError(testCase.fragment, testCase.code);
+      } catch (error) {
+        throw new Error(`case ${testCase.name} failed`, { cause: error });
+      }
+      expect(getVocabularyGeneration()).toBe(generation);
+    }
+  });
+
+  it('rejects reuse of a registered content hash with a different payload before mutation', () => {
+    const registered = makeFragment([{ text: 'registry content hash owner' }]);
+    registerVocabularyFragment(registered);
+    const generation = getVocabularyGeneration();
+    const conflictingPayload = makeFragment([{ text: 'registry content hash impostor' }]);
+
+    expectRegistrationError(
+      { ...conflictingPayload, contentHash: registered.contentHash },
+      'LMAO_VOCABULARY_CONTENT_HASH_COLLISION',
+    );
+    expect(getVocabularyGeneration()).toBe(generation);
+    expect(decodedBinding(registerVocabularyFragment(registered))).toEqual(['registry content hash owner']);
+  });
+
+  it('rejects a stable-ID collision without partially registering the fragment', () => {
+    const firstRecord = { text: 'stable collision fixture 585' };
+    const secondRecord = { text: 'stable collision fixture 4248' };
+    const first = makeFragment([firstRecord]);
+    const second = makeFragment([secondRecord]);
+
+    expect(first.ids[0]).toBe(15_813_490);
+    expect(second.ids[0]).toBe(first.ids[0]);
+    registerVocabularyFragment(first);
+    const generation = getVocabularyGeneration();
+    expectRegistrationError(second, 'LMAO_VOCABULARY_ID_COLLISION');
+    expect(getVocabularyGeneration()).toBe(generation);
+    expect(decodedBinding(registerVocabularyFragment(first))).toEqual([firstRecord.text]);
+  });
+});

package/src/lib/arrow/lease.ts

@@ -0,0 +1,43 @@
+import type { Table } from '@uwdata/flechette';
+
+export interface ArrowLease {
+  readonly table: Table;
+  readonly released: boolean;
+  release(): void;
+  [Symbol.dispose](): void;
+}
+
+/** Internal constructor for a lease over borrowed Arrow source chunks. */
+export function createArrowLease(
+  table: Table,
+  pinned: readonly unknown[],
+  releases: readonly (() => void)[],
+): ArrowLease {
+  let released = false;
+  // Retained by the closures for the full lease lifetime. Clearing this array on
+  // release makes the backing epochs and dictionary generation collectible.
+  const retained = [...pinned];
+  return Object.freeze({
+    table,
+    get released() {
+      return released;
+    },
+    release(): void {
+      if (released) return;
+      released = true;
+      let failure: unknown;
+      for (let index = releases.length - 1; index >= 0; index--) {
+        try {
+          releases[index]();
+        } catch (error) {
+          failure ??= error;
+        }
+      }
+      retained.length = 0;
+      if (failure !== undefined) throw failure;
+    },
+    [Symbol.dispose](): void {
+      this.release();
+    },
+  });
+}

package/src/lib/arrow/utils.ts

@@ -3,6 +3,7 @@
  */
 
 import { copyBits, countNulls } from '@smoothbricks/arrow-builder';
+import { Column } from '@uwdata/flechette';
 import type { AnySpanBuffer } from '../types.js';
 
 const UTF8_ENCODER = new TextEncoder();
@@ -104,23 +105,18 @@ export function concatenateNullBitmaps(buffers: AnySpanBuffer[], columnName: str
   return { nullBitmap, nullCount };
 }
 
+/** Combine Arrow-native batches by copying only batch references, never values. */
+export function combineColumnChunks(chunks: readonly Column<unknown>[]): Column<unknown> {
+  if (chunks.length === 0) throw new Error('Cannot combine an empty Arrow chunk list');
+  if (chunks.length === 1) return chunks[0];
+  const batches = [];
+  for (const chunk of chunks) batches.push(...chunk.data);
+  return new Column(batches);
+}
+
 /**
  * Get Arrow field name from LMAO field name
  */
 export function getArrowFieldName(fieldName: string): string {
   return fieldName;
 }
-
-/**
- * Walk a SpanBuffer tree (including overflow chains and children)
- */
-export function walkSpanTree(root: AnySpanBuffer, visitor: (buffer: AnySpanBuffer) => void): void {
-  let current: AnySpanBuffer | undefined = root;
-  while (current) {
-    visitor(current);
-    current = current._overflow;
-  }
-  for (const child of root._children) {
-    walkSpanTree(child, visitor);
-  }
-}

package/src/lib/arrow/vocabularyDictionary.ts

@@ -0,0 +1,123 @@
+import { Column, utf8 } from '@uwdata/flechette';
+import { ENTRY_TYPE_NAMES } from '../schema/systemSchema.js';
+import type { VocabularyGeneration } from '../vocabularyRegistry.js';
+import { makeArrowColumn } from './flechette.js';
+
+const ENTRY_TYPE_UTF8 = new TextEncoder().encode(ENTRY_TYPE_NAMES.join(''));
+const ENTRY_TYPE_OFFSETS = new Int32Array(ENTRY_TYPE_NAMES.length + 1);
+for (let index = 0, offset = 0; index < ENTRY_TYPE_NAMES.length; index++) {
+  ENTRY_TYPE_OFFSETS[index] = offset;
+  offset += ENTRY_TYPE_NAMES[index].length;
+  ENTRY_TYPE_OFFSETS[index + 1] = offset;
+}
+
+function makeUtf8Dictionary(length: number, offsets: Int32Array, values: Uint8Array): Column<unknown> {
+  return makeArrowColumn({
+    type: utf8(),
+    length,
+    nullCount: 0,
+    offsets,
+    values,
+  });
+}
+
+const EMPTY_DICTIONARY_BYTES = new Uint8Array(0);
+const EMPTY_DICTIONARY_OFFSETS = new Int32Array([0, 0]);
+const decoder = new TextDecoder('utf-8', { fatal: true });
+
+export interface VocabularyDictionaryPrefix {
+  readonly column: Column<unknown>;
+  readonly length: number;
+  readonly valueToDenseIndex: ReadonlyMap<string, number>;
+}
+
+const vocabularyDictionaries = new WeakMap<VocabularyGeneration, VocabularyDictionaryPrefix>();
+
+/**
+ * Return the immutable Arrow dictionary prefix for one pinned vocabulary generation.
+ * Record text is decoded and copied into Arrow UTF-8 buffers once per generation;
+ * every flush thereafter reuses the same column and backing buffers.
+ */
+export function getVocabularyDictionaryPrefix(generation: VocabularyGeneration): VocabularyDictionaryPrefix {
+  const cached = vocabularyDictionaries.get(generation);
+  if (cached !== undefined) return cached;
+
+  if (generation.ids.length === 0) {
+    const emptyPrefix = Object.freeze({
+      column: makeUtf8Dictionary(1, EMPTY_DICTIONARY_OFFSETS, EMPTY_DICTIONARY_BYTES),
+      length: 1,
+      valueToDenseIndex: new Map<string, number>([['', 0]]),
+    });
+    emptyPrefix.column.cache();
+    vocabularyDictionaries.set(generation, emptyPrefix);
+    return emptyPrefix;
+  }
+
+  const count = generation.ids.length;
+  const offsets = new Int32Array(count + 1);
+  const textLengths = new Uint32Array(count);
+  let byteLength = 0;
+  for (let denseIndex = 0; denseIndex < count; denseIndex++) {
+    const recordOffset = generation.offsets[denseIndex];
+    const textLength = new DataView(
+      generation.records.buffer,
+      generation.records.byteOffset + recordOffset,
+      4,
+    ).getUint32(0, true);
+    textLengths[denseIndex] = textLength;
+    byteLength += textLength;
+    offsets[denseIndex + 1] = byteLength;
+  }
+
+  const values = new Uint8Array(byteLength);
+  const valueToDenseIndex = new Map<string, number>();
+  for (let denseIndex = 0; denseIndex < count; denseIndex++) {
+    const textStart = generation.offsets[denseIndex] + 4;
+    const text = generation.records.subarray(textStart, textStart + textLengths[denseIndex]);
+    values.set(text, offsets[denseIndex]);
+    const value = decoder.decode(text);
+    if (!valueToDenseIndex.has(value)) valueToDenseIndex.set(value, denseIndex);
+  }
+
+  const prefix = Object.freeze({
+    column: makeUtf8Dictionary(count, offsets, values),
+    length: count,
+    valueToDenseIndex,
+  });
+  prefix.column.cache();
+  vocabularyDictionaries.set(generation, prefix);
+  return prefix;
+}
+
+/**
+ * Append one UTF-8 dictionary batch without copying the prefix buffers or
+ * scanning its cached strings. Flechette serializes later dictionary batches
+ * as Arrow dictionary deltas.
+ */
+export function appendVocabularyDictionarySuffix(
+  prefix: VocabularyDictionaryPrefix,
+  suffix: Column<unknown>,
+  suffixValues: readonly string[],
+): Column<unknown> {
+  const dictionary = new Column([...prefix.column.data, ...suffix.data]);
+  const prefixCache = prefix.column.cache();
+  const cacheTarget = new Array<unknown>(prefix.length + suffixValues.length);
+  dictionary._cache = new Proxy(cacheTarget, {
+    get(target, property, receiver) {
+      if (typeof property === 'string') {
+        const index = Number(property);
+        if (Number.isInteger(index) && index >= 0 && index < target.length && String(index) === property) {
+          return index < prefix.length ? prefixCache[index] : suffixValues[index - prefix.length];
+        }
+      }
+      return Reflect.get(target, property, receiver);
+    },
+  });
+  return dictionary;
+}
+
+export const PREBUILT_ENTRY_TYPE_DICTIONARY = makeUtf8Dictionary(
+  ENTRY_TYPE_NAMES.length,
+  ENTRY_TYPE_OFFSETS,
+  ENTRY_TYPE_UTF8,
+);

package/src/lib/bufferStrategy.ts

@@ -22,6 +22,7 @@
 import type { Table } from '@uwdata/flechette';
 import type { OpMetadata } from './opContext/opTypes.js';
 import type { LogSchema } from './schema/LogSchema.js';
+import type { SpanBufferConstructor } from './spanBuffer.js';
 import type { ITraceRoot } from './traceRoot.js';
 import type { AnySpanBuffer, SpanBuffer } from './types.js';
 
@@ -45,7 +46,13 @@ export interface BufferStrategy<T extends LogSchema = LogSchema> {
    * @param capacity - Optional capacity override
    * @returns New SpanBuffer for the root span
    */
-  createSpanBuffer(schema: T, traceRoot: ITraceRoot, opMetadata: OpMetadata, capacity?: number): SpanBuffer<T>;
+  createSpanBuffer(
+    schema: T,
+    traceRoot: ITraceRoot,
+    opMetadata: OpMetadata,
+    capacity?: number,
+    SpanBufferClass?: SpanBufferConstructor<T>,
+  ): SpanBuffer<T>;
 
   /**
    * Create a child span buffer linked to a parent.
@@ -65,6 +72,7 @@ export interface BufferStrategy<T extends LogSchema = LogSchema> {
     opMetadata: OpMetadata,
     capacity?: number,
     schema?: T,
+    SpanBufferClass?: SpanBufferConstructor<T>,
   ): SpanBuffer<T>;
 
   /**

package/src/lib/cloudflare/traceRows.ts

@@ -10,6 +10,7 @@
  * buffer's user schema columns, flattened so downstream SQL transforms need no unnesting.
  */
 
+import { resolveEntryType, resolveMessage } from '../resolveMessage.js';
 import { readSpanFieldValue, walkSpanSegments } from '../sqlite/sqlite-common.js';
 import type { AnySpanBuffer } from '../types.js';
 
@@ -46,9 +47,9 @@ export function spanBufferToTraceRows(rootBuffer: AnySpanBuffer): TraceRow[] {
         span_id: spanId,
         parent_span_id: parentSpanId,
         row_index: rowOffset + row,
-        entry_type: buffer.entry_type[row],
+        entry_type: resolveEntryType(buffer, row),
         timestamp_ns: Number(buffer.timestamp[row]),
-        message: buffer.message_values[row] ?? null,
+        message: toTraceRowValue(resolveMessage(buffer, row)),
       };
       for (const fieldName of fieldNames) {
         record[fieldName] = toTraceRowValue(readSpanFieldValue(buffer, fieldName, row));