@smoothbricks/lmao 0.1.0

package/src/lib/archive/chunkEnvelope.ts

@@ -0,0 +1,79 @@
+export interface TraceChunkEnvelopeInput {
+  readonly file_ref: string;
+  readonly chunk_ref: string;
+  readonly row_count: number;
+  readonly started_at_ms: number;
+  readonly ended_at_ms: number;
+  readonly partition_keys?: readonly string[];
+  readonly metadata?: Record<string, unknown>;
+}
+
+export interface TraceChunkEnvelope {
+  readonly chunk_id: string;
+  readonly file_ref: string;
+  readonly chunk_ref: string;
+  readonly row_count: number;
+  readonly started_at_ms: number;
+  readonly ended_at_ms: number;
+  readonly partition_keys: readonly string[];
+  readonly metadata?: Record<string, unknown>;
+}
+
+export function buildTraceChunkEnvelope(input: TraceChunkEnvelopeInput): TraceChunkEnvelope {
+  const canonical = {
+    chunk_ref: input.chunk_ref,
+    ended_at_ms: input.ended_at_ms,
+    file_ref: input.file_ref,
+    metadata: input.metadata ?? null,
+    partition_keys: [...(input.partition_keys ?? [])].sort(),
+    row_count: input.row_count,
+    started_at_ms: input.started_at_ms,
+  };
+
+  const chunk_id = `chunk_${hashDeterministic(canonical)}`;
+  return {
+    chunk_id,
+    file_ref: input.file_ref,
+    chunk_ref: input.chunk_ref,
+    row_count: input.row_count,
+    started_at_ms: input.started_at_ms,
+    ended_at_ms: input.ended_at_ms,
+    partition_keys: canonical.partition_keys,
+    metadata: input.metadata,
+  };
+}
+
+function hashDeterministic(value: unknown): string {
+  return fnv1a64(stableSerialize(value));
+}
+
+function stableSerialize(value: unknown): string {
+  if (value === null) return 'null';
+  if (typeof value === 'number' || typeof value === 'boolean') return JSON.stringify(value);
+  if (typeof value === 'string') return JSON.stringify(value);
+  if (typeof value === 'bigint') {
+    return `"${value.toString()}n"`;
+  }
+  if (Array.isArray(value)) {
+    return `[${value.map((item) => stableSerialize(item)).join(',')}]`;
+  }
+  if (typeof value === 'object') {
+    // null and array excluded above — TS narrows to `object` via direct typeof check
+    const entries = Object.entries(value).sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0));
+    return `{${entries.map(([k, v]) => `${JSON.stringify(k)}:${stableSerialize(v)}`).join(',')}}`;
+  }
+  return JSON.stringify(String(value));
+}
+
+function fnv1a64(input: string): string {
+  let hash = 0xcbf29ce484222325n;
+  const prime = 0x100000001b3n;
+  const mod = 0xffffffffffffffffn;
+
+  for (let i = 0; i < input.length; i++) {
+    hash ^= BigInt(input.charCodeAt(i));
+    hash = (hash * prime) & mod;
+  }
+
+  return hash.toString(16).padStart(16, '0');
+}

package/src/lib/archive/chunkRouting.ts

@@ -0,0 +1,70 @@
+export type PartitionCardinality = 'single' | 'mixed' | 'unknown';
+
+export interface PartitionSplit<T> {
+  readonly partition: string;
+  readonly selector: string;
+  readonly row_indexes: readonly number[];
+  readonly rows: readonly T[];
+}
+
+export function inspectPartitionCardinality<T>(
+  rows: readonly T[],
+  partitionOf: (row: T) => string | null | undefined,
+): PartitionCardinality {
+  if (rows.length === 0) {
+    return 'unknown';
+  }
+
+  const partitions = new Set<string>();
+  for (const row of rows) {
+    const partition = partitionOf(row);
+    if (partition === undefined || partition === null || partition.length === 0) {
+      return 'unknown';
+    }
+    partitions.add(partition);
+    if (partitions.size > 1) {
+      return 'mixed';
+    }
+  }
+
+  return partitions.size === 1 ? 'single' : 'unknown';
+}
+
+export function splitChunkByPartition<T>(
+  rows: readonly T[],
+  partitionOf: (row: T) => string | null | undefined,
+  partitionField = 'group',
+): readonly PartitionSplit<T>[] {
+  const grouped = new Map<string, { indexes: number[]; rows: T[] }>();
+
+  for (let index = 0; index < rows.length; index++) {
+    const row = rows[index];
+    const partition = partitionOf(row);
+    if (partition === undefined || partition === null || partition.length === 0) {
+      continue;
+    }
+
+    const group = grouped.get(partition);
+    if (group) {
+      group.indexes.push(index);
+      group.rows.push(row);
+      continue;
+    }
+
+    grouped.set(partition, { indexes: [index], rows: [row] });
+  }
+
+  const keys = [...grouped.keys()].sort();
+  return keys.map((partition) => {
+    const group = grouped.get(partition);
+    if (!group) {
+      throw new Error(`invariant throw: missing partition group for ${partition}`);
+    }
+    return {
+      partition,
+      selector: `${partitionField} == ${JSON.stringify(partition)}`,
+      row_indexes: group.indexes,
+      rows: group.rows,
+    };
+  });
+}

package/src/lib/archive/chunkStats.ts

@@ -0,0 +1,25 @@
+import type { TraceChunkEnvelope } from './chunkEnvelope.js';
+import type { PartitionSplit } from './chunkRouting.js';
+
+export interface TraceChunkStats {
+  readonly chunk_id: string;
+  readonly row_count: number;
+  readonly partition_count: number;
+  readonly started_at_ms: number;
+  readonly ended_at_ms: number;
+  readonly duration_ms: number;
+}
+
+export function extractChunkStats<T>(
+  envelope: TraceChunkEnvelope,
+  partitions: readonly PartitionSplit<T>[] = [],
+): TraceChunkStats {
+  return {
+    chunk_id: envelope.chunk_id,
+    row_count: envelope.row_count,
+    partition_count: partitions.length,
+    started_at_ms: envelope.started_at_ms,
+    ended_at_ms: envelope.ended_at_ms,
+    duration_ms: Math.max(0, envelope.ended_at_ms - envelope.started_at_ms),
+  };
+}

package/src/lib/archive/index.ts

@@ -0,0 +1,4 @@
+export * from './chunkCompaction.js';
+export * from './chunkEnvelope.js';
+export * from './chunkRouting.js';
+export * from './chunkStats.js';

package/src/lib/arrow/capacityStats.ts

@@ -0,0 +1,340 @@
+import {
+  bool,
+  type Column,
+  dictionary,
+  float64,
+  type IntType,
+  int8,
+  type Table,
+  TimeUnit,
+  tableFromColumns,
+  timestamp,
+  uint8,
+  uint32,
+  uint64,
+  utf8,
+} from '@uwdata/flechette';
+import { SYSTEM_SCHEMA_FIELD_NAMES } from '../schema/systemSchema.js';
+import { getSchemaType } from '../schema/typeGuards.js';
+import type { SpanBufferConstructor } from '../spanBuffer.js';
+import type { OpMetadata } from '../types.js';
+import { getArrowIndexTypeOr, makeArrowColumn } from './flechette.js';
+import { getArrowFieldName } from './utils.js';
+
+export interface CapacityStatsEntry {
+  bufferClass: SpanBufferConstructor;
+  metadata: OpMetadata;
+}
+
+const F64Array = Float64Array;
+
+type Utf8Encoded = {
+  data: Uint8Array;
+  offsets: Int32Array;
+};
+
+function encodeUtf8(strings: string[]): Utf8Encoded {
+  const enc = new TextEncoder();
+  const bytes = strings.map((s) => enc.encode(s));
+  const total = bytes.reduce((n, b) => n + b.length, 0);
+  const data = new Uint8Array(total);
+  const offsets = new Int32Array(strings.length + 1);
+  let pos = 0;
+  for (let i = 0; i < bytes.length; i++) {
+    data.set(bytes[i], pos);
+    offsets[i] = pos;
+    pos += bytes[i].length;
+  }
+  offsets[strings.length] = pos;
+  return { data, offsets };
+}
+
+function indexTypeForCount(count: number): IntType {
+  return count <= 255 ? uint8() : uint32();
+}
+
+function append(entries: [string, Column<unknown>][], name: string, column: Column<unknown>): void {
+  entries.push([name, column]);
+}
+
+export function createCapacityStatsTable(
+  entriesToLog: CapacityStatsEntry[],
+  periodStartNs: bigint,
+  mergedSchema: Record<string, unknown>,
+): Table {
+  if (entriesToLog.length === 0) return tableFromColumns({});
+
+  const rowsPerModule = 4;
+  const totalRows = entriesToLog.length * rowsPerModule;
+
+  const entryTypeStrings = ['period-start', 'buffer-writes', 'buffer-spans', 'buffer-capacity'];
+  const [periodStartIdx, writesIdx, spansIdx, capacityIdx] = [0, 1, 2, 3];
+
+  const traceIdStrings = [''];
+  const packageNameStrings = Array.from(new Set(entriesToLog.map((e) => e.metadata.package_name)));
+  const packageFileStrings = Array.from(new Set(entriesToLog.map((e) => e.metadata.package_file)));
+  const gitShaStrings = Array.from(new Set(entriesToLog.map((e) => e.metadata.git_sha)));
+  const messageStrings = [''];
+
+  const packageNameMap = new Map(packageNameStrings.map((v, i) => [v, i]));
+  const packageFileMap = new Map(packageFileStrings.map((v, i) => [v, i]));
+  const gitShaMap = new Map(gitShaStrings.map((v, i) => [v, i]));
+
+  const entryTypes = new Int8Array(totalRows);
+  const traceIds = new Uint8Array(totalRows);
+  const packageNames = new Uint32Array(totalRows);
+  const packageFiles = new Uint32Array(totalRows);
+  const gitShas = new Uint32Array(totalRows);
+  const messages = new Uint8Array(totalRows);
+
+  const timestamps = new BigInt64Array(totalRows);
+  const threadIds = new BigUint64Array(totalRows);
+  const spanIds = new Uint32Array(totalRows);
+  const parentThreadIds = new BigUint64Array(totalRows);
+  const parentSpanIds = new Uint32Array(totalRows);
+  const uint64Values = new BigUint64Array(totalRows);
+
+  const parentThreadValidity = new Uint8Array(Math.ceil(totalRows / 8));
+  const parentSpanValidity = new Uint8Array(Math.ceil(totalRows / 8));
+
+  for (let m = 0; m < entriesToLog.length; m++) {
+    const base = m * rowsPerModule;
+    const { bufferClass, metadata } = entriesToLog[m];
+
+    const pName = packageNameMap.get(metadata.package_name) ?? 0;
+    const pFile = packageFileMap.get(metadata.package_file) ?? 0;
+    const gSha = gitShaMap.get(metadata.git_sha) ?? 0;
+
+    const stats = bufferClass.stats;
+    const values = [periodStartNs, BigInt(stats.totalWrites), BigInt(stats.spansCreated), BigInt(stats.capacity)];
+    const kinds = [periodStartIdx, writesIdx, spansIdx, capacityIdx];
+
+    for (let i = 0; i < rowsPerModule; i++) {
+      const row = base + i;
+      entryTypes[row] = kinds[i];
+      traceIds[row] = 0;
+      packageNames[row] = pName;
+      packageFiles[row] = pFile;
+      gitShas[row] = gSha;
+      messages[row] = 0;
+      uint64Values[row] = values[i];
+      timestamps[row] = periodStartNs;
+      threadIds[row] = 0n;
+      spanIds[row] = 0;
+      parentThreadIds[row] = 0n;
+      parentSpanIds[row] = 0;
+    }
+  }
+
+  const entryTypeUtf8 = encodeUtf8(entryTypeStrings);
+  const traceIdUtf8 = encodeUtf8(traceIdStrings);
+  const packageNameUtf8 = encodeUtf8(packageNameStrings);
+  const packageFileUtf8 = encodeUtf8(packageFileStrings);
+  const gitShaUtf8 = encodeUtf8(gitShaStrings);
+  const messageUtf8 = encodeUtf8(messageStrings);
+
+  const cols: [string, Column<unknown>][] = [];
+
+  append(
+    cols,
+    'timestamp',
+    makeArrowColumn({ type: timestamp(TimeUnit.NANOSECOND), length: totalRows, nullCount: 0, values: timestamps }),
+  );
+
+  append(
+    cols,
+    'trace_id',
+    makeArrowColumn({
+      type: dictionary(utf8(), indexTypeForCount(traceIdStrings.length), false, 0),
+      length: totalRows,
+      nullCount: 0,
+      values: traceIds,
+      dictionary: makeArrowColumn({
+        type: utf8(),
+        length: traceIdStrings.length,
+        nullCount: 0,
+        values: traceIdUtf8.data,
+        offsets: traceIdUtf8.offsets,
+      }),
+    }),
+  );
+
+  append(cols, 'thread_id', makeArrowColumn({ type: uint64(), length: totalRows, nullCount: 0, values: threadIds }));
+  append(cols, 'span_id', makeArrowColumn({ type: uint32(), length: totalRows, nullCount: 0, values: spanIds }));
+  append(
+    cols,
+    'parent_thread_id',
+    makeArrowColumn({
+      type: uint64(),
+      length: totalRows,
+      nullCount: totalRows,
+      values: parentThreadIds,
+      validity: parentThreadValidity,
+    }),
+  );
+  append(
+    cols,
+    'parent_span_id',
+    makeArrowColumn({
+      type: uint32(),
+      length: totalRows,
+      nullCount: totalRows,
+      values: parentSpanIds,
+      validity: parentSpanValidity,
+    }),
+  );
+
+  append(
+    cols,
+    'entry_type',
+    makeArrowColumn({
+      type: dictionary(utf8(), int8(), false, 1),
+      length: totalRows,
+      nullCount: 0,
+      values: entryTypes,
+      dictionary: makeArrowColumn({
+        type: utf8(),
+        length: entryTypeStrings.length,
+        nullCount: 0,
+        values: entryTypeUtf8.data,
+        offsets: entryTypeUtf8.offsets,
+      }),
+    }),
+  );
+
+  append(
+    cols,
+    'package_name',
+    makeArrowColumn({
+      type: dictionary(utf8(), indexTypeForCount(packageNameStrings.length), false, 2),
+      length: totalRows,
+      nullCount: 0,
+      values: packageNames,
+      dictionary: makeArrowColumn({
+        type: utf8(),
+        length: packageNameStrings.length,
+        nullCount: 0,
+        values: packageNameUtf8.data,
+        offsets: packageNameUtf8.offsets,
+      }),
+    }),
+  );
+  append(
+    cols,
+    'package_file',
+    makeArrowColumn({
+      type: dictionary(utf8(), indexTypeForCount(packageFileStrings.length), false, 3),
+      length: totalRows,
+      nullCount: 0,
+      values: packageFiles,
+      dictionary: makeArrowColumn({
+        type: utf8(),
+        length: packageFileStrings.length,
+        nullCount: 0,
+        values: packageFileUtf8.data,
+        offsets: packageFileUtf8.offsets,
+      }),
+    }),
+  );
+  append(
+    cols,
+    'git_sha',
+    makeArrowColumn({
+      type: dictionary(utf8(), indexTypeForCount(gitShaStrings.length), false, 4),
+      length: totalRows,
+      nullCount: 0,
+      values: gitShas,
+      dictionary: makeArrowColumn({
+        type: utf8(),
+        length: gitShaStrings.length,
+        nullCount: 0,
+        values: gitShaUtf8.data,
+        offsets: gitShaUtf8.offsets,
+      }),
+    }),
+  );
+  append(
+    cols,
+    'message',
+    makeArrowColumn({
+      type: dictionary(utf8(), indexTypeForCount(messageStrings.length), false, 5),
+      length: totalRows,
+      nullCount: 0,
+      values: messages,
+      dictionary: makeArrowColumn({
+        type: utf8(),
+        length: messageStrings.length,
+        nullCount: 0,
+        values: messageUtf8.data,
+        offsets: messageUtf8.offsets,
+      }),
+    }),
+  );
+  append(
+    cols,
+    'uint64_value',
+    makeArrowColumn({ type: uint64(), length: totalRows, nullCount: 0, values: uint64Values }),
+  );
+
+  for (const [fieldName, fieldSchema] of Object.entries(mergedSchema)) {
+    if (SYSTEM_SCHEMA_FIELD_NAMES.has(fieldName)) continue;
+    const lmaoType = getSchemaType(fieldSchema);
+    const arrowFieldName = getArrowFieldName(fieldName);
+
+    if (lmaoType === 'number') {
+      append(
+        cols,
+        arrowFieldName,
+        makeArrowColumn({
+          type: float64(),
+          length: totalRows,
+          nullCount: totalRows,
+          values: new F64Array(totalRows),
+          validity: new Uint8Array(Math.ceil(totalRows / 8)),
+        }),
+      );
+      continue;
+    }
+
+    if (lmaoType === 'boolean') {
+      append(
+        cols,
+        arrowFieldName,
+        makeArrowColumn({
+          type: bool(),
+          length: totalRows,
+          nullCount: totalRows,
+          values: new Uint8Array(Math.ceil(totalRows / 8)),
+          validity: new Uint8Array(Math.ceil(totalRows / 8)),
+        }),
+      );
+      continue;
+    }
+
+    let idxType: IntType = uint8();
+    if (lmaoType === 'enum') {
+      idxType = getArrowIndexTypeOr(fieldSchema, uint8());
+    }
+    const dictType = dictionary(utf8(), idxType);
+    append(
+      cols,
+      arrowFieldName,
+      makeArrowColumn({
+        type: dictType,
+        length: totalRows,
+        nullCount: totalRows,
+        values: new Uint8Array(totalRows),
+        validity: new Uint8Array(Math.ceil(totalRows / 8)),
+        dictionary: makeArrowColumn({
+          type: utf8(),
+          length: 1,
+          nullCount: 0,
+          values: new Uint8Array(0),
+          offsets: new Int32Array([0, 0]),
+        }),
+      }),
+    );
+  }
+
+  return tableFromColumns(cols);
+}

package/src/lib/arrow/dictionaries.ts

@@ -0,0 +1,159 @@
+/**
+ * Dictionary building for Arrow conversion
+ */
+
+import { type IntType, uint8, uint16, uint32 } from '@uwdata/flechette';
+import type { AnySpanBuffer } from '../types.js';
+import { concatenateNullBitmaps } from './utils.js';
+
+/**
+ * Dictionary build result with consistent shape for V8 optimization.
+ * Using a class ensures all instances share the same hidden class.
+ */
+export class DictionaryBuildResult {
+  constructor(
+    public readonly dictionary: string[],
+    public readonly indices: Uint8Array | Uint16Array | Uint32Array,
+    public readonly indexArrayCtor: Uint8ArrayConstructor | Uint16ArrayConstructor | Uint32ArrayConstructor,
+    public readonly arrowIndexType: IntType,
+    public readonly nullBitmap: Uint8Array | undefined,
+    public readonly nullCount: number,
+  ) {}
+}
+
+/**
+ * Build a sorted category dictionary from buffers
+ */
+export function buildSortedCategoryDictionary(
+  buffers: AnySpanBuffer[],
+  columnName: string,
+  maskTransform?: (value: string) => string,
+): DictionaryBuildResult {
+  const totalRows = buffers.reduce((sum, buf) => sum + buf._writeIndex, 0);
+
+  // Build mapping from original value to masked value (for dictionary lookup)
+  // and collect unique masked values for the dictionary
+  const uniqueMaskedStrings = new Set<string>();
+  const originalToMasked = new Map<string, string>();
+
+  for (const buf of buffers) {
+    const column = buf.getColumnIfAllocated(columnName);
+    if (column && Array.isArray(column)) {
+      for (let i = 0; i < buf._writeIndex; i++) {
+        const value = column[i];
+        if (typeof value === 'string' && !originalToMasked.has(value)) {
+          const maskedValue = maskTransform ? maskTransform(value) : value;
+          originalToMasked.set(value, maskedValue);
+          uniqueMaskedStrings.add(maskedValue);
+        }
+      }
+    }
+  }
+
+  const dictionary = Array.from(uniqueMaskedStrings).sort();
+  const maskedToIndex = new Map(dictionary.map((s, i) => [s, i]));
+
+  // Determine index type constructors based on dictionary size
+  const uniqueCount = dictionary.length;
+  const indexArrayCtor = uniqueCount <= 255 ? Uint8Array : uniqueCount <= 65535 ? Uint16Array : Uint32Array;
+  const arrowIndexType = uniqueCount <= 255 ? uint8() : uniqueCount <= 65535 ? uint16() : uint32();
+  const indices = new indexArrayCtor(totalRows);
+  const { nullBitmap, nullCount } = concatenateNullBitmaps(buffers, columnName);
+
+  let rowOffset = 0;
+  for (const buf of buffers) {
+    const column = buf.getColumnIfAllocated(columnName);
+    if (column && Array.isArray(column)) {
+      for (let i = 0; i < buf._writeIndex; i++) {
+        const value = column[i];
+        if (typeof value === 'string') {
+          const maskedValue = originalToMasked.get(value) ?? value;
+          indices[rowOffset + i] = maskedToIndex.get(maskedValue) ?? 0;
+        } else {
+          indices[rowOffset + i] = 0;
+        }
+      }
+    }
+    rowOffset += buf._writeIndex;
+  }
+
+  return new DictionaryBuildResult(dictionary, indices, indexArrayCtor, arrowIndexType, nullBitmap, nullCount);
+}
+
+/**
+ * Build a text dictionary from buffers (not sorted, uses frequency-based optimization)
+ */
+export function buildTextDictionary(
+  buffers: AnySpanBuffer[],
+  columnName: string,
+  maskTransform?: (value: string) => string,
+): DictionaryBuildResult | null {
+  const totalRows = buffers.reduce((sum, buf) => sum + buf._writeIndex, 0);
+
+  // Build mapping from original value to masked value and track frequency of masked values
+  const frequencyMap = new Map<string, number>();
+  const originalToMasked = new Map<string, string>();
+
+  for (const buf of buffers) {
+    const column = buf.getColumnIfAllocated(columnName);
+    if (column && Array.isArray(column)) {
+      for (let i = 0; i < buf._writeIndex; i++) {
+        const value = column[i];
+        if (typeof value === 'string') {
+          let maskedValue: string;
+          if (originalToMasked.has(value)) {
+            const masked = originalToMasked.get(value);
+            if (masked === undefined) {
+              throw new Error(`Masked value not found for: ${value}`);
+            }
+            maskedValue = masked;
+          } else {
+            maskedValue = maskTransform ? maskTransform(value) : value;
+            originalToMasked.set(value, maskedValue);
+          }
+          frequencyMap.set(maskedValue, (frequencyMap.get(maskedValue) ?? 0) + 1);
+        }
+      }
+    }
+  }
+
+  if (frequencyMap.size === 0) {
+    return new DictionaryBuildResult(
+      [],
+      new Uint32Array(totalRows),
+      Uint32Array,
+      uint32(),
+      new Uint8Array(Math.ceil(totalRows / 8)),
+      totalRows,
+    );
+  }
+
+  const dictionary = Array.from(frequencyMap.keys());
+  const maskedToIndex = new Map(dictionary.map((s, i) => [s, i]));
+
+  // Determine index type constructors based on dictionary size
+  const uniqueCount = dictionary.length;
+  const indexArrayCtor = uniqueCount <= 255 ? Uint8Array : uniqueCount <= 65535 ? Uint16Array : Uint32Array;
+  const arrowIndexType = uniqueCount <= 255 ? uint8() : uniqueCount <= 65535 ? uint16() : uint32();
+  const indices = new indexArrayCtor(totalRows);
+  const { nullBitmap, nullCount } = concatenateNullBitmaps(buffers, columnName);
+
+  let rowOffset = 0;
+  for (const buf of buffers) {
+    const column = buf.getColumnIfAllocated(columnName);
+    if (column && Array.isArray(column)) {
+      for (let i = 0; i < buf._writeIndex; i++) {
+        const value = column[i];
+        if (typeof value === 'string') {
+          const maskedValue = originalToMasked.get(value) ?? value;
+          indices[rowOffset + i] = maskedToIndex.get(maskedValue) ?? 0;
+        } else {
+          indices[rowOffset + i] = 0;
+        }
+      }
+    }
+    rowOffset += buf._writeIndex;
+  }
+
+  return new DictionaryBuildResult(dictionary, indices, indexArrayCtor, arrowIndexType, nullBitmap, nullCount);
+}