@smoothbricks/lmao 0.1.0

package/src/lib/arrow/flechette.ts

@@ -0,0 +1,114 @@
+import { batchType, bool, Column, type DataType, type dictionary, type IntType, type utf8 } from '@uwdata/flechette';
+
+const EMPTY_VALIDITY = new Uint8Array(0);
+const BOOL_TYPE_ID = bool().typeId;
+
+export type ArrowIndexArray = Uint8Array | Uint16Array | Uint32Array;
+export type ArrowIndexArrayConstructor = Uint8ArrayConstructor | Uint16ArrayConstructor | Uint32ArrayConstructor;
+export type BoolType = ReturnType<typeof bool>;
+export type DictionaryType = ReturnType<typeof dictionary>;
+export type Utf8Type = ReturnType<typeof utf8>;
+
+type ColumnDataBase<TType, TValues = unknown> = {
+  type: TType;
+  length: number;
+  nullCount: number;
+  values?: TValues;
+  validity?: Uint8Array;
+};
+
+export type DictionaryColumnData<TValues = unknown> = ColumnDataBase<DictionaryType, TValues> & {
+  dictionary: Column<unknown>;
+};
+export type Utf8ColumnData = ColumnDataBase<Utf8Type, Uint8Array> & { offsets: Int32Array };
+export type BoolColumnData = ColumnDataBase<BoolType, Uint8Array>;
+export type GenericColumnData<TType extends DataType = DataType, TValues = unknown> = ColumnDataBase<TType, TValues>;
+
+export function hasTypeId(value: unknown): value is { readonly typeId: unknown } {
+  return typeof value === 'object' && value !== null && 'typeId' in value;
+}
+
+export function isBoolType(value: unknown): value is BoolType {
+  return hasTypeId(value) && value.typeId === BOOL_TYPE_ID;
+}
+
+function isArrowIndexArrayConstructor(value: unknown): value is ArrowIndexArrayConstructor {
+  return value === Uint8Array || value === Uint16Array || value === Uint32Array;
+}
+
+export function hasArrowIndexType(value: unknown): value is { readonly __arrow_index_type: IntType } {
+  return typeof value === 'object' && value !== null && '__arrow_index_type' in value;
+}
+
+export function hasArrowIndexMetadata(
+  value: unknown,
+): value is { readonly __arrow_index_type: IntType; readonly __index_array_ctor: ArrowIndexArrayConstructor } {
+  return (
+    hasArrowIndexType(value) && '__index_array_ctor' in value && isArrowIndexArrayConstructor(value.__index_array_ctor)
+  );
+}
+
+export function getArrowIndexTypeOr(value: unknown, fallback: IntType): IntType {
+  return hasArrowIndexType(value) ? value.__arrow_index_type : fallback;
+}
+
+export function getArrowIndexArrayConstructorOr(
+  value: unknown,
+  fallback: ArrowIndexArrayConstructor = Uint8Array,
+): ArrowIndexArrayConstructor {
+  return hasArrowIndexMetadata(value) ? value.__index_array_ctor : fallback;
+}
+
+export function makeArrowColumn(data: DictionaryColumnData): Column<unknown>;
+export function makeArrowColumn(data: Utf8ColumnData): Column<unknown>;
+export function makeArrowColumn(data: BoolColumnData): Column<unknown>;
+export function makeArrowColumn(data: GenericColumnData): Column<unknown>;
+export function makeArrowColumn(
+  data: DictionaryColumnData | Utf8ColumnData | BoolColumnData | GenericColumnData,
+): Column<unknown> {
+  const validity = data.validity ?? EMPTY_VALIDITY;
+
+  if ('dictionary' in data) {
+    const batch = new (batchType(data.type))({
+      type: data.type,
+      length: data.length,
+      nullCount: data.nullCount,
+      validity,
+      values: data.values,
+    });
+    batch.setDictionary(data.dictionary);
+    return new Column([batch]);
+  }
+
+  if ('offsets' in data) {
+    const batch = new (batchType(data.type))({
+      type: data.type,
+      length: data.length,
+      nullCount: data.nullCount,
+      validity,
+      offsets: data.offsets,
+      values: data.values,
+    });
+    return new Column([batch]);
+  }
+
+  if (isBoolType(data.type)) {
+    const batch = new (batchType(data.type))({
+      type: data.type,
+      length: data.length,
+      nullCount: data.nullCount,
+      validity,
+      values: data.values,
+    });
+    return new Column([batch]);
+  }
+
+  const batch = new (batchType(data.type))({
+    type: data.type,
+    length: data.length,
+    nullCount: data.nullCount,
+    validity,
+    values: data.values,
+  });
+  return new Column([batch]);
+}

package/src/lib/arrow/utils.ts

@@ -0,0 +1,126 @@
+/**
+ * Utility functions for Arrow conversion
+ */
+
+import { copyBits, countNulls } from '@smoothbricks/arrow-builder';
+import type { AnySpanBuffer } from '../types.js';
+
+const UTF8_ENCODER = new TextEncoder();
+
+type ConcatenatedNullBitmaps = { nullBitmap: Uint8Array | undefined; nullCount: number };
+
+/**
+ * Encode an array of strings into a single UTF-8 Uint8Array
+ */
+export function encodeUtf8Strings(strings: readonly string[]): Uint8Array {
+  const encoded = strings.map((s) => UTF8_ENCODER.encode(s));
+  const totalLength = encoded.reduce((sum, arr) => sum + arr.length, 0);
+  const result = new Uint8Array(totalLength);
+  let offset = 0;
+  for (const arr of encoded) {
+    result.set(arr, offset);
+    offset += arr.length;
+  }
+  return result;
+}
+
+/**
+ * Calculate UTF-8 byte offsets for an array of strings
+ */
+export function calculateUtf8Offsets(strings: readonly string[]): Int32Array {
+  const offsets = new Int32Array(strings.length + 1);
+  offsets[0] = 0;
+  for (let i = 0; i < strings.length; i++) {
+    const byteLength = UTF8_ENCODER.encode(strings[i]).length;
+    offsets[i + 1] = offsets[i] + byteLength;
+  }
+  return offsets;
+}
+
+/**
+ * Concatenate Uint8 arrays without type casting.
+ */
+export function concatenateUint8Arrays(arrays: Uint8Array[]): Uint8Array {
+  if (arrays.length === 0) throw new Error('Cannot concatenate empty array list');
+  if (arrays.length === 1) return arrays[0];
+  const totalLength = arrays.reduce((sum, arr) => sum + arr.length, 0);
+  const result = new Uint8Array(totalLength);
+  let offset = 0;
+  for (const arr of arrays) {
+    result.set(arr, offset);
+    offset += arr.length;
+  }
+  return result;
+}
+
+/**
+ * Concatenate Float64 arrays without type casting.
+ */
+export function concatenateFloat64Arrays(arrays: Float64Array[]): Float64Array {
+  if (arrays.length === 0) throw new Error('Cannot concatenate empty array list');
+  if (arrays.length === 1) return arrays[0];
+  const totalLength = arrays.reduce((sum, arr) => sum + arr.length, 0);
+  const result = new Float64Array(totalLength);
+  let offset = 0;
+  for (const arr of arrays) {
+    result.set(arr, offset);
+    offset += arr.length;
+  }
+  return result;
+}
+
+/**
+ * Concatenate null bitmaps from multiple buffers
+ */
+export function concatenateNullBitmaps(buffers: AnySpanBuffer[], columnName: string): ConcatenatedNullBitmaps {
+  const hasAnyNulls = buffers.some((buf) => buf.getNullsIfAllocated(columnName) !== undefined);
+
+  if (!hasAnyNulls) return { nullBitmap: undefined, nullCount: 0 };
+
+  const totalRows = buffers.reduce((sum, buf) => sum + buf._writeIndex, 0);
+  const bitmapBytes = Math.ceil(totalRows / 8);
+  const nullBitmap = new Uint8Array(bitmapBytes);
+  nullBitmap.fill(0xff); // Default all valid
+
+  // Buffer chains: all buffers except the last are full (writeIndex == capacity).
+  // If capacity is a multiple of 8, each buffer starts at a byte boundary.
+  let rowOffset = 0;
+  let nullCount = 0;
+
+  for (const buf of buffers) {
+    const sourceBitmap = buf.getNullsIfAllocated(columnName);
+    const rowCount = buf._writeIndex;
+
+    if (sourceBitmap) {
+      copyBits(nullBitmap, rowOffset, sourceBitmap, 0, rowCount);
+      // Count nulls using countNulls from arrow-builder
+      nullCount += countNulls(sourceBitmap, rowCount);
+    }
+    // If no sourceBitmap, leave as 0xff (all valid)
+
+    rowOffset += rowCount;
+  }
+
+  return { nullBitmap, nullCount };
+}
+
+/**
+ * Get Arrow field name from LMAO field name
+ */
+export function getArrowFieldName(fieldName: string): string {
+  return fieldName;
+}
+
+/**
+ * Walk a SpanBuffer tree (including overflow chains and children)
+ */
+export function walkSpanTree(root: AnySpanBuffer, visitor: (buffer: AnySpanBuffer) => void): void {
+  let current: AnySpanBuffer | undefined = root;
+  while (current) {
+    visitor(current);
+    current = current._overflow;
+  }
+  for (const child of root._children) {
+    walkSpanTree(child, visitor);
+  }
+}

package/src/lib/buffer/Nanoseconds.ts

@@ -0,0 +1,131 @@
+/**
+ * Nanoseconds branded type for type-safe timestamp handling
+ *
+ * Prevents accidental unit mixing (milliseconds vs nanoseconds) and provides
+ * a centralized location for timestamp conversion utilities.
+ *
+ * This is a generic buffer concept used for high-precision time-series data.
+ *
+ * **Benefits:**
+ * - Type safety prevents mixing time units
+ * - Self-documenting code - units clear from type
+ * - Zero runtime overhead on hot path (BigInt operations are fast)
+ * - Conversion functions centralized and tested
+ * - Works with Arrow's TimestampNanosecond type
+ *
+ * **Precision:**
+ * - BigInt can represent arbitrarily large integers
+ * - Nanoseconds since epoch will never overflow
+ * - Node.js: Native nanosecond precision via process.hrtime.bigint()
+ * - Browser: Microsecond precision via performance.now() (last 3 digits are 000)
+ *
+ * @module nanoseconds
+ *
+ * @example
+ * ```typescript
+ * import { Nanoseconds } from '@smoothbricks/arrow-builder';
+ *
+ * // Node.js: Native nanoseconds
+ * const timestamp = Nanoseconds.now();
+ *
+ * // Store in buffer (BigInt64Array)
+ * buffer.timestamps[idx] = timestamp; // Type-safe!
+ * ```
+ */
+
+/**
+ * Nanoseconds branded type
+ *
+ * A `bigint` with a compile-time brand that prevents accidental mixing of time units.
+ * The brand is erased at runtime, so there is zero overhead.
+ */
+declare const nanosecondsBrand: unique symbol;
+
+export type Nanoseconds = bigint & { readonly [nanosecondsBrand]: 'Nanoseconds' };
+
+function toNanoseconds(value: bigint): Nanoseconds;
+function toNanoseconds(value: bigint): bigint {
+  return value;
+}
+
+/**
+ * Utility functions for working with Nanoseconds
+ */
+export namespace Nanoseconds {
+  /**
+   * Get current timestamp in nanoseconds since Unix epoch.
+   *
+   * Uses the most efficient method available:
+   * - Browser/generic: performance.timeOrigin + performance.now() (microsecond precision)
+   * - Browsers typically provide 5-20 microsecond precision
+   *
+   * For Node.js with true nanosecond precision, use the /node entry point.
+   *
+   * **Precision handling:**
+   * - Epoch milliseconds (~1.7 trillion) is safe as Number
+   * - Multiply by 1000 to get microseconds (~1.7 quadrillion) - still safe
+   * - Convert to BigInt, then multiply by 1000n for nanoseconds
+   *
+   * @returns Current timestamp in nanoseconds since Unix epoch
+   *
+   * @example
+   * ```typescript
+   * buffer.timestamps[idx] = Nanoseconds.now();
+   * ```
+   */
+  export function now(): Nanoseconds {
+    // performance.timeOrigin + performance.now() gives epoch time in milliseconds
+    // with sub-millisecond precision (e.g., 1702789123456.789)
+    // Convert to microseconds first (safe as Number), then to nanoseconds as BigInt
+    const epochMicros = Math.round((performance.timeOrigin + performance.now()) * 1000);
+    return toNanoseconds(BigInt(epochMicros) * 1000n);
+  }
+
+  /**
+   * Convert milliseconds to nanoseconds.
+   *
+   * @param millis - Time in milliseconds (e.g., from Date.now())
+   * @returns Time in nanoseconds
+   *
+   * @example
+   * ```typescript
+   * const epochNanos = Nanoseconds.fromMillis(Date.now());
+   * ```
+   */
+  export function fromMillis(millis: number): Nanoseconds {
+    return toNanoseconds(BigInt(Math.trunc(millis)) * 1_000_000n);
+  }
+
+  /**
+   * Cast a raw bigint to Nanoseconds (use with caution).
+   *
+   * **Warning:** This bypasses type safety. Only use when you're certain
+   * the value is already in nanoseconds.
+   *
+   * @param value - Raw bigint value in nanoseconds
+   * @returns Branded Nanoseconds value
+   */
+  export function unsafe(value: bigint): Nanoseconds {
+    return toNanoseconds(value);
+  }
+
+  /**
+   * Convert nanoseconds to milliseconds.
+   *
+   * @param nanos - Time in nanoseconds
+   * @returns Time in milliseconds (as number)
+   */
+  export function toMillis(nanos: Nanoseconds): number {
+    return Number(nanos / 1_000_000n);
+  }
+
+  /**
+   * Convert nanoseconds to microseconds.
+   *
+   * @param nanos - Time in nanoseconds
+   * @returns Time in microseconds (as bigint)
+   */
+  export function toMicros(nanos: Nanoseconds): bigint {
+    return nanos / 1_000n;
+  }
+}

package/src/lib/bufferStrategy.ts

@@ -0,0 +1,106 @@
+/**
+ * BufferStrategy - Abstraction for SpanBuffer memory management and Arrow conversion.
+ *
+ * This interface separates memory management concerns from sink concerns (Tracer):
+ * - BufferStrategy: HOW buffers are allocated and converted (JS TypedArrays vs WASM)
+ * - Tracer (sink): WHAT happens with completed buffers (stdio, queue, test accumulation)
+ *
+ * Key responsibilities:
+ * 1. Buffer creation (root, child, overflow)
+ * 2. Arrow conversion (strategy knows optimal path)
+ * 3. Buffer release (GC for JS, freelist for WASM)
+ *
+ * The sink (Tracer subclass) controls WHEN to convert and release:
+ * - StdioTracer: prints then releases immediately
+ * - TestTracer: holds buffers until clear()
+ * - SqsTracer: accumulates, converts on flush(), then releases
+ *
+ * @module bufferStrategy
+ */
+
+import type { Table } from '@uwdata/flechette';
+import type { OpMetadata } from './opContext/opTypes.js';
+import type { LogSchema } from './schema/LogSchema.js';
+import type { ITraceRoot } from './traceRoot.js';
+import type { AnySpanBuffer, SpanBuffer } from './types.js';
+
+/**
+ * BufferStrategy interface for memory management and Arrow conversion.
+ *
+ * Implementations:
+ * - JsBufferStrategy: GC-managed JS TypedArrays, zero-copy Arrow via flechette
+ * - WasmBufferStrategy: WASM memory with freelist, Arrow built in WASM
+ * - (future) NapiBufferStrategy: Native code with SharedArrayBuffer
+ *
+ * @typeParam T - LogSchema type for type-safe buffer access
+ */
+export interface BufferStrategy<T extends LogSchema = LogSchema> {
+  /**
+   * Create a root SpanBuffer for a new trace.
+   *
+   * @param schema - LogSchema defining column types
+   * @param traceRoot - Platform-specific trace root with timestamp anchors
+   * @param opMetadata - Metadata for the executing op
+   * @param capacity - Optional capacity override
+   * @returns New SpanBuffer for the root span
+   */
+  createSpanBuffer(schema: T, traceRoot: ITraceRoot, opMetadata: OpMetadata, capacity?: number): SpanBuffer<T>;
+
+  /**
+   * Create a child span buffer linked to a parent.
+   *
+   * @param parentBuffer - Parent buffer for tree linkage
+   * Note: Caller must call writeSpanStart() after this to set span name in message_values[0].
+   *
+   * @param callsiteMetadata - Metadata for WHERE span() was called (row 0)
+   * @param opMetadata - Metadata for WHAT op is executing (rows 1+)
+   * @param capacity - Optional capacity override
+   * @param schema - Optional schema for the child (if different from parent, e.g., library ops)
+   * @returns New SpanBuffer linked to parent
+   */
+  createChildSpanBuffer(
+    parentBuffer: SpanBuffer<T>,
+    callsiteMetadata: OpMetadata,
+    opMetadata: OpMetadata,
+    capacity?: number,
+    schema?: T,
+  ): SpanBuffer<T>;
+
+  /**
+   * Create an overflow buffer when current buffer is full.
+   *
+   * @param buffer - Full buffer that needs overflow storage
+   * @returns New SpanBuffer linked via buffer._overflow
+   */
+  createOverflowBuffer(buffer: SpanBuffer<T>): SpanBuffer<T>;
+
+  /**
+   * Convert a SpanBuffer tree to Arrow Table.
+   *
+   * Each strategy knows the optimal conversion path:
+   * - JS: Zero-copy wrap TypedArrays via flechette
+   * - WASM: Build Arrow in WASM memory, return table
+   * - NAPI: Call into native code for parallel conversion
+   *
+   * @param buffer - Root buffer to convert (includes children and overflow chain)
+   * @returns Arrow Table (sync) or Promise<Table> (async)
+   */
+  toArrowTable(buffer: AnySpanBuffer): Table | Promise<Table>;
+
+  /**
+   * Release a SpanBuffer's memory resources.
+   *
+   * Called by the sink when it's done with a buffer.
+   *
+   * - JsBufferStrategy: No-op (GC handles it)
+   * - WasmBufferStrategy: Returns blocks to freelist
+   *
+   * The sink decides WHEN to release:
+   * - After printing (StdioTracer)
+   * - On clear() (TestTracer)
+   * - After Arrow conversion (SqsTracer)
+   *
+   * @param buffer - Buffer to release (and its entire tree)
+   */
+  releaseBuffer(buffer: AnySpanBuffer): void;
+}

package/src/lib/capacityTuning.ts

@@ -0,0 +1,102 @@
+/**
+ * Capacity Tuning - Self-tuning buffer capacity based on utilization
+ *
+ * Per specs/lmao/01b2_buffer_self_tuning.md:
+ * - Track row utilization = totalWrites / (spansCreated * usableRowsPerSpan)
+ * - Increase capacity if utilization > 150% (regularly overflowing)
+ * - Decrease capacity if utilization < 50% (wasting space)
+ * - Maintain power-of-2 capacities between 8 and 1024
+ *
+ * ## Utilization Model
+ *
+ * - 100% = exactly filling first buffer per span (ideal)
+ * - 150% = filling 1.5x first buffer (some spans overflow)
+ * - 200% = filling 2x first buffer (every span overflows once)
+ * - 50% = only using half the first buffer (wasting space)
+ *
+ * The 3x gap between shrink (50%) and grow (150%) thresholds prevents flip-flopping:
+ * - After grow: utilization halves → lands in stable zone
+ * - After shrink: utilization doubles → lands in stable zone
+ *
+ * ## Architecture Note
+ *
+ * Stats are on SpanBufferClass.stats (static property), NOT on LogBinding.
+ * This reduces per-instance memory and avoids sync bugs from having two stat objects.
+ */
+
+import type { SpanBufferStats } from './spanBufferStats.js';
+
+/** Minimum spans needed before tuning decisions */
+const MIN_SPANS_FOR_TUNING = 10;
+
+/** Grow capacity when utilization exceeds this (150% = regularly overflowing) */
+const GROW_THRESHOLD = 1.5;
+
+/** Shrink capacity when utilization falls below this (50% = wasting space) */
+const SHRINK_THRESHOLD = 0.5;
+
+/** Minimum capacity (must be power of 2, >= 8 for alignment) */
+const MIN_CAPACITY = 8;
+
+/** Maximum capacity (must be power of 2) */
+const MAX_CAPACITY = 1024;
+
+/**
+ * Check if capacity should be tuned after an overflow event.
+ *
+ * Called when a buffer overflows and needs to chain to a new buffer.
+ *
+ * @param stats - SpanBufferStats from SpanBufferClass.stats
+ * @internal
+ */
+export function checkCapacityTuning(stats: SpanBufferStats): void {
+  shouldTuneCapacity(stats);
+}
+
+/**
+ * Reset stats after capacity tuning.
+ *
+ * Called after a tuning decision to start fresh measurement window.
+ * This ensures the tuning algorithm responds to recent behavior, not cumulative history.
+ *
+ * Note: Tracer.onStatsWillResetFor() is called BEFORE this reset (in spanLoggerGenerator's
+ * _getNextBuffer method) to capture stats for observability before they're lost.
+ */
+function resetStats(stats: SpanBufferStats): void {
+  stats.totalWrites = 0;
+  stats.spansCreated = 0;
+}
+
+/**
+ * Check if capacity should be tuned based on utilization.
+ *
+ * Algorithm:
+ * - Requires minimum 10 spans before tuning
+ * - Compute utilization = totalWrites / (spansCreated * usableRowsPerSpan)
+ * - Increase capacity (×2, max 1024) if utilization > 150%
+ * - Decrease capacity (÷2, min 8) if utilization < 50%
+ * - Reset stats after tuning to start fresh measurement window
+ *
+ * @param stats - SpanBufferStats from SpanBufferClass.stats
+ * @internal Exported for testing
+ */
+export function shouldTuneCapacity(stats: SpanBufferStats): void {
+  if (stats.spansCreated < MIN_SPANS_FOR_TUNING) return;
+
+  // Usable rows per span = capacity - 2 (rows 0-1 reserved for span-start/end)
+  const usableRowsPerSpan = stats.capacity - 2;
+  const utilization = stats.totalWrites / (stats.spansCreated * usableRowsPerSpan);
+
+  // Grow if utilization > 150% (regularly overflowing)
+  if (utilization > GROW_THRESHOLD && stats.capacity < MAX_CAPACITY) {
+    stats.capacity = Math.min(stats.capacity * 2, MAX_CAPACITY);
+    resetStats(stats);
+    return;
+  }
+
+  // Shrink if utilization < 50% (wasting space)
+  if (utilization < SHRINK_THRESHOLD && stats.capacity > MIN_CAPACITY) {
+    stats.capacity = Math.max(MIN_CAPACITY, stats.capacity / 2);
+    resetStats(stats);
+  }
+}