@smoothbricks/lmao 0.1.0

package/src/lib/sqlite/__tests__/sqlite-common.test.ts

@@ -0,0 +1,89 @@
+import { describe, expect, it } from 'bun:test';
+import { createTestTracerOptions } from '../../__tests__/test-helpers.js';
+import { defineOpContext } from '../../defineOpContext.js';
+import { S } from '../../schema/builder.js';
+import { defineLogSchema } from '../../schema/defineLogSchema.js';
+import { TestTracer } from '../../tracers/TestTracer.js';
+import type { AnySpanBuffer } from '../../types.js';
+import { buildInsertParams, parseSqliteTableInfoRows, type SpanSegment } from '../sqlite-common.js';
+
+const opContext = defineOpContext({
+  logSchema: defineLogSchema({
+    scoped_text: S.category(),
+    scoped_num: S.number(),
+    mixed: S.category(),
+  }),
+});
+
+async function createSegment(): Promise<{ buffer: AnySpanBuffer; segment: SpanSegment }> {
+  const tracer = new TestTracer(opContext, createTestTracerOptions());
+
+  await tracer.trace('sqlite-scope-fallback', async (ctx) => {
+    ctx.setScope({
+      scoped_text: 'scope-text',
+      scoped_num: 7,
+      mixed: 'scope-mixed',
+    });
+
+    // Direct row-0 write should override scope for row 0 only.
+    ctx.tag.mixed('direct-mixed');
+    return ctx.ok(null);
+  });
+
+  const buffer = tracer.rootBuffers[0] as AnySpanBuffer;
+  const segment: SpanSegment = {
+    buffer,
+    traceId: buffer.trace_id,
+    spanId: buffer.span_id,
+    parentSpanId: buffer.parent_span_id,
+    rowOffset: 0,
+  };
+
+  return { buffer, segment };
+}
+
+describe('sqlite-common scope fallback', () => {
+  it('fills user columns from scope when row value is missing', async () => {
+    const { segment } = await createSegment();
+
+    const activeUserFields = ['scoped_text', 'scoped_num', 'mixed'];
+
+    const row0 = buildInsertParams(segment, 0, activeUserFields);
+    const row1 = buildInsertParams(segment, 1, activeUserFields);
+
+    expect(row0.slice(7)).toEqual(['scope-text', 7, 'direct-mixed']);
+    expect(row1.slice(7)).toEqual(['scope-text', 7, 'scope-mixed']);
+  });
+});
+
+describe('parseSqliteTableInfoRows', () => {
+  it('accepts bigint PRAGMA integers from SQLite adapters', () => {
+    expect(
+      parseSqliteTableInfoRows([
+        {
+          cid: 0n,
+          name: 'trace_id',
+          type: 'TEXT',
+          notnull: 1n,
+          dflt_value: null,
+          pk: 1n,
+        },
+      ]),
+    ).toEqual([
+      {
+        cid: 0,
+        name: 'trace_id',
+        type: 'TEXT',
+        notnull: 1,
+        dflt_value: null,
+        pk: 1,
+      },
+    ]);
+  });
+
+  it('throws a descriptive error for invalid row shapes', () => {
+    expect(() =>
+      parseSqliteTableInfoRows([{ cid: 0, name: 'trace_id', type: 1, notnull: 1, dflt_value: null, pk: 1 }]),
+    ).toThrow('PRAGMA table_info(spans) returned an invalid SQLite row 0: type expected string, got number');
+  });
+});

package/src/lib/sqlite/index.ts

@@ -0,0 +1,26 @@
+/**
+ * SQLite persistence for LMAO span buffers.
+ *
+ * General-purpose writer API for persisting span buffer trees to SQLite.
+ * Platform-agnostic — accepts any SyncSQLiteDatabase (bun:sqlite or better-sqlite3).
+ *
+ * @module sqlite
+ */
+
+export { SQLiteAsyncTraceWriter } from './sqlite-async-writer.js';
+export {
+  buildAddColumnSql,
+  extractSqliteColumnsFromTableInfo,
+  getMissingSqliteColumns,
+  quoteSqlIdentifier,
+  SPANS_TABLE_INFO_SQL,
+  SPANS_TABLE_INIT_SQL,
+} from './sqlite-common.js';
+export { createD1SQLiteDatabase, type D1LikeDatabase, type D1LikePreparedStatement } from './sqlite-d1.js';
+export type {
+  AsyncSQLiteDatabase,
+  AsyncSQLiteStatement,
+  SyncSQLiteDatabase,
+  SyncSQLiteStatement,
+} from './sqlite-db.js';
+export { SQLiteTraceWriter, type SQLiteWriterConfig } from './sqlite-writer.js';

package/src/lib/sqlite/sqlite-async-writer.ts

@@ -0,0 +1,101 @@
+/**
+ * SQLiteAsyncTraceWriter - Async SQLite persistence for span buffer trees.
+ *
+ * Works with async SQLite drivers (for example Cloudflare D1 adapters) while
+ * preserving the same `spans` table shape as the sync SQLiteTraceWriter.
+ *
+ * @module sqlite-async-writer
+ */
+
+import type { LogSchema } from '../schema/LogSchema.js';
+import type { AnySpanBuffer } from '../types.js';
+import {
+  buildAddColumnSql,
+  buildInsertParams,
+  buildInsertSql,
+  extractSqliteColumnsFromTableInfo,
+  getActiveUserFields,
+  getInsertStatementCacheKey,
+  getMissingSchemaColumns,
+  isSqliteDuplicateColumnError,
+  parseSqliteTableInfoRows,
+  SPANS_TABLE_INFO_SQL,
+  SPANS_TABLE_INIT_SQL,
+  walkSpanSegments,
+} from './sqlite-common.js';
+import type { AsyncSQLiteDatabase, AsyncSQLiteStatement } from './sqlite-db.js';
+
+export class SQLiteAsyncTraceWriter {
+  private knownColumns = new Set<string>();
+  private insertStmtCache = new Map<string, AsyncSQLiteStatement>();
+  private initialized = false;
+
+  constructor(private db: AsyncSQLiteDatabase) {}
+
+  private async init(): Promise<void> {
+    if (this.initialized) {
+      return;
+    }
+
+    await this.db.exec(SPANS_TABLE_INIT_SQL);
+
+    await this.refreshKnownColumns();
+
+    this.initialized = true;
+  }
+
+  private async refreshKnownColumns(): Promise<void> {
+    this.knownColumns.clear();
+    // WHY: async drivers are still a real schema boundary; validate PRAGMA rows before caching column names that drive
+    // later DDL and inserts.
+    const rows = parseSqliteTableInfoRows(await this.db.prepare(SPANS_TABLE_INFO_SQL).all());
+    for (const column of extractSqliteColumnsFromTableInfo(rows)) {
+      this.knownColumns.add(column.name);
+    }
+  }
+
+  private async ensureColumns(schema: LogSchema): Promise<void> {
+    for (const column of getMissingSchemaColumns(schema, this.knownColumns)) {
+      try {
+        await this.db.exec(buildAddColumnSql(column));
+      } catch (error) {
+        if (!isSqliteDuplicateColumnError(error, column.name)) {
+          throw error;
+        }
+      }
+
+      await this.refreshKnownColumns();
+      this.knownColumns.add(column.name);
+    }
+  }
+
+  private getInsertStatement(activeUserFields: readonly string[]): AsyncSQLiteStatement {
+    const key = getInsertStatementCacheKey(activeUserFields);
+    const cached = this.insertStmtCache.get(key);
+    if (cached) {
+      return cached;
+    }
+
+    const stmt = this.db.prepare(buildInsertSql(activeUserFields));
+    this.insertStmtCache.set(key, stmt);
+    return stmt;
+  }
+
+  async flush(rootBuffer: AnySpanBuffer): Promise<void> {
+    await this.init();
+    for (const segment of walkSpanSegments(rootBuffer)) {
+      await this.ensureColumns(segment.buffer._logSchema);
+
+      const activeUserFields = getActiveUserFields(segment.buffer._logSchema, this.knownColumns);
+      const insertStmt = this.getInsertStatement(activeUserFields);
+
+      for (let row = 0; row < segment.buffer._writeIndex; row++) {
+        await insertStmt.run(...buildInsertParams(segment, row, activeUserFields));
+      }
+    }
+  }
+
+  async close(): Promise<void> {
+    await this.db.close();
+  }
+}

package/src/lib/sqlite/sqlite-common.ts

@@ -0,0 +1,309 @@
+import type { SchemaType } from '@smoothbricks/arrow-builder';
+import { hasOwn, hasOwnString, isRecord } from '@smoothbricks/validation';
+import type { LogSchema } from '../schema/LogSchema.js';
+import { getSchemaType } from '../schema/typeGuards.js';
+import type { AnySpanBuffer } from '../types.js';
+
+export const SPANS_TABLE_INIT_SQL = `
+  CREATE TABLE IF NOT EXISTS spans (
+    trace_id TEXT NOT NULL,
+    span_id INTEGER NOT NULL,
+    parent_span_id INTEGER NOT NULL,
+    row_index INTEGER NOT NULL,
+    entry_type INTEGER NOT NULL,
+    timestamp_ns INTEGER NOT NULL,
+    message TEXT,
+    PRIMARY KEY (trace_id, span_id, row_index)
+  );
+
+  CREATE INDEX IF NOT EXISTS idx_spans_trace ON spans(trace_id);
+  CREATE INDEX IF NOT EXISTS idx_spans_parent ON spans(trace_id, parent_span_id);
+`;
+
+export const SPANS_TABLE_INFO_SQL = 'PRAGMA table_info(spans)';
+
+type SQLiteTableInfoIntegerField = keyof Pick<SQLiteTableInfoRow, 'cid' | 'notnull' | 'pk'>;
+
+export interface SQLiteColumnDefinition {
+  name: string;
+  sqliteType: string;
+}
+
+export interface SQLiteTableInfoRow {
+  cid: number;
+  name: string;
+  type: string;
+  notnull: number;
+  dflt_value: string | null;
+  pk: number;
+}
+
+export interface SpanSegment {
+  buffer: AnySpanBuffer;
+  traceId: string;
+  spanId: number;
+  parentSpanId: number;
+  rowOffset: number;
+}
+
+/** Check if bit at index is set in Arrow null bitmap (1 = valid/present, 0 = null) */
+function isBitSet(bitmap: Uint8Array, idx: number): boolean {
+  return (bitmap[idx >>> 3] & (1 << (idx & 7))) !== 0;
+}
+
+/**
+ * Read a user column value at the given row.
+ *
+ * Typed arrays are pre-allocated with zeros, so null bitmaps distinguish
+ * "set to zero" from "never written". JS arrays use `undefined` for unwritten slots.
+ */
+function readUserValue(buffer: AnySpanBuffer, fieldName: string, row: number): unknown {
+  const values = Reflect.get(buffer, `${fieldName}_values`);
+  if (!values) return undefined;
+
+  if (ArrayBuffer.isView(values)) {
+    const nulls = Reflect.get(buffer, `${fieldName}_nulls`);
+    if (!(nulls instanceof Uint8Array) || !('length' in values)) {
+      return undefined;
+    }
+
+    return isBitSet(nulls, row) ? Reflect.get(values, row) : undefined;
+  }
+
+  return Array.isArray(values) ? values[row] : undefined;
+}
+
+function readScopeValue(buffer: AnySpanBuffer, fieldName: string): unknown {
+  return isRecord(buffer._scopeValues) ? buffer._scopeValues[fieldName] : undefined;
+}
+
+function describeSqliteBoundaryType(value: unknown): string {
+  return typeof value;
+}
+
+function invalidSqliteBoundaryRow(boundary: string, rowIndex: number, message: string): Error {
+  return new Error(`${boundary} returned an invalid SQLite row ${rowIndex}: ${message}`);
+}
+
+function normalizeSqliteTableInfoInteger(
+  value: unknown,
+  boundary: string,
+  rowIndex: number,
+  fieldName: SQLiteTableInfoIntegerField,
+): number {
+  if (typeof value === 'number') {
+    return value;
+  }
+  if (typeof value === 'bigint') {
+    return Number(value);
+  }
+
+  throw invalidSqliteBoundaryRow(
+    boundary,
+    rowIndex,
+    `${fieldName} expected number, got ${describeSqliteBoundaryType(value)}`,
+  );
+}
+
+function parseSqliteTableInfoRow(row: unknown, boundary: string, rowIndex: number): SQLiteTableInfoRow {
+  if (!isRecord(row)) {
+    throw invalidSqliteBoundaryRow(boundary, rowIndex, `expected record, got ${describeSqliteBoundaryType(row)}`);
+  }
+
+  const rowRecord = row;
+
+  if (!hasOwn(rowRecord, 'cid')) {
+    throw invalidSqliteBoundaryRow(boundary, rowIndex, 'missing cid');
+  }
+  if (!hasOwnString(rowRecord, 'name')) {
+    throw invalidSqliteBoundaryRow(
+      boundary,
+      rowIndex,
+      `name expected string, got ${describeSqliteBoundaryType(rowRecord.name)}`,
+    );
+  }
+  if (!hasOwnString(rowRecord, 'type')) {
+    throw invalidSqliteBoundaryRow(
+      boundary,
+      rowIndex,
+      `type expected string, got ${describeSqliteBoundaryType(rowRecord.type)}`,
+    );
+  }
+  if (!hasOwn(rowRecord, 'notnull')) {
+    throw invalidSqliteBoundaryRow(boundary, rowIndex, 'missing notnull');
+  }
+  if (!hasOwn(rowRecord, 'dflt_value')) {
+    throw invalidSqliteBoundaryRow(boundary, rowIndex, 'missing dflt_value');
+  }
+  if (typeof rowRecord.dflt_value !== 'string' && rowRecord.dflt_value !== null) {
+    throw invalidSqliteBoundaryRow(
+      boundary,
+      rowIndex,
+      `dflt_value expected string|null, got ${describeSqliteBoundaryType(rowRecord.dflt_value)}`,
+    );
+  }
+  if (!hasOwn(rowRecord, 'pk')) {
+    throw invalidSqliteBoundaryRow(boundary, rowIndex, 'missing pk');
+  }
+
+  return {
+    // WHY: some SQLite adapters promote PRAGMA integer metadata to bigint, but the shared table-info contract uses
+    // numeric counters plus string column descriptors.
+    cid: normalizeSqliteTableInfoInteger(rowRecord.cid, boundary, rowIndex, 'cid'),
+    name: rowRecord.name,
+    type: rowRecord.type,
+    notnull: normalizeSqliteTableInfoInteger(rowRecord.notnull, boundary, rowIndex, 'notnull'),
+    dflt_value: rowRecord.dflt_value,
+    pk: normalizeSqliteTableInfoInteger(rowRecord.pk, boundary, rowIndex, 'pk'),
+  };
+}
+
+/** Map arrow-builder SchemaType to SQLite column type */
+function schemaTypeToSqlite(schemaType: SchemaType): string {
+  switch (schemaType) {
+    case 'category':
+    case 'text':
+    case 'enum':
+      return 'TEXT';
+    case 'number':
+      return 'REAL';
+    case 'bigUint64':
+    case 'boolean':
+      return 'INTEGER';
+    case 'binary':
+      return 'BLOB';
+  }
+
+  throw new Error(`Unsupported schema type: ${String(schemaType)}`);
+}
+
+export function normalizeSqliteColumnType(type: string): string {
+  return type.trim() === '' ? 'TEXT' : type;
+}
+
+export function quoteSqlIdentifier(name: string): string {
+  if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(name)) {
+    throw new Error(`Unsafe SQLite identifier: ${name}`);
+  }
+  return `"${name}"`;
+}
+
+export function buildAddColumnSql(column: SQLiteColumnDefinition): string {
+  return `ALTER TABLE spans ADD COLUMN ${quoteSqlIdentifier(column.name)} ${normalizeSqliteColumnType(column.sqliteType)}`;
+}
+
+export function isSqliteDuplicateColumnError(error: unknown, columnName: string): boolean {
+  if (!(error instanceof Error)) {
+    return false;
+  }
+
+  return error.message.toLowerCase().includes(`duplicate column name: ${columnName.toLowerCase()}`);
+}
+
+export function getMissingSqliteColumns(
+  columns: readonly SQLiteColumnDefinition[],
+  knownColumns: ReadonlySet<string>,
+): SQLiteColumnDefinition[] {
+  const missing: SQLiteColumnDefinition[] = [];
+  for (const column of columns) {
+    if (!knownColumns.has(column.name)) {
+      missing.push(column);
+    }
+  }
+  return missing;
+}
+
+export function extractSqliteColumnsFromTableInfo(columns: readonly SQLiteTableInfoRow[]): SQLiteColumnDefinition[] {
+  return columns.map((column) => ({
+    name: column.name,
+    sqliteType: normalizeSqliteColumnType(column.type),
+  }));
+}
+
+export function parseSqliteTableInfoRows(
+  rows: readonly unknown[],
+  boundary = 'PRAGMA table_info(spans)',
+): SQLiteTableInfoRow[] {
+  return rows.map((row, rowIndex) => parseSqliteTableInfoRow(row, boundary, rowIndex));
+}
+
+function* walkOverflowSegments(
+  buffer: AnySpanBuffer,
+  traceId: string,
+  spanId: number,
+  parentSpanId: number,
+  rowOffset: number,
+): Generator<SpanSegment> {
+  yield {
+    buffer,
+    traceId,
+    spanId,
+    parentSpanId,
+    rowOffset,
+  };
+
+  if (buffer._overflow) {
+    yield* walkOverflowSegments(buffer._overflow, traceId, spanId, parentSpanId, rowOffset + buffer._writeIndex);
+  }
+}
+
+function* walkTree(buffer: AnySpanBuffer): Generator<SpanSegment> {
+  yield* walkOverflowSegments(buffer, buffer.trace_id, buffer.span_id, buffer.parent_span_id, 0);
+
+  for (const child of buffer._children) {
+    yield* walkTree(child);
+  }
+}
+
+export function* walkSpanSegments(rootBuffer: AnySpanBuffer): Generator<SpanSegment> {
+  yield* walkTree(rootBuffer);
+}
+
+export function getMissingSchemaColumns(
+  schema: LogSchema,
+  knownColumns: ReadonlySet<string>,
+): SQLiteColumnDefinition[] {
+  const schemaColumns: SQLiteColumnDefinition[] = [];
+
+  for (const [name, field] of Object.entries(schema.fields)) {
+    const schemaType = getSchemaType(field);
+    if (!schemaType) continue;
+
+    schemaColumns.push({ name, sqliteType: schemaTypeToSqlite(schemaType) });
+  }
+
+  return getMissingSqliteColumns(schemaColumns, knownColumns);
+}
+
+export function getActiveUserFields(schema: LogSchema, knownColumns: ReadonlySet<string>): string[] {
+  return schema._columnNames.filter((fieldName) => knownColumns.has(fieldName));
+}
+
+export function getInsertStatementCacheKey(activeUserFields: readonly string[]): string {
+  return activeUserFields.join(',');
+}
+
+export function buildInsertSql(activeUserFields: readonly string[]): string {
+  const userColsSql = activeUserFields.length > 0 ? `, ${activeUserFields.join(', ')}` : '';
+  const userPlaceholders = activeUserFields.length > 0 ? `, ${activeUserFields.map(() => '?').join(', ')}` : '';
+  return `INSERT INTO spans (trace_id, span_id, parent_span_id, row_index, entry_type, timestamp_ns, message${userColsSql})
+   VALUES (?, ?, ?, ?, ?, ?, ?${userPlaceholders})`;
+}
+
+export function buildInsertParams(segment: SpanSegment, row: number, activeUserFields: readonly string[]): unknown[] {
+  const { buffer, traceId, spanId, parentSpanId, rowOffset } = segment;
+  const entryType = buffer.entry_type[row];
+  const timestampNs = Number(buffer.timestamp[row]);
+  const message = buffer.message_values[row] ?? null;
+
+  const userValues = activeUserFields.map((fieldName) => {
+    const directValue = readUserValue(buffer, fieldName, row);
+    const val = directValue ?? readScopeValue(buffer, fieldName);
+    if (val === undefined || val === null) {
+      return null;
+    }
+    return typeof val === 'bigint' ? Number(val) : val;
+  });
+
+  return [traceId, spanId, parentSpanId, rowOffset + row, entryType, timestampNs, message, ...userValues];
+}

package/src/lib/sqlite/sqlite-d1.ts

@@ -0,0 +1,104 @@
+/**
+ * D1 adapter for LMAO async SQLite interfaces.
+ *
+ * This keeps D1-specific behavior in one place while exposing the generic
+ * AsyncSQLiteDatabase contract used by SQLiteAsyncTracer/SQLiteAsyncTraceWriter.
+ *
+ * @module sqlite-d1
+ */
+
+import type { AsyncSQLiteDatabase, AsyncSQLiteStatement } from './sqlite-db.js';
+
+type D1BindValue = null | number | string | ArrayBuffer | ArrayBufferView;
+
+export interface D1LikePreparedStatement {
+  bind(...values: D1BindValue[]): D1LikePreparedStatement;
+  run(): Promise<unknown>;
+  all<T = unknown>(): Promise<{ results?: T[] }>;
+  first<T = unknown>(): Promise<T | null>;
+}
+
+export interface D1LikeDatabase {
+  prepare(query: string): D1LikePreparedStatement;
+  exec(query: string): Promise<unknown>;
+}
+
+function toD1BindValue(value: unknown): D1BindValue {
+  if (value == null) {
+    return null;
+  }
+
+  if (typeof value === 'boolean') {
+    return value ? 1 : 0;
+  }
+
+  if (typeof value === 'bigint') {
+    return Number(value);
+  }
+
+  if (typeof value === 'number' || typeof value === 'string') {
+    return value;
+  }
+
+  if (value instanceof ArrayBuffer) {
+    return value;
+  }
+
+  if (ArrayBuffer.isView(value)) {
+    return value;
+  }
+
+  throw new Error(`Unsupported D1 bind param type: ${typeof value}`);
+}
+
+class D1StatementAdapter implements AsyncSQLiteStatement {
+  constructor(
+    private readonly db: D1LikeDatabase,
+    private readonly query: string,
+  ) {}
+
+  private bind(params: unknown[]): D1LikePreparedStatement {
+    const statement = this.db.prepare(this.query);
+    return statement.bind(...params.map(toD1BindValue));
+  }
+
+  async run(...params: unknown[]): Promise<void> {
+    await this.bind(params).run();
+  }
+
+  async all(...params: unknown[]): Promise<unknown[]> {
+    const result = await this.bind(params).all();
+    return result.results ?? [];
+  }
+
+  async get(...params: unknown[]): Promise<unknown> {
+    return (await this.bind(params).first()) ?? undefined;
+  }
+}
+
+class D1DatabaseAdapter implements AsyncSQLiteDatabase {
+  constructor(private readonly db: D1LikeDatabase) {}
+
+  async exec(sql: string): Promise<void> {
+    const statements = sql
+      .split(';')
+      .map((statement) => statement.trim())
+      .filter((statement) => statement.length > 0);
+
+    for (const statement of statements) {
+      await this.db.prepare(statement).run();
+    }
+  }
+
+  prepare(sql: string): AsyncSQLiteStatement {
+    return new D1StatementAdapter(this.db, sql);
+  }
+
+  async close(): Promise<void> {
+    // D1 bindings are managed by the worker runtime.
+  }
+}
+
+export function createD1SQLiteDatabase(db: D1LikeDatabase): AsyncSQLiteDatabase {
+  return new D1DatabaseAdapter(db);
+}

package/src/lib/sqlite/sqlite-db.ts

@@ -0,0 +1,32 @@
+/**
+ * Minimal SQLite interfaces used by trace persistence.
+ *
+ * Sync APIs match bun:sqlite / better-sqlite3 style drivers.
+ * Async APIs match worker-style drivers (for example D1 adapters).
+ *
+ * @module sqlite-db
+ */
+
+export interface SyncSQLiteDatabase {
+  exec(sql: string): void;
+  prepare(sql: string): SyncSQLiteStatement;
+  close(): void;
+}
+
+export interface SyncSQLiteStatement {
+  run(...params: unknown[]): void;
+  all(...params: unknown[]): unknown[];
+  get(...params: unknown[]): unknown;
+}
+
+export interface AsyncSQLiteDatabase {
+  exec(sql: string): Promise<void>;
+  prepare(sql: string): AsyncSQLiteStatement;
+  close(): Promise<void>;
+}
+
+export interface AsyncSQLiteStatement {
+  run(...params: unknown[]): Promise<void>;
+  all(...params: unknown[]): Promise<unknown[]>;
+  get(...params: unknown[]): Promise<unknown>;
+}