@smoothbricks/lmao 0.1.0

package/src/lib/__tests__/arrow-builder-integration/lazy-columns.test.ts

@@ -0,0 +1,389 @@
+import { describe, expect, it } from 'bun:test';
+import { createColumnBuffer } from '@smoothbricks/arrow-builder';
+import { createTagWriter, S } from '@smoothbricks/lmao';
+import { createSpanBuffer } from '../../spanBuffer.js';
+
+import { createTestOpMetadata, createTestSchema, createTestTraceRoot } from '../test-helpers.js';
+
+/**
+ * Tests for lazy column initialization
+ *
+ * Per GitHub review feedback: Columns should only be allocated when accessed,
+ * not eagerly during buffer creation. This saves memory for sparse columns.
+ *
+ * COLUMN TYPES:
+ *
+ * 1. SpanBuffer-specific columns (NOT in schema, created in preamble):
+ *    - timestamp, entry_type
+ *    → ONLY exist on SpanBuffer, NOT on ColumnBuffer from createColumnBuffer()
+ *    → Already tested in integration.test.ts lines 45-46
+ *
+ * 2. System SCHEMA columns (merged via createTestSchema → mergeWithSystemSchema):
+ *    - message: S.category().eager() ← EAGER (allocated immediately)
+ *    - line, error_code, etc. ← LAZY (default)
+ *
+ * 3. User SCHEMA columns:
+ *    - Default: LAZY (allocated on first access)
+ *    - Can be .eager() to allocate immediately
+ */
+
+describe('Eager vs Lazy Column Allocation', () => {
+  it('eager schema columns allocated immediately, lazy user columns deferred', () => {
+    const schema = createTestSchema({
+      userId: S.category(),
+      requestId: S.category(),
+      count: S.number(),
+      active: S.boolean(),
+    });
+    // createColumnBuffer is generic and returns ColumnBuffer<T>
+    const buffer = createColumnBuffer(schema, 64);
+
+    // System schema's message column is EAGER - always allocated (via mergeWithSystemSchema)
+    expect(Array.isArray(buffer.message_values)).toBe(true);
+
+    // Access ONE user column via _values suffix (category = Array now, not Uint32Array)
+    // Note: buffer.userId is a setter method, buffer.userId_values is the getter
+    const userIdColumn = buffer.userId_values;
+    expect(Array.isArray(userIdColumn)).toBe(true);
+
+    // Now check that OTHER user columns were NOT allocated
+    // Getters are on the prototype, not the instance
+    const proto = Object.getPrototypeOf(buffer);
+    const descriptor = Object.getOwnPropertyDescriptor(proto, 'requestId_values');
+    expect(descriptor?.get).toBeDefined(); // Should be a getter on prototype
+
+    // After access, the getter should be replaced with the value
+    const requestIdColumn = buffer.requestId_values;
+    expect(Array.isArray(requestIdColumn)).toBe(true);
+  });
+
+  it('should allocate null bitmaps lazily', () => {
+    const schema = createTestSchema({
+      userId: S.category(),
+      requestId: S.category(),
+    });
+    // createColumnBuffer expects ColumnSchema instance (schema extends LogSchema extends ColumnSchema)
+    const buffer = createColumnBuffer(schema, 64);
+
+    // Access userId column via _values suffix (category = Array now)
+    const userIdColumn = buffer.userId_values;
+    expect(Array.isArray(userIdColumn)).toBe(true);
+
+    // Check that userId's null bitmap was allocated (by accessing it)
+    const userIdBitmap = buffer.userId_nulls;
+    expect(userIdBitmap).toBeInstanceOf(Uint8Array);
+
+    // Check requestId's null bitmap is still a getter on prototype (not accessed yet)
+    const proto = Object.getPrototypeOf(buffer);
+    const bitmapDescriptor = Object.getOwnPropertyDescriptor(proto, 'requestId_nulls');
+    expect(bitmapDescriptor?.get).toBeDefined();
+  });
+
+  it('should save memory with sparse column access', () => {
+    const schema = createTestSchema({
+      col1: S.number(),
+      col2: S.number(),
+      col3: S.number(),
+      col4: S.number(),
+      col5: S.number(),
+    });
+    // createColumnBuffer expects ColumnSchema instance (schema extends LogSchema extends ColumnSchema)
+    const buffer = createColumnBuffer(schema, 1024);
+
+    // Only access 2 of 5 columns (use _values suffix for the array)
+    buffer.col1_values[0] = 1;
+    buffer.col3_values[0] = 3;
+
+    // col2, col4, col5 should still be getters on prototype (not accessed yet)
+    // Note: use _values suffix to check the getter (not the setter method)
+    const proto = Object.getPrototypeOf(buffer);
+    const col2Desc = Object.getOwnPropertyDescriptor(proto, 'col2_values');
+    const col4Desc = Object.getOwnPropertyDescriptor(proto, 'col4_values');
+    const col5Desc = Object.getOwnPropertyDescriptor(proto, 'col5_values');
+
+    expect(col2Desc?.get).toBeDefined();
+    expect(col4Desc?.get).toBeDefined();
+    expect(col5Desc?.get).toBeDefined();
+  });
+
+  it('should have getters that become values after first access', () => {
+    const schema = createTestSchema({
+      field1: S.category(),
+      field2: S.number(),
+    });
+    // createColumnBuffer expects ColumnSchema instance (schema extends LogSchema extends ColumnSchema)
+    const buffer = createColumnBuffer(schema, 64);
+
+    // Before access: should be getter on prototype (use _values suffix)
+    const proto = Object.getPrototypeOf(buffer);
+    const beforeDesc = Object.getOwnPropertyDescriptor(proto, 'field1_values');
+    expect(beforeDesc?.get).toBeDefined();
+
+    // Access the column via _values suffix (category = Array now)
+    const column = buffer.field1_values;
+    expect(Array.isArray(column)).toBe(true);
+
+    // After access: the property should still work (getter returns cached value from symbol-keyed storage)
+    const afterValue = buffer.field1_values;
+    expect(afterValue).toBe(column); // Same instance returned
+    expect(Array.isArray(afterValue)).toBe(true);
+  });
+});
+
+describe('Lazy Column Initialization', () => {
+  it('should not allocate attribute columns until accessed', () => {
+    const schema = createTestSchema({
+      userId: S.category(),
+      requestId: S.category(),
+      operation: S.enum(['GET', 'POST', 'PUT', 'DELETE']),
+      duration: S.number(),
+    });
+
+    // createColumnBuffer expects ColumnSchema instance (schema extends LogSchema extends ColumnSchema)
+    const buffer = createColumnBuffer(schema, 64);
+
+    // System schema's message column is EAGER - always allocated (via mergeWithSystemSchema)
+    // Note: timestamp/entry_type are SpanBuffer-specific, NOT in ColumnBuffer
+    expect(Array.isArray(buffer.message_values)).toBe(true);
+
+    // Access one attribute column via _values suffix - should allocate it lazily on first access
+    const userIdColumn = buffer.userId_values;
+    // Category stores raw strings in Array
+    expect(Array.isArray(userIdColumn)).toBe(true);
+    expect(userIdColumn.length).toBeGreaterThan(0);
+
+    // Access null bitmap - should allocate it lazily on first access
+    const userIdNulls = buffer.userId_nulls;
+    expect(userIdNulls).toBeInstanceOf(Uint8Array);
+    expect(userIdNulls.length).toBeGreaterThan(0);
+
+    // Verify lazy allocation worked by checking that multiple accesses return same object
+    expect(buffer.userId_values).toBe(userIdColumn);
+    expect(buffer.userId_nulls).toBe(userIdNulls);
+  });
+
+  it('should allocate correct TypedArray type for different schema types', () => {
+    const schema = createTestSchema({
+      smallEnum: S.enum(['A', 'B', 'C']), // < 256 values -> Uint8Array
+      category: S.category(), // -> Array<string> (no hot-path interning)
+      text: S.text(), // -> Array<string> (no hot-path interning)
+      num: S.number(), // -> Float64Array
+      bool: S.boolean(), // -> Uint8Array
+    });
+
+    // createColumnBuffer expects ColumnSchema instance (schema extends LogSchema extends ColumnSchema)
+    const buffer = createColumnBuffer(schema, 64);
+
+    // Access each column via _values suffix and verify type
+    // Per spec 01a:
+    // - enum: Uint8Array (small enums) or Uint16Array/Uint32Array (larger)
+    // - category: Array<string> (raw strings, no hot-path interning)
+    // - text: Array<string> (raw strings)
+    expect(buffer.smallEnum_values).toBeInstanceOf(Uint8Array);
+    expect(Array.isArray(buffer.category_values)).toBe(true); // Category stores raw strings
+    expect(Array.isArray(buffer.text_values)).toBe(true); // Text stores raw strings
+    expect(buffer.num_values).toBeInstanceOf(Float64Array);
+    expect(buffer.bool_values).toBeInstanceOf(Uint8Array);
+  });
+
+  it('should allocate small enum as Uint8Array', () => {
+    // Create enum with < 256 values (at the limit)
+    const enumValues = Array.from({ length: 256 }, (_, i) => `VALUE_${i}`);
+    const schema = createTestSchema({
+      maxEnum: S.enum(enumValues),
+    });
+
+    // createColumnBuffer expects ColumnSchema instance (schema extends LogSchema extends ColumnSchema)
+    const buffer = createColumnBuffer(schema, 64);
+
+    // Should use Uint8Array for <= 255 values (use _values suffix)
+    expect(buffer.maxEnum_values).toBeInstanceOf(Uint8Array);
+  });
+
+  it('should work correctly when writing to lazily-allocated columns', () => {
+    const schema = createTestSchema({
+      userId: S.category(),
+      count: S.number(),
+    });
+
+    // createColumnBuffer expects ColumnSchema instance (schema extends LogSchema extends ColumnSchema)
+    const buffer = createColumnBuffer(schema, 64);
+
+    // Write to columns via _values suffix (triggering lazy allocation)
+    // Category stores strings in Array (no hot-path interning)
+    const userIdColumn = buffer.userId_values;
+    const countColumn = buffer.count_values;
+
+    // Write a string value for category (raw strings stored directly)
+    userIdColumn[0] = 'user-123';
+    countColumn[0] = 45.67;
+
+    // Verify values
+    expect(userIdColumn[0]).toBe('user-123');
+    expect(countColumn[0]).toBe(45.67);
+  });
+
+  it('should allow multiple accesses to the same column without re-allocation', () => {
+    const schema = createTestSchema({
+      userId: S.category(),
+    });
+
+    // createColumnBuffer expects ColumnSchema instance (schema extends LogSchema extends ColumnSchema)
+    const buffer = createColumnBuffer(schema, 64);
+
+    // First access (use _values suffix)
+    const column1 = buffer.userId_values;
+
+    // Second access - should return same array
+    const column2 = buffer.userId_values;
+
+    // Should be the exact same object
+    expect(column1).toBe(column2);
+  });
+
+  it('should handle null bitmap lazy initialization correctly', () => {
+    const schema = createTestSchema({
+      userId: S.category(),
+      requestId: S.category(),
+    });
+
+    // createColumnBuffer expects ColumnSchema instance (schema extends LogSchema extends ColumnSchema)
+    const buffer = createColumnBuffer(schema, 64);
+
+    // Access one null bitmap
+    const userIdNulls = buffer.userId_nulls;
+    expect(userIdNulls).toBeInstanceOf(Uint8Array);
+
+    // Access another null bitmap
+    const requestIdNulls = buffer.requestId_nulls;
+    expect(requestIdNulls).toBeInstanceOf(Uint8Array);
+
+    // Should be different arrays
+    expect(userIdNulls).not.toBe(requestIdNulls);
+
+    // Multiple accesses should return same array
+    expect(buffer.userId_nulls).toBe(userIdNulls);
+  });
+});
+
+describe('SpanBuffer Lazy Column Allocation', () => {
+  it('should have eager columns allocated immediately and lazy columns undefined until accessed', () => {
+    const schema = createTestSchema({
+      userId: S.category(),
+      requestId: S.category(),
+      count: S.number(),
+      operation: S.enum(['CREATE', 'READ', 'UPDATE', 'DELETE']),
+    });
+
+    const buffer = createSpanBuffer(schema, createTestTraceRoot('test-trace'), createTestOpMetadata(), 8);
+
+    // Eager columns (system columns) should be allocated immediately
+    expect(buffer.timestamp).toBeDefined();
+    expect(buffer.timestamp).toBeInstanceOf(BigInt64Array);
+    expect(buffer.entry_type).toBeDefined();
+    expect(buffer.entry_type).toBeInstanceOf(Uint8Array);
+    // Also check public aliases
+    expect(buffer.timestamp).toBe(buffer.timestamp);
+    expect(buffer.entry_type).toBe(buffer.entry_type);
+
+    // System schema's message column is EAGER - always allocated
+    expect(Array.isArray(buffer.message_values)).toBe(true);
+
+    // Lazy columns (user attributes) should be undefined before access
+    expect(buffer.getColumnIfAllocated('userId')).toBeUndefined();
+    expect(buffer.getColumnIfAllocated('requestId')).toBeUndefined();
+    expect(buffer.getColumnIfAllocated('count')).toBeUndefined();
+    expect(buffer.getColumnIfAllocated('operation')).toBeUndefined();
+
+    // Access one lazy column - should trigger allocation
+    const userIdColumn = buffer.userId_values;
+    expect(Array.isArray(userIdColumn)).toBe(true);
+
+    // After access, the column should be allocated
+    expect(buffer.getColumnIfAllocated('userId')).toBeDefined();
+    expect(buffer.getColumnIfAllocated('userId')).toBe(userIdColumn);
+
+    // Other columns should still be undefined
+    expect(buffer.getColumnIfAllocated('requestId')).toBeUndefined();
+    expect(buffer.getColumnIfAllocated('count')).toBeUndefined();
+    expect(buffer.getColumnIfAllocated('operation')).toBeUndefined();
+
+    // Access another column
+    const countColumn = buffer.count_values;
+    expect(countColumn).toBeInstanceOf(Float64Array);
+
+    // Now both accessed columns should be allocated
+    expect(buffer.getColumnIfAllocated('userId')).toBeDefined();
+    expect(buffer.getColumnIfAllocated('count')).toBeDefined();
+    expect(buffer.getColumnIfAllocated('count')).toBe(countColumn);
+
+    // Unaccessed columns should still be undefined
+    expect(buffer.getColumnIfAllocated('requestId')).toBeUndefined();
+    expect(buffer.getColumnIfAllocated('operation')).toBeUndefined();
+  });
+
+  it('should allocate lazy columns when TagWriter writes to them', () => {
+    const schema = createTestSchema({
+      userId: S.category(),
+      requestId: S.category(),
+      count: S.number(),
+      operation: S.enum(['CREATE', 'READ', 'UPDATE', 'DELETE']),
+    });
+
+    const buffer = createSpanBuffer(schema, createTestTraceRoot('test-trace'), createTestOpMetadata(), 8);
+
+    // Lazy columns should be undefined before TagWriter access
+    expect(buffer.getColumnIfAllocated('userId')).toBeUndefined();
+    expect(buffer.getColumnIfAllocated('requestId')).toBeUndefined();
+    expect(buffer.getColumnIfAllocated('count')).toBeUndefined();
+    expect(buffer.getColumnIfAllocated('operation')).toBeUndefined();
+
+    // Create TagWriter and write to one column (createTagWriter expects LogSchema instance)
+    const tagWriter = createTagWriter(schema, buffer);
+    tagWriter.userId('user-123');
+
+    // After TagWriter write, the column should be allocated
+    expect(buffer.getColumnIfAllocated('userId')).toBeDefined();
+    const userIdColumn = buffer.getColumnIfAllocated('userId');
+    expect(Array.isArray(userIdColumn)).toBe(true);
+    if (!Array.isArray(userIdColumn)) {
+      throw new Error('Expected userId column to allocate as a string array');
+    }
+    expect(userIdColumn[0]).toBe('user-123');
+
+    // Other columns should still be undefined
+    expect(buffer.getColumnIfAllocated('requestId')).toBeUndefined();
+    expect(buffer.getColumnIfAllocated('count')).toBeUndefined();
+    expect(buffer.getColumnIfAllocated('operation')).toBeUndefined();
+
+    // Write to another column via TagWriter
+    tagWriter.count(42);
+
+    // Now both columns should be allocated
+    expect(buffer.getColumnIfAllocated('userId')).toBeDefined();
+    expect(buffer.getColumnIfAllocated('count')).toBeDefined();
+    const countColumn = buffer.getColumnIfAllocated('count');
+    expect(countColumn).toBeInstanceOf(Float64Array);
+    if (!(countColumn instanceof Float64Array)) {
+      throw new Error('Expected count column to allocate as Float64Array');
+    }
+    expect(countColumn[0]).toBe(42);
+
+    // Unaccessed columns should still be undefined
+    expect(buffer.getColumnIfAllocated('requestId')).toBeUndefined();
+    expect(buffer.getColumnIfAllocated('operation')).toBeUndefined();
+
+    // Write to enum column via TagWriter
+    tagWriter.operation('CREATE');
+
+    // Enum column should now be allocated
+    expect(buffer.getColumnIfAllocated('operation')).toBeDefined();
+    const operationColumn = buffer.getColumnIfAllocated('operation');
+    expect(operationColumn).toBeInstanceOf(Uint8Array);
+    if (!(operationColumn instanceof Uint8Array)) {
+      throw new Error('Expected operation column to allocate as Uint8Array');
+    }
+    // Enum values are mapped to indices (CREATE = 0, READ = 1, etc.)
+    expect(operationColumn[0]).toBe(0);
+  });
+});

package/src/lib/__tests__/arrow-conversion-algorithms.test.ts

@@ -0,0 +1,54 @@
+import { describe, expect, it } from 'bun:test';
+import type { ColumnValueType } from '@smoothbricks/arrow-builder';
+import { buildTextDictionary } from '../arrow/dictionaries.js';
+import { concatenateNullBitmaps } from '../arrow/utils.js';
+import { S } from '../schema/builder.js';
+import { defineLogSchema } from '../schema/defineLogSchema.js';
+import type { AnySpanBuffer } from '../types.js';
+import { createBuffer } from './test-helpers.js';
+
+function mockBuffer(
+  writeIndex: number,
+  column: ColumnValueType | undefined,
+  nulls: Uint8Array | undefined,
+): AnySpanBuffer {
+  const buffer = createBuffer(
+    defineLogSchema({
+      field: S.text(),
+    }),
+  );
+  buffer._writeIndex = writeIndex;
+  buffer.getColumnIfAllocated = (columnName: string): ColumnValueType | undefined =>
+    columnName === 'field' ? column : undefined;
+  buffer.getNullsIfAllocated = (columnName: string): Uint8Array | undefined =>
+    columnName === 'field' ? nulls : undefined;
+  return buffer;
+}
+
+describe('arrow conversion algorithms', () => {
+  it('concatenateNullBitmaps handles non-byte-aligned buffer offsets', () => {
+    const buffers: AnySpanBuffer[] = [
+      mockBuffer(3, ['a', undefined, 'b'], new Uint8Array([0b00000101])),
+      mockBuffer(5, ['c', 'd', undefined, 'e', undefined], new Uint8Array([0b00001011])),
+    ];
+
+    const { nullBitmap, nullCount } = concatenateNullBitmaps(buffers, 'field');
+
+    expect(nullBitmap).toBeDefined();
+    expect(nullBitmap?.[0]).toBe(0b01011101);
+    expect(nullCount).toBe(3);
+  });
+
+  it('buildTextDictionary marks all rows null when text column is never allocated', () => {
+    const buffers: AnySpanBuffer[] = [mockBuffer(2, undefined, undefined), mockBuffer(1, undefined, undefined)];
+
+    const result = buildTextDictionary(buffers, 'field');
+
+    expect(result).not.toBeNull();
+    expect(result?.dictionary).toEqual([]);
+    expect(result?.indices.length).toBe(3);
+    expect(result?.nullCount).toBe(3);
+    expect(result?.nullBitmap).toBeDefined();
+    expect(result?.nullBitmap?.[0]).toBe(0);
+  });
+});

package/src/lib/__tests__/arrow-test-helpers.ts

@@ -0,0 +1,71 @@
+import { isRecord } from '@smoothbricks/validation';
+import type { Column, Table } from '@uwdata/flechette';
+import type { AnySpanBuffer } from '../types.js';
+
+export function requireColumn(table: Table, columnName: string): Column<unknown> {
+  const column = table.getChild(columnName);
+  if (!column) {
+    throw new Error(`Column not found: ${columnName}`);
+  }
+  return column;
+}
+
+export function getColumnValue(table: Table, columnName: string, rowIndex: number): unknown {
+  return requireColumn(table, columnName).get(rowIndex);
+}
+
+export function getRawTimestamp(table: Table, rowIndex: number): bigint {
+  const column = requireColumn(table, 'timestamp');
+  const batch = column.data[0];
+  if (!batch || !(batch.values instanceof BigInt64Array)) {
+    throw new Error('timestamp column does not expose raw BigInt64Array values');
+  }
+  return batch.values[rowIndex];
+}
+
+export function requireBinaryCell(column: Column<unknown>, rowIndex: number): Uint8Array {
+  const value = column.at(rowIndex);
+  if (!(value instanceof Uint8Array)) {
+    throw new Error(`Expected binary cell at row ${rowIndex}`);
+  }
+  return value;
+}
+
+export function nextWriterRow(writer: { nextRow(): unknown }): object {
+  const row = writer.nextRow();
+  if (!isRecord(row)) {
+    throw new Error('Expected createColumnWriter().nextRow() to return a row writer object');
+  }
+  return row;
+}
+
+export function invokeWriterMethod(writer: object, methodName: string, value: unknown): object {
+  const method = Reflect.get(writer, methodName);
+  if (typeof method !== 'function') {
+    throw new Error(`Expected row writer method '${methodName}'`);
+  }
+  const next = Reflect.apply(method, writer, [value]);
+  if (!isRecord(next)) {
+    throw new Error(`Expected row writer method '${methodName}' to return a row writer object`);
+  }
+  return next;
+}
+
+export function callBufferWriter(buffer: object, methodName: string, rowIndex: number, value: unknown): void {
+  const method = Reflect.get(buffer, methodName);
+  if (typeof method !== 'function') {
+    throw new Error(`Expected buffer writer '${methodName}'`);
+  }
+  Reflect.apply(method, buffer, [rowIndex, value]);
+}
+
+export function isAnySpanBuffer(value: unknown): value is AnySpanBuffer {
+  return isRecord(value) && 'timestamp' in value && 'entry_type' in value && '_writeIndex' in value;
+}
+
+export function requireAnySpanBuffer(value: unknown, message: string): AnySpanBuffer {
+  if (!isAnySpanBuffer(value)) {
+    throw new Error(message);
+  }
+  return value;
+}