@smoothbricks/lmao 0.2.2 → 0.3.0

package/src/lib/__tests__/arrow-builder-integration/arrow-conversion.test.ts

@@ -11,7 +11,12 @@
 
 import { describe, expect, test } from 'bun:test';
 import type { CapacityStatsEntry } from '../../arrow/capacityStats.js';
-import { convertSpanTreeToArrowTable, convertToArrowTable } from '../../convertToArrow.js';
+import {
+  convertSpanTreeToArrowTable,
+  convertSpanTreeToLeasedArrowTable,
+  convertToArrowTable,
+  convertToLeasedArrowTable,
+} from '../../convertToArrow.js';
 import { DEFAULT_METADATA } from '../../opContext/defineOp.js';
 import { S } from '../../schema/builder.js';
 import {
@@ -28,7 +33,6 @@ import {
   ENTRY_TYPE_WARN,
 } from '../../schema/systemSchema.js';
 import { createChildSpanBuffer, createOverflowBuffer, createSpanBuffer, getSpanBufferClass } from '../../spanBuffer.js';
-import type { SpanBuffer } from '../../types.js';
 import { getColumnValue, getRawTimestamp } from '../arrow-test-helpers.js';
 import { createTestOpMetadata, createTestSchema, createTestTraceRoot } from '../test-helpers.js';
 
@@ -54,7 +58,11 @@ describe('Arrow Table Conversion', () => {
 
       // Write some test data
       buffer.timestamp[0] = 1000n;
-      buffer.entry_type[0] = ENTRY_TYPE_SPAN_START;
+      {
+        const entryTypes = buffer.entry_type;
+        if (entryTypes === undefined) throw new Error('Expected split entry-type lane');
+        entryTypes[0] = ENTRY_TYPE_SPAN_START;
+      }
       buffer.message(0, 'test-span');
       buffer.httpStatus(0, 200);
       buffer.userId(0, 'user-123');
@@ -82,17 +90,29 @@ describe('Arrow Table Conversion', () => {
 
       // Write multiple rows
       buffer.timestamp[0] = 1000n;
-      buffer.entry_type[0] = ENTRY_TYPE_SPAN_START;
+      {
+        const entryTypes = buffer.entry_type;
+        if (entryTypes === undefined) throw new Error('Expected split entry-type lane');
+        entryTypes[0] = ENTRY_TYPE_SPAN_START;
+      }
       buffer.message(0, 'span-1');
       buffer.level(0, 1); // INFO
 
       buffer.timestamp[1] = 2000n;
-      buffer.entry_type[1] = ENTRY_TYPE_INFO;
+      {
+        const entryTypes = buffer.entry_type;
+        if (entryTypes === undefined) throw new Error('Expected split entry-type lane');
+        entryTypes[1] = ENTRY_TYPE_INFO;
+      }
       buffer.message(1, 'Log message');
       buffer.level(1, 0); // DEBUG
 
       buffer.timestamp[2] = 3000n;
-      buffer.entry_type[2] = ENTRY_TYPE_SPAN_OK;
+      {
+        const entryTypes = buffer.entry_type;
+        if (entryTypes === undefined) throw new Error('Expected split entry-type lane');
+        entryTypes[2] = ENTRY_TYPE_SPAN_OK;
+      }
       buffer.message(2, 'span-1');
       buffer.level(2, 2); // WARN
 
@@ -125,7 +145,11 @@ describe('Arrow Table Conversion', () => {
 
       // Write row with only requiredField set
       buffer.timestamp[0] = 1000n;
-      buffer.entry_type[0] = ENTRY_TYPE_SPAN_START;
+      {
+        const entryTypes = buffer.entry_type;
+        if (entryTypes === undefined) throw new Error('Expected split entry-type lane');
+        entryTypes[0] = ENTRY_TYPE_SPAN_START;
+      }
       buffer.message(0, 'test-span');
       buffer.requiredField(0, 42);
       // optionalField not set - should be null
@@ -150,7 +174,11 @@ describe('Arrow Table Conversion', () => {
 
       // Fill first buffer
       buffer1.timestamp[0] = 1000n;
-      buffer1.entry_type[0] = ENTRY_TYPE_SPAN_START;
+      {
+        const entryTypes = buffer1.entry_type;
+        if (entryTypes === undefined) throw new Error('Expected split entry-type lane');
+        entryTypes[0] = ENTRY_TYPE_SPAN_START;
+      }
       buffer1.message(0, 'test-span');
       buffer1.counter(0, 1);
       buffer1._writeIndex = 1;
@@ -158,7 +186,11 @@ describe('Arrow Table Conversion', () => {
       // Create chained buffer
       const buffer2 = createOverflowBuffer(buffer1);
       buffer2.timestamp[0] = 2000n;
-      buffer2.entry_type[0] = ENTRY_TYPE_INFO;
+      {
+        const entryTypes = buffer2.entry_type;
+        if (entryTypes === undefined) throw new Error('Expected split entry-type lane');
+        entryTypes[0] = ENTRY_TYPE_INFO;
+      }
       buffer2.message(0, 'Log in chain');
       buffer2.counter(0, 2);
       buffer2._writeIndex = 1;
@@ -182,21 +214,33 @@ describe('Arrow Table Conversion', () => {
       const buffer1 = createSpanBuffer(schema, createTestTraceRoot('trace-multi-chain'), DEFAULT_METADATA, undefined);
 
       buffer1.timestamp[0] = 1000n;
-      buffer1.entry_type[0] = ENTRY_TYPE_SPAN_START;
+      {
+        const entryTypes = buffer1.entry_type;
+        if (entryTypes === undefined) throw new Error('Expected split entry-type lane');
+        entryTypes[0] = ENTRY_TYPE_SPAN_START;
+      }
       buffer1.message(0, 'test');
       buffer1.value(0, 1);
       buffer1._writeIndex = 1;
 
       const buffer2 = createOverflowBuffer(buffer1);
       buffer2.timestamp[0] = 2000n;
-      buffer2.entry_type[0] = ENTRY_TYPE_INFO;
+      {
+        const entryTypes = buffer2.entry_type;
+        if (entryTypes === undefined) throw new Error('Expected split entry-type lane');
+        entryTypes[0] = ENTRY_TYPE_INFO;
+      }
       buffer2.message(0, 'log-1');
       buffer2.value(0, 2);
       buffer2._writeIndex = 1;
 
       const buffer3 = createOverflowBuffer(buffer2);
       buffer3.timestamp[0] = 3000n;
-      buffer3.entry_type[0] = ENTRY_TYPE_SPAN_OK;
+      {
+        const entryTypes = buffer3.entry_type;
+        if (entryTypes === undefined) throw new Error('Expected split entry-type lane');
+        entryTypes[0] = ENTRY_TYPE_SPAN_OK;
+      }
       buffer3.message(0, 'test');
       buffer3.value(0, 3);
       buffer3._writeIndex = 1;
@@ -244,35 +288,45 @@ describe('Arrow Table Conversion', () => {
       const parentBuffer = createSpanBuffer(schema, createTestTraceRoot('trace-tree'), DEFAULT_METADATA, undefined);
 
       parentBuffer.timestamp[0] = 1000n;
-      parentBuffer.entry_type[0] = ENTRY_TYPE_SPAN_START;
+      {
+        const entryTypes = parentBuffer.entry_type;
+        if (entryTypes === undefined) throw new Error('Expected split entry-type lane');
+        entryTypes[0] = ENTRY_TYPE_SPAN_START;
+      }
       parentBuffer.message(0, 'parent-span');
       parentBuffer.spanType(0, 'parent');
       parentBuffer._writeIndex = 1;
 
-      // Create child span and register with parent
-      const childBuffer = createChildSpanBuffer(
-        parentBuffer,
-        SpanBufferClass,
-        DEFAULT_METADATA,
-        DEFAULT_METADATA,
-      ) as SpanBuffer<typeof schema>;
-      parentBuffer._children.push(childBuffer); // Explicit registration per spanBuffer.ts
+      // Create child span; the factory registers it with the parent topology.
+      const childBuffer = createChildSpanBuffer(parentBuffer, SpanBufferClass, DEFAULT_METADATA, DEFAULT_METADATA);
 
       childBuffer.timestamp[0] = 1500n;
-      childBuffer.entry_type[0] = ENTRY_TYPE_SPAN_START;
+      {
+        const entryTypes = childBuffer.entry_type;
+        if (entryTypes === undefined) throw new Error('Expected split entry-type lane');
+        entryTypes[0] = ENTRY_TYPE_SPAN_START;
+      }
       childBuffer.message(0, 'child-span');
       childBuffer.spanType(0, 'child');
       childBuffer._writeIndex = 1;
 
       // Complete child span
       childBuffer.timestamp[1] = 1800n;
-      childBuffer.entry_type[1] = ENTRY_TYPE_SPAN_OK;
+      {
+        const entryTypes = childBuffer.entry_type;
+        if (entryTypes === undefined) throw new Error('Expected split entry-type lane');
+        entryTypes[1] = ENTRY_TYPE_SPAN_OK;
+      }
       childBuffer.message(1, 'child-span');
       childBuffer._writeIndex = 2;
 
       // Complete parent span
       parentBuffer.timestamp[1] = 2000n;
-      parentBuffer.entry_type[1] = ENTRY_TYPE_SPAN_OK;
+      {
+        const entryTypes = parentBuffer.entry_type;
+        if (entryTypes === undefined) throw new Error('Expected split entry-type lane');
+        entryTypes[1] = ENTRY_TYPE_SPAN_OK;
+      }
       parentBuffer.message(1, 'parent-span');
       parentBuffer._writeIndex = 2;
 
@@ -302,35 +356,57 @@ describe('Arrow Table Conversion', () => {
       const parentBuffer = createSpanBuffer(schema, createTestTraceRoot('trace-siblings'), DEFAULT_METADATA, undefined);
 
       parentBuffer.timestamp[0] = 1000n;
-      parentBuffer.entry_type[0] = ENTRY_TYPE_SPAN_START;
+      {
+        const entryTypes = parentBuffer.entry_type;
+        if (entryTypes === undefined) throw new Error('Expected split entry-type lane');
+        entryTypes[0] = ENTRY_TYPE_SPAN_START;
+      }
       parentBuffer.message(0, 'parent');
       parentBuffer._writeIndex = 1;
 
-      // Create first child and register with parent
+      // Create first child; the factory registers it with the parent topology.
       const child1Buffer = createChildSpanBuffer(parentBuffer, SpanBufferClass, DEFAULT_METADATA, DEFAULT_METADATA);
-      parentBuffer._children.push(child1Buffer); // Explicit registration per spanBuffer.ts
       child1Buffer.timestamp[0] = 1100n;
-      child1Buffer.entry_type[0] = ENTRY_TYPE_SPAN_START;
+      {
+        const entryTypes = child1Buffer.entry_type;
+        if (entryTypes === undefined) throw new Error('Expected split entry-type lane');
+        entryTypes[0] = ENTRY_TYPE_SPAN_START;
+      }
       child1Buffer.message(0, 'child-1');
       child1Buffer.timestamp[1] = 1200n;
-      child1Buffer.entry_type[1] = ENTRY_TYPE_SPAN_OK;
+      {
+        const entryTypes = child1Buffer.entry_type;
+        if (entryTypes === undefined) throw new Error('Expected split entry-type lane');
+        entryTypes[1] = ENTRY_TYPE_SPAN_OK;
+      }
       child1Buffer.message(1, 'child-1');
       child1Buffer._writeIndex = 2;
 
-      // Create second child and register with parent
+      // Create second child; the factory preserves sibling insertion order.
       const child2Buffer = createChildSpanBuffer(parentBuffer, SpanBufferClass, DEFAULT_METADATA, DEFAULT_METADATA);
-      parentBuffer._children.push(child2Buffer); // Explicit registration per spanBuffer.ts
       child2Buffer.timestamp[0] = 1300n;
-      child2Buffer.entry_type[0] = ENTRY_TYPE_SPAN_START;
+      {
+        const entryTypes = child2Buffer.entry_type;
+        if (entryTypes === undefined) throw new Error('Expected split entry-type lane');
+        entryTypes[0] = ENTRY_TYPE_SPAN_START;
+      }
       child2Buffer.message(0, 'child-2');
       child2Buffer.timestamp[1] = 1400n;
-      child2Buffer.entry_type[1] = ENTRY_TYPE_SPAN_OK;
+      {
+        const entryTypes = child2Buffer.entry_type;
+        if (entryTypes === undefined) throw new Error('Expected split entry-type lane');
+        entryTypes[1] = ENTRY_TYPE_SPAN_OK;
+      }
       child2Buffer.message(1, 'child-2');
       child2Buffer._writeIndex = 2;
 
       // Complete parent
       parentBuffer.timestamp[1] = 1500n;
-      parentBuffer.entry_type[1] = ENTRY_TYPE_SPAN_OK;
+      {
+        const entryTypes = parentBuffer.entry_type;
+        if (entryTypes === undefined) throw new Error('Expected split entry-type lane');
+        entryTypes[1] = ENTRY_TYPE_SPAN_OK;
+      }
       parentBuffer.message(1, 'parent');
       parentBuffer._writeIndex = 2;
 
@@ -379,7 +455,11 @@ describe('Arrow Table Conversion', () => {
       // Write one row for each entry type
       for (let i = 0; i < entryTypes.length; i++) {
         buffer.timestamp[i] = BigInt(1000 + i * 100);
-        buffer.entry_type[i] = entryTypes[i];
+        {
+          const physicalEntryTypes = buffer.entry_type;
+          if (physicalEntryTypes === undefined) throw new Error('Expected split entry-type lane');
+          physicalEntryTypes[i] = entryTypes[i];
+        }
         buffer.message(i, `entry-${i}`);
       }
       buffer._writeIndex = entryTypes.length;
@@ -404,12 +484,20 @@ describe('Arrow Table Conversion', () => {
 
       // Write some span entries
       buffer.timestamp[0] = 1000n;
-      buffer.entry_type[0] = ENTRY_TYPE_SPAN_START;
+      {
+        const entryTypes = buffer.entry_type;
+        if (entryTypes === undefined) throw new Error('Expected split entry-type lane');
+        entryTypes[0] = ENTRY_TYPE_SPAN_START;
+      }
       buffer.message(0, 'test-span');
       buffer._writeIndex = 1;
 
       buffer.timestamp[1] = 2000n;
-      buffer.entry_type[1] = ENTRY_TYPE_INFO;
+      {
+        const entryTypes = buffer.entry_type;
+        if (entryTypes === undefined) throw new Error('Expected split entry-type lane');
+        entryTypes[1] = ENTRY_TYPE_INFO;
+      }
       buffer.message(1, 'Test log message');
       buffer._writeIndex = 2;
 
@@ -482,4 +570,214 @@ describe('Arrow Table Conversion', () => {
       expect(toBigInt(getColumnValue(table, 'uint64_value', 3))).toBe(64n); // capacity
     });
   });
+
+  describe('leased chunked conversion', () => {
+    test('pins source chunks until idempotent release and preserves exact schema, nulls, and tree preorder', () => {
+      const schema = createTestSchema({ value: S.number(), label: S.category() });
+      const SpanBufferClass = getSpanBufferClass(schema);
+      const root = createSpanBuffer(schema, createTestTraceRoot('leased-tree'), DEFAULT_METADATA, 8);
+      root.timestamp[0] = 1000n;
+      {
+        const entryTypes = root.entry_type;
+        if (entryTypes === undefined) throw new Error('Expected split entry-type lane');
+        entryTypes[0] = ENTRY_TYPE_SPAN_START;
+      }
+      root.message(0, 'root-start');
+      root.value(0, 11);
+      root.label(0, 'root');
+      root.timestamp[1] = 1100n;
+      {
+        const entryTypes = root.entry_type;
+        if (entryTypes === undefined) throw new Error('Expected split entry-type lane');
+        entryTypes[1] = ENTRY_TYPE_SPAN_OK;
+      }
+      root._writeIndex = 2;
+
+      const overflow = createOverflowBuffer(root);
+      overflow.timestamp[0] = 1200n;
+      {
+        const entryTypes = overflow.entry_type;
+        if (entryTypes === undefined) throw new Error('Expected split entry-type lane');
+        entryTypes[0] = ENTRY_TYPE_INFO;
+      }
+      overflow.message(0, 'overflow-log');
+      overflow.value(0, 22);
+      overflow._writeIndex = 1;
+
+      const child = createChildSpanBuffer(root, SpanBufferClass, DEFAULT_METADATA, DEFAULT_METADATA);
+      child.timestamp[0] = 1300n;
+      {
+        const entryTypes = child.entry_type;
+        if (entryTypes === undefined) throw new Error('Expected split entry-type lane');
+        entryTypes[0] = ENTRY_TYPE_SPAN_START;
+      }
+      child.message(0, 'child-start');
+      child.label(0, 'child');
+      child.value(0, 0);
+      const childValueNulls = child.getNullsIfAllocated('value');
+      if (!(childValueNulls instanceof Uint8Array)) throw new Error('Expected child value validity');
+      childValueNulls[0] &= ~1;
+      child._writeIndex = 1;
+
+      const topology = root._traceRoot._topology;
+      const generation = topology.generation;
+      const lease = convertSpanTreeToLeasedArrowTable(root);
+      expect(lease.released).toBe(false);
+      expect(lease.table.names).toEqual([
+        'timestamp',
+        'trace_id',
+        'thread_id',
+        'span_id',
+        'parent_thread_id',
+        'parent_span_id',
+        'entry_type',
+        'package_name',
+        'package_file',
+        'git_sha',
+        'message',
+        'uint64_value',
+        'value',
+        'label',
+      ]);
+      const timestampColumn = lease.table.getChild('timestamp');
+      if (!timestampColumn) throw new Error('Missing leased timestamp column');
+      const rawTimestamps = timestampColumn.data.flatMap((batch) => {
+        if (!(batch.values instanceof BigInt64Array)) throw new Error('Expected raw BigInt64Array timestamps');
+        return Array.from(batch.values.subarray(0, batch.length));
+      });
+      expect(rawTimestamps).toEqual([1000n, 1100n, 1200n, 1300n]);
+      expect(
+        Array.from({ length: lease.table.numRows }, (_, row) => getColumnValue(lease.table, 'message', row)),
+      ).toEqual(['root-start', null, 'overflow-log', 'child-start']);
+      expect(
+        Array.from({ length: lease.table.numRows }, (_, row) => getColumnValue(lease.table, 'value', row)),
+      ).toEqual([11, null, 22, null]);
+      expect(
+        Array.from({ length: lease.table.numRows }, (_, row) => getColumnValue(lease.table, 'label', row)),
+      ).toEqual(['root', null, null, 'child']);
+
+      const valueColumn = lease.table.getChild('value');
+      if (!valueColumn) throw new Error('Missing leased value column');
+      const sourceColumns = [root, overflow, child].map((buffer) => {
+        const source = buffer.getColumnIfAllocated('value');
+        if (!(source instanceof Float64Array)) throw new Error('Expected source Float64Array');
+        return source;
+      });
+      expect(valueColumn.data).toHaveLength(3);
+      expect(valueColumn.data.map((batch) => batch.values.buffer)).toEqual(
+        sourceColumns.map((source) => source.buffer),
+      );
+      expect(topology.generation).toBe(generation);
+      expect(() => topology.assertLive(root)).not.toThrow();
+
+      root._traceRoot.tracer.bufferStrategy.releaseBuffer(root);
+      expect(topology.generation).toBe(generation);
+      expect(() => topology.assertLive(root)).toThrow('stale');
+      expect(getColumnValue(lease.table, 'message', 2)).toBe('overflow-log');
+      lease.release();
+      expect(lease.released).toBe(true);
+      expect(topology.generation).toBe(generation + 1);
+      expect(() => topology.assertLive(root)).toThrow('stale');
+      expect(() => lease.release()).not.toThrow();
+      expect(() => lease[Symbol.dispose]()).not.toThrow();
+    });
+
+    test('borrows nonpacked entry types but derives packed entry types without aliasing row headers', () => {
+      const schema = createTestSchema({ value: S.number() });
+      for (const physicalLayout of ['current', 'specialized', 'packed'] as const) {
+        const SpanBufferClass = getSpanBufferClass(schema, 'mixed', physicalLayout);
+        const root = createSpanBuffer(
+          schema,
+          createTestTraceRoot(`leased-entry-${physicalLayout}`),
+          DEFAULT_METADATA,
+          8,
+          SpanBufferClass,
+        );
+        root.timestamp[0] = 1000n;
+        root.timestamp[1] = 1100n;
+        root.timestamp[2] = 1200n;
+        if (physicalLayout !== 'packed') {
+          if (root.entry_type === undefined) throw new Error('Expected split entry-type lane');
+          {
+            const entryTypes = root.entry_type;
+            if (entryTypes === undefined) throw new Error('Expected split entry-type lane');
+            entryTypes[0] = ENTRY_TYPE_SPAN_START;
+          }
+          {
+            const entryTypes = root.entry_type;
+            if (entryTypes === undefined) throw new Error('Expected split entry-type lane');
+            entryTypes[1] = ENTRY_TYPE_INFO;
+          }
+          {
+            const entryTypes = root.entry_type;
+            if (entryTypes === undefined) throw new Error('Expected split entry-type lane');
+            entryTypes[2] = ENTRY_TYPE_SPAN_OK;
+          }
+        } else {
+          if (root._rowHeaders === undefined) throw new Error('Expected packed row-header lane');
+          root._rowHeaders[0] = ENTRY_TYPE_SPAN_START;
+          root._rowHeaders[1] = ENTRY_TYPE_INFO;
+          root._rowHeaders[2] = ENTRY_TYPE_SPAN_OK;
+        }
+        root.message(0, `${physicalLayout}-root`);
+        root.message(1, `${physicalLayout}-raw`);
+        root.value(1, 42);
+        root._writeIndex = 3;
+
+        const lease = convertToLeasedArrowTable(root);
+        expect(Array.from({ length: 3 }, (_, row) => getColumnValue(lease.table, 'entry_type', row))).toEqual([
+          'span-start',
+          'info',
+          'span-ok',
+        ]);
+        expect(Array.from({ length: 3 }, (_, row) => getColumnValue(lease.table, 'message', row))).toEqual([
+          `${physicalLayout}-root`,
+          `${physicalLayout}-raw`,
+          null,
+        ]);
+        const entryTypes = lease.table.getChild('entry_type');
+        if (!entryTypes) throw new Error('Missing leased entry_type column');
+        const values = entryTypes.data[0]?.values;
+        if (!(values instanceof Int8Array)) throw new Error('Expected dictionary entry-type indices');
+        if (physicalLayout !== 'packed') {
+          if (root.entry_type === undefined) throw new Error('Expected split entry-type lane');
+          expect(values.buffer).toBe(root.entry_type.buffer);
+        } else {
+          if (root._rowHeaders === undefined) throw new Error('Expected packed row-header lane');
+          expect(values.buffer).not.toBe(root._rowHeaders.buffer);
+        }
+        lease.release();
+      }
+    });
+
+    test('keeps the single-buffer Table API semantically identical while exposing leased source reuse', () => {
+      const schema = createTestSchema({ value: S.number() });
+      const root = createSpanBuffer(schema, createTestTraceRoot('leased-single'), DEFAULT_METADATA, 8);
+      root.timestamp[0] = 2000n;
+      {
+        const entryTypes = root.entry_type;
+        if (entryTypes === undefined) throw new Error('Expected split entry-type lane');
+        entryTypes[0] = ENTRY_TYPE_INFO;
+      }
+      root.message(0, 'single');
+      root.value(0, 42);
+      root._writeIndex = 1;
+
+      const legacy = convertToArrowTable(root);
+      const lease = convertToLeasedArrowTable(root);
+      expect(lease.table.names).toEqual(legacy.names);
+      expect(lease.table.numRows).toBe(legacy.numRows);
+      for (const name of ['timestamp', 'entry_type', 'message', 'value']) {
+        expect(getColumnValue(lease.table, name, 0)).toEqual(getColumnValue(legacy, name, 0));
+      }
+      const source = root.getColumnIfAllocated('value');
+      if (!(source instanceof Float64Array)) throw new Error('Expected source Float64Array');
+      const valueColumn = lease.table.getChild('value');
+      if (!valueColumn) throw new Error('Missing leased value column');
+      expect(valueColumn.data[0]?.values.buffer).toBe(source.buffer);
+      root._traceRoot.tracer.bufferStrategy.releaseBuffer(root);
+      lease[Symbol.dispose]();
+      expect(lease.released).toBe(true);
+    });
+  });
 });