@smoothbricks/lmao 0.1.0

package/dist/lib/archive/chunkEnvelope.d.ts

@@ -0,0 +1,21 @@
+export interface TraceChunkEnvelopeInput {
+    readonly file_ref: string;
+    readonly chunk_ref: string;
+    readonly row_count: number;
+    readonly started_at_ms: number;
+    readonly ended_at_ms: number;
+    readonly partition_keys?: readonly string[];
+    readonly metadata?: Record<string, unknown>;
+}
+export interface TraceChunkEnvelope {
+    readonly chunk_id: string;
+    readonly file_ref: string;
+    readonly chunk_ref: string;
+    readonly row_count: number;
+    readonly started_at_ms: number;
+    readonly ended_at_ms: number;
+    readonly partition_keys: readonly string[];
+    readonly metadata?: Record<string, unknown>;
+}
+export declare function buildTraceChunkEnvelope(input: TraceChunkEnvelopeInput): TraceChunkEnvelope;
+//# sourceMappingURL=chunkEnvelope.d.ts.map

package/dist/lib/archive/chunkEnvelope.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"chunkEnvelope.d.ts","sourceRoot":"","sources":["../../../src/lib/archive/chunkEnvelope.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,uBAAuB;IACtC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,cAAc,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAC5C,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAC7C;AAED,MAAM,WAAW,kBAAkB;IACjC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,cAAc,EAAE,SAAS,MAAM,EAAE,CAAC;IAC3C,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAC7C;AAED,wBAAgB,uBAAuB,CAAC,KAAK,EAAE,uBAAuB,GAAG,kBAAkB,CAsB1F"}

package/dist/lib/archive/chunkEnvelope.js

@@ -0,0 +1,55 @@
+export function buildTraceChunkEnvelope(input) {
+    const canonical = {
+        chunk_ref: input.chunk_ref,
+        ended_at_ms: input.ended_at_ms,
+        file_ref: input.file_ref,
+        metadata: input.metadata ?? null,
+        partition_keys: [...(input.partition_keys ?? [])].sort(),
+        row_count: input.row_count,
+        started_at_ms: input.started_at_ms,
+    };
+    const chunk_id = `chunk_${hashDeterministic(canonical)}`;
+    return {
+        chunk_id,
+        file_ref: input.file_ref,
+        chunk_ref: input.chunk_ref,
+        row_count: input.row_count,
+        started_at_ms: input.started_at_ms,
+        ended_at_ms: input.ended_at_ms,
+        partition_keys: canonical.partition_keys,
+        metadata: input.metadata,
+    };
+}
+function hashDeterministic(value) {
+    return fnv1a64(stableSerialize(value));
+}
+function stableSerialize(value) {
+    if (value === null)
+        return 'null';
+    if (typeof value === 'number' || typeof value === 'boolean')
+        return JSON.stringify(value);
+    if (typeof value === 'string')
+        return JSON.stringify(value);
+    if (typeof value === 'bigint') {
+        return `"${value.toString()}n"`;
+    }
+    if (Array.isArray(value)) {
+        return `[${value.map((item) => stableSerialize(item)).join(',')}]`;
+    }
+    if (typeof value === 'object') {
+        // null and array excluded above — TS narrows to `object` via direct typeof check
+        const entries = Object.entries(value).sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0));
+        return `{${entries.map(([k, v]) => `${JSON.stringify(k)}:${stableSerialize(v)}`).join(',')}}`;
+    }
+    return JSON.stringify(String(value));
+}
+function fnv1a64(input) {
+    let hash = 0xcbf29ce484222325n;
+    const prime = 0x100000001b3n;
+    const mod = 0xffffffffffffffffn;
+    for (let i = 0; i < input.length; i++) {
+        hash ^= BigInt(input.charCodeAt(i));
+        hash = (hash * prime) & mod;
+    }
+    return hash.toString(16).padStart(16, '0');
+}

package/dist/lib/archive/chunkRouting.d.ts

@@ -0,0 +1,10 @@
+export type PartitionCardinality = 'single' | 'mixed' | 'unknown';
+export interface PartitionSplit<T> {
+    readonly partition: string;
+    readonly selector: string;
+    readonly row_indexes: readonly number[];
+    readonly rows: readonly T[];
+}
+export declare function inspectPartitionCardinality<T>(rows: readonly T[], partitionOf: (row: T) => string | null | undefined): PartitionCardinality;
+export declare function splitChunkByPartition<T>(rows: readonly T[], partitionOf: (row: T) => string | null | undefined, partitionField?: string): readonly PartitionSplit<T>[];
+//# sourceMappingURL=chunkRouting.d.ts.map

package/dist/lib/archive/chunkRouting.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"chunkRouting.d.ts","sourceRoot":"","sources":["../../../src/lib/archive/chunkRouting.ts"],"names":[],"mappings":"AAAA,MAAM,MAAM,oBAAoB,GAAG,QAAQ,GAAG,OAAO,GAAG,SAAS,CAAC;AAElE,MAAM,WAAW,cAAc,CAAC,CAAC;IAC/B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,WAAW,EAAE,SAAS,MAAM,EAAE,CAAC;IACxC,QAAQ,CAAC,IAAI,EAAE,SAAS,CAAC,EAAE,CAAC;CAC7B;AAED,wBAAgB,2BAA2B,CAAC,CAAC,EAC3C,IAAI,EAAE,SAAS,CAAC,EAAE,EAClB,WAAW,EAAE,CAAC,GAAG,EAAE,CAAC,KAAK,MAAM,GAAG,IAAI,GAAG,SAAS,GACjD,oBAAoB,CAkBtB;AAED,wBAAgB,qBAAqB,CAAC,CAAC,EACrC,IAAI,EAAE,SAAS,CAAC,EAAE,EAClB,WAAW,EAAE,CAAC,GAAG,EAAE,CAAC,KAAK,MAAM,GAAG,IAAI,GAAG,SAAS,EAClD,cAAc,SAAU,GACvB,SAAS,cAAc,CAAC,CAAC,CAAC,EAAE,CAiC9B"}

package/dist/lib/archive/chunkRouting.js

@@ -0,0 +1,47 @@
+export function inspectPartitionCardinality(rows, partitionOf) {
+    if (rows.length === 0) {
+        return 'unknown';
+    }
+    const partitions = new Set();
+    for (const row of rows) {
+        const partition = partitionOf(row);
+        if (partition === undefined || partition === null || partition.length === 0) {
+            return 'unknown';
+        }
+        partitions.add(partition);
+        if (partitions.size > 1) {
+            return 'mixed';
+        }
+    }
+    return partitions.size === 1 ? 'single' : 'unknown';
+}
+export function splitChunkByPartition(rows, partitionOf, partitionField = 'group') {
+    const grouped = new Map();
+    for (let index = 0; index < rows.length; index++) {
+        const row = rows[index];
+        const partition = partitionOf(row);
+        if (partition === undefined || partition === null || partition.length === 0) {
+            continue;
+        }
+        const group = grouped.get(partition);
+        if (group) {
+            group.indexes.push(index);
+            group.rows.push(row);
+            continue;
+        }
+        grouped.set(partition, { indexes: [index], rows: [row] });
+    }
+    const keys = [...grouped.keys()].sort();
+    return keys.map((partition) => {
+        const group = grouped.get(partition);
+        if (!group) {
+            throw new Error(`invariant throw: missing partition group for ${partition}`);
+        }
+        return {
+            partition,
+            selector: `${partitionField} == ${JSON.stringify(partition)}`,
+            row_indexes: group.indexes,
+            rows: group.rows,
+        };
+    });
+}

package/dist/lib/archive/chunkStats.d.ts

@@ -0,0 +1,12 @@
+import type { TraceChunkEnvelope } from './chunkEnvelope.js';
+import type { PartitionSplit } from './chunkRouting.js';
+export interface TraceChunkStats {
+    readonly chunk_id: string;
+    readonly row_count: number;
+    readonly partition_count: number;
+    readonly started_at_ms: number;
+    readonly ended_at_ms: number;
+    readonly duration_ms: number;
+}
+export declare function extractChunkStats<T>(envelope: TraceChunkEnvelope, partitions?: readonly PartitionSplit<T>[]): TraceChunkStats;
+//# sourceMappingURL=chunkStats.d.ts.map

package/dist/lib/archive/chunkStats.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"chunkStats.d.ts","sourceRoot":"","sources":["../../../src/lib/archive/chunkStats.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AAC7D,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAExD,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;CAC9B;AAED,wBAAgB,iBAAiB,CAAC,CAAC,EACjC,QAAQ,EAAE,kBAAkB,EAC5B,UAAU,GAAE,SAAS,cAAc,CAAC,CAAC,CAAC,EAAO,GAC5C,eAAe,CASjB"}

package/dist/lib/archive/chunkStats.js

@@ -0,0 +1,10 @@
+export function extractChunkStats(envelope, partitions = []) {
+    return {
+        chunk_id: envelope.chunk_id,
+        row_count: envelope.row_count,
+        partition_count: partitions.length,
+        started_at_ms: envelope.started_at_ms,
+        ended_at_ms: envelope.ended_at_ms,
+        duration_ms: Math.max(0, envelope.ended_at_ms - envelope.started_at_ms),
+    };
+}

package/dist/lib/archive/index.d.ts

@@ -0,0 +1,5 @@
+export * from './chunkCompaction.js';
+export * from './chunkEnvelope.js';
+export * from './chunkRouting.js';
+export * from './chunkStats.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/lib/archive/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/lib/archive/index.ts"],"names":[],"mappings":"AAAA,cAAc,sBAAsB,CAAC;AACrC,cAAc,oBAAoB,CAAC;AACnC,cAAc,mBAAmB,CAAC;AAClC,cAAc,iBAAiB,CAAC"}

package/dist/lib/archive/index.js

@@ -0,0 +1,4 @@
+export * from './chunkCompaction.js';
+export * from './chunkEnvelope.js';
+export * from './chunkRouting.js';
+export * from './chunkStats.js';

package/dist/lib/arrow/capacityStats.d.ts

@@ -0,0 +1,9 @@
+import { type Table } from '@uwdata/flechette';
+import type { SpanBufferConstructor } from '../spanBuffer.js';
+import type { OpMetadata } from '../types.js';
+export interface CapacityStatsEntry {
+    bufferClass: SpanBufferConstructor;
+    metadata: OpMetadata;
+}
+export declare function createCapacityStatsTable(entriesToLog: CapacityStatsEntry[], periodStartNs: bigint, mergedSchema: Record<string, unknown>): Table;
+//# sourceMappingURL=capacityStats.d.ts.map

package/dist/lib/arrow/capacityStats.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"capacityStats.d.ts","sourceRoot":"","sources":["../../../src/lib/arrow/capacityStats.ts"],"names":[],"mappings":"AAAA,OAAO,EAOL,KAAK,KAAK,EAQX,MAAM,mBAAmB,CAAC;AAG3B,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,kBAAkB,CAAC;AAC9D,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAI9C,MAAM,WAAW,kBAAkB;IACjC,WAAW,EAAE,qBAAqB,CAAC;IACnC,QAAQ,EAAE,UAAU,CAAC;CACtB;AAiCD,wBAAgB,wBAAwB,CACtC,YAAY,EAAE,kBAAkB,EAAE,EAClC,aAAa,EAAE,MAAM,EACrB,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GACpC,KAAK,CAoRP"}

package/dist/lib/arrow/capacityStats.js

@@ -0,0 +1,231 @@
+import { bool, dictionary, float64, int8, TimeUnit, tableFromColumns, timestamp, uint8, uint32, uint64, utf8, } from '@uwdata/flechette';
+import { SYSTEM_SCHEMA_FIELD_NAMES } from '../schema/systemSchema.js';
+import { getSchemaType } from '../schema/typeGuards.js';
+import { getArrowIndexTypeOr, makeArrowColumn } from './flechette.js';
+import { getArrowFieldName } from './utils.js';
+const F64Array = Float64Array;
+function encodeUtf8(strings) {
+    const enc = new TextEncoder();
+    const bytes = strings.map((s) => enc.encode(s));
+    const total = bytes.reduce((n, b) => n + b.length, 0);
+    const data = new Uint8Array(total);
+    const offsets = new Int32Array(strings.length + 1);
+    let pos = 0;
+    for (let i = 0; i < bytes.length; i++) {
+        data.set(bytes[i], pos);
+        offsets[i] = pos;
+        pos += bytes[i].length;
+    }
+    offsets[strings.length] = pos;
+    return { data, offsets };
+}
+function indexTypeForCount(count) {
+    return count <= 255 ? uint8() : uint32();
+}
+function append(entries, name, column) {
+    entries.push([name, column]);
+}
+export function createCapacityStatsTable(entriesToLog, periodStartNs, mergedSchema) {
+    if (entriesToLog.length === 0)
+        return tableFromColumns({});
+    const rowsPerModule = 4;
+    const totalRows = entriesToLog.length * rowsPerModule;
+    const entryTypeStrings = ['period-start', 'buffer-writes', 'buffer-spans', 'buffer-capacity'];
+    const [periodStartIdx, writesIdx, spansIdx, capacityIdx] = [0, 1, 2, 3];
+    const traceIdStrings = [''];
+    const packageNameStrings = Array.from(new Set(entriesToLog.map((e) => e.metadata.package_name)));
+    const packageFileStrings = Array.from(new Set(entriesToLog.map((e) => e.metadata.package_file)));
+    const gitShaStrings = Array.from(new Set(entriesToLog.map((e) => e.metadata.git_sha)));
+    const messageStrings = [''];
+    const packageNameMap = new Map(packageNameStrings.map((v, i) => [v, i]));
+    const packageFileMap = new Map(packageFileStrings.map((v, i) => [v, i]));
+    const gitShaMap = new Map(gitShaStrings.map((v, i) => [v, i]));
+    const entryTypes = new Int8Array(totalRows);
+    const traceIds = new Uint8Array(totalRows);
+    const packageNames = new Uint32Array(totalRows);
+    const packageFiles = new Uint32Array(totalRows);
+    const gitShas = new Uint32Array(totalRows);
+    const messages = new Uint8Array(totalRows);
+    const timestamps = new BigInt64Array(totalRows);
+    const threadIds = new BigUint64Array(totalRows);
+    const spanIds = new Uint32Array(totalRows);
+    const parentThreadIds = new BigUint64Array(totalRows);
+    const parentSpanIds = new Uint32Array(totalRows);
+    const uint64Values = new BigUint64Array(totalRows);
+    const parentThreadValidity = new Uint8Array(Math.ceil(totalRows / 8));
+    const parentSpanValidity = new Uint8Array(Math.ceil(totalRows / 8));
+    for (let m = 0; m < entriesToLog.length; m++) {
+        const base = m * rowsPerModule;
+        const { bufferClass, metadata } = entriesToLog[m];
+        const pName = packageNameMap.get(metadata.package_name) ?? 0;
+        const pFile = packageFileMap.get(metadata.package_file) ?? 0;
+        const gSha = gitShaMap.get(metadata.git_sha) ?? 0;
+        const stats = bufferClass.stats;
+        const values = [periodStartNs, BigInt(stats.totalWrites), BigInt(stats.spansCreated), BigInt(stats.capacity)];
+        const kinds = [periodStartIdx, writesIdx, spansIdx, capacityIdx];
+        for (let i = 0; i < rowsPerModule; i++) {
+            const row = base + i;
+            entryTypes[row] = kinds[i];
+            traceIds[row] = 0;
+            packageNames[row] = pName;
+            packageFiles[row] = pFile;
+            gitShas[row] = gSha;
+            messages[row] = 0;
+            uint64Values[row] = values[i];
+            timestamps[row] = periodStartNs;
+            threadIds[row] = 0n;
+            spanIds[row] = 0;
+            parentThreadIds[row] = 0n;
+            parentSpanIds[row] = 0;
+        }
+    }
+    const entryTypeUtf8 = encodeUtf8(entryTypeStrings);
+    const traceIdUtf8 = encodeUtf8(traceIdStrings);
+    const packageNameUtf8 = encodeUtf8(packageNameStrings);
+    const packageFileUtf8 = encodeUtf8(packageFileStrings);
+    const gitShaUtf8 = encodeUtf8(gitShaStrings);
+    const messageUtf8 = encodeUtf8(messageStrings);
+    const cols = [];
+    append(cols, 'timestamp', makeArrowColumn({ type: timestamp(TimeUnit.NANOSECOND), length: totalRows, nullCount: 0, values: timestamps }));
+    append(cols, 'trace_id', makeArrowColumn({
+        type: dictionary(utf8(), indexTypeForCount(traceIdStrings.length), false, 0),
+        length: totalRows,
+        nullCount: 0,
+        values: traceIds,
+        dictionary: makeArrowColumn({
+            type: utf8(),
+            length: traceIdStrings.length,
+            nullCount: 0,
+            values: traceIdUtf8.data,
+            offsets: traceIdUtf8.offsets,
+        }),
+    }));
+    append(cols, 'thread_id', makeArrowColumn({ type: uint64(), length: totalRows, nullCount: 0, values: threadIds }));
+    append(cols, 'span_id', makeArrowColumn({ type: uint32(), length: totalRows, nullCount: 0, values: spanIds }));
+    append(cols, 'parent_thread_id', makeArrowColumn({
+        type: uint64(),
+        length: totalRows,
+        nullCount: totalRows,
+        values: parentThreadIds,
+        validity: parentThreadValidity,
+    }));
+    append(cols, 'parent_span_id', makeArrowColumn({
+        type: uint32(),
+        length: totalRows,
+        nullCount: totalRows,
+        values: parentSpanIds,
+        validity: parentSpanValidity,
+    }));
+    append(cols, 'entry_type', makeArrowColumn({
+        type: dictionary(utf8(), int8(), false, 1),
+        length: totalRows,
+        nullCount: 0,
+        values: entryTypes,
+        dictionary: makeArrowColumn({
+            type: utf8(),
+            length: entryTypeStrings.length,
+            nullCount: 0,
+            values: entryTypeUtf8.data,
+            offsets: entryTypeUtf8.offsets,
+        }),
+    }));
+    append(cols, 'package_name', makeArrowColumn({
+        type: dictionary(utf8(), indexTypeForCount(packageNameStrings.length), false, 2),
+        length: totalRows,
+        nullCount: 0,
+        values: packageNames,
+        dictionary: makeArrowColumn({
+            type: utf8(),
+            length: packageNameStrings.length,
+            nullCount: 0,
+            values: packageNameUtf8.data,
+            offsets: packageNameUtf8.offsets,
+        }),
+    }));
+    append(cols, 'package_file', makeArrowColumn({
+        type: dictionary(utf8(), indexTypeForCount(packageFileStrings.length), false, 3),
+        length: totalRows,
+        nullCount: 0,
+        values: packageFiles,
+        dictionary: makeArrowColumn({
+            type: utf8(),
+            length: packageFileStrings.length,
+            nullCount: 0,
+            values: packageFileUtf8.data,
+            offsets: packageFileUtf8.offsets,
+        }),
+    }));
+    append(cols, 'git_sha', makeArrowColumn({
+        type: dictionary(utf8(), indexTypeForCount(gitShaStrings.length), false, 4),
+        length: totalRows,
+        nullCount: 0,
+        values: gitShas,
+        dictionary: makeArrowColumn({
+            type: utf8(),
+            length: gitShaStrings.length,
+            nullCount: 0,
+            values: gitShaUtf8.data,
+            offsets: gitShaUtf8.offsets,
+        }),
+    }));
+    append(cols, 'message', makeArrowColumn({
+        type: dictionary(utf8(), indexTypeForCount(messageStrings.length), false, 5),
+        length: totalRows,
+        nullCount: 0,
+        values: messages,
+        dictionary: makeArrowColumn({
+            type: utf8(),
+            length: messageStrings.length,
+            nullCount: 0,
+            values: messageUtf8.data,
+            offsets: messageUtf8.offsets,
+        }),
+    }));
+    append(cols, 'uint64_value', makeArrowColumn({ type: uint64(), length: totalRows, nullCount: 0, values: uint64Values }));
+    for (const [fieldName, fieldSchema] of Object.entries(mergedSchema)) {
+        if (SYSTEM_SCHEMA_FIELD_NAMES.has(fieldName))
+            continue;
+        const lmaoType = getSchemaType(fieldSchema);
+        const arrowFieldName = getArrowFieldName(fieldName);
+        if (lmaoType === 'number') {
+            append(cols, arrowFieldName, makeArrowColumn({
+                type: float64(),
+                length: totalRows,
+                nullCount: totalRows,
+                values: new F64Array(totalRows),
+                validity: new Uint8Array(Math.ceil(totalRows / 8)),
+            }));
+            continue;
+        }
+        if (lmaoType === 'boolean') {
+            append(cols, arrowFieldName, makeArrowColumn({
+                type: bool(),
+                length: totalRows,
+                nullCount: totalRows,
+                values: new Uint8Array(Math.ceil(totalRows / 8)),
+                validity: new Uint8Array(Math.ceil(totalRows / 8)),
+            }));
+            continue;
+        }
+        let idxType = uint8();
+        if (lmaoType === 'enum') {
+            idxType = getArrowIndexTypeOr(fieldSchema, uint8());
+        }
+        const dictType = dictionary(utf8(), idxType);
+        append(cols, arrowFieldName, makeArrowColumn({
+            type: dictType,
+            length: totalRows,
+            nullCount: totalRows,
+            values: new Uint8Array(totalRows),
+            validity: new Uint8Array(Math.ceil(totalRows / 8)),
+            dictionary: makeArrowColumn({
+                type: utf8(),
+                length: 1,
+                nullCount: 0,
+                values: new Uint8Array(0),
+                offsets: new Int32Array([0, 0]),
+            }),
+        }));
+    }
+    return tableFromColumns(cols);
+}

package/dist/lib/arrow/dictionaries.d.ts

@@ -0,0 +1,27 @@
+/**
+ * Dictionary building for Arrow conversion
+ */
+import { type IntType } from '@uwdata/flechette';
+import type { AnySpanBuffer } from '../types.js';
+/**
+ * Dictionary build result with consistent shape for V8 optimization.
+ * Using a class ensures all instances share the same hidden class.
+ */
+export declare class DictionaryBuildResult {
+    readonly dictionary: string[];
+    readonly indices: Uint8Array | Uint16Array | Uint32Array;
+    readonly indexArrayCtor: Uint8ArrayConstructor | Uint16ArrayConstructor | Uint32ArrayConstructor;
+    readonly arrowIndexType: IntType;
+    readonly nullBitmap: Uint8Array | undefined;
+    readonly nullCount: number;
+    constructor(dictionary: string[], indices: Uint8Array | Uint16Array | Uint32Array, indexArrayCtor: Uint8ArrayConstructor | Uint16ArrayConstructor | Uint32ArrayConstructor, arrowIndexType: IntType, nullBitmap: Uint8Array | undefined, nullCount: number);
+}
+/**
+ * Build a sorted category dictionary from buffers
+ */
+export declare function buildSortedCategoryDictionary(buffers: AnySpanBuffer[], columnName: string, maskTransform?: (value: string) => string): DictionaryBuildResult;
+/**
+ * Build a text dictionary from buffers (not sorted, uses frequency-based optimization)
+ */
+export declare function buildTextDictionary(buffers: AnySpanBuffer[], columnName: string, maskTransform?: (value: string) => string): DictionaryBuildResult | null;
+//# sourceMappingURL=dictionaries.d.ts.map

package/dist/lib/arrow/dictionaries.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"dictionaries.d.ts","sourceRoot":"","sources":["../../../src/lib/arrow/dictionaries.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,KAAK,OAAO,EAAyB,MAAM,mBAAmB,CAAC;AACxE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAGjD;;;GAGG;AACH,qBAAa,qBAAqB;aAEd,UAAU,EAAE,MAAM,EAAE;aACpB,OAAO,EAAE,UAAU,GAAG,WAAW,GAAG,WAAW;aAC/C,cAAc,EAAE,qBAAqB,GAAG,sBAAsB,GAAG,sBAAsB;aACvF,cAAc,EAAE,OAAO;aACvB,UAAU,EAAE,UAAU,GAAG,SAAS;aAClC,SAAS,EAAE,MAAM;gBALjB,UAAU,EAAE,MAAM,EAAE,EACpB,OAAO,EAAE,UAAU,GAAG,WAAW,GAAG,WAAW,EAC/C,cAAc,EAAE,qBAAqB,GAAG,sBAAsB,GAAG,sBAAsB,EACvF,cAAc,EAAE,OAAO,EACvB,UAAU,EAAE,UAAU,GAAG,SAAS,EAClC,SAAS,EAAE,MAAM;CAEpC;AAED;;GAEG;AACH,wBAAgB,6BAA6B,CAC3C,OAAO,EAAE,aAAa,EAAE,EACxB,UAAU,EAAE,MAAM,EAClB,aAAa,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,MAAM,GACxC,qBAAqB,CAkDvB;AAED;;GAEG;AACH,wBAAgB,mBAAmB,CACjC,OAAO,EAAE,aAAa,EAAE,EACxB,UAAU,EAAE,MAAM,EAClB,aAAa,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,MAAM,GACxC,qBAAqB,GAAG,IAAI,CAqE9B"}

package/dist/lib/arrow/dictionaries.js

@@ -0,0 +1,135 @@
+/**
+ * Dictionary building for Arrow conversion
+ */
+import { uint8, uint16, uint32 } from '@uwdata/flechette';
+import { concatenateNullBitmaps } from './utils.js';
+/**
+ * Dictionary build result with consistent shape for V8 optimization.
+ * Using a class ensures all instances share the same hidden class.
+ */
+export class DictionaryBuildResult {
+    dictionary;
+    indices;
+    indexArrayCtor;
+    arrowIndexType;
+    nullBitmap;
+    nullCount;
+    constructor(dictionary, indices, indexArrayCtor, arrowIndexType, nullBitmap, nullCount) {
+        this.dictionary = dictionary;
+        this.indices = indices;
+        this.indexArrayCtor = indexArrayCtor;
+        this.arrowIndexType = arrowIndexType;
+        this.nullBitmap = nullBitmap;
+        this.nullCount = nullCount;
+    }
+}
+/**
+ * Build a sorted category dictionary from buffers
+ */
+export function buildSortedCategoryDictionary(buffers, columnName, maskTransform) {
+    const totalRows = buffers.reduce((sum, buf) => sum + buf._writeIndex, 0);
+    // Build mapping from original value to masked value (for dictionary lookup)
+    // and collect unique masked values for the dictionary
+    const uniqueMaskedStrings = new Set();
+    const originalToMasked = new Map();
+    for (const buf of buffers) {
+        const column = buf.getColumnIfAllocated(columnName);
+        if (column && Array.isArray(column)) {
+            for (let i = 0; i < buf._writeIndex; i++) {
+                const value = column[i];
+                if (typeof value === 'string' && !originalToMasked.has(value)) {
+                    const maskedValue = maskTransform ? maskTransform(value) : value;
+                    originalToMasked.set(value, maskedValue);
+                    uniqueMaskedStrings.add(maskedValue);
+                }
+            }
+        }
+    }
+    const dictionary = Array.from(uniqueMaskedStrings).sort();
+    const maskedToIndex = new Map(dictionary.map((s, i) => [s, i]));
+    // Determine index type constructors based on dictionary size
+    const uniqueCount = dictionary.length;
+    const indexArrayCtor = uniqueCount <= 255 ? Uint8Array : uniqueCount <= 65535 ? Uint16Array : Uint32Array;
+    const arrowIndexType = uniqueCount <= 255 ? uint8() : uniqueCount <= 65535 ? uint16() : uint32();
+    const indices = new indexArrayCtor(totalRows);
+    const { nullBitmap, nullCount } = concatenateNullBitmaps(buffers, columnName);
+    let rowOffset = 0;
+    for (const buf of buffers) {
+        const column = buf.getColumnIfAllocated(columnName);
+        if (column && Array.isArray(column)) {
+            for (let i = 0; i < buf._writeIndex; i++) {
+                const value = column[i];
+                if (typeof value === 'string') {
+                    const maskedValue = originalToMasked.get(value) ?? value;
+                    indices[rowOffset + i] = maskedToIndex.get(maskedValue) ?? 0;
+                }
+                else {
+                    indices[rowOffset + i] = 0;
+                }
+            }
+        }
+        rowOffset += buf._writeIndex;
+    }
+    return new DictionaryBuildResult(dictionary, indices, indexArrayCtor, arrowIndexType, nullBitmap, nullCount);
+}
+/**
+ * Build a text dictionary from buffers (not sorted, uses frequency-based optimization)
+ */
+export function buildTextDictionary(buffers, columnName, maskTransform) {
+    const totalRows = buffers.reduce((sum, buf) => sum + buf._writeIndex, 0);
+    // Build mapping from original value to masked value and track frequency of masked values
+    const frequencyMap = new Map();
+    const originalToMasked = new Map();
+    for (const buf of buffers) {
+        const column = buf.getColumnIfAllocated(columnName);
+        if (column && Array.isArray(column)) {
+            for (let i = 0; i < buf._writeIndex; i++) {
+                const value = column[i];
+                if (typeof value === 'string') {
+                    let maskedValue;
+                    if (originalToMasked.has(value)) {
+                        const masked = originalToMasked.get(value);
+                        if (masked === undefined) {
+                            throw new Error(`Masked value not found for: ${value}`);
+                        }
+                        maskedValue = masked;
+                    }
+                    else {
+                        maskedValue = maskTransform ? maskTransform(value) : value;
+                        originalToMasked.set(value, maskedValue);
+                    }
+                    frequencyMap.set(maskedValue, (frequencyMap.get(maskedValue) ?? 0) + 1);
+                }
+            }
+        }
+    }
+    if (frequencyMap.size === 0) {
+        return new DictionaryBuildResult([], new Uint32Array(totalRows), Uint32Array, uint32(), new Uint8Array(Math.ceil(totalRows / 8)), totalRows);
+    }
+    const dictionary = Array.from(frequencyMap.keys());
+    const maskedToIndex = new Map(dictionary.map((s, i) => [s, i]));
+    // Determine index type constructors based on dictionary size
+    const uniqueCount = dictionary.length;
+    const indexArrayCtor = uniqueCount <= 255 ? Uint8Array : uniqueCount <= 65535 ? Uint16Array : Uint32Array;
+    const arrowIndexType = uniqueCount <= 255 ? uint8() : uniqueCount <= 65535 ? uint16() : uint32();
+    const indices = new indexArrayCtor(totalRows);
+    const { nullBitmap, nullCount } = concatenateNullBitmaps(buffers, columnName);
+    let rowOffset = 0;
+    for (const buf of buffers) {
+        const column = buf.getColumnIfAllocated(columnName);
+        if (column && Array.isArray(column)) {
+            for (let i = 0; i < buf._writeIndex; i++) {
+                const value = column[i];
+                if (typeof value === 'string') {
+                    const maskedValue = originalToMasked.get(value) ?? value;
+                    indices[rowOffset + i] = maskedToIndex.get(maskedValue) ?? 0;
+                }
+                else {
+                    indices[rowOffset + i] = 0;
+                }
+            }
+        }
+        rowOffset += buf._writeIndex;
+    }
+    return new DictionaryBuildResult(dictionary, indices, indexArrayCtor, arrowIndexType, nullBitmap, nullCount);
+}

package/dist/lib/arrow/flechette.d.ts

@@ -0,0 +1,40 @@
+import { bool, Column, type DataType, type dictionary, type IntType, type utf8 } from '@uwdata/flechette';
+export type ArrowIndexArray = Uint8Array | Uint16Array | Uint32Array;
+export type ArrowIndexArrayConstructor = Uint8ArrayConstructor | Uint16ArrayConstructor | Uint32ArrayConstructor;
+export type BoolType = ReturnType<typeof bool>;
+export type DictionaryType = ReturnType<typeof dictionary>;
+export type Utf8Type = ReturnType<typeof utf8>;
+type ColumnDataBase<TType, TValues = unknown> = {
+    type: TType;
+    length: number;
+    nullCount: number;
+    values?: TValues;
+    validity?: Uint8Array;
+};
+export type DictionaryColumnData<TValues = unknown> = ColumnDataBase<DictionaryType, TValues> & {
+    dictionary: Column<unknown>;
+};
+export type Utf8ColumnData = ColumnDataBase<Utf8Type, Uint8Array> & {
+    offsets: Int32Array;
+};
+export type BoolColumnData = ColumnDataBase<BoolType, Uint8Array>;
+export type GenericColumnData<TType extends DataType = DataType, TValues = unknown> = ColumnDataBase<TType, TValues>;
+export declare function hasTypeId(value: unknown): value is {
+    readonly typeId: unknown;
+};
+export declare function isBoolType(value: unknown): value is BoolType;
+export declare function hasArrowIndexType(value: unknown): value is {
+    readonly __arrow_index_type: IntType;
+};
+export declare function hasArrowIndexMetadata(value: unknown): value is {
+    readonly __arrow_index_type: IntType;
+    readonly __index_array_ctor: ArrowIndexArrayConstructor;
+};
+export declare function getArrowIndexTypeOr(value: unknown, fallback: IntType): IntType;
+export declare function getArrowIndexArrayConstructorOr(value: unknown, fallback?: ArrowIndexArrayConstructor): ArrowIndexArrayConstructor;
+export declare function makeArrowColumn(data: DictionaryColumnData): Column<unknown>;
+export declare function makeArrowColumn(data: Utf8ColumnData): Column<unknown>;
+export declare function makeArrowColumn(data: BoolColumnData): Column<unknown>;
+export declare function makeArrowColumn(data: GenericColumnData): Column<unknown>;
+export {};
+//# sourceMappingURL=flechette.d.ts.map

package/dist/lib/arrow/flechette.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"flechette.d.ts","sourceRoot":"","sources":["../../../src/lib/arrow/flechette.ts"],"names":[],"mappings":"AAAA,OAAO,EAAa,IAAI,EAAE,MAAM,EAAE,KAAK,QAAQ,EAAE,KAAK,UAAU,EAAE,KAAK,OAAO,EAAE,KAAK,IAAI,EAAE,MAAM,mBAAmB,CAAC;AAKrH,MAAM,MAAM,eAAe,GAAG,UAAU,GAAG,WAAW,GAAG,WAAW,CAAC;AACrE,MAAM,MAAM,0BAA0B,GAAG,qBAAqB,GAAG,sBAAsB,GAAG,sBAAsB,CAAC;AACjH,MAAM,MAAM,QAAQ,GAAG,UAAU,CAAC,OAAO,IAAI,CAAC,CAAC;AAC/C,MAAM,MAAM,cAAc,GAAG,UAAU,CAAC,OAAO,UAAU,CAAC,CAAC;AAC3D,MAAM,MAAM,QAAQ,GAAG,UAAU,CAAC,OAAO,IAAI,CAAC,CAAC;AAE/C,KAAK,cAAc,CAAC,KAAK,EAAE,OAAO,GAAG,OAAO,IAAI;IAC9C,IAAI,EAAE,KAAK,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,QAAQ,CAAC,EAAE,UAAU,CAAC;CACvB,CAAC;AAEF,MAAM,MAAM,oBAAoB,CAAC,OAAO,GAAG,OAAO,IAAI,cAAc,CAAC,cAAc,EAAE,OAAO,CAAC,GAAG;IAC9F,UAAU,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC;CAC7B,CAAC;AACF,MAAM,MAAM,cAAc,GAAG,cAAc,CAAC,QAAQ,EAAE,UAAU,CAAC,GAAG;IAAE,OAAO,EAAE,UAAU,CAAA;CAAE,CAAC;AAC5F,MAAM,MAAM,cAAc,GAAG,cAAc,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;AAClE,MAAM,MAAM,iBAAiB,CAAC,KAAK,SAAS,QAAQ,GAAG,QAAQ,EAAE,OAAO,GAAG,OAAO,IAAI,cAAc,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;AAErH,wBAAgB,SAAS,CAAC,KAAK,EAAE,OAAO,GAAG,KAAK,IAAI;IAAE,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAA;CAAE,CAE/E;AAED,wBAAgB,UAAU,CAAC,KAAK,EAAE,OAAO,GAAG,KAAK,IAAI,QAAQ,CAE5D;AAMD,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,OAAO,GAAG,KAAK,IAAI;IAAE,QAAQ,CAAC,kBAAkB,EAAE,OAAO,CAAA;CAAE,CAEnG;AAED,wBAAgB,qBAAqB,CACnC,KAAK,EAAE,OAAO,GACb,KAAK,IAAI;IAAE,QAAQ,CAAC,kBAAkB,EAAE,OAAO,CAAC;IAAC,QAAQ,CAAC,kBAAkB,EAAE,0BAA0B,CAAA;CAAE,CAI5G;AAED,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,OAAO,GAAG,OAAO,CAE9E;AAED,wBAAgB,+BAA+B,CAC7C,KAAK,EAAE,OAAO,EACd,QAAQ,GAAE,0BAAuC,GAChD,0BAA0B,CAE5B;AAED,wBAAgB,eAAe,CAAC,IAAI,EAAE,oBAAoB,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC;AAC7E,wBAAgB,eAAe,CAAC,IAAI,EAAE,cAAc,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC;AACvE,wBAAgB,eAAe,CAAC,IAAI,EAAE,cAAc,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC;AACvE,wBAAgB,eAAe,CAAC,IAAI,EAAE,iBAAiB,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC"}