@smoothbricks/lmao 0.2.2 → 0.2.3

package/dist/lib/convertToArrow.js

@@ -7,14 +7,19 @@
  * - Text columns: Plain strings without dictionary
  * - Zero-copy wrap TypedArrays as Arrow vectors
  */
-import { clearBitRange, compareStrings, createSortedDictionary, DictionaryBuilder, getMaskTransform, sortInPlace, } from '@smoothbricks/arrow-builder';
+import { clearBitRange, compareStrings, countNulls, createSortedDictionary, DictionaryBuilder, getMaskTransform, sortInPlace, } from '@smoothbricks/arrow-builder';
 import { batchType, binary, bool, Column, dictionary, float64, int8, TimeUnit, tableFromColumns, timestamp, uint8, uint16, uint32, uint64, utf8, } from '@uwdata/flechette';
 import { createCapacityStatsTable } from './arrow/capacityStats.js';
 import { buildSortedCategoryDictionary, buildTextDictionary } from './arrow/dictionaries.js';
-import { getArrowIndexArrayConstructorOr, getArrowIndexTypeOr, makeArrowColumn, } from './arrow/flechette.js';
-import { calculateUtf8Offsets, concatenateFloat64Arrays, concatenateNullBitmaps, concatenateUint8Arrays, encodeUtf8Strings, getArrowFieldName, walkSpanTree, } from './arrow/utils.js';
-import { ENTRY_TYPE_NAMES, SYSTEM_SCHEMA_FIELD_NAMES } from './schema/systemSchema.js';
-import { getBinaryEncoder, getEnumUtf8, getEnumValues, getSchemaType } from './schema/typeGuards.js';
+import { makeArrowColumn, } from './arrow/flechette.js';
+import { createArrowLease } from './arrow/lease.js';
+import { calculateUtf8Offsets, combineColumnChunks, concatenateFloat64Arrays, concatenateNullBitmaps, concatenateUint8Arrays, encodeUtf8Strings, getArrowFieldName, } from './arrow/utils.js';
+import { appendVocabularyDictionarySuffix, getVocabularyDictionaryPrefix, PREBUILT_ENTRY_TYPE_DICTIONARY, } from './arrow/vocabularyDictionary.js';
+import { resolveEnumLookupDescriptor } from './enumMetadata.js';
+import { resolveEntryType } from './resolveMessage.js';
+import { ENTRY_TYPE_ERROR, ENTRY_TYPE_SPAN_START, ENTRY_TYPE_TRACE, SYSTEM_SCHEMA_FIELD_NAMES, } from './schema/systemSchema.js';
+import { getBinaryEncoder, getEnumUtf8, getSchemaType } from './schema/typeGuards.js';
+import { walkSpanTree } from './traceTopology.js';
 import { globalUtf8Cache } from './utf8Cache.js';
 const DictBuilder = DictionaryBuilder;
 const F64Array = Float64Array;
@@ -83,15 +88,20 @@ function appendTables(base, extra) {
     }
     return tableFromColumns(columns);
 }
-function getAllocatedStringColumn(buffer, columnName) {
-    const column = buffer.getColumnIfAllocated(columnName);
+function resolveColumnName(buffer, outputName, remaps) {
+    return remaps.get(buffer)?.sourceNames[outputName] ?? outputName;
+}
+function getAllocatedStringColumn(buffer, columnName, remaps) {
+    const sourceName = remaps ? resolveColumnName(buffer, columnName, remaps) : columnName;
+    const column = buffer.getColumnIfAllocated(sourceName);
     if (!Array.isArray(column)) {
         return undefined;
     }
     return isStringArray(column) ? column : undefined;
 }
-function getAllocatedBinaryColumn(buffer, columnName) {
-    const column = buffer.getColumnIfAllocated(columnName);
+function getAllocatedBinaryColumn(buffer, columnName, remaps) {
+    const sourceName = remaps ? resolveColumnName(buffer, columnName, remaps) : columnName;
+    const column = buffer.getColumnIfAllocated(sourceName);
     return Array.isArray(column) ? column : undefined;
 }
 //#region smoo/lmao!n/scope-attributes.arrow-fill
@@ -99,16 +109,19 @@ function getAllocatedBinaryColumn(buffer, columnName) {
 // Arrow conversion. These typed readers pull from the buffer's immutable _scopeValues; the
 // per-column fill loops below call them so a row with a direct write keeps its value and scope
 // only fills the gaps.
-function getStringScopeValue(buffer, fieldName) {
-    const value = buffer._scopeValues?.[fieldName];
+function getStringScopeValue(buffer, fieldName, remaps) {
+    const sourceName = remaps ? resolveColumnName(buffer, fieldName, remaps) : fieldName;
+    const value = buffer._scopeValues?.[sourceName];
     return typeof value === 'string' ? value : undefined;
 }
-function getNumberScopeValue(buffer, fieldName) {
-    const value = buffer._scopeValues?.[fieldName];
+function getNumberScopeValue(buffer, fieldName, remaps) {
+    const sourceName = remaps ? resolveColumnName(buffer, fieldName, remaps) : fieldName;
+    const value = buffer._scopeValues?.[sourceName];
     return typeof value === 'number' ? value : undefined;
 }
-function getBooleanScopeValue(buffer, fieldName) {
-    const value = buffer._scopeValues?.[fieldName];
+function getBooleanScopeValue(buffer, fieldName, remaps) {
+    const sourceName = remaps ? resolveColumnName(buffer, fieldName, remaps) : fieldName;
+    const value = buffer._scopeValues?.[sourceName];
     return typeof value === 'boolean' ? value : undefined;
 }
 //#endregion smoo/lmao!n/scope-attributes.arrow-fill
@@ -122,12 +135,166 @@ function isStringArray(value) {
     return Array.isArray(value) && value.every((entry) => typeof entry === 'string' || entry == null);
 }
 function indexTypeForCount(count) {
-    if (count <= 255)
+    if (count <= 0x100)
         return uint8();
-    if (count <= 65535)
+    if (count <= 65536)
         return uint16();
     return uint32();
 }
+function assertStaticMessageKind(buffer, row, entryType, denseIndex) {
+    const generation = buffer._vocabularyGeneration;
+    if (denseIndex >= generation.ids.length) {
+        throw new Error(`Invalid vocabulary dense index ${denseIndex} for generation ${generation.generation}`);
+    }
+    const expectedKind = entryType === ENTRY_TYPE_SPAN_START ? 2 : entryType >= ENTRY_TYPE_TRACE && entryType <= ENTRY_TYPE_ERROR ? 1 : 0;
+    if (expectedKind === 0) {
+        throw new Error(`Entry type ${entryType} at row ${row} cannot carry a static vocabulary identity`);
+    }
+    const actualKind = generation.kindTags[denseIndex];
+    if (actualKind !== expectedKind) {
+        throw new Error(`Vocabulary kind ${actualKind} at dense index ${denseIndex} does not match entry type ${entryType} at row ${row}`);
+    }
+}
+function buildMessageColumn(buffers, totalRows, dictionaryId, materializeEntryTypes = true) {
+    let generation = buffers[0]._vocabularyGeneration;
+    for (let index = 1; index < buffers.length; index++) {
+        const candidate = buffers[index]._vocabularyGeneration;
+        if (candidate.generation > generation.generation)
+            generation = candidate;
+    }
+    const prefix = getVocabularyDictionaryPrefix(generation);
+    const indices = new Uint32Array(totalRows);
+    const entryTypeIndices = materializeEntryTypes ? new Int8Array(totalRows) : undefined;
+    const nullBitmap = new Uint8Array(Math.ceil(totalRows / 8));
+    let nullCount = 0;
+    let rowOffset = 0;
+    let suffixValues;
+    let suffixIndices;
+    for (const buffer of buffers) {
+        const plannedStorage = buffer._opMetadata._physicalLayoutPlan?.arrowExposure.messageIdentityStorage;
+        const messageIdentityStorage = plannedStorage ??
+            (buffer._messagePhysicalLayout === 'current'
+                ? 'local-u16'
+                : buffer._messagePhysicalLayout === 'specialized'
+                    ? 'global-u32'
+                    : 'packed-row-headers');
+        for (let row = 0; row < buffer._writeIndex; row++) {
+            const outputRow = rowOffset + row;
+            const hasDedicatedMessage = (row === 0 && buffer._spanName !== undefined) || (row === 1 && buffer._terminalMessage !== undefined);
+            let dictionaryIndex;
+            let entryType;
+            if (messageIdentityStorage === 'local-u16') {
+                entryType = resolveEntryType(buffer, row);
+                const localId = buffer._messageIds?.[row];
+                const rawMessage = buffer.message_values?.[row];
+                if (row === 0 && typeof buffer._spanName === 'number') {
+                    if ((localId !== undefined && localId > 0) || rawMessage !== undefined) {
+                        throw new Error(`Local row ${row} has conflicting static message identities`);
+                    }
+                    dictionaryIndex = buffer._spanName;
+                }
+                else if (localId !== undefined && localId > 0) {
+                    if (hasDedicatedMessage || rawMessage !== undefined) {
+                        throw new Error(`Local row ${row} has conflicting static and raw message storage`);
+                    }
+                    dictionaryIndex = buffer._opMetadata._physicalLayoutPlan?.localMessageDictionary[localId - 1];
+                    if (dictionaryIndex === undefined)
+                        throw new Error(`Missing local message dictionary entry ${localId}`);
+                }
+            }
+            else if (messageIdentityStorage === 'global-u32') {
+                entryType = resolveEntryType(buffer, row);
+                const encodedDenseIndex = buffer._logHeaders?.[row];
+                if (encodedDenseIndex !== undefined && encodedDenseIndex > 0) {
+                    if (hasDedicatedMessage || buffer.message_values?.[row] !== undefined) {
+                        throw new Error(`Global row ${row} has conflicting static and raw message storage`);
+                    }
+                    dictionaryIndex = encodedDenseIndex - 1;
+                }
+            }
+            else {
+                const rowHeaders = buffer._rowHeaders;
+                if (rowHeaders === undefined)
+                    throw new TypeError('Packed message layout is missing row headers');
+                const header = rowHeaders[row];
+                entryType = header & 0xff;
+                const encodedDenseIndex = header >>> 8;
+                if (encodedDenseIndex !== 0) {
+                    if (hasDedicatedMessage) {
+                        throw new Error(`Packed row ${row} has conflicting static and dedicated message storage`);
+                    }
+                    dictionaryIndex = encodedDenseIndex - 1;
+                }
+            }
+            const dedicatedDenseIndex = row === 0 && typeof buffer._spanName === 'number' ? buffer._spanName : undefined;
+            if (dedicatedDenseIndex !== undefined) {
+                if (buffer.message_values?.[row] !== undefined) {
+                    throw new Error(`Row ${row} has conflicting dedicated static and raw message storage`);
+                }
+                dictionaryIndex ??= dedicatedDenseIndex;
+            }
+            if (entryTypeIndices !== undefined)
+                entryTypeIndices[outputRow] = entryType;
+            if (dictionaryIndex !== undefined) {
+                assertStaticMessageKind(buffer, row, entryType, dictionaryIndex);
+            }
+            else {
+                const message = row === 0 && typeof buffer._spanName === 'string'
+                    ? buffer._spanName
+                    : row === 1 && buffer._terminalMessage !== undefined
+                        ? buffer._terminalMessage
+                        : buffer.message_values?.[row];
+                if (message === undefined) {
+                    nullCount++;
+                    continue;
+                }
+                const staticIndex = prefix.valueToDenseIndex.get(message);
+                if (staticIndex !== undefined) {
+                    dictionaryIndex = staticIndex;
+                }
+                else {
+                    let dynamicIndex = suffixIndices?.get(message);
+                    if (dynamicIndex === undefined) {
+                        suffixValues ??= [];
+                        suffixIndices ??= new Map();
+                        dynamicIndex = suffixValues.length;
+                        suffixValues.push(message);
+                        suffixIndices.set(message, dynamicIndex);
+                    }
+                    dictionaryIndex = prefix.length + dynamicIndex;
+                }
+            }
+            indices[outputRow] = dictionaryIndex;
+            nullBitmap[outputRow >>> 3] |= 1 << (outputRow & 7);
+        }
+        rowOffset += buffer._writeIndex;
+    }
+    let messageDictionary = prefix.column;
+    if (suffixValues !== undefined) {
+        const encoded = globalUtf8Cache.encodeMany(suffixValues);
+        const suffix = buildColumn(buildData({
+            type: utf8(),
+            offset: 0,
+            length: suffixValues.length,
+            nullCount: 0,
+            valueOffsets: encoded.offsets,
+            data: encoded.data,
+        }));
+        messageDictionary = appendVocabularyDictionarySuffix(prefix, suffix, suffixValues);
+    }
+    return {
+        column: buildColumn(buildData({
+            type: dictionaryId === undefined ? dictionary(utf8(), uint32()) : dictionary(utf8(), uint32(), false, dictionaryId),
+            offset: 0,
+            length: totalRows,
+            nullCount,
+            data: indices,
+            nullBitmap: nullCount > 0 ? nullBitmap : undefined,
+            dictionary: messageDictionary,
+        })),
+        entryTypeIndices,
+    };
+}
 // ═══════════════════════════════════════════════════════════════════════════════
 // Shared metadata column helpers (used by both buildMetadataColumnsWithFields
 // and convertBuffersWithSharedDicts to avoid duplication)
@@ -209,57 +376,73 @@ function buildNonDictIdentityColumns(buffers, totalRows, fields, vectors) {
         nullBitmap: parentSpanIdNullCount > 0 ? parentSpanIdNulls : undefined,
     })));
 }
-/**
- * Build timestamp column from buffers.
- */
-function buildTimestampColumn(buffers, totalRows, fields, vectors) {
+function buildTimestampColumn(buffers, totalRows, fields, vectors, borrowChunks = false) {
+    const timestampType = timestamp(TimeUnit.NANOSECOND);
+    fields.push('timestamp');
+    if (borrowChunks) {
+        const chunks = buffers.map((buffer) => buildColumn(buildData({
+            type: timestampType,
+            offset: 0,
+            length: buffer._writeIndex,
+            nullCount: 0,
+            data: buffer.timestamp.subarray(0, buffer._writeIndex),
+        })));
+        vectors.push(combineColumnChunks(chunks));
+        return;
+    }
     const allTimestamps = new BigInt64Array(totalRows);
     let offset = 0;
-    for (const buf of buffers) {
-        if (!buf.timestamp) {
-            throw new Error(`Buffer missing timestamps property (_writeIndex: ${buf._writeIndex})`);
-        }
-        allTimestamps.set(buf.timestamp.subarray(0, buf._writeIndex), offset);
-        offset += buf._writeIndex;
+    for (const buffer of buffers) {
+        allTimestamps.set(buffer.timestamp.subarray(0, buffer._writeIndex), offset);
+        offset += buffer._writeIndex;
     }
-    fields.push('timestamp');
-    vectors.push(buildColumn(buildData({
-        type: timestamp(TimeUnit.NANOSECOND),
-        offset: 0,
-        length: totalRows,
-        nullCount: 0,
-        data: allTimestamps,
-    })));
+    vectors.push(buildColumn(buildData({ type: timestampType, offset: 0, length: totalRows, nullCount: 0, data: allTimestamps })));
 }
-/**
- * Build entry_type dictionary column.
- */
-function buildEntryTypeColumn(buffers, totalRows, fields, vectors, dictType) {
-    const entryTypeIndices = new Int8Array(totalRows);
-    let offset = 0;
-    for (const buf of buffers) {
-        if (!buf.entry_type) {
-            throw new Error(`Buffer missing operations property (_writeIndex: ${buf._writeIndex})`);
+function canBorrowEntryTypeChunks(buffers, borrowChunks) {
+    return (borrowChunks &&
+        buffers.every((buffer) => {
+            const declared = buffer._opMetadata._physicalLayoutPlan?.arrowExposure.entryTypeStorage;
+            return declared === 'borrowed-u8' || (declared === undefined && buffer.entry_type !== undefined);
+        }));
+}
+function buildEntryTypeColumn(buffers, totalRows, fields, vectors, dictType, borrowChunks = false, classifiedEntryTypes) {
+    const entryTypeDictionary = PREBUILT_ENTRY_TYPE_DICTIONARY;
+    const type = dictType ?? dictionary(utf8(), int8());
+    fields.push('entry_type');
+    if (canBorrowEntryTypeChunks(buffers, borrowChunks)) {
+        const chunks = buffers.map((buffer) => {
+            const source = buffer.entry_type;
+            if (source === undefined)
+                throw new TypeError('Current/specialized buffer is missing entry types');
+            const values = new Int8Array(source.buffer, source.byteOffset, buffer._writeIndex);
+            return buildColumn(buildData({
+                type,
+                offset: 0,
+                length: buffer._writeIndex,
+                nullCount: 0,
+                data: values,
+                dictionary: entryTypeDictionary,
+            }));
+        });
+        vectors.push(combineColumnChunks(chunks));
+        return;
+    }
+    const entryTypeIndices = classifiedEntryTypes ?? new Int8Array(totalRows);
+    if (classifiedEntryTypes === undefined) {
+        let offset = 0;
+        for (const buffer of buffers) {
+            for (let row = 0; row < buffer._writeIndex; row++) {
+                entryTypeIndices[offset++] = resolveEntryType(buffer, row);
+            }
         }
-        entryTypeIndices.set(buf.entry_type.subarray(0, buf._writeIndex), offset);
-        offset += buf._writeIndex;
     }
-    const entryTypeDictData = buildData({
-        type: utf8(),
-        offset: 0,
-        length: ENTRY_TYPE_NAMES.length,
-        nullCount: 0,
-        valueOffsets: calculateUtf8Offsets(ENTRY_TYPE_NAMES),
-        data: encodeUtf8Strings(ENTRY_TYPE_NAMES),
-    });
-    fields.push('entry_type');
     vectors.push(buildColumn(buildData({
-        type: dictType ?? dictionary(utf8(), int8()),
+        type,
         offset: 0,
         length: totalRows,
         nullCount: 0,
         data: entryTypeIndices,
-        dictionary: buildColumn(entryTypeDictData),
+        dictionary: entryTypeDictionary,
     })));
 }
 /**
@@ -305,13 +488,14 @@ function buildPerBufferDictColumn(buffers, totalRows, fieldName, metadataGetter,
  * Binary columns store arbitrary bytes (raw Uint8Array or encoder-encoded objects).
  * At flush time, the encoder (if present) transforms stored values into Uint8Array.
  */
-function buildBinaryColumnFromBuffers(buffers, columnName, totalRows, fieldSchema) {
+function buildBinaryColumnFromBuffers(buffers, columnName, totalRows, fieldSchema, remaps) {
     const encoder = getBinaryEncoder(fieldSchema);
     // Collect all values from buffers, encode each if encoder is present
     const allValues = [];
     for (const buf of buffers) {
-        const col = getAllocatedBinaryColumn(buf, columnName);
-        const srcNulls = buf.getNullsIfAllocated(columnName);
+        const col = getAllocatedBinaryColumn(buf, columnName, remaps);
+        const sourceName = remaps ? resolveColumnName(buf, columnName, remaps) : columnName;
+        const srcNulls = buf.getNullsIfAllocated(sourceName);
         for (let i = 0; i < buf._writeIndex; i++) {
             if (srcNulls) {
                 const isValid = (srcNulls[i >>> 3] & (1 << (i & 7))) !== 0;
@@ -388,14 +572,14 @@ function buildBinaryColumnFromBuffers(buffers, columnName, totalRows, fieldSchem
  * Dictionaries are built from the data in this batch only and are not shared
  * with other batches, even when combined into a Table.
  */
-export function convertToTable(buffer, systemColumnBuilder) {
+export function convertToTable(buffer, systemColumnBuilder, borrowChunks = false) {
     const buffers = [];
     let currentBuffer = buffer;
     while (currentBuffer) {
         buffers.push(currentBuffer);
         currentBuffer = currentBuffer._overflow;
     }
-    return convertBuffersToTable(buffers, systemColumnBuilder);
+    return convertBuffersToTable(buffers, systemColumnBuilder, borrowChunks);
 }
 /**
  * Convert multiple buffers to a single table.
@@ -404,13 +588,14 @@ export function convertToTable(buffer, systemColumnBuilder) {
  * Dictionaries are built from the data in this batch only and are not shared
  * with other batches, even when combined into a Table.
  */
-function convertBuffersToTable(buffers, systemColumnBuilder) {
+function convertBuffersToTable(buffers, systemColumnBuilder, borrowChunks = false) {
     if (buffers.length === 0)
         return tableFromColumns({});
     const totalRows = buffers.reduce((sum, buf) => sum + buf._writeIndex, 0);
     if (totalRows === 0)
         return tableFromColumns({});
     const schema = buffers[0]._logSchema;
+    const enumLookup = resolveEnumLookupDescriptor(schema);
     // Build vectors first, then derive schema from them
     // This ensures Field types and vector data types are identical (Arrow IPC requirement)
     const fields = [];
@@ -421,46 +606,16 @@ function convertBuffersToTable(buffers, systemColumnBuilder) {
         vectors.push(...systemColumns.vectors);
     }
     else {
-        // Build metadata columns - returns both fields and vectors with matching types
-        const metadataResult = buildMetadataColumnsWithFields(buffers, totalRows);
+        const borrowEntryTypes = canBorrowEntryTypeChunks(buffers, borrowChunks);
+        const messageResult = buildMessageColumn(buffers, totalRows, undefined, !borrowEntryTypes);
+        // Build metadata columns - returns both fields and vectors with matching types.
+        const metadataResult = buildMetadataColumnsWithFields(buffers, totalRows, borrowChunks, messageResult.entryTypeIndices);
         fields.push(...metadataResult.fields);
         vectors.push(...metadataResult.vectors);
-        // System attribute column: message (eager category)
-        const maskTransform = undefined; // message has no masking
-        let { dictionary: dictValues, indices, arrowIndexType, nullBitmap, } = buildSortedCategoryDictionary(buffers, 'message', maskTransform);
-        // Ensure dictionary has at least one entry (empty string) if no messages were written
-        if (dictValues.length === 0) {
-            dictValues = [''];
-        }
-        // Ensure nullBitmap is all 1s (all valid) for eager column
-        if (!nullBitmap) {
-            const bitmapBytes = Math.ceil(totalRows / 8);
-            nullBitmap = new Uint8Array(bitmapBytes);
-            nullBitmap.fill(0xff);
-        }
-        // CRITICAL: Create ONE Dictionary type instance for field and batch data
-        // Arrow Schema validates that fields with same dictionary ID have identical type references
-        const messageDictType = dictionary(utf8(), arrowIndexType);
+        // Static vocabulary values form an immutable prefix. Only raw messages
+        // missing from that prefix are interned into a first-seen suffix.
         fields.push('message');
-        const { data: messageUtf8Data, offsets: messageUtf8Offsets } = globalUtf8Cache.encodeMany(dictValues);
-        const messageDictData = buildData({
-            type: utf8(),
-            offset: 0,
-            length: dictValues.length,
-            nullCount: 0,
-            valueOffsets: messageUtf8Offsets,
-            data: messageUtf8Data,
-        });
-        const messageData = buildData({
-            type: messageDictType,
-            offset: 0,
-            length: totalRows,
-            nullCount: 0, // message is eager, never null
-            data: indices,
-            nullBitmap, // Always include nullBitmap for message (even though nullCount is 0)
-            dictionary: buildColumn(messageDictData),
-        });
-        vectors.push(buildColumn(messageData));
+        vectors.push(messageResult.column);
     }
     // Build user attribute vectors
     // Skip system schema fields - they are handled separately as system columns
@@ -471,17 +626,19 @@ function convertBuffersToTable(buffers, systemColumnBuilder) {
         const arrowFieldName = getArrowFieldName(fieldName);
         const columnName = fieldName; // User columns have no prefix
         if (lmaoType === 'enum') {
-            const enumValues = getEnumValues(fieldSchema) || [];
+            const descriptor = enumLookup.byField[fieldName];
+            if (!descriptor)
+                throw new TypeError(`Enum lookup missing for field '${fieldName}'`);
+            const enumValues = descriptor.values;
             const enumUtf8 = getEnumUtf8(fieldSchema);
-            // Get constructors from schema metadata
-            const indexArrayCtor = getArrowIndexArrayConstructorOr(fieldSchema);
-            const arrowIndexType = getArrowIndexTypeOr(fieldSchema, uint8());
+            const indexArrayCtor = descriptor.indexArrayConstructor;
+            const arrowIndexType = indexTypeForCount(enumValues.length);
             // Collect value arrays - need to handle different TypedArray types
             const valueArrays = [];
             for (const buf of buffers) {
                 const column = buf.getColumnIfAllocated(columnName);
                 if (column && column instanceof indexArrayCtor) {
-                    valueArrays.push(column.subarray(0, buf._writeIndex));
+                    valueArrays.push(column.subarray(0, buf._writeIndex).slice());
                 }
                 else {
                     valueArrays.push(new indexArrayCtor(buf._writeIndex));
@@ -589,29 +746,46 @@ function convertBuffersToTable(buffers, systemColumnBuilder) {
             }
         }
         else if (lmaoType === 'number') {
-            const valueArrays = [];
-            for (const buf of buffers) {
-                const column = buf.getColumnIfAllocated(columnName);
-                if (column && column instanceof Float64Array) {
-                    valueArrays.push(column.subarray(0, buf._writeIndex));
-                }
-                else {
-                    valueArrays.push(new F64Array(buf._writeIndex));
-                }
-            }
-            const allValues = concatenateFloat64Arrays(valueArrays);
-            const { nullBitmap, nullCount } = concatenateNullBitmaps(buffers, columnName);
             const numberType = float64();
             fields.push(arrowFieldName);
-            const numberData = buildData({
-                type: numberType,
-                offset: 0,
-                length: totalRows,
-                nullCount,
-                data: allValues,
-                nullBitmap,
-            });
-            vectors.push(buildColumn(numberData));
+            const borrowed = borrowChunks
+                ? buffers.map((buffer) => {
+                    const column = buffer.getColumnIfAllocated(columnName);
+                    const validity = buffer.getNullsIfAllocated(columnName);
+                    if (!(column instanceof Float64Array) || !(validity instanceof Uint8Array))
+                        return undefined;
+                    return buildColumn(buildData({
+                        type: numberType,
+                        offset: 0,
+                        length: buffer._writeIndex,
+                        nullCount: countNulls(validity, buffer._writeIndex),
+                        data: column.subarray(0, buffer._writeIndex),
+                        nullBitmap: validity.subarray(0, Math.ceil(buffer._writeIndex / 8)),
+                    }));
+                })
+                : undefined;
+            if (borrowed?.every((chunk) => chunk !== undefined)) {
+                vectors.push(combineColumnChunks(borrowed));
+            }
+            else {
+                const valueArrays = [];
+                for (const buffer of buffers) {
+                    const column = buffer.getColumnIfAllocated(columnName);
+                    valueArrays.push(column instanceof Float64Array
+                        ? column.subarray(0, buffer._writeIndex).slice()
+                        : new F64Array(buffer._writeIndex));
+                }
+                const allValues = concatenateFloat64Arrays(valueArrays);
+                const { nullBitmap, nullCount } = concatenateNullBitmaps(buffers, columnName);
+                vectors.push(buildColumn(buildData({
+                    type: numberType,
+                    offset: 0,
+                    length: totalRows,
+                    nullCount,
+                    data: allValues,
+                    nullBitmap,
+                })));
+            }
         }
         else if (lmaoType === 'boolean') {
             const valueArrays = [];
@@ -619,7 +793,7 @@ function convertBuffersToTable(buffers, systemColumnBuilder) {
                 const column = buf.getColumnIfAllocated(columnName);
                 if (column && column instanceof Uint8Array) {
                     const requiredBytes = Math.ceil(buf._writeIndex / 8);
-                    valueArrays.push(column.subarray(0, requiredBytes));
+                    valueArrays.push(column.subarray(0, requiredBytes).slice());
                 }
                 else {
                     const requiredBytes = Math.ceil(buf._writeIndex / 8);
@@ -655,13 +829,13 @@ function convertBuffersToTable(buffers, systemColumnBuilder) {
  * This is required for Arrow IPC serialization - Schema validates that
  * fields with same dictionary ID have identical type object references.
  */
-function buildMetadataColumnsWithFields(buffers, totalRows) {
+function buildMetadataColumnsWithFields(buffers, totalRows, borrowChunks = false, classifiedEntryTypes) {
     const fields = [];
     const vectors = [];
     // Column order: timestamp, trace_id, thread_id, span_id, parent_thread_id, parent_span_id,
     //               entry_type, package_name, package_file, git_sha
     // 1. Timestamp
-    buildTimestampColumn(buffers, totalRows, fields, vectors);
+    buildTimestampColumn(buffers, totalRows, fields, vectors, borrowChunks);
     // 2. Trace ID dictionary (built inline from buffers)
     const traceIdSet = new Set();
     for (const buf of buffers)
@@ -700,7 +874,7 @@ function buildMetadataColumnsWithFields(buffers, totalRows) {
     // 3-6. thread_id, span_id, parent_thread_id, parent_span_id
     buildNonDictIdentityColumns(buffers, totalRows, fields, vectors);
     // 7. Entry type
-    buildEntryTypeColumn(buffers, totalRows, fields, vectors);
+    buildEntryTypeColumn(buffers, totalRows, fields, vectors, undefined, borrowChunks, classifiedEntryTypes);
     // 8-10. Per-buffer metadata columns (package_name, package_file, git_sha)
     for (const [fieldName, metadataGetter] of [
         ['package_name', (m) => m.package_name],
@@ -721,6 +895,43 @@ function buildMetadataColumnsWithFields(buffers, totalRows) {
     }
     return { fields, vectors };
 }
+function isEpochPinnableBuffer(buffer) {
+    if (!('_allocator' in buffer))
+        return false;
+    const allocator = Reflect.get(buffer, '_allocator');
+    return (typeof allocator === 'object' &&
+        allocator !== null &&
+        typeof Reflect.get(allocator, 'pinMemoryEpoch') === 'function');
+}
+function createLeasedConversion(root, buffers, build) {
+    const topology = root._traceRoot._topology;
+    topology.assertLive(root);
+    const releaseTopology = topology.acquireLease();
+    const generations = new Set();
+    const allocators = new Set();
+    for (const buffer of buffers) {
+        generations.add(buffer._vocabularyGeneration);
+        if (isEpochPinnableBuffer(buffer))
+            allocators.add(buffer._allocator);
+    }
+    const epochPins = Array.from(allocators, (allocator) => allocator.pinMemoryEpoch());
+    // WASM linear-memory growth cannot preserve borrowed JS TypedArray views, so
+    // its leased result uses the existing Arrow-owned copy path. JS heap buffers
+    // expose their physical chunks directly under the topology lease.
+    const borrowChunks = epochPins.length === 0;
+    try {
+        const table = build(borrowChunks);
+        for (const pin of epochPins)
+            pin.release();
+        return createArrowLease(table, [...buffers, ...generations], [releaseTopology]);
+    }
+    catch (error) {
+        for (const pin of epochPins)
+            pin.release();
+        releaseTopology();
+        throw error;
+    }
+}
 /**
  * Convert SpanBuffer (and its overflow chain) to Arrow Table
  */
@@ -730,6 +941,19 @@ export function convertToArrowTable(buffer, systemColumnBuilder) {
         return tableFromColumns({});
     return batch;
 }
+/** Borrow JS physical Arrow chunks until the returned lease is released. */
+export function convertToLeasedArrowTable(buffer, systemColumnBuilder) {
+    const buffers = [];
+    let segment = buffer;
+    while (segment) {
+        buffers.push(segment);
+        segment = segment._overflow;
+    }
+    return createLeasedConversion(buffer, buffers, (borrowChunks) => {
+        const table = convertToTable(buffer, systemColumnBuilder, borrowChunks);
+        return table.numRows === 0 ? tableFromColumns({}) : table;
+    });
+}
 // ═══════════════════════════════════════════════════════════════════════════════
 // Tree Conversion with Shared Dictionaries
 // ═══════════════════════════════════════════════════════════════════════════════
@@ -740,25 +964,35 @@ export function convertToArrowTable(buffer, systemColumnBuilder) {
  * - Pass 1: Walk tree, build dictionaries (collect unique strings, cache UTF-8)
  * - Pass 2: Walk tree, convert each buffer to a table using shared dictionaries
  */
-export function convertSpanTreeToArrowTable(rootBuffer, _systemColumnBuilder, modulesToLogStats, periodStartNs) {
+export function convertSpanTreeToArrowTable(rootBuffer, _systemColumnBuilder, modulesToLogStats, periodStartNs, borrowChunks = false) {
     // ═══════════════════════════════════════════════════════════════════════════
     // PASS 0: Collect ALL unique schema fields from ALL buffers in the tree
     // Per specs/lmao/01k_tree_walker_and_arrow_conversion.md - child spans may have different schemas
     // (e.g., library prefixed schemas like db_query, http_status, etc.)
     // ═══════════════════════════════════════════════════════════════════════════
     const mergedSchemaFields = new Map();
-    walkSpanTree(rootBuffer, (buffer) => {
-        // Use buffer._columns directly - works for both SpanBuffer and RemappedBufferView
-        const fields = buffer._columns;
-        for (const [fieldName, fieldSchema] of fields) {
-            // Skip system schema fields - they are handled separately as system columns
-            // (message, line, error_code, exception_stack, ff_value, uint64_value)
-            if (SYSTEM_SCHEMA_FIELD_NAMES.has(fieldName)) {
-                continue;
+    const remaps = new WeakMap();
+    const enumLookupsByField = new Map();
+    walkSpanTree(rootBuffer, (buffer, remapDescriptor) => {
+        const enumLookup = resolveEnumLookupDescriptor(buffer._logSchema);
+        if (remapDescriptor) {
+            remaps.set(buffer, remapDescriptor);
+            for (const [fieldName, sourceName, fieldSchema] of remapDescriptor.columns) {
+                if (!SYSTEM_SCHEMA_FIELD_NAMES.has(fieldName) && !mergedSchemaFields.has(fieldName)) {
+                    mergedSchemaFields.set(fieldName, fieldSchema);
+                    const descriptor = enumLookup.byField[sourceName];
+                    if (descriptor)
+                        enumLookupsByField.set(fieldName, descriptor);
+                }
             }
-            // Only add if not already present (first buffer with this field wins)
-            if (!mergedSchemaFields.has(fieldName)) {
+            return;
+        }
+        for (const [fieldName, fieldSchema] of buffer._columns) {
+            if (!SYSTEM_SCHEMA_FIELD_NAMES.has(fieldName) && !mergedSchemaFields.has(fieldName)) {
                 mergedSchemaFields.set(fieldName, fieldSchema);
+                const descriptor = enumLookup.byField[fieldName];
+                if (descriptor)
+                    enumLookupsByField.set(fieldName, descriptor);
             }
         }
     });
@@ -790,11 +1024,6 @@ export function convertSpanTreeToArrowTable(rootBuffer, _systemColumnBuilder, mo
             textOriginalToMasked.set(fieldName, new Map());
         }
     }
-    // System attribute column: message (eager category, not in user schema)
-    // message is always present (eager allocation), so we need to handle it explicitly
-    categoryBuilders.set('message', new DictBuilder(globalUtf8Cache));
-    categoryMaskTransforms.set('message', undefined); // message has no masking
-    categoryOriginalToMasked.set('message', new Map());
     // Metadata column dictionaries
     // For traceId (metadata), use DictionaryBuilder (values not pre-encoded)
     const traceIdBuilder = new DictBuilder(globalUtf8Cache);
@@ -812,12 +1041,12 @@ export function convertSpanTreeToArrowTable(rootBuffer, _systemColumnBuilder, mo
             uniqueModules.add(buffer._callsiteMetadata);
         }
         for (const [fieldName, builder] of categoryBuilders) {
-            const col = getAllocatedStringColumn(buffer, fieldName);
             const maskTransform = categoryMaskTransforms.get(fieldName);
             const originalToMasked = categoryOriginalToMasked.get(fieldName);
             if (!originalToMasked) {
                 throw new Error(`Category originalToMasked map not found for field: ${fieldName}`);
             }
+            const col = getAllocatedStringColumn(buffer, fieldName, remaps);
             if (col) {
                 for (let i = 0; i < buffer._writeIndex; i++) {
                     const originalValue = col[i];
@@ -833,7 +1062,7 @@ export function convertSpanTreeToArrowTable(rootBuffer, _systemColumnBuilder, mo
             }
             // Also add scope values to dictionary (they may not be in columns)
             // Per specs/lmao/01i_span_scope_attributes.md: scope values fill NULL cells at Arrow conversion
-            const scopeValue = getStringScopeValue(buffer, fieldName);
+            const scopeValue = getStringScopeValue(buffer, fieldName, remaps);
             if (scopeValue !== undefined) {
                 let maskedScopeValue = originalToMasked.get(scopeValue);
                 if (maskedScopeValue === undefined) {
@@ -844,7 +1073,7 @@ export function convertSpanTreeToArrowTable(rootBuffer, _systemColumnBuilder, mo
             }
         }
         for (const [fieldName, builder] of textBuilders) {
-            const col = getAllocatedStringColumn(buffer, fieldName);
+            const col = getAllocatedStringColumn(buffer, fieldName, remaps);
             const maskTransform = textMaskTransforms.get(fieldName);
             const originalToMasked = textOriginalToMasked.get(fieldName);
             if (!originalToMasked) {
@@ -865,7 +1094,7 @@ export function convertSpanTreeToArrowTable(rootBuffer, _systemColumnBuilder, mo
             }
             // Also add scope values to dictionary (they may not be in columns)
             // Per specs/lmao/01i_span_scope_attributes.md: scope values fill NULL cells at Arrow conversion
-            const scopeValue = getStringScopeValue(buffer, fieldName);
+            const scopeValue = getStringScopeValue(buffer, fieldName, remaps);
             if (scopeValue !== undefined) {
                 let maskedScopeValue = originalToMasked.get(scopeValue);
                 if (maskedScopeValue === undefined) {
@@ -909,14 +1138,6 @@ export function convertSpanTreeToArrowTable(rootBuffer, _systemColumnBuilder, mo
     for (const [name, builder] of textBuilders) {
         textDicts.set(name, builder.finalize(false)); // not sorted
     }
-    // Get messageDict from categoryDicts (message is handled as a category column)
-    // If no messages were written, create an empty dictionary
-    let messageDict = categoryDicts.get('message');
-    if (!messageDict) {
-        const messageBuilder = new DictBuilder(globalUtf8Cache);
-        messageBuilder.add(''); // Single empty string entry
-        messageDict = messageBuilder.finalize(true); // sorted
-    }
     // ═══════════════════════════════════════════════════════════════════════════
     // PASS 2: Collect all buffers with non-zero rows
     // ═══════════════════════════════════════════════════════════════════════════
@@ -928,7 +1149,7 @@ export function convertSpanTreeToArrowTable(rootBuffer, _systemColumnBuilder, mo
     });
     // Build a single table from all buffers
     const batch = allBuffers.length > 0
-        ? convertBuffersWithSharedDicts(allBuffers, traceIdDict, packageDict, packagePathDict, gitShaDict, categoryDicts, textDicts, schemaFields, categoryOriginalToMasked, textOriginalToMasked)
+        ? convertBuffersWithSharedDicts(allBuffers, traceIdDict, packageDict, packagePathDict, gitShaDict, categoryDicts, textDicts, schemaFields, enumLookupsByField, categoryOriginalToMasked, textOriginalToMasked, remaps, borrowChunks)
         : undefined;
     // Create capacity stats table if needed
     if (modulesToLogStats && modulesToLogStats.length > 0) {
@@ -946,6 +1167,15 @@ export function convertSpanTreeToArrowTable(rootBuffer, _systemColumnBuilder, mo
     }
     return tableFromColumns({});
 }
+/** Convert a logical trace tree while pinning every borrowed physical chunk. */
+export function convertSpanTreeToLeasedArrowTable(rootBuffer, systemColumnBuilder, modulesToLogStats, periodStartNs) {
+    const buffers = [];
+    walkSpanTree(rootBuffer, (buffer) => {
+        if (buffer._writeIndex > 0)
+            buffers.push(buffer);
+    });
+    return createLeasedConversion(rootBuffer, buffers, (borrowChunks) => convertSpanTreeToArrowTable(rootBuffer, systemColumnBuilder, modulesToLogStats, periodStartNs, borrowChunks));
+}
 /**
  * Convert multiple buffers to a single table using pre-built dictionaries.
  *
@@ -954,10 +1184,12 @@ export function convertSpanTreeToArrowTable(rootBuffer, _systemColumnBuilder, mo
  * a single table. The resulting batches contain their own dictionary data and
  * are not shared with other batches, even when combined into a Table.
  */
-function convertBuffersWithSharedDicts(buffers, traceIdDict, packageDict, packagePathDict, gitShaDict, categoryDicts, textDicts, schemaFields, categoryOriginalToMasked, textOriginalToMasked) {
+function convertBuffersWithSharedDicts(buffers, traceIdDict, packageDict, packagePathDict, gitShaDict, categoryDicts, textDicts, schemaFields, enumLookupsByField, categoryOriginalToMasked, textOriginalToMasked, remaps, borrowChunks = false) {
     const totalRows = buffers.reduce((sum, buf) => sum + buf._writeIndex, 0);
     const fields = [];
     const vectors = [];
+    const borrowEntryTypes = canBorrowEntryTypeChunks(buffers, borrowChunks);
+    const messageResult = buildMessageColumn(buffers, totalRows, 5, !borrowEntryTypes);
     // Dictionary types with explicit IDs for IPC serialization
     const traceIdType = dictionary(utf8(), traceIdDict.arrowIndexType, false, 0);
     const entryTypeDictType = dictionary(utf8(), int8(), false, 1);
@@ -967,7 +1199,7 @@ function convertBuffersWithSharedDicts(buffers, traceIdDict, packageDict, packag
     // Column order: timestamp, trace_id, thread_id, span_id, parent_thread_id, parent_span_id,
     //               entry_type, package_name, package_file, git_sha
     // 1. Timestamp
-    buildTimestampColumn(buffers, totalRows, fields, vectors);
+    buildTimestampColumn(buffers, totalRows, fields, vectors, borrowChunks);
     // 2. Trace ID (using shared pre-built dictionary)
     const traceIdIndices = new traceIdDict.indexArrayCtor(totalRows);
     let offset = 0;
@@ -995,7 +1227,7 @@ function convertBuffersWithSharedDicts(buffers, traceIdDict, packageDict, packag
     // 3-6. thread_id, span_id, parent_thread_id, parent_span_id
     buildNonDictIdentityColumns(buffers, totalRows, fields, vectors);
     // 7. Entry type (with explicit dictionary ID for IPC)
-    buildEntryTypeColumn(buffers, totalRows, fields, vectors, entryTypeDictType);
+    buildEntryTypeColumn(buffers, totalRows, fields, vectors, entryTypeDictType, borrowChunks, messageResult.entryTypeIndices);
     // 8-10. Per-buffer metadata columns using shared pre-built dictionaries
     // Per specs/lmao/01c_context_flow_and_op_wrappers.md:
     // - Row 0 (span-start): uses callsiteModule for gitSha/packageName/packagePath
@@ -1012,46 +1244,7 @@ function convertBuffersWithSharedDicts(buffers, traceIdDict, packageDict, packag
     // System attribute and user attribute columns - concatenate from all buffers
     // ═══════════════════════════════════════════════════════════════════════════
     // System attribute column: message (at fixed position 10, before user attributes)
-    const messageDict = categoryDicts.get('message');
-    const messageOriginalToMasked = categoryOriginalToMasked.get('message');
-    if (!messageDict || !messageOriginalToMasked) {
-        throw new Error('Message dictionary or mapping not found');
-    }
-    const messageFieldType = dictionary(utf8(), messageDict.arrowIndexType, false, 5);
-    const messageIndices = new messageDict.indexArrayCtor(totalRows);
-    const messageNullBitmap = new Uint8Array(Math.ceil(totalRows / 8));
-    let messageRowOffset = 0;
-    for (const buf of buffers) {
-        const col = getAllocatedStringColumn(buf, 'message');
-        if (col) {
-            for (let i = 0; i < buf._writeIndex; i++) {
-                const v = col[i];
-                const rowIdx = messageRowOffset + i;
-                if (v != null) {
-                    const maskedValue = messageOriginalToMasked.get(v) ?? v;
-                    messageIndices[rowIdx] = messageDict.indexMap.get(maskedValue) ?? 0;
-                    messageNullBitmap[rowIdx >>> 3] |= 1 << (rowIdx & 7);
-                }
-            }
-        }
-        messageRowOffset += buf._writeIndex;
-    }
-    vectors.push(buildColumn(buildData({
-        type: messageFieldType,
-        offset: 0,
-        length: totalRows,
-        nullCount: 0, // message is eager, never null
-        data: messageIndices,
-        nullBitmap: messageNullBitmap,
-        dictionary: buildColumn(buildData({
-            type: utf8(),
-            offset: 0,
-            length: messageDict.indexMap.size,
-            nullCount: 0,
-            valueOffsets: messageDict.offsets,
-            data: messageDict.data,
-        })),
-    })));
+    vectors.push(messageResult.column);
     fields.push('message');
     // System attribute column: uint64_value (for buffer metrics - op durations, counts, etc.)
     // For span data, this is all null - only buffer metrics use it
@@ -1082,9 +1275,9 @@ function convertBuffersWithSharedDicts(buffers, traceIdDict, packageDict, packag
             let rowOffset = 0;
             // Per specs/lmao/01i_span_scope_attributes.md: scope values fill NULL cells at Arrow conversion
             for (const buf of buffers) {
-                const col = getAllocatedStringColumn(buf, columnName);
+                const col = getAllocatedStringColumn(buf, columnName, remaps);
                 // Check if this buffer has a scope value for this field
-                const scopeValue = getStringScopeValue(buf, fieldName);
+                const scopeValue = getStringScopeValue(buf, fieldName, remaps);
                 let scopeEncodedValue;
                 if (scopeValue !== undefined) {
                     const maskedScopeValue = originalToMasked.get(scopeValue) ?? scopeValue;
@@ -1153,9 +1346,9 @@ function convertBuffersWithSharedDicts(buffers, traceIdDict, packageDict, packag
             let nullCount = 0;
             let rowOffset = 0;
             for (const buf of buffers) {
-                const col = getAllocatedStringColumn(buf, columnName);
+                const col = getAllocatedStringColumn(buf, columnName, remaps);
                 // Check if this buffer has a scope value for this field
-                const scopeValue = getStringScopeValue(buf, fieldName);
+                const scopeValue = getStringScopeValue(buf, fieldName, remaps);
                 let scopeEncodedValue;
                 if (scopeValue !== undefined) {
                     const maskedScopeValue = originalToMasked.get(scopeValue) ?? scopeValue;
@@ -1214,76 +1407,101 @@ function convertBuffersWithSharedDicts(buffers, traceIdDict, packageDict, packag
             fields.push(arrowFieldName);
         }
         else if (lmaoType === 'number') {
-            const allValues = new F64Array(totalRows);
-            const nullBitmap = new Uint8Array(Math.ceil(totalRows / 8));
-            // Start with all nulls (bits cleared) - we'll set bits as we find values
-            let nullCount = 0;
-            let rowOffset = 0;
-            for (const buf of buffers) {
-                const col = buf.getColumnIfAllocated(columnName);
-                const srcNulls = buf.getNullsIfAllocated(columnName);
-                // Check if this buffer has a scope value for this field
-                const scopeValue = getNumberScopeValue(buf, fieldName);
-                if (col instanceof Float64Array) {
-                    allValues.set(col.subarray(0, buf._writeIndex), rowOffset);
-                    if (srcNulls) {
-                        // Process each row - direct write wins, then scope, then null
-                        for (let i = 0; i < buf._writeIndex; i++) {
-                            const srcByte = i >>> 3;
-                            const srcBit = i & 7;
-                            const isValid = (srcNulls[srcByte] & (1 << srcBit)) !== 0;
-                            const rowIdx = rowOffset + i;
-                            if (isValid) {
-                                // Direct write - mark as valid
-                                nullBitmap[rowIdx >>> 3] |= 1 << (rowIdx & 7);
-                            }
-                            else if (scopeValue !== undefined) {
-                                // No direct write, but have scope value - use it
-                                allValues[rowIdx] = scopeValue;
-                                nullBitmap[rowIdx >>> 3] |= 1 << (rowIdx & 7);
+            const borrowed = borrowChunks
+                ? buffers.map((buffer) => {
+                    const sourceName = resolveColumnName(buffer, columnName, remaps);
+                    if (getNumberScopeValue(buffer, fieldName, remaps) !== undefined)
+                        return undefined;
+                    const column = buffer.getColumnIfAllocated(sourceName);
+                    const validity = buffer.getNullsIfAllocated(sourceName);
+                    if (!(column instanceof Float64Array) || !(validity instanceof Uint8Array))
+                        return undefined;
+                    return buildColumn(buildData({
+                        type: float64(),
+                        offset: 0,
+                        length: buffer._writeIndex,
+                        nullCount: countNulls(validity, buffer._writeIndex),
+                        data: column.subarray(0, buffer._writeIndex),
+                        nullBitmap: validity.subarray(0, Math.ceil(buffer._writeIndex / 8)),
+                    }));
+                })
+                : undefined;
+            if (borrowed?.every((chunk) => chunk !== undefined)) {
+                vectors.push(combineColumnChunks(borrowed));
+            }
+            else {
+                const allValues = new F64Array(totalRows);
+                const nullBitmap = new Uint8Array(Math.ceil(totalRows / 8));
+                // Start with all nulls (bits cleared) - we'll set bits as we find values
+                let nullCount = 0;
+                let rowOffset = 0;
+                for (const buf of buffers) {
+                    const sourceName = resolveColumnName(buf, columnName, remaps);
+                    const col = buf.getColumnIfAllocated(sourceName);
+                    const srcNulls = buf.getNullsIfAllocated(sourceName);
+                    // Check if this buffer has a scope value for this field
+                    const scopeValue = getNumberScopeValue(buf, fieldName, remaps);
+                    if (col instanceof Float64Array) {
+                        allValues.set(col.subarray(0, buf._writeIndex), rowOffset);
+                        if (srcNulls) {
+                            // Process each row - direct write wins, then scope, then null
+                            for (let i = 0; i < buf._writeIndex; i++) {
+                                const srcByte = i >>> 3;
+                                const srcBit = i & 7;
+                                const isValid = (srcNulls[srcByte] & (1 << srcBit)) !== 0;
+                                const rowIdx = rowOffset + i;
+                                if (isValid) {
+                                    // Direct write - mark as valid
+                                    nullBitmap[rowIdx >>> 3] |= 1 << (rowIdx & 7);
+                                }
+                                else if (scopeValue !== undefined) {
+                                    // No direct write, but have scope value - use it
+                                    allValues[rowIdx] = scopeValue;
+                                    nullBitmap[rowIdx >>> 3] |= 1 << (rowIdx & 7);
+                                }
+                                else {
+                                    // No direct write and no scope - null
+                                    nullCount++;
+                                }
                             }
-                            else {
-                                // No direct write and no scope - null
-                                nullCount++;
+                        }
+                        else {
+                            // No null bitmap means column was allocated but no writes - check scope
+                            for (let i = 0; i < buf._writeIndex; i++) {
+                                const rowIdx = rowOffset + i;
+                                if (scopeValue !== undefined) {
+                                    allValues[rowIdx] = scopeValue;
+                                    nullBitmap[rowIdx >>> 3] |= 1 << (rowIdx & 7);
+                                }
+                                else {
+                                    nullCount++;
+                                }
                             }
                         }
                     }
-                    else {
-                        // No null bitmap means column was allocated but no writes - check scope
+                    else if (scopeValue !== undefined) {
+                        // Column not allocated but we have scope - fill all rows with scope value
                         for (let i = 0; i < buf._writeIndex; i++) {
                             const rowIdx = rowOffset + i;
-                            if (scopeValue !== undefined) {
-                                allValues[rowIdx] = scopeValue;
-                                nullBitmap[rowIdx >>> 3] |= 1 << (rowIdx & 7);
-                            }
-                            else {
-                                nullCount++;
-                            }
+                            allValues[rowIdx] = scopeValue;
+                            nullBitmap[rowIdx >>> 3] |= 1 << (rowIdx & 7);
                         }
                     }
-                }
-                else if (scopeValue !== undefined) {
-                    // Column not allocated but we have scope - fill all rows with scope value
-                    for (let i = 0; i < buf._writeIndex; i++) {
-                        const rowIdx = rowOffset + i;
-                        allValues[rowIdx] = scopeValue;
-                        nullBitmap[rowIdx >>> 3] |= 1 << (rowIdx & 7);
+                    else {
+                        // Column doesn't exist for this buffer and no scope - all nulls
+                        nullCount += buf._writeIndex;
                     }
+                    rowOffset += buf._writeIndex;
                 }
-                else {
-                    // Column doesn't exist for this buffer and no scope - all nulls
-                    nullCount += buf._writeIndex;
-                }
-                rowOffset += buf._writeIndex;
+                vectors.push(buildColumn(buildData({
+                    type: float64(),
+                    offset: 0,
+                    length: totalRows,
+                    nullCount,
+                    data: allValues,
+                    nullBitmap: nullCount > 0 ? nullBitmap : undefined,
+                })));
             }
-            vectors.push(buildColumn(buildData({
-                type: float64(),
-                offset: 0,
-                length: totalRows,
-                nullCount,
-                data: allValues,
-                nullBitmap: nullCount > 0 ? nullBitmap : undefined,
-            })));
             fields.push(arrowFieldName);
         }
         else if (lmaoType === 'boolean') {
@@ -1294,10 +1512,11 @@ function convertBuffersWithSharedDicts(buffers, traceIdDict, packageDict, packag
             let nullCount = 0;
             let rowOffset = 0;
             for (const buf of buffers) {
-                const col = buf.getColumnIfAllocated(columnName);
-                const srcNulls = buf.getNullsIfAllocated(columnName);
+                const sourceName = resolveColumnName(buf, columnName, remaps);
+                const col = buf.getColumnIfAllocated(sourceName);
+                const srcNulls = buf.getNullsIfAllocated(sourceName);
                 // Check if this buffer has a scope value for this field
-                const scopeValue = getBooleanScopeValue(buf, fieldName);
+                const scopeValue = getBooleanScopeValue(buf, fieldName, remaps);
                 if (col instanceof Uint8Array) {
                     // Copy boolean values bit by bit - can't avoid loop for bit-level operations
                     for (let i = 0; i < buf._writeIndex; i++) {
@@ -1367,30 +1586,25 @@ function convertBuffersWithSharedDicts(buffers, traceIdDict, packageDict, packag
             fields.push(arrowFieldName);
         }
         else if (lmaoType === 'enum') {
-            const enumValues = getEnumValues(fieldSchema) || [];
+            const descriptor = enumLookupsByField.get(fieldName);
+            if (!descriptor)
+                throw new TypeError(`Enum lookup missing for field '${fieldName}'`);
+            const enumValues = descriptor.values;
             const enumUtf8 = getEnumUtf8(fieldSchema);
-            // Get constructors from schema metadata
-            const indexArrayCtor = getArrowIndexArrayConstructorOr(fieldSchema);
-            const arrowIndexType = getArrowIndexTypeOr(fieldSchema, uint8());
+            const indexArrayCtor = descriptor.indexArrayConstructor;
+            const arrowIndexType = indexTypeForCount(enumValues.length);
             const allIndices = new indexArrayCtor(totalRows);
             const nullBitmap = new Uint8Array(Math.ceil(totalRows / 8));
             // Start with all nulls (bits cleared) - we'll set bits as we find values
             let nullCount = 0;
             let rowOffset = 0;
-            // Build enum value to index mapping for scope lookup
-            const enumValueToIndex = new Map();
-            for (let i = 0; i < enumValues.length; i++) {
-                enumValueToIndex.set(enumValues[i], i);
-            }
             for (const buf of buffers) {
-                const col = buf.getColumnIfAllocated(columnName);
-                const srcNulls = buf.getNullsIfAllocated(columnName);
+                const sourceName = resolveColumnName(buf, columnName, remaps);
+                const col = buf.getColumnIfAllocated(sourceName);
+                const srcNulls = buf.getNullsIfAllocated(sourceName);
                 // Check if this buffer has a scope value for this field
-                const scopeValue = getStringScopeValue(buf, fieldName);
-                let scopeEncodedValue;
-                if (scopeValue !== undefined) {
-                    scopeEncodedValue = enumValueToIndex.get(scopeValue);
-                }
+                const scopeValue = getStringScopeValue(buf, fieldName, remaps);
+                const scopeEncodedValue = scopeValue === undefined ? undefined : descriptor.encode(scopeValue);
                 if (col instanceof indexArrayCtor) {
                     allIndices.set(col.subarray(0, buf._writeIndex), rowOffset);
                     if (srcNulls) {
@@ -1464,7 +1678,7 @@ function convertBuffersWithSharedDicts(buffers, traceIdDict, packageDict, packag
         else if (lmaoType === 'binary') {
             // Binary columns: raw Uint8Array or encoder-wrapped values (e.g. msgpack)
             // Binary columns don't use dictionaries and are not scope-fillable
-            const { column } = buildBinaryColumnFromBuffers(buffers, columnName, totalRows, fieldSchema);
+            const { column } = buildBinaryColumnFromBuffers(buffers, columnName, totalRows, fieldSchema, remaps);
             fields.push(arrowFieldName);
             vectors.push(column);
         }