@smoothbricks/lmao 0.2.2 → 0.3.0

package/src/lib/convertToArrow.ts

@@ -11,6 +11,7 @@
 import {
   clearBitRange,
   compareStrings,
+  countNulls,
   createSortedDictionary,
   DictionaryBuilder,
   type FinalizedDictionary,
@@ -43,31 +44,47 @@ import { buildSortedCategoryDictionary, buildTextDictionary } from './arrow/dict
 import {
   type DictionaryColumnData,
   type GenericColumnData,
-  getArrowIndexArrayConstructorOr,
-  getArrowIndexTypeOr,
   makeArrowColumn,
   type Utf8ColumnData,
 } from './arrow/flechette.js';
+import { type ArrowLease, createArrowLease } from './arrow/lease.js';
 import {
   calculateUtf8Offsets,
+  combineColumnChunks,
   concatenateFloat64Arrays,
   concatenateNullBitmaps,
   concatenateUint8Arrays,
   encodeUtf8Strings,
   getArrowFieldName,
-  walkSpanTree,
 } from './arrow/utils.js';
-import { ENTRY_TYPE_NAMES, SYSTEM_SCHEMA_FIELD_NAMES } from './schema/systemSchema.js';
-import { getBinaryEncoder, getEnumUtf8, getEnumValues, getSchemaType } from './schema/typeGuards.js';
+import {
+  appendVocabularyDictionarySuffix,
+  getVocabularyDictionaryPrefix,
+  PREBUILT_ENTRY_TYPE_DICTIONARY,
+} from './arrow/vocabularyDictionary.js';
+import { type EnumLookupDescriptor, resolveEnumLookupDescriptor } from './enumMetadata.js';
+import type { RemapDescriptor } from './logBinding.js';
+import { resolveEntryType } from './resolveMessage.js';
+import {
+  ENTRY_TYPE_ERROR,
+  ENTRY_TYPE_SPAN_START,
+  ENTRY_TYPE_TRACE,
+  SYSTEM_SCHEMA_FIELD_NAMES,
+} from './schema/systemSchema.js';
+import { getBinaryEncoder, getEnumUtf8, getSchemaType } from './schema/typeGuards.js';
 import type { LogSchema } from './schema/types.js';
+import { walkSpanTree } from './traceTopology.js';
 import type { AnySpanBuffer, OpMetadata } from './types.js';
 import { globalUtf8Cache } from './utf8Cache.js';
+import type { VocabularyGeneration } from './vocabularyRegistry.js';
 
 const DictBuilder = DictionaryBuilder;
 const F64Array = Float64Array;
 
 type BuiltArrowColumn = { column: Column<unknown>; nullCount: number };
 type BuiltMetadataColumns = { fields: string[]; vectors: Column<unknown>[] };
+type BuiltMessageColumn = { column: Column<unknown>; entryTypeIndices?: Int8Array };
+type RemapsByBuffer = WeakMap<AnySpanBuffer, RemapDescriptor>;
 
 export type SystemColumnBuilder = (
   buffer: AnySpanBuffer,
@@ -173,16 +190,30 @@ function appendTables(base: Table, extra: Table): Table {
   return tableFromColumns(columns);
 }
 
-function getAllocatedStringColumn(buffer: AnySpanBuffer, columnName: string): string[] | undefined {
-  const column = buffer.getColumnIfAllocated(columnName);
+function resolveColumnName(buffer: AnySpanBuffer, outputName: string, remaps: RemapsByBuffer): string {
+  return remaps.get(buffer)?.sourceNames[outputName] ?? outputName;
+}
+
+function getAllocatedStringColumn(
+  buffer: AnySpanBuffer,
+  columnName: string,
+  remaps?: RemapsByBuffer,
+): string[] | undefined {
+  const sourceName = remaps ? resolveColumnName(buffer, columnName, remaps) : columnName;
+  const column = buffer.getColumnIfAllocated(sourceName);
   if (!Array.isArray(column)) {
     return undefined;
   }
   return isStringArray(column) ? column : undefined;
 }
 
-function getAllocatedBinaryColumn(buffer: AnySpanBuffer, columnName: string): unknown[] | undefined {
-  const column = buffer.getColumnIfAllocated(columnName);
+function getAllocatedBinaryColumn(
+  buffer: AnySpanBuffer,
+  columnName: string,
+  remaps?: RemapsByBuffer,
+): unknown[] | undefined {
+  const sourceName = remaps ? resolveColumnName(buffer, columnName, remaps) : columnName;
+  const column = buffer.getColumnIfAllocated(sourceName);
   return Array.isArray(column) ? column : undefined;
 }
 
@@ -191,18 +222,21 @@ function getAllocatedBinaryColumn(buffer: AnySpanBuffer, columnName: string): un
 // Arrow conversion. These typed readers pull from the buffer's immutable _scopeValues; the
 // per-column fill loops below call them so a row with a direct write keeps its value and scope
 // only fills the gaps.
-function getStringScopeValue(buffer: AnySpanBuffer, fieldName: string): string | undefined {
-  const value = buffer._scopeValues?.[fieldName];
+function getStringScopeValue(buffer: AnySpanBuffer, fieldName: string, remaps?: RemapsByBuffer): string | undefined {
+  const sourceName = remaps ? resolveColumnName(buffer, fieldName, remaps) : fieldName;
+  const value = buffer._scopeValues?.[sourceName];
   return typeof value === 'string' ? value : undefined;
 }
 
-function getNumberScopeValue(buffer: AnySpanBuffer, fieldName: string): number | undefined {
-  const value = buffer._scopeValues?.[fieldName];
+function getNumberScopeValue(buffer: AnySpanBuffer, fieldName: string, remaps?: RemapsByBuffer): number | undefined {
+  const sourceName = remaps ? resolveColumnName(buffer, fieldName, remaps) : fieldName;
+  const value = buffer._scopeValues?.[sourceName];
   return typeof value === 'number' ? value : undefined;
 }
 
-function getBooleanScopeValue(buffer: AnySpanBuffer, fieldName: string): boolean | undefined {
-  const value = buffer._scopeValues?.[fieldName];
+function getBooleanScopeValue(buffer: AnySpanBuffer, fieldName: string, remaps?: RemapsByBuffer): boolean | undefined {
+  const sourceName = remaps ? resolveColumnName(buffer, fieldName, remaps) : fieldName;
+  const value = buffer._scopeValues?.[sourceName];
   return typeof value === 'boolean' ? value : undefined;
 }
 //#endregion smoo/lmao!n/scope-attributes.arrow-fill
@@ -219,11 +253,179 @@ function isStringArray(value: unknown): value is string[] {
 }
 
 function indexTypeForCount(count: number): IntType {
-  if (count <= 255) return uint8();
-  if (count <= 65535) return uint16();
+  if (count <= 0x100) return uint8();
+  if (count <= 0x1_0000) return uint16();
   return uint32();
 }
 
+function assertStaticMessageKind(buffer: AnySpanBuffer, row: number, entryType: number, denseIndex: number): void {
+  const generation = buffer._vocabularyGeneration;
+  if (denseIndex >= generation.ids.length) {
+    throw new Error(`Invalid vocabulary dense index ${denseIndex} for generation ${generation.generation}`);
+  }
+  const expectedKind =
+    entryType === ENTRY_TYPE_SPAN_START ? 2 : entryType >= ENTRY_TYPE_TRACE && entryType <= ENTRY_TYPE_ERROR ? 1 : 0;
+  if (expectedKind === 0) {
+    throw new Error(`Entry type ${entryType} at row ${row} cannot carry a static vocabulary identity`);
+  }
+  const actualKind = generation.kindTags[denseIndex];
+  if (actualKind !== expectedKind) {
+    throw new Error(
+      `Vocabulary kind ${actualKind} at dense index ${denseIndex} does not match entry type ${entryType} at row ${row}`,
+    );
+  }
+}
+
+function buildMessageColumn(
+  buffers: AnySpanBuffer[],
+  totalRows: number,
+  dictionaryId?: number,
+  materializeEntryTypes = true,
+): BuiltMessageColumn {
+  let generation = buffers[0]._vocabularyGeneration;
+  for (let index = 1; index < buffers.length; index++) {
+    const candidate = buffers[index]._vocabularyGeneration;
+    if (candidate.generation > generation.generation) generation = candidate;
+  }
+  const prefix = getVocabularyDictionaryPrefix(generation);
+  const indices = new Uint32Array(totalRows);
+  const entryTypeIndices = materializeEntryTypes ? new Int8Array(totalRows) : undefined;
+  const nullBitmap = new Uint8Array(Math.ceil(totalRows / 8));
+  let nullCount = 0;
+  let rowOffset = 0;
+  let suffixValues: string[] | undefined;
+  let suffixIndices: Map<string, number> | undefined;
+
+  for (const buffer of buffers) {
+    const plannedStorage = buffer._opMetadata._physicalLayoutPlan?.arrowExposure.messageIdentityStorage;
+    const messageIdentityStorage =
+      plannedStorage ??
+      (buffer._messagePhysicalLayout === 'current'
+        ? 'local-u16'
+        : buffer._messagePhysicalLayout === 'specialized'
+          ? 'global-u32'
+          : 'packed-row-headers');
+    for (let row = 0; row < buffer._writeIndex; row++) {
+      const outputRow = rowOffset + row;
+      const hasDedicatedMessage =
+        (row === 0 && buffer._spanName !== undefined) || (row === 1 && buffer._terminalMessage !== undefined);
+      let dictionaryIndex: number | undefined;
+      let entryType: number;
+
+      if (messageIdentityStorage === 'local-u16') {
+        entryType = resolveEntryType(buffer, row);
+        const localId = buffer._messageIds?.[row];
+        const rawMessage = buffer.message_values?.[row];
+        if (row === 0 && typeof buffer._spanName === 'number') {
+          if ((localId !== undefined && localId > 0) || rawMessage !== undefined) {
+            throw new Error(`Local row ${row} has conflicting static message identities`);
+          }
+          dictionaryIndex = buffer._spanName;
+        } else if (localId !== undefined && localId > 0) {
+          if (hasDedicatedMessage || rawMessage !== undefined) {
+            throw new Error(`Local row ${row} has conflicting static and raw message storage`);
+          }
+          dictionaryIndex = buffer._opMetadata._physicalLayoutPlan?.localMessageDictionary[localId - 1];
+          if (dictionaryIndex === undefined) throw new Error(`Missing local message dictionary entry ${localId}`);
+        }
+      } else if (messageIdentityStorage === 'global-u32') {
+        entryType = resolveEntryType(buffer, row);
+        const encodedDenseIndex = buffer._logHeaders?.[row];
+        if (encodedDenseIndex !== undefined && encodedDenseIndex > 0) {
+          if (hasDedicatedMessage || buffer.message_values?.[row] !== undefined) {
+            throw new Error(`Global row ${row} has conflicting static and raw message storage`);
+          }
+          dictionaryIndex = encodedDenseIndex - 1;
+        }
+      } else {
+        const rowHeaders = buffer._rowHeaders;
+        if (rowHeaders === undefined) throw new TypeError('Packed message layout is missing row headers');
+        const header = rowHeaders[row];
+        entryType = header & 0xff;
+        const encodedDenseIndex = header >>> 8;
+        if (encodedDenseIndex !== 0) {
+          if (hasDedicatedMessage) {
+            throw new Error(`Packed row ${row} has conflicting static and dedicated message storage`);
+          }
+          dictionaryIndex = encodedDenseIndex - 1;
+        }
+      }
+      const dedicatedDenseIndex = row === 0 && typeof buffer._spanName === 'number' ? buffer._spanName : undefined;
+      if (dedicatedDenseIndex !== undefined) {
+        if (buffer.message_values?.[row] !== undefined) {
+          throw new Error(`Row ${row} has conflicting dedicated static and raw message storage`);
+        }
+        dictionaryIndex ??= dedicatedDenseIndex;
+      }
+
+      if (entryTypeIndices !== undefined) entryTypeIndices[outputRow] = entryType;
+      if (dictionaryIndex !== undefined) {
+        assertStaticMessageKind(buffer, row, entryType, dictionaryIndex);
+      } else {
+        const message =
+          row === 0 && typeof buffer._spanName === 'string'
+            ? buffer._spanName
+            : row === 1 && buffer._terminalMessage !== undefined
+              ? buffer._terminalMessage
+              : buffer.message_values?.[row];
+        if (message === undefined) {
+          nullCount++;
+          continue;
+        }
+        const staticIndex = prefix.valueToDenseIndex.get(message);
+        if (staticIndex !== undefined) {
+          dictionaryIndex = staticIndex;
+        } else {
+          let dynamicIndex = suffixIndices?.get(message);
+          if (dynamicIndex === undefined) {
+            suffixValues ??= [];
+            suffixIndices ??= new Map<string, number>();
+            dynamicIndex = suffixValues.length;
+            suffixValues.push(message);
+            suffixIndices.set(message, dynamicIndex);
+          }
+          dictionaryIndex = prefix.length + dynamicIndex;
+        }
+      }
+      indices[outputRow] = dictionaryIndex;
+      nullBitmap[outputRow >>> 3] |= 1 << (outputRow & 7);
+    }
+    rowOffset += buffer._writeIndex;
+  }
+
+  let messageDictionary = prefix.column;
+  if (suffixValues !== undefined) {
+    const encoded = globalUtf8Cache.encodeMany(suffixValues);
+    const suffix = buildColumn(
+      buildData({
+        type: utf8(),
+        offset: 0,
+        length: suffixValues.length,
+        nullCount: 0,
+        valueOffsets: encoded.offsets,
+        data: encoded.data,
+      }),
+    );
+    messageDictionary = appendVocabularyDictionarySuffix(prefix, suffix, suffixValues);
+  }
+
+  return {
+    column: buildColumn(
+      buildData({
+        type:
+          dictionaryId === undefined ? dictionary(utf8(), uint32()) : dictionary(utf8(), uint32(), false, dictionaryId),
+        offset: 0,
+        length: totalRows,
+        nullCount,
+        data: indices,
+        nullBitmap: nullCount > 0 ? nullBitmap : undefined,
+        dictionary: messageDictionary,
+      }),
+    ),
+    entryTypeIndices,
+  };
+}
+
 // ═══════════════════════════════════════════════════════════════════════════════
 // Shared metadata column helpers (used by both buildMetadataColumnsWithFields
 // and convertBuffersWithSharedDicts to avoid duplication)
@@ -321,75 +523,100 @@ function buildNonDictIdentityColumns(
   );
 }
 
-/**
- * Build timestamp column from buffers.
- */
 function buildTimestampColumn(
   buffers: AnySpanBuffer[],
   totalRows: number,
   fields: string[],
   vectors: Column<unknown>[],
+  borrowChunks = false,
 ): void {
+  const timestampType = timestamp(TimeUnit.NANOSECOND);
+  fields.push('timestamp');
+  if (borrowChunks) {
+    const chunks = buffers.map((buffer) =>
+      buildColumn(
+        buildData({
+          type: timestampType,
+          offset: 0,
+          length: buffer._writeIndex,
+          nullCount: 0,
+          data: buffer.timestamp.subarray(0, buffer._writeIndex),
+        }),
+      ),
+    );
+    vectors.push(combineColumnChunks(chunks));
+    return;
+  }
   const allTimestamps = new BigInt64Array(totalRows);
   let offset = 0;
-  for (const buf of buffers) {
-    if (!buf.timestamp) {
-      throw new Error(`Buffer missing timestamps property (_writeIndex: ${buf._writeIndex})`);
-    }
-    allTimestamps.set(buf.timestamp.subarray(0, buf._writeIndex), offset);
-    offset += buf._writeIndex;
+  for (const buffer of buffers) {
+    allTimestamps.set(buffer.timestamp.subarray(0, buffer._writeIndex), offset);
+    offset += buffer._writeIndex;
   }
-  fields.push('timestamp');
   vectors.push(
-    buildColumn(
-      buildData({
-        type: timestamp(TimeUnit.NANOSECOND),
-        offset: 0,
-        length: totalRows,
-        nullCount: 0,
-        data: allTimestamps,
-      }),
-    ),
+    buildColumn(buildData({ type: timestampType, offset: 0, length: totalRows, nullCount: 0, data: allTimestamps })),
+  );
+}
+
+function canBorrowEntryTypeChunks(buffers: AnySpanBuffer[], borrowChunks: boolean): boolean {
+  return (
+    borrowChunks &&
+    buffers.every((buffer) => {
+      const declared = buffer._opMetadata._physicalLayoutPlan?.arrowExposure.entryTypeStorage;
+      return declared === 'borrowed-u8' || (declared === undefined && buffer.entry_type !== undefined);
+    })
   );
 }
 
-/**
- * Build entry_type dictionary column.
- */
 function buildEntryTypeColumn(
   buffers: AnySpanBuffer[],
   totalRows: number,
   fields: string[],
   vectors: Column<unknown>[],
   dictType?: ReturnType<typeof dictionary>,
+  borrowChunks = false,
+  classifiedEntryTypes?: Int8Array,
 ): void {
-  const entryTypeIndices = new Int8Array(totalRows);
-  let offset = 0;
-  for (const buf of buffers) {
-    if (!buf.entry_type) {
-      throw new Error(`Buffer missing operations property (_writeIndex: ${buf._writeIndex})`);
+  const entryTypeDictionary = PREBUILT_ENTRY_TYPE_DICTIONARY;
+  const type = dictType ?? dictionary(utf8(), int8());
+  fields.push('entry_type');
+  if (canBorrowEntryTypeChunks(buffers, borrowChunks)) {
+    const chunks = buffers.map((buffer) => {
+      const source = buffer.entry_type;
+      if (source === undefined) throw new TypeError('Current/specialized buffer is missing entry types');
+      const values = new Int8Array(source.buffer, source.byteOffset, buffer._writeIndex);
+      return buildColumn(
+        buildData({
+          type,
+          offset: 0,
+          length: buffer._writeIndex,
+          nullCount: 0,
+          data: values,
+          dictionary: entryTypeDictionary,
+        }),
+      );
+    });
+    vectors.push(combineColumnChunks(chunks));
+    return;
+  }
+  const entryTypeIndices = classifiedEntryTypes ?? new Int8Array(totalRows);
+  if (classifiedEntryTypes === undefined) {
+    let offset = 0;
+    for (const buffer of buffers) {
+      for (let row = 0; row < buffer._writeIndex; row++) {
+        entryTypeIndices[offset++] = resolveEntryType(buffer, row);
+      }
     }
-    entryTypeIndices.set(buf.entry_type.subarray(0, buf._writeIndex), offset);
-    offset += buf._writeIndex;
   }
-  const entryTypeDictData = buildData({
-    type: utf8(),
-    offset: 0,
-    length: ENTRY_TYPE_NAMES.length,
-    nullCount: 0,
-    valueOffsets: calculateUtf8Offsets(ENTRY_TYPE_NAMES),
-    data: encodeUtf8Strings(ENTRY_TYPE_NAMES),
-  });
-  fields.push('entry_type');
   vectors.push(
     buildColumn(
       buildData({
-        type: dictType ?? dictionary(utf8(), int8()),
+        type,
         offset: 0,
         length: totalRows,
         nullCount: 0,
         data: entryTypeIndices,
-        dictionary: buildColumn(entryTypeDictData),
+        dictionary: entryTypeDictionary,
       }),
     ),
   );
@@ -462,14 +689,16 @@ function buildBinaryColumnFromBuffers(
   columnName: string,
   totalRows: number,
   fieldSchema: unknown,
+  remaps?: RemapsByBuffer,
 ): BuiltArrowColumn {
   const encoder = getBinaryEncoder(fieldSchema);
 
   // Collect all values from buffers, encode each if encoder is present
   const allValues: (Uint8Array | null)[] = [];
   for (const buf of buffers) {
-    const col = getAllocatedBinaryColumn(buf, columnName);
-    const srcNulls = buf.getNullsIfAllocated(columnName);
+    const col = getAllocatedBinaryColumn(buf, columnName, remaps);
+    const sourceName = remaps ? resolveColumnName(buf, columnName, remaps) : columnName;
+    const srcNulls = buf.getNullsIfAllocated(sourceName);
     for (let i = 0; i < buf._writeIndex; i++) {
       if (srcNulls) {
         const isValid = (srcNulls[i >>> 3] & (1 << (i & 7))) !== 0;
@@ -549,16 +778,18 @@ function buildBinaryColumnFromBuffers(
  * Dictionaries are built from the data in this batch only and are not shared
  * with other batches, even when combined into a Table.
  */
-export function convertToTable(buffer: AnySpanBuffer, systemColumnBuilder?: SystemColumnBuilder): Table {
+export function convertToTable(
+  buffer: AnySpanBuffer,
+  systemColumnBuilder?: SystemColumnBuilder,
+  borrowChunks = false,
+): Table {
   const buffers: AnySpanBuffer[] = [];
   let currentBuffer: AnySpanBuffer | undefined = buffer;
-
   while (currentBuffer) {
     buffers.push(currentBuffer);
     currentBuffer = currentBuffer._overflow;
   }
-
-  return convertBuffersToTable(buffers, systemColumnBuilder);
+  return convertBuffersToTable(buffers, systemColumnBuilder, borrowChunks);
 }
 
 /**
@@ -568,13 +799,18 @@ export function convertToTable(buffer: AnySpanBuffer, systemColumnBuilder?: Syst
  * Dictionaries are built from the data in this batch only and are not shared
  * with other batches, even when combined into a Table.
  */
-function convertBuffersToTable(buffers: AnySpanBuffer[], systemColumnBuilder?: SystemColumnBuilder): Table {
+function convertBuffersToTable(
+  buffers: AnySpanBuffer[],
+  systemColumnBuilder?: SystemColumnBuilder,
+  borrowChunks = false,
+): Table {
   if (buffers.length === 0) return tableFromColumns({});
 
   const totalRows = buffers.reduce((sum, buf) => sum + buf._writeIndex, 0);
   if (totalRows === 0) return tableFromColumns({});
 
   const schema: LogSchema = buffers[0]._logSchema;
+  const enumLookup = resolveEnumLookupDescriptor(schema);
 
   // Build vectors first, then derive schema from them
   // This ensures Field types and vector data types are identical (Arrow IPC requirement)
@@ -586,57 +822,22 @@ function convertBuffersToTable(buffers: AnySpanBuffer[], systemColumnBuilder?: S
     fields.push(...systemColumns.fields);
     vectors.push(...systemColumns.vectors);
   } else {
-    // Build metadata columns - returns both fields and vectors with matching types
-    const metadataResult = buildMetadataColumnsWithFields(buffers, totalRows);
+    const borrowEntryTypes = canBorrowEntryTypeChunks(buffers, borrowChunks);
+    const messageResult = buildMessageColumn(buffers, totalRows, undefined, !borrowEntryTypes);
+    // Build metadata columns - returns both fields and vectors with matching types.
+    const metadataResult = buildMetadataColumnsWithFields(
+      buffers,
+      totalRows,
+      borrowChunks,
+      messageResult.entryTypeIndices,
+    );
     fields.push(...metadataResult.fields);
     vectors.push(...metadataResult.vectors);
 
-    // System attribute column: message (eager category)
-    const maskTransform = undefined; // message has no masking
-    let {
-      dictionary: dictValues,
-      indices,
-      arrowIndexType,
-      nullBitmap,
-    } = buildSortedCategoryDictionary(buffers, 'message', maskTransform);
-    // Ensure dictionary has at least one entry (empty string) if no messages were written
-    if (dictValues.length === 0) {
-      dictValues = [''];
-    }
-    // Ensure nullBitmap is all 1s (all valid) for eager column
-    if (!nullBitmap) {
-      const bitmapBytes = Math.ceil(totalRows / 8);
-      nullBitmap = new Uint8Array(bitmapBytes);
-      nullBitmap.fill(0xff);
-    }
-
-    // CRITICAL: Create ONE Dictionary type instance for field and batch data
-    // Arrow Schema validates that fields with same dictionary ID have identical type references
-    const messageDictType = dictionary(utf8(), arrowIndexType);
+    // Static vocabulary values form an immutable prefix. Only raw messages
+    // missing from that prefix are interned into a first-seen suffix.
     fields.push('message');
-
-    const { data: messageUtf8Data, offsets: messageUtf8Offsets } = globalUtf8Cache.encodeMany(dictValues);
-
-    const messageDictData = buildData({
-      type: utf8(),
-      offset: 0,
-      length: dictValues.length,
-      nullCount: 0,
-      valueOffsets: messageUtf8Offsets,
-      data: messageUtf8Data,
-    });
-
-    const messageData = buildData({
-      type: messageDictType,
-      offset: 0,
-      length: totalRows,
-      nullCount: 0, // message is eager, never null
-      data: indices,
-      nullBitmap, // Always include nullBitmap for message (even though nullCount is 0)
-      dictionary: buildColumn(messageDictData),
-    });
-
-    vectors.push(buildColumn(messageData));
+    vectors.push(messageResult.column);
   }
 
   // Build user attribute vectors
@@ -649,17 +850,18 @@ function convertBuffersToTable(buffers: AnySpanBuffer[], systemColumnBuilder?: S
     const columnName = fieldName; // User columns have no prefix
 
     if (lmaoType === 'enum') {
-      const enumValues = getEnumValues(fieldSchema) || [];
+      const descriptor = enumLookup.byField[fieldName];
+      if (!descriptor) throw new TypeError(`Enum lookup missing for field '${fieldName}'`);
+      const enumValues = descriptor.values;
       const enumUtf8 = getEnumUtf8(fieldSchema);
-      // Get constructors from schema metadata
-      const indexArrayCtor = getArrowIndexArrayConstructorOr(fieldSchema);
-      const arrowIndexType = getArrowIndexTypeOr(fieldSchema, uint8());
+      const indexArrayCtor = descriptor.indexArrayConstructor;
+      const arrowIndexType = indexTypeForCount(enumValues.length);
       // Collect value arrays - need to handle different TypedArray types
       const valueArrays: (Uint8Array | Uint16Array | Uint32Array)[] = [];
       for (const buf of buffers) {
         const column = buf.getColumnIfAllocated(columnName);
         if (column && column instanceof indexArrayCtor) {
-          valueArrays.push(column.subarray(0, buf._writeIndex));
+          valueArrays.push(column.subarray(0, buf._writeIndex).slice());
         } else {
           valueArrays.push(new indexArrayCtor(buf._writeIndex));
         }
@@ -784,33 +986,52 @@ function convertBuffersToTable(buffers: AnySpanBuffer[], systemColumnBuilder?: S
         vectors.push(buildColumn(textData));
       }
     } else if (lmaoType === 'number') {
-      const valueArrays: Float64Array[] = [];
-
-      for (const buf of buffers) {
-        const column = buf.getColumnIfAllocated(columnName);
-        if (column && column instanceof Float64Array) {
-          valueArrays.push(column.subarray(0, buf._writeIndex));
-        } else {
-          valueArrays.push(new F64Array(buf._writeIndex));
-        }
-      }
-
-      const allValues = concatenateFloat64Arrays(valueArrays);
-      const { nullBitmap, nullCount } = concatenateNullBitmaps(buffers, columnName);
-
       const numberType = float64();
       fields.push(arrowFieldName);
-
-      const numberData = buildData({
-        type: numberType,
-        offset: 0,
-        length: totalRows,
-        nullCount,
-        data: allValues,
-        nullBitmap,
-      });
-
-      vectors.push(buildColumn(numberData));
+      const borrowed = borrowChunks
+        ? buffers.map((buffer) => {
+            const column = buffer.getColumnIfAllocated(columnName);
+            const validity = buffer.getNullsIfAllocated(columnName);
+            if (!(column instanceof Float64Array) || !(validity instanceof Uint8Array)) return undefined;
+            return buildColumn(
+              buildData({
+                type: numberType,
+                offset: 0,
+                length: buffer._writeIndex,
+                nullCount: countNulls(validity, buffer._writeIndex),
+                data: column.subarray(0, buffer._writeIndex),
+                nullBitmap: validity.subarray(0, Math.ceil(buffer._writeIndex / 8)),
+              }),
+            );
+          })
+        : undefined;
+      if (borrowed?.every((chunk): chunk is Column<unknown> => chunk !== undefined)) {
+        vectors.push(combineColumnChunks(borrowed));
+      } else {
+        const valueArrays: Float64Array[] = [];
+        for (const buffer of buffers) {
+          const column = buffer.getColumnIfAllocated(columnName);
+          valueArrays.push(
+            column instanceof Float64Array
+              ? column.subarray(0, buffer._writeIndex).slice()
+              : new F64Array(buffer._writeIndex),
+          );
+        }
+        const allValues = concatenateFloat64Arrays(valueArrays);
+        const { nullBitmap, nullCount } = concatenateNullBitmaps(buffers, columnName);
+        vectors.push(
+          buildColumn(
+            buildData({
+              type: numberType,
+              offset: 0,
+              length: totalRows,
+              nullCount,
+              data: allValues,
+              nullBitmap,
+            }),
+          ),
+        );
+      }
     } else if (lmaoType === 'boolean') {
       const valueArrays: Uint8Array[] = [];
 
@@ -818,7 +1039,7 @@ function convertBuffersToTable(buffers: AnySpanBuffer[], systemColumnBuilder?: S
         const column = buf.getColumnIfAllocated(columnName);
         if (column && column instanceof Uint8Array) {
           const requiredBytes = Math.ceil(buf._writeIndex / 8);
-          valueArrays.push(column.subarray(0, requiredBytes));
+          valueArrays.push(column.subarray(0, requiredBytes).slice());
         } else {
           const requiredBytes = Math.ceil(buf._writeIndex / 8);
           valueArrays.push(new Uint8Array(requiredBytes));
@@ -858,7 +1079,12 @@ function convertBuffersToTable(buffers: AnySpanBuffer[], systemColumnBuilder?: S
  * This is required for Arrow IPC serialization - Schema validates that
  * fields with same dictionary ID have identical type object references.
  */
-function buildMetadataColumnsWithFields(buffers: AnySpanBuffer[], totalRows: number): BuiltMetadataColumns {
+function buildMetadataColumnsWithFields(
+  buffers: AnySpanBuffer[],
+  totalRows: number,
+  borrowChunks = false,
+  classifiedEntryTypes?: Int8Array,
+): BuiltMetadataColumns {
   const fields: string[] = [];
   const vectors: Column<unknown>[] = [];
 
@@ -866,7 +1092,7 @@ function buildMetadataColumnsWithFields(buffers: AnySpanBuffer[], totalRows: num
   //               entry_type, package_name, package_file, git_sha
 
   // 1. Timestamp
-  buildTimestampColumn(buffers, totalRows, fields, vectors);
+  buildTimestampColumn(buffers, totalRows, fields, vectors, borrowChunks);
 
   // 2. Trace ID dictionary (built inline from buffers)
   const traceIdSet = new Set<string>();
@@ -915,7 +1141,7 @@ function buildMetadataColumnsWithFields(buffers: AnySpanBuffer[], totalRows: num
   buildNonDictIdentityColumns(buffers, totalRows, fields, vectors);
 
   // 7. Entry type
-  buildEntryTypeColumn(buffers, totalRows, fields, vectors);
+  buildEntryTypeColumn(buffers, totalRows, fields, vectors, undefined, borrowChunks, classifiedEntryTypes);
 
   // 8-10. Per-buffer metadata columns (package_name, package_file, git_sha)
   for (const [fieldName, metadataGetter] of [
@@ -951,6 +1177,51 @@ function buildMetadataColumnsWithFields(buffers: AnySpanBuffer[], totalRows: num
   return { fields, vectors };
 }
 
+type MemoryEpochPin = { readonly buffer: ArrayBuffer; release(): void };
+type EpochPinnableAllocator = { pinMemoryEpoch(): MemoryEpochPin };
+type EpochPinnableBuffer = AnySpanBuffer & { readonly _allocator: EpochPinnableAllocator };
+
+function isEpochPinnableBuffer(buffer: AnySpanBuffer): buffer is EpochPinnableBuffer {
+  if (!('_allocator' in buffer)) return false;
+  const allocator = Reflect.get(buffer, '_allocator');
+  return (
+    typeof allocator === 'object' &&
+    allocator !== null &&
+    typeof Reflect.get(allocator, 'pinMemoryEpoch') === 'function'
+  );
+}
+
+function createLeasedConversion(
+  root: AnySpanBuffer,
+  buffers: readonly AnySpanBuffer[],
+  build: (borrowChunks: boolean) => Table,
+): ArrowLease {
+  const topology = root._traceRoot._topology;
+  topology.assertLive(root);
+  const releaseTopology = topology.acquireLease();
+  const generations = new Set<VocabularyGeneration>();
+  const allocators = new Set<EpochPinnableAllocator>();
+  for (const buffer of buffers) {
+    generations.add(buffer._vocabularyGeneration);
+    if (isEpochPinnableBuffer(buffer)) allocators.add(buffer._allocator);
+  }
+
+  const epochPins = Array.from(allocators, (allocator) => allocator.pinMemoryEpoch());
+  // WASM linear-memory growth cannot preserve borrowed JS TypedArray views, so
+  // its leased result uses the existing Arrow-owned copy path. JS heap buffers
+  // expose their physical chunks directly under the topology lease.
+  const borrowChunks = epochPins.length === 0;
+  try {
+    const table = build(borrowChunks);
+    for (const pin of epochPins) pin.release();
+    return createArrowLease(table, [...buffers, ...generations], [releaseTopology]);
+  } catch (error) {
+    for (const pin of epochPins) pin.release();
+    releaseTopology();
+    throw error;
+  }
+}
+
 /**
  * Convert SpanBuffer (and its overflow chain) to Arrow Table
  */
@@ -963,6 +1234,23 @@ export function convertToArrowTable<_T extends LogSchema = LogSchema>(
   return batch;
 }
 
+/** Borrow JS physical Arrow chunks until the returned lease is released. */
+export function convertToLeasedArrowTable(
+  buffer: AnySpanBuffer,
+  systemColumnBuilder?: SystemColumnBuilder,
+): ArrowLease {
+  const buffers: AnySpanBuffer[] = [];
+  let segment: AnySpanBuffer | undefined = buffer;
+  while (segment) {
+    buffers.push(segment);
+    segment = segment._overflow;
+  }
+  return createLeasedConversion(buffer, buffers, (borrowChunks) => {
+    const table = convertToTable(buffer, systemColumnBuilder, borrowChunks);
+    return table.numRows === 0 ? tableFromColumns({}) : table;
+  });
+}
+
 // ═══════════════════════════════════════════════════════════════════════════════
 // Tree Conversion with Shared Dictionaries
 // ═══════════════════════════════════════════════════════════════════════════════
@@ -979,6 +1267,7 @@ export function convertSpanTreeToArrowTable(
   _systemColumnBuilder?: SystemColumnBuilder,
   modulesToLogStats?: CapacityStatsEntry[],
   periodStartNs?: bigint,
+  borrowChunks = false,
 ): Table {
   // ═══════════════════════════════════════════════════════════════════════════
   // PASS 0: Collect ALL unique schema fields from ALL buffers in the tree
@@ -986,19 +1275,27 @@ export function convertSpanTreeToArrowTable(
   // (e.g., library prefixed schemas like db_query, http_status, etc.)
   // ═══════════════════════════════════════════════════════════════════════════
   const mergedSchemaFields = new Map<string, unknown>();
-
-  walkSpanTree(rootBuffer, (buffer) => {
-    // Use buffer._columns directly - works for both SpanBuffer and RemappedBufferView
-    const fields = buffer._columns;
-    for (const [fieldName, fieldSchema] of fields) {
-      // Skip system schema fields - they are handled separately as system columns
-      // (message, line, error_code, exception_stack, ff_value, uint64_value)
-      if (SYSTEM_SCHEMA_FIELD_NAMES.has(fieldName)) {
-        continue;
+  const remaps: RemapsByBuffer = new WeakMap();
+  const enumLookupsByField = new Map<string, EnumLookupDescriptor>();
+
+  walkSpanTree(rootBuffer, (buffer, remapDescriptor) => {
+    const enumLookup = resolveEnumLookupDescriptor(buffer._logSchema);
+    if (remapDescriptor) {
+      remaps.set(buffer, remapDescriptor);
+      for (const [fieldName, sourceName, fieldSchema] of remapDescriptor.columns) {
+        if (!SYSTEM_SCHEMA_FIELD_NAMES.has(fieldName) && !mergedSchemaFields.has(fieldName)) {
+          mergedSchemaFields.set(fieldName, fieldSchema);
+          const descriptor = enumLookup.byField[sourceName];
+          if (descriptor) enumLookupsByField.set(fieldName, descriptor);
+        }
       }
-      // Only add if not already present (first buffer with this field wins)
-      if (!mergedSchemaFields.has(fieldName)) {
+      return;
+    }
+    for (const [fieldName, fieldSchema] of buffer._columns) {
+      if (!SYSTEM_SCHEMA_FIELD_NAMES.has(fieldName) && !mergedSchemaFields.has(fieldName)) {
         mergedSchemaFields.set(fieldName, fieldSchema);
+        const descriptor = enumLookup.byField[fieldName];
+        if (descriptor) enumLookupsByField.set(fieldName, descriptor);
       }
     }
   });
@@ -1035,12 +1332,6 @@ export function convertSpanTreeToArrowTable(
     }
   }
 
-  // System attribute column: message (eager category, not in user schema)
-  // message is always present (eager allocation), so we need to handle it explicitly
-  categoryBuilders.set('message', new DictBuilder(globalUtf8Cache));
-  categoryMaskTransforms.set('message', undefined); // message has no masking
-  categoryOriginalToMasked.set('message', new Map());
-
   // Metadata column dictionaries
   // For traceId (metadata), use DictionaryBuilder (values not pre-encoded)
   const traceIdBuilder = new DictBuilder(globalUtf8Cache);
@@ -1062,12 +1353,12 @@ export function convertSpanTreeToArrowTable(
     }
 
     for (const [fieldName, builder] of categoryBuilders) {
-      const col = getAllocatedStringColumn(buffer, fieldName);
       const maskTransform = categoryMaskTransforms.get(fieldName);
       const originalToMasked = categoryOriginalToMasked.get(fieldName);
       if (!originalToMasked) {
         throw new Error(`Category originalToMasked map not found for field: ${fieldName}`);
       }
+      const col = getAllocatedStringColumn(buffer, fieldName, remaps);
       if (col) {
         for (let i = 0; i < buffer._writeIndex; i++) {
           const originalValue = col[i];
@@ -1083,7 +1374,7 @@ export function convertSpanTreeToArrowTable(
       }
       // Also add scope values to dictionary (they may not be in columns)
       // Per specs/lmao/01i_span_scope_attributes.md: scope values fill NULL cells at Arrow conversion
-      const scopeValue = getStringScopeValue(buffer, fieldName);
+      const scopeValue = getStringScopeValue(buffer, fieldName, remaps);
       if (scopeValue !== undefined) {
         let maskedScopeValue = originalToMasked.get(scopeValue);
         if (maskedScopeValue === undefined) {
@@ -1095,7 +1386,7 @@ export function convertSpanTreeToArrowTable(
     }
 
     for (const [fieldName, builder] of textBuilders) {
-      const col = getAllocatedStringColumn(buffer, fieldName);
+      const col = getAllocatedStringColumn(buffer, fieldName, remaps);
       const maskTransform = textMaskTransforms.get(fieldName);
       const originalToMasked = textOriginalToMasked.get(fieldName);
       if (!originalToMasked) {
@@ -1116,7 +1407,7 @@ export function convertSpanTreeToArrowTable(
       }
       // Also add scope values to dictionary (they may not be in columns)
       // Per specs/lmao/01i_span_scope_attributes.md: scope values fill NULL cells at Arrow conversion
-      const scopeValue = getStringScopeValue(buffer, fieldName);
+      const scopeValue = getStringScopeValue(buffer, fieldName, remaps);
       if (scopeValue !== undefined) {
         let maskedScopeValue = originalToMasked.get(scopeValue);
         if (maskedScopeValue === undefined) {
@@ -1166,15 +1457,6 @@ export function convertSpanTreeToArrowTable(
     textDicts.set(name, builder.finalize(false)); // not sorted
   }
 
-  // Get messageDict from categoryDicts (message is handled as a category column)
-  // If no messages were written, create an empty dictionary
-  let messageDict = categoryDicts.get('message');
-  if (!messageDict) {
-    const messageBuilder = new DictBuilder(globalUtf8Cache);
-    messageBuilder.add(''); // Single empty string entry
-    messageDict = messageBuilder.finalize(true); // sorted
-  }
-
   // ═══════════════════════════════════════════════════════════════════════════
   // PASS 2: Collect all buffers with non-zero rows
   // ═══════════════════════════════════════════════════════════════════════════
@@ -1198,8 +1480,11 @@ export function convertSpanTreeToArrowTable(
           categoryDicts,
           textDicts,
           schemaFields,
+          enumLookupsByField,
           categoryOriginalToMasked,
           textOriginalToMasked,
+          remaps,
+          borrowChunks,
         )
       : undefined;
 
@@ -1221,6 +1506,22 @@ export function convertSpanTreeToArrowTable(
   return tableFromColumns({});
 }
 
+/** Convert a logical trace tree while pinning every borrowed physical chunk. */
+export function convertSpanTreeToLeasedArrowTable(
+  rootBuffer: AnySpanBuffer,
+  systemColumnBuilder?: SystemColumnBuilder,
+  modulesToLogStats?: CapacityStatsEntry[],
+  periodStartNs?: bigint,
+): ArrowLease {
+  const buffers: AnySpanBuffer[] = [];
+  walkSpanTree(rootBuffer, (buffer) => {
+    if (buffer._writeIndex > 0) buffers.push(buffer);
+  });
+  return createLeasedConversion(rootBuffer, buffers, (borrowChunks) =>
+    convertSpanTreeToArrowTable(rootBuffer, systemColumnBuilder, modulesToLogStats, periodStartNs, borrowChunks),
+  );
+}
+
 /**
  * Convert multiple buffers to a single table using pre-built dictionaries.
  *
@@ -1238,12 +1539,17 @@ function convertBuffersWithSharedDicts(
   categoryDicts: Map<string, FinalizedDictionary>,
   textDicts: Map<string, FinalizedDictionary>,
   schemaFields: Array<[string, unknown]>,
+  enumLookupsByField: ReadonlyMap<string, EnumLookupDescriptor>,
   categoryOriginalToMasked: Map<string, Map<string, string>>,
   textOriginalToMasked: Map<string, Map<string, string>>,
+  remaps: RemapsByBuffer,
+  borrowChunks = false,
 ): Table {
   const totalRows = buffers.reduce((sum, buf) => sum + buf._writeIndex, 0);
   const fields: string[] = [];
   const vectors: Column<unknown>[] = [];
+  const borrowEntryTypes = canBorrowEntryTypeChunks(buffers, borrowChunks);
+  const messageResult = buildMessageColumn(buffers, totalRows, 5, !borrowEntryTypes);
 
   // Dictionary types with explicit IDs for IPC serialization
   const traceIdType = dictionary(utf8(), traceIdDict.arrowIndexType, false, 0);
@@ -1256,7 +1562,7 @@ function convertBuffersWithSharedDicts(
   //               entry_type, package_name, package_file, git_sha
 
   // 1. Timestamp
-  buildTimestampColumn(buffers, totalRows, fields, vectors);
+  buildTimestampColumn(buffers, totalRows, fields, vectors, borrowChunks);
 
   // 2. Trace ID (using shared pre-built dictionary)
   const traceIdIndices = new traceIdDict.indexArrayCtor(totalRows);
@@ -1293,7 +1599,15 @@ function convertBuffersWithSharedDicts(
   buildNonDictIdentityColumns(buffers, totalRows, fields, vectors);
 
   // 7. Entry type (with explicit dictionary ID for IPC)
-  buildEntryTypeColumn(buffers, totalRows, fields, vectors, entryTypeDictType);
+  buildEntryTypeColumn(
+    buffers,
+    totalRows,
+    fields,
+    vectors,
+    entryTypeDictType,
+    borrowChunks,
+    messageResult.entryTypeIndices,
+  );
 
   // 8-10. Per-buffer metadata columns using shared pre-built dictionaries
   // Per specs/lmao/01c_context_flow_and_op_wrappers.md:
@@ -1325,52 +1639,7 @@ function convertBuffersWithSharedDicts(
   // ═══════════════════════════════════════════════════════════════════════════
 
   // System attribute column: message (at fixed position 10, before user attributes)
-  const messageDict = categoryDicts.get('message');
-  const messageOriginalToMasked = categoryOriginalToMasked.get('message');
-  if (!messageDict || !messageOriginalToMasked) {
-    throw new Error('Message dictionary or mapping not found');
-  }
-  const messageFieldType = dictionary(utf8(), messageDict.arrowIndexType, false, 5);
-  const messageIndices = new messageDict.indexArrayCtor(totalRows);
-  const messageNullBitmap = new Uint8Array(Math.ceil(totalRows / 8));
-  let messageRowOffset = 0;
-  for (const buf of buffers) {
-    const col = getAllocatedStringColumn(buf, 'message');
-    if (col) {
-      for (let i = 0; i < buf._writeIndex; i++) {
-        const v = col[i];
-        const rowIdx = messageRowOffset + i;
-        if (v != null) {
-          const maskedValue = messageOriginalToMasked.get(v) ?? v;
-          messageIndices[rowIdx] = messageDict.indexMap.get(maskedValue) ?? 0;
-          messageNullBitmap[rowIdx >>> 3] |= 1 << (rowIdx & 7);
-        }
-      }
-    }
-    messageRowOffset += buf._writeIndex;
-  }
-  vectors.push(
-    buildColumn(
-      buildData({
-        type: messageFieldType,
-        offset: 0,
-        length: totalRows,
-        nullCount: 0, // message is eager, never null
-        data: messageIndices,
-        nullBitmap: messageNullBitmap,
-        dictionary: buildColumn(
-          buildData({
-            type: utf8(),
-            offset: 0,
-            length: messageDict.indexMap.size,
-            nullCount: 0,
-            valueOffsets: messageDict.offsets,
-            data: messageDict.data,
-          }),
-        ),
-      }),
-    ),
-  );
+  vectors.push(messageResult.column);
   fields.push('message');
 
   // System attribute column: uint64_value (for buffer metrics - op durations, counts, etc.)
@@ -1409,9 +1678,9 @@ function convertBuffersWithSharedDicts(
 
       // Per specs/lmao/01i_span_scope_attributes.md: scope values fill NULL cells at Arrow conversion
       for (const buf of buffers) {
-        const col = getAllocatedStringColumn(buf, columnName);
+        const col = getAllocatedStringColumn(buf, columnName, remaps);
         // Check if this buffer has a scope value for this field
-        const scopeValue = getStringScopeValue(buf, fieldName);
+        const scopeValue = getStringScopeValue(buf, fieldName, remaps);
         let scopeEncodedValue: number | undefined;
         if (scopeValue !== undefined) {
           const maskedScopeValue = originalToMasked.get(scopeValue) ?? scopeValue;
@@ -1484,9 +1753,9 @@ function convertBuffersWithSharedDicts(
       let rowOffset = 0;
 
       for (const buf of buffers) {
-        const col = getAllocatedStringColumn(buf, columnName);
+        const col = getAllocatedStringColumn(buf, columnName, remaps);
         // Check if this buffer has a scope value for this field
-        const scopeValue = getStringScopeValue(buf, fieldName);
+        const scopeValue = getStringScopeValue(buf, fieldName, remaps);
         let scopeEncodedValue: number | undefined;
         if (scopeValue !== undefined) {
           const maskedScopeValue = originalToMasked.get(scopeValue) ?? scopeValue;
@@ -1548,77 +1817,101 @@ function convertBuffersWithSharedDicts(
       );
       fields.push(arrowFieldName);
     } else if (lmaoType === 'number') {
-      const allValues = new F64Array(totalRows);
-      const nullBitmap = new Uint8Array(Math.ceil(totalRows / 8));
-      // Start with all nulls (bits cleared) - we'll set bits as we find values
-      let nullCount = 0;
-      let rowOffset = 0;
-
-      for (const buf of buffers) {
-        const col = buf.getColumnIfAllocated(columnName);
-        const srcNulls = buf.getNullsIfAllocated(columnName);
-        // Check if this buffer has a scope value for this field
-        const scopeValue = getNumberScopeValue(buf, fieldName);
-
-        if (col instanceof Float64Array) {
-          allValues.set(col.subarray(0, buf._writeIndex), rowOffset);
-          if (srcNulls) {
-            // Process each row - direct write wins, then scope, then null
-            for (let i = 0; i < buf._writeIndex; i++) {
-              const srcByte = i >>> 3;
-              const srcBit = i & 7;
-              const isValid = (srcNulls[srcByte] & (1 << srcBit)) !== 0;
-              const rowIdx = rowOffset + i;
-              if (isValid) {
-                // Direct write - mark as valid
-                nullBitmap[rowIdx >>> 3] |= 1 << (rowIdx & 7);
-              } else if (scopeValue !== undefined) {
-                // No direct write, but have scope value - use it
-                allValues[rowIdx] = scopeValue;
-                nullBitmap[rowIdx >>> 3] |= 1 << (rowIdx & 7);
-              } else {
-                // No direct write and no scope - null
-                nullCount++;
+      const borrowed = borrowChunks
+        ? buffers.map((buffer) => {
+            const sourceName = resolveColumnName(buffer, columnName, remaps);
+            if (getNumberScopeValue(buffer, fieldName, remaps) !== undefined) return undefined;
+            const column = buffer.getColumnIfAllocated(sourceName);
+            const validity = buffer.getNullsIfAllocated(sourceName);
+            if (!(column instanceof Float64Array) || !(validity instanceof Uint8Array)) return undefined;
+            return buildColumn(
+              buildData({
+                type: float64(),
+                offset: 0,
+                length: buffer._writeIndex,
+                nullCount: countNulls(validity, buffer._writeIndex),
+                data: column.subarray(0, buffer._writeIndex),
+                nullBitmap: validity.subarray(0, Math.ceil(buffer._writeIndex / 8)),
+              }),
+            );
+          })
+        : undefined;
+      if (borrowed?.every((chunk): chunk is Column<unknown> => chunk !== undefined)) {
+        vectors.push(combineColumnChunks(borrowed));
+      } else {
+        const allValues = new F64Array(totalRows);
+        const nullBitmap = new Uint8Array(Math.ceil(totalRows / 8));
+        // Start with all nulls (bits cleared) - we'll set bits as we find values
+        let nullCount = 0;
+        let rowOffset = 0;
+
+        for (const buf of buffers) {
+          const sourceName = resolveColumnName(buf, columnName, remaps);
+          const col = buf.getColumnIfAllocated(sourceName);
+          const srcNulls = buf.getNullsIfAllocated(sourceName);
+          // Check if this buffer has a scope value for this field
+          const scopeValue = getNumberScopeValue(buf, fieldName, remaps);
+
+          if (col instanceof Float64Array) {
+            allValues.set(col.subarray(0, buf._writeIndex), rowOffset);
+            if (srcNulls) {
+              // Process each row - direct write wins, then scope, then null
+              for (let i = 0; i < buf._writeIndex; i++) {
+                const srcByte = i >>> 3;
+                const srcBit = i & 7;
+                const isValid = (srcNulls[srcByte] & (1 << srcBit)) !== 0;
+                const rowIdx = rowOffset + i;
+                if (isValid) {
+                  // Direct write - mark as valid
+                  nullBitmap[rowIdx >>> 3] |= 1 << (rowIdx & 7);
+                } else if (scopeValue !== undefined) {
+                  // No direct write, but have scope value - use it
+                  allValues[rowIdx] = scopeValue;
+                  nullBitmap[rowIdx >>> 3] |= 1 << (rowIdx & 7);
+                } else {
+                  // No direct write and no scope - null
+                  nullCount++;
+                }
+              }
+            } else {
+              // No null bitmap means column was allocated but no writes - check scope
+              for (let i = 0; i < buf._writeIndex; i++) {
+                const rowIdx = rowOffset + i;
+                if (scopeValue !== undefined) {
+                  allValues[rowIdx] = scopeValue;
+                  nullBitmap[rowIdx >>> 3] |= 1 << (rowIdx & 7);
+                } else {
+                  nullCount++;
+                }
               }
             }
-          } else {
-            // No null bitmap means column was allocated but no writes - check scope
+          } else if (scopeValue !== undefined) {
+            // Column not allocated but we have scope - fill all rows with scope value
             for (let i = 0; i < buf._writeIndex; i++) {
               const rowIdx = rowOffset + i;
-              if (scopeValue !== undefined) {
-                allValues[rowIdx] = scopeValue;
-                nullBitmap[rowIdx >>> 3] |= 1 << (rowIdx & 7);
-              } else {
-                nullCount++;
-              }
+              allValues[rowIdx] = scopeValue;
+              nullBitmap[rowIdx >>> 3] |= 1 << (rowIdx & 7);
             }
+          } else {
+            // Column doesn't exist for this buffer and no scope - all nulls
+            nullCount += buf._writeIndex;
           }
-        } else if (scopeValue !== undefined) {
-          // Column not allocated but we have scope - fill all rows with scope value
-          for (let i = 0; i < buf._writeIndex; i++) {
-            const rowIdx = rowOffset + i;
-            allValues[rowIdx] = scopeValue;
-            nullBitmap[rowIdx >>> 3] |= 1 << (rowIdx & 7);
-          }
-        } else {
-          // Column doesn't exist for this buffer and no scope - all nulls
-          nullCount += buf._writeIndex;
+          rowOffset += buf._writeIndex;
         }
-        rowOffset += buf._writeIndex;
-      }
 
-      vectors.push(
-        buildColumn(
-          buildData({
-            type: float64(),
-            offset: 0,
-            length: totalRows,
-            nullCount,
-            data: allValues,
-            nullBitmap: nullCount > 0 ? nullBitmap : undefined,
-          }),
-        ),
-      );
+        vectors.push(
+          buildColumn(
+            buildData({
+              type: float64(),
+              offset: 0,
+              length: totalRows,
+              nullCount,
+              data: allValues,
+              nullBitmap: nullCount > 0 ? nullBitmap : undefined,
+            }),
+          ),
+        );
+      }
       fields.push(arrowFieldName);
     } else if (lmaoType === 'boolean') {
       const requiredBytes = Math.ceil(totalRows / 8);
@@ -1629,10 +1922,11 @@ function convertBuffersWithSharedDicts(
       let rowOffset = 0;
 
       for (const buf of buffers) {
-        const col = buf.getColumnIfAllocated(columnName);
-        const srcNulls = buf.getNullsIfAllocated(columnName);
+        const sourceName = resolveColumnName(buf, columnName, remaps);
+        const col = buf.getColumnIfAllocated(sourceName);
+        const srcNulls = buf.getNullsIfAllocated(sourceName);
         // Check if this buffer has a scope value for this field
-        const scopeValue = getBooleanScopeValue(buf, fieldName);
+        const scopeValue = getBooleanScopeValue(buf, fieldName, remaps);
 
         if (col instanceof Uint8Array) {
           // Copy boolean values bit by bit - can't avoid loop for bit-level operations
@@ -1702,32 +1996,25 @@ function convertBuffersWithSharedDicts(
       );
       fields.push(arrowFieldName);
     } else if (lmaoType === 'enum') {
-      const enumValues = getEnumValues(fieldSchema) || [];
+      const descriptor = enumLookupsByField.get(fieldName);
+      if (!descriptor) throw new TypeError(`Enum lookup missing for field '${fieldName}'`);
+      const enumValues = descriptor.values;
       const enumUtf8 = getEnumUtf8(fieldSchema);
-      // Get constructors from schema metadata
-      const indexArrayCtor = getArrowIndexArrayConstructorOr(fieldSchema);
-      const arrowIndexType = getArrowIndexTypeOr(fieldSchema, uint8());
+      const indexArrayCtor = descriptor.indexArrayConstructor;
+      const arrowIndexType = indexTypeForCount(enumValues.length);
       const allIndices = new indexArrayCtor(totalRows);
       const nullBitmap = new Uint8Array(Math.ceil(totalRows / 8));
       // Start with all nulls (bits cleared) - we'll set bits as we find values
       let nullCount = 0;
       let rowOffset = 0;
 
-      // Build enum value to index mapping for scope lookup
-      const enumValueToIndex = new Map<string, number>();
-      for (let i = 0; i < enumValues.length; i++) {
-        enumValueToIndex.set(enumValues[i], i);
-      }
-
       for (const buf of buffers) {
-        const col = buf.getColumnIfAllocated(columnName);
-        const srcNulls = buf.getNullsIfAllocated(columnName);
+        const sourceName = resolveColumnName(buf, columnName, remaps);
+        const col = buf.getColumnIfAllocated(sourceName);
+        const srcNulls = buf.getNullsIfAllocated(sourceName);
         // Check if this buffer has a scope value for this field
-        const scopeValue = getStringScopeValue(buf, fieldName);
-        let scopeEncodedValue: number | undefined;
-        if (scopeValue !== undefined) {
-          scopeEncodedValue = enumValueToIndex.get(scopeValue);
-        }
+        const scopeValue = getStringScopeValue(buf, fieldName, remaps);
+        const scopeEncodedValue = scopeValue === undefined ? undefined : descriptor.encode(scopeValue);
 
         if (col instanceof indexArrayCtor) {
           allIndices.set(col.subarray(0, buf._writeIndex), rowOffset);
@@ -1801,7 +2088,7 @@ function convertBuffersWithSharedDicts(
     } else if (lmaoType === 'binary') {
       // Binary columns: raw Uint8Array or encoder-wrapped values (e.g. msgpack)
       // Binary columns don't use dictionaries and are not scope-fillable
-      const { column } = buildBinaryColumnFromBuffers(buffers, columnName, totalRows, fieldSchema);
+      const { column } = buildBinaryColumnFromBuffers(buffers, columnName, totalRows, fieldSchema, remaps);
       fields.push(arrowFieldName);
       vectors.push(column);
     }