@loaders.gl/parquet 4.5.3 → 5.0.0-alpha.10

package/src/lib/parquet-predicate.ts

@@ -0,0 +1,410 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {ArrayType} from '@loaders.gl/schema';
+
+import type {
+  ParquetColumnChunkStatistics,
+  ParquetComparisonPredicate,
+  ParquetInPredicate,
+  ParquetLogicalPredicate,
+  ParquetNotPredicate,
+  ParquetNullPredicate,
+  ParquetPredicate,
+  ParquetPredicateProperty,
+  ParquetPredicateValue,
+  ParquetRowGroupMetadata
+} from '../parquet-source-types';
+import {encodeUtf8} from '../parquetjs/utils/binary-utils';
+
+/** Returns the unique top-level columns referenced by a predicate. */
+export function getParquetPredicateColumns(predicate: ParquetPredicate): string[] {
+  return [...new Set(getParquetPredicatePaths(predicate).map(path => path[0]))];
+}
+
+/** Returns the unique physical schema paths referenced by a predicate. */
+export function getParquetPredicatePaths(predicate: ParquetPredicate): string[][] {
+  const paths = new Map<string, string[]>();
+  visitParquetPredicate(predicate, child => {
+    const property = getPredicateProperty(child);
+    if (property) {
+      const path = [...getParquetPredicatePath(property)];
+      paths.set(path.join('\0'), path);
+    }
+  });
+  return [...paths.values()];
+}
+
+/** Copies a predicate tree and mutable scalar values for read-scoped option snapshots. */
+export function copyParquetPredicate(predicate: ParquetPredicate): ParquetPredicate {
+  if (isParquetLogicalPredicate(predicate)) {
+    return {
+      op: predicate.op,
+      args: predicate.args.map(copyParquetPredicate)
+    };
+  }
+  if (isParquetNotPredicate(predicate)) {
+    return {
+      op: 'not',
+      args: [copyParquetPredicate(predicate.args[0])]
+    };
+  }
+  const property = copyPredicateProperty(predicate.args[0]);
+  if (predicate.op === 'isNull') {
+    return {op: 'isNull', args: [property]};
+  }
+  if (predicate.op === 'in') {
+    return {
+      op: 'in',
+      args: [property, predicate.args[1].map(copyPredicateValue)]
+    };
+  }
+  return {
+    op: predicate.op,
+    args: [property, copyPredicateValue(predicate.args[1])]
+  };
+}
+
+/** Validates a predicate and every referenced top-level column. */
+export function validateParquetPredicate(
+  predicate: ParquetPredicate,
+  availableColumns: ReadonlySet<string>
+): void {
+  visitParquetPredicate(predicate, child => {
+    if (isParquetLogicalPredicate(child)) {
+      if (child.args.length < 2) {
+        throw new Error(`Parquet predicate ${child.op} requires at least two child predicates`);
+      }
+      return;
+    }
+    if (isParquetNotPredicate(child)) {
+      return;
+    }
+    const property = child.args[0];
+    const path = getParquetPredicatePath(property);
+    if (!path.length || path.some(pathComponent => !pathComponent)) {
+      throw new Error('Parquet predicate property path must contain non-empty strings');
+    }
+    if (!availableColumns.has(path[0])) {
+      throw new Error(`Parquet predicate column not found: ${path.join('.')}`);
+    }
+    if (child.op === 'in' && child.args[1].length === 0) {
+      throw new Error('Parquet predicate in requires at least one value');
+    }
+  });
+}
+
+/** Conservatively determines whether a row group can contain a matching row. */
+export function canParquetRowGroupMatch(
+  predicate: ParquetPredicate,
+  rowGroup: ParquetRowGroupMetadata
+): boolean {
+  if (isParquetLogicalPredicate(predicate)) {
+    return predicate.op === 'and'
+      ? predicate.args.every(child => canParquetRowGroupMatch(child, rowGroup))
+      : predicate.args.some(child => canParquetRowGroupMatch(child, rowGroup));
+  }
+  if (isParquetNotPredicate(predicate)) {
+    const child = predicate.args[0];
+    if (child.op !== 'isNull') {
+      return true;
+    }
+    const column = getRowGroupColumn(rowGroup, getParquetPredicatePath(child.args[0]));
+    return !column?.statistics || column.statistics.nullCount !== rowGroup.rowCount;
+  }
+
+  const column = getRowGroupColumn(rowGroup, getParquetPredicatePath(predicate.args[0]));
+  const statistics = column?.statistics;
+  if (!statistics) {
+    return true;
+  }
+  return canParquetStatisticsMatch(predicate, statistics, rowGroup.rowCount);
+}
+
+/** Conservatively determines whether one predicate leaf can match supplied statistics. */
+export function canParquetStatisticsMatch(
+  predicate: ParquetComparisonPredicate | ParquetInPredicate | ParquetNullPredicate,
+  statistics: ParquetColumnChunkStatistics,
+  rowCount: number
+): boolean {
+  if (statistics.nullCount === rowCount) {
+    return predicate.op === 'isNull';
+  }
+  if (predicate.op === 'isNull') {
+    return statistics.nullCount !== 0;
+  }
+  const minimum = statistics.minIsExact === false ? undefined : statistics.min;
+  const maximum = statistics.maxIsExact === false ? undefined : statistics.max;
+  if (predicate.op === 'in') {
+    return predicate.args[1].some(value => canComparisonMatch('=', value, minimum, maximum));
+  }
+  return canComparisonMatch(predicate.op, predicate.args[1], minimum, maximum);
+}
+
+/** Returns exact source row indexes matching a predicate. */
+export function filterParquetRowIndices(
+  predicate: ParquetPredicate | undefined,
+  columns: Record<string, ArrayType>,
+  rowCount: number
+): number[] {
+  if (!predicate) {
+    return Array.from({length: rowCount}, (_, index) => index);
+  }
+  const rowIndices: number[] = [];
+  for (let rowIndex = 0; rowIndex < rowCount; rowIndex++) {
+    if (evaluateParquetPredicate(predicate, columns, rowIndex) === true) {
+      rowIndices.push(rowIndex);
+    }
+  }
+  return rowIndices;
+}
+
+/** Gathers selected source rows without constructing row objects. */
+export function gatherParquetColumns(
+  columns: Record<string, ArrayType>,
+  rowIndices: readonly number[],
+  selectedColumns?: ReadonlySet<string>
+): Record<string, ArrayType> {
+  const gatheredColumns: Record<string, ArrayType> = {};
+  for (const [name, column] of Object.entries(columns)) {
+    if (selectedColumns && !selectedColumns.has(name)) {
+      continue;
+    }
+    gatheredColumns[name] = rowIndices.map(rowIndex => (column as ArrayLike<unknown>)[rowIndex]);
+  }
+  return gatheredColumns;
+}
+
+/** Applies an exact predicate to one materialized row. */
+function evaluateParquetPredicate(
+  predicate: ParquetPredicate,
+  columns: Record<string, ArrayType>,
+  rowIndex: number
+): boolean | null {
+  if (isParquetLogicalPredicate(predicate)) {
+    const results = predicate.args.map(child => evaluateParquetPredicate(child, columns, rowIndex));
+    if (predicate.op === 'and') {
+      return results.includes(false) ? false : results.includes(null) ? null : true;
+    }
+    return results.includes(true) ? true : results.includes(null) ? null : false;
+  }
+  if (isParquetNotPredicate(predicate)) {
+    const result = evaluateParquetPredicate(predicate.args[0], columns, rowIndex);
+    return result === null ? null : !result;
+  }
+  const value = getColumnValue(columns, getParquetPredicatePath(predicate.args[0]), rowIndex);
+  if (predicate.op === 'isNull') {
+    return value === null || value === undefined;
+  }
+  if (value === null || value === undefined) {
+    return null;
+  }
+  if (predicate.op === 'in') {
+    return predicate.args[1].some(candidate => comparePredicateValues(value, candidate) === 0);
+  }
+  const comparison = comparePredicateValues(value, predicate.args[1]);
+  switch (predicate.op) {
+    case '=':
+      return comparison === 0;
+    case '<>':
+      return comparison !== 0;
+    case '<':
+      return comparison < 0;
+    case '<=':
+      return comparison <= 0;
+    case '>':
+      return comparison > 0;
+    case '>=':
+      return comparison >= 0;
+  }
+}
+
+/** Returns whether footer min/max leave any possible match for one comparison. */
+function canComparisonMatch(
+  operator: '=' | '<>' | '<' | '<=' | '>' | '>=',
+  value: ParquetPredicateValue,
+  minimum: unknown,
+  maximum: unknown
+): boolean {
+  try {
+    switch (operator) {
+      case '=':
+        return !(
+          (minimum !== undefined && compareStatisticsValues(value, minimum) < 0) ||
+          (maximum !== undefined && compareStatisticsValues(value, maximum) > 0)
+        );
+      case '<>':
+        return !(
+          minimum !== undefined &&
+          maximum !== undefined &&
+          compareStatisticsValues(value, minimum) === 0 &&
+          compareStatisticsValues(value, maximum) === 0
+        );
+      case '<':
+        return minimum === undefined || compareStatisticsValues(minimum, value) < 0;
+      case '<=':
+        return minimum === undefined || compareStatisticsValues(minimum, value) <= 0;
+      case '>':
+        return maximum === undefined || compareStatisticsValues(maximum, value) > 0;
+      case '>=':
+        return maximum === undefined || compareStatisticsValues(maximum, value) >= 0;
+    }
+  } catch {
+    return true;
+  }
+}
+
+/** Compares statistics using Parquet's unsigned UTF-8 byte ordering for strings. */
+function compareStatisticsValues(left: unknown, right: unknown): number {
+  if (typeof left === 'string' && typeof right === 'string') {
+    return compareUint8Arrays(encodeUtf8(left), encodeUtf8(right));
+  }
+  return comparePredicateValues(left, right);
+}
+
+/** Compares predicate-compatible scalar values and rejects incomparable types. */
+function comparePredicateValues(left: unknown, right: unknown): number {
+  if (left instanceof Uint8Array && right instanceof Uint8Array) {
+    return compareUint8Arrays(left, right);
+  }
+  if (left instanceof Uint8Array || right instanceof Uint8Array) {
+    throw new Error('Parquet binary predicates require binary values on both sides');
+  }
+  if (left instanceof Date || right instanceof Date) {
+    if (!(left instanceof Date) || !(right instanceof Date)) {
+      throw new Error('Parquet date predicates require date values on both sides');
+    }
+    const leftTime = left.getTime();
+    const rightTime = right.getTime();
+    if (!Number.isFinite(leftTime) || !Number.isFinite(rightTime)) {
+      throw new Error('Invalid dates cannot be ordered by a Parquet predicate');
+    }
+    return leftTime < rightTime ? -1 : leftTime > rightTime ? 1 : 0;
+  }
+  const leftType = typeof left;
+  const rightType = typeof right;
+  const bothNumeric =
+    (leftType === 'number' || leftType === 'bigint') &&
+    (rightType === 'number' || rightType === 'bigint');
+  if (!bothNumeric && leftType !== rightType) {
+    throw new Error(`Parquet predicate cannot compare ${leftType} with ${rightType}`);
+  }
+  if (!bothNumeric && leftType !== 'string' && leftType !== 'boolean') {
+    throw new Error(`Parquet predicate does not support ${leftType} values`);
+  }
+  const comparableLeft = left as string | boolean | number | bigint;
+  const comparableRight = right as string | boolean | number | bigint;
+  if (
+    (typeof comparableLeft === 'number' && !Number.isFinite(comparableLeft)) ||
+    (typeof comparableRight === 'number' && !Number.isFinite(comparableRight))
+  ) {
+    throw new Error('Non-finite numbers cannot be ordered by a Parquet predicate');
+  }
+  if ((comparableLeft as never) < (comparableRight as never)) {
+    return -1;
+  }
+  if ((comparableLeft as never) > (comparableRight as never)) {
+    return 1;
+  }
+  return 0;
+}
+
+/** Compares two byte sequences using unsigned lexicographic ordering. */
+function compareUint8Arrays(left: Uint8Array, right: Uint8Array): number {
+  const length = Math.min(left.length, right.length);
+  for (let index = 0; index < length; index++) {
+    if (left[index] !== right[index]) {
+      return left[index] < right[index] ? -1 : 1;
+    }
+  }
+  return Math.sign(left.length - right.length);
+}
+
+/** Reads one value from an array-like materialized Parquet column. */
+function getColumnValue(
+  columns: Record<string, ArrayType>,
+  path: readonly string[],
+  rowIndex: number
+): unknown {
+  const column = columns[path[0]];
+  if (!column) {
+    throw new Error('Parquet predicate column was not decoded');
+  }
+  let value = (column as ArrayLike<unknown>)[rowIndex];
+  for (let pathIndex = 1; pathIndex < path.length; pathIndex++) {
+    if (value === null || value === undefined || typeof value !== 'object') {
+      return undefined;
+    }
+    const property = path[pathIndex];
+    value =
+      typeof (value as {get?: unknown}).get === 'function'
+        ? (value as {get: (name: string) => unknown}).get(property)
+        : (value as Record<string, unknown>)[property];
+  }
+  return value;
+}
+
+/** Copies mutable binary and date predicate values. */
+function copyPredicateValue(value: ParquetPredicateValue): ParquetPredicateValue {
+  if (value instanceof Uint8Array) {
+    return value.slice();
+  }
+  if (value instanceof Date) {
+    return new Date(value.getTime());
+  }
+  return value;
+}
+
+/** Copies a predicate property reference. */
+function copyPredicateProperty(property: ParquetPredicateProperty): ParquetPredicateProperty {
+  return {
+    property: Array.isArray(property.property) ? [...property.property] : property.property
+  };
+}
+
+/** Returns one predicate property as an explicit Parquet schema path. */
+export function getParquetPredicatePath(property: ParquetPredicateProperty): readonly string[] {
+  return Array.isArray(property.property) ? property.property : [property.property as string];
+}
+
+/** Returns the property reference from a leaf predicate. */
+function getPredicateProperty(predicate: ParquetPredicate): ParquetPredicateProperty | undefined {
+  return isParquetLogicalPredicate(predicate) || isParquetNotPredicate(predicate)
+    ? undefined
+    : predicate.args[0];
+}
+
+/** Returns one top-level column chunk from normalized row-group metadata. */
+function getRowGroupColumn(rowGroup: ParquetRowGroupMetadata, path: readonly string[]) {
+  return rowGroup.columns.find(
+    columnChunk =>
+      columnChunk.path.length === path.length &&
+      columnChunk.path.every((pathComponent, index) => pathComponent === path[index])
+  );
+}
+
+/** Visits every predicate node depth-first. */
+function visitParquetPredicate(
+  predicate: ParquetPredicate,
+  visit: (predicate: ParquetPredicate) => void
+): void {
+  visit(predicate);
+  if (isParquetLogicalPredicate(predicate) || isParquetNotPredicate(predicate)) {
+    for (const child of predicate.args) {
+      visitParquetPredicate(child, visit);
+    }
+  }
+}
+
+/** Returns whether a predicate combines multiple child predicates. */
+function isParquetLogicalPredicate(
+  predicate: ParquetPredicate
+): predicate is ParquetLogicalPredicate {
+  return predicate.op === 'and' || predicate.op === 'or';
+}
+
+/** Returns whether a predicate negates one child predicate. */
+function isParquetNotPredicate(predicate: ParquetPredicate): predicate is ParquetNotPredicate {
+  return predicate.op === 'not';
+}

package/src/lib/parquet-source-worker-client.ts

@@ -0,0 +1,41 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {Loader} from '@loaders.gl/loader-utils';
+import {canParseWithWorker, isBrowser, parseWithWorker} from '@loaders.gl/loader-utils';
+
+import {PARQUET_LOADER_BASE} from '../parquet-loader-base';
+import {PARQUET_SOURCE_WORKER_LOAD_WORKER} from '../parquet-source-worker-factory';
+import {PARQUET_SOURCE_WORKER_URL} from '../parquet-source-worker-url';
+import type {
+  ParquetSourceWorkerInput,
+  ParquetSourceWorkerOptions,
+  ParquetSourceWorkerResult
+} from './parquet-source-worker-types';
+
+/** Private worker descriptor using the package-local selective source worker bundle. */
+const PARQUET_SOURCE_WORKER: Loader = {
+  ...PARQUET_LOADER_BASE,
+  id: 'parquet-source',
+  name: 'ParquetSource',
+  worker: PARQUET_SOURCE_WORKER_URL,
+  loadWorker: PARQUET_SOURCE_WORKER_LOAD_WORKER
+};
+
+/** Returns whether this runtime and option set can decode Parquet source rows on a worker. */
+export function canDecodeParquetSourceOnWorker(options: ParquetSourceWorkerOptions): boolean {
+  return isBrowser && canParseWithWorker(PARQUET_SOURCE_WORKER, options);
+}
+
+/** Sends selected compressed Parquet chunks to a worker and returns transferable Arrow buffers. */
+export async function decodeParquetSourceRowGroupOnWorker(
+  input: ParquetSourceWorkerInput,
+  options: ParquetSourceWorkerOptions
+): Promise<ParquetSourceWorkerResult> {
+  return (await parseWithWorker(
+    PARQUET_SOURCE_WORKER,
+    input,
+    options
+  )) as ParquetSourceWorkerResult;
+}

package/src/lib/parquet-source-worker-decoder.ts

@@ -0,0 +1,252 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import {dehydrateArrowTable} from '@loaders.gl/arrow/transport';
+import type {ArrayType} from '@loaders.gl/schema';
+import {convertTable} from '@loaders.gl/schema-utils';
+
+import {preloadCompressions} from '../parquetjs/compression';
+import {filterParquetRowIndices, gatherParquetColumns} from './parquet-predicate';
+import type {RowGroup} from '../parquetjs/parquet-thrift/index';
+import {ParquetReader} from '../parquetjs/parser/parquet-reader';
+import {ParquetSchema} from '../parquetjs/schema/schema';
+import {ParquetSourceWorkerFile} from './parquet-source-worker-file';
+import type {ParquetKeyRetriever} from './parquet-encryption';
+import type {
+  ParquetSourceWorkerBatch,
+  ParquetSourceWorkerColumnChunk,
+  ParquetSourceWorkerInput,
+  ParquetSourceWorkerResult
+} from './parquet-source-worker-types';
+
+/** Decodes one transferred row group entirely inside the worker. */
+export async function decodeParquetSourceWorkerInput(
+  input: ParquetSourceWorkerInput
+): Promise<ParquetSourceWorkerResult> {
+  await preloadCompressions();
+  const file = new ParquetSourceWorkerFile(input.fileByteLength, input.ranges);
+  const schema = new ParquetSchema(input.schemaDefinition);
+  const keyRetriever = input.encryption
+    ? createWorkerKeyRetriever(
+        input.columnChunks,
+        input.encryption.aadPrefix?.byteLength ?? 0,
+        input.encryption.fileUnique.byteLength
+      )
+    : undefined;
+  const reader = new ParquetReader(file, {
+    preserveBinary: input.preserveBinary,
+    int96AsTimestamp: input.int96AsTimestamp,
+    verifyPageChecksums: input.verifyPageChecksums,
+    encryptionContext: input.encryption
+      ? {
+          algorithm: input.encryption.algorithm,
+          aadPrefix: input.encryption.aadPrefix
+            ? new Uint8Array(input.encryption.aadPrefix)
+            : undefined,
+          fileUnique: new Uint8Array(input.encryption.fileUnique)
+        }
+      : undefined,
+    keyRetriever
+  });
+  const rowGroup = createWorkerRowGroup(input);
+
+  const decodeStartTime = getCurrentTime();
+  let decodedRowGroups;
+  try {
+    decodedRowGroups = input.pagePlan
+      ? await Promise.all(
+          input.pagePlan.rowRanges.map(rowRange =>
+            reader.readRowGroupRange(schema, rowGroup, [], rowRange, input.pagePlan!.pageLocations)
+          )
+        )
+      : [await reader.readRowGroup(schema, rowGroup, [])];
+  } catch (error) {
+    throw new Error(
+      `Parquet worker decode failed: ${
+        error instanceof Error ? `${error.name}: ${error.message}` : String(error)
+      }`
+    );
+  }
+  const columns = concatenateMaterializedColumns(
+    decodedRowGroups.map(decodedRowGroup => schema.materializeColumns(decodedRowGroup))
+  );
+  const sourceRowIndices = input.pagePlan
+    ? input.pagePlan.rowRanges.flatMap(rowRange =>
+        Array.from({length: rowRange.end - rowRange.start}, (_, index) => rowRange.start + index)
+      )
+    : undefined;
+  const sourceRowCount = sourceRowIndices?.length ?? input.rowCount;
+  const localRowIndices = input.predicate
+    ? filterParquetRowIndices(input.predicate, columns, sourceRowCount)
+    : undefined;
+  const rowIndices = localRowIndices?.map(rowIndex => sourceRowIndices?.[rowIndex] ?? rowIndex);
+  const decodeDurationMs = getCurrentTime() - decodeStartTime;
+
+  const conversionStartTime = getCurrentTime();
+  const batches: ParquetSourceWorkerBatch[] = [];
+  const outputColumns = new Set(input.projectedSchema.fields.map(field => field.name));
+  const outputRowCount = rowIndices?.length ?? sourceRowCount;
+  for (
+    let outputRowOffset = 0;
+    outputRowOffset < outputRowCount;
+    outputRowOffset += input.batchSize
+  ) {
+    const rowCount = Math.min(input.batchSize, outputRowCount - outputRowOffset);
+    const batchRowIndices = rowIndices?.slice(outputRowOffset, outputRowOffset + rowCount);
+    const batchLocalRowIndices = localRowIndices?.slice(
+      outputRowOffset,
+      outputRowOffset + rowCount
+    );
+    const rowGroupRowOffset = batchRowIndices?.[0] ?? outputRowOffset;
+    const batchColumns = batchLocalRowIndices
+      ? gatherParquetColumns(columns, batchLocalRowIndices, outputColumns)
+      : sliceColumns(columns, outputRowOffset, outputRowOffset + rowCount);
+    const arrowTable = convertTable(
+      {
+        shape: 'columnar-table',
+        schema: input.projectedSchema,
+        data: batchColumns
+      },
+      'arrow-table'
+    );
+    batches.push({
+      rowGroupRowOffset,
+      rowCount,
+      rowGroupRowIndices: batchRowIndices,
+      arrowTable: dehydrateArrowTable(arrowTable.data)
+    });
+  }
+  const arrowConversionDurationMs = getCurrentTime() - conversionStartTime;
+  return {
+    sourceRowCount,
+    rowCount: outputRowCount,
+    batches,
+    decodeDurationMs,
+    arrowConversionDurationMs
+  };
+}
+
+/** Concatenates materialized page-range fragments without constructing row objects. */
+function concatenateMaterializedColumns(
+  fragments: readonly Record<string, ArrayType>[]
+): Record<string, ArrayType> {
+  const columns: Record<string, unknown[]> = {};
+  for (const fragment of fragments) {
+    for (const [name, values] of Object.entries(fragment)) {
+      columns[name] ||= [];
+      const destination = columns[name];
+      for (let index = 0; index < values.length; index++) {
+        destination.push(values[index]);
+      }
+    }
+  }
+  return columns as Record<string, ArrayType>;
+}
+
+/** Returns a contiguous row slice of every decoded column without constructing row objects. */
+function sliceColumns(
+  columns: Record<string, ArrayType>,
+  start: number,
+  end: number
+): Record<string, ArrayType> {
+  const slicedColumns: Record<string, ArrayType> = {};
+  for (const [name, column] of Object.entries(columns)) {
+    const slice = (column as ArrayType & {slice?: (start: number, end: number) => ArrayType}).slice;
+    slicedColumns[name] = slice
+      ? slice.call(column, start, end)
+      : Array.prototype.slice.call(column, start, end);
+  }
+  return slicedColumns;
+}
+
+/** Reconstructs the minimal Thrift-compatible row-group object used by the decoder. */
+function createWorkerRowGroup(input: ParquetSourceWorkerInput): RowGroup {
+  return {
+    num_rows: input.rowCount,
+    total_byte_size: input.uncompressedByteLength,
+    columns: input.columnChunks.map((columnChunk, index) => {
+      const result = {
+        file_path: columnChunk.filePath,
+        parquetColumnOrdinal: columnChunk.columnOrdinal ?? index,
+        meta_data: {
+          type: columnChunk.physicalType,
+          codec: columnChunk.compressionCodec,
+          path_in_schema: columnChunk.path,
+          num_values: columnChunk.valueCount,
+          total_compressed_size: columnChunk.compressedByteLength,
+          total_uncompressed_size: columnChunk.uncompressedByteLength,
+          data_page_offset: columnChunk.dataPageOffset,
+          dictionary_page_offset: columnChunk.dictionaryPageOffset,
+          encodings: []
+        }
+      } as unknown as RowGroup['columns'][number];
+      if (columnChunk.encrypted) {
+        result.crypto_metadata = (columnChunk.keyMetadata
+          ? {
+              ENCRYPTION_WITH_COLUMN_KEY: {
+                path_in_schema: columnChunk.path,
+                key_metadata: new Uint8Array(columnChunk.keyMetadata)
+              }
+            }
+          : {
+              ENCRYPTION_WITH_FOOTER_KEY: {}
+            }) as unknown as RowGroup['columns'][number]['crypto_metadata'];
+      }
+      return result;
+    })
+  } as unknown as RowGroup;
+}
+
+/** Creates a worker-local key retriever from caller-resolved transferable keys. */
+export function createWorkerKeyRetriever(
+  columnChunks: readonly ParquetSourceWorkerColumnChunk[],
+  aadPrefixByteLength: number,
+  fileUniqueByteLength: number
+): ParquetKeyRetriever {
+  return (keyMetadata, context) => {
+    const columnOrdinal = getColumnOrdinalFromAad(
+      context.aad,
+      aadPrefixByteLength,
+      fileUniqueByteLength
+    );
+    const metadataMatches = columnChunks.filter(columnChunk =>
+      equalBytes(keyMetadata, columnChunk.keyMetadata && new Uint8Array(columnChunk.keyMetadata))
+    );
+    const matchingColumn =
+      metadataMatches.find(columnChunk => columnChunk.columnOrdinal === columnOrdinal) ??
+      (metadataMatches.length === 1 ? metadataMatches[0] : undefined);
+    if (!matchingColumn?.keyMaterial) {
+      throw new Error('Encrypted Parquet worker key was not transferred for the selected column');
+    }
+    return new Uint8Array(matchingColumn.keyMaterial);
+  };
+}
+
+/** Extracts the physical column ordinal encoded in a page-module AAD suffix. */
+function getColumnOrdinalFromAad(
+  aad: Uint8Array,
+  aadPrefixByteLength: number,
+  fileUniqueByteLength: number
+): number | undefined {
+  const columnOrdinalOffset = aadPrefixByteLength + fileUniqueByteLength + 3;
+  if (columnOrdinalOffset + 2 > aad.byteLength) return undefined;
+  return new DataView(aad.buffer, aad.byteOffset + columnOrdinalOffset, 2).getInt16(0, true);
+}
+
+/** Compares optional key metadata without exposing key material in structured-clone state. */
+function equalBytes(
+  firstBytes: Uint8Array | undefined,
+  secondBytes: Uint8Array | undefined
+): boolean {
+  if (!firstBytes || !secondBytes) return !firstBytes && !secondBytes;
+  return (
+    firstBytes.length === secondBytes.length &&
+    firstBytes.every((value, index) => value === secondBytes[index])
+  );
+}
+
+/** Returns a monotonic timestamp when available and falls back to wall-clock time. */
+function getCurrentTime(): number {
+  return globalThis.performance?.now() ?? Date.now();
+}