@loaders.gl/parquet 4.4.5 → 5.0.0-alpha.2

package/src/lib/parquet-source-worker-types.ts

@@ -0,0 +1,107 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {DehydratedArrowTable} from '@loaders.gl/arrow';
+import type {StrictLoaderOptions} from '@loaders.gl/loader-utils';
+import type {Schema} from '@loaders.gl/schema';
+
+import type {ParquetPredicate} from '../parquet-source-types';
+import type {ParquetPagePruningPlan} from './parquet-page-index';
+import type {SchemaDefinition} from '../parquetjs/schema/declare';
+
+/** One transferable range containing a selected Parquet column chunk. */
+export type ParquetSourceWorkerRange = {
+  /** Absolute byte offset of this range in the source object. */
+  offset: number;
+  /** Exact selected bytes transferred to the worker. */
+  data: ArrayBuffer;
+};
+
+/** Serializable metadata required to decode one selected Parquet column chunk. */
+export type ParquetSourceWorkerColumnChunk = {
+  /** Optional external file path declared by the column chunk. */
+  filePath?: string;
+  /** Numeric Parquet physical type. */
+  physicalType: number;
+  /** Numeric Parquet compression codec. */
+  compressionCodec: number;
+  /** Nested path in the Parquet schema. */
+  path: string[];
+  /** Encoded value count. */
+  valueCount: number;
+  /** Compressed byte length of the complete column chunk. */
+  compressedByteLength: number;
+  /** Uncompressed byte length declared by the column chunk. */
+  uncompressedByteLength: number;
+  /** Absolute offset of the first data page. */
+  dataPageOffset: number;
+  /** Absolute offset of the dictionary page, when present. */
+  dictionaryPageOffset?: number;
+};
+
+/** Transferable input for one worker-backed Parquet source row-group decode. */
+export type ParquetSourceWorkerInput = {
+  /** Total source byte length used to validate virtual file reads. */
+  fileByteLength: number;
+  /** Logical row count in the selected row group. */
+  rowCount: number;
+  /** Total uncompressed byte length declared by the row group. */
+  uncompressedByteLength: number;
+  /** TypeScript decoder schema definition cached from the footer. */
+  schemaDefinition: SchemaDefinition;
+  /** Projected loaders.gl schema used for Arrow conversion. */
+  projectedSchema: Schema;
+  /** Selected column chunks represented without Thrift class instances. */
+  columnChunks: ParquetSourceWorkerColumnChunk[];
+  /** Selected compressed ranges transferred from the main thread. */
+  ranges: ParquetSourceWorkerRange[];
+  /** Maximum rows per returned Arrow table batch. */
+  batchSize: number;
+  /** Serializable exact predicate evaluated after decoding. */
+  predicate?: ParquetPredicate;
+  /** Selective data-page plan prepared from column and offset indexes on the caller thread. */
+  pagePlan?: ParquetPagePruningPlan;
+  /** Whether BYTE_ARRAY values stay binary during logical conversion. */
+  preserveBinary: boolean;
+};
+
+/** One directly transferable Arrow batch decoded by a Parquet source worker. */
+export type ParquetSourceWorkerBatch = {
+  /** Logical offset of this batch within its row group. */
+  rowGroupRowOffset: number;
+  /** Logical rows in this batch. */
+  rowCount: number;
+  /** Exact source row indexes represented by this batch. */
+  rowGroupRowIndices?: number[];
+  /** Dehydrated Arrow table whose primitive buffers are directly transferable. */
+  arrowTable: DehydratedArrowTable;
+};
+
+/** Result returned by one worker-backed Parquet source row-group decode. */
+export type ParquetSourceWorkerResult = {
+  /** Logical row count in the decoded source row group before filtering. */
+  sourceRowCount: number;
+  /** Logical row count in the decoded row group. */
+  rowCount: number;
+  /** Directly transferable Arrow batches produced in row order. */
+  batches: ParquetSourceWorkerBatch[];
+  /** Worker time spent decompressing, decoding, and materializing columns. */
+  decodeDurationMs: number;
+  /** Worker time spent converting columns and preparing transferable Arrow buffers. */
+  arrowConversionDurationMs: number;
+};
+
+/** Worker options used by selective Parquet source jobs. */
+export type ParquetSourceWorkerOptions = StrictLoaderOptions & {
+  /** Private descriptor options keyed by the selective worker id. */
+  'parquet-source'?: {
+    /** Overrides the package-local selective Parquet worker asset. */
+    workerUrl?: string;
+  };
+  /** Parquet cancellation controls. */
+  parquet?: {
+    /** Terminates active worker jobs when aborted. */
+    signal?: AbortSignal;
+  };
+};

package/src/lib/parquet-worker-transport.ts

@@ -0,0 +1,57 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {SerializedArrowTableIPC} from '@loaders.gl/arrow';
+import {deserializeArrowTableFromIPC, serializeArrowTableToIPC} from '@loaders.gl/arrow';
+import type {ArrowTable, ObjectRowTable} from '@loaders.gl/schema';
+
+type ParquetWorkerResult = ArrowTable | ObjectRowTable;
+
+type SerializedArrowTable = Omit<ArrowTable, 'data'> & {
+  /** Transferable Arrow IPC payload produced by the worker. */
+  data: SerializedArrowTableIPC;
+};
+
+/** Serializes a Parquet worker result so Arrow buffers are transferred without cloning. */
+export function serializeParquetWorkerResult(result: unknown): unknown {
+  if (!isArrowTable(result)) {
+    return result;
+  }
+  return {
+    ...result,
+    data: serializeArrowTableToIPC(result.data)
+  };
+}
+
+/** Rehydrates a transferred Parquet Arrow result into main-thread Arrow class instances. */
+export function deserializeParquetWorkerResult(result: unknown): ParquetWorkerResult {
+  if (isSerializedArrowTable(result)) {
+    return {...result, data: deserializeArrowTableFromIPC(result.data)};
+  }
+  return result as ParquetWorkerResult;
+}
+
+/** Returns true when a parser result wraps an Apache Arrow table. */
+function isArrowTable(value: unknown): value is ArrowTable {
+  const table = value as ArrowTable;
+  return Boolean(
+    table &&
+      typeof table === 'object' &&
+      table.shape === 'arrow-table' &&
+      table.data &&
+      'batches' in table.data
+  );
+}
+
+/** Returns true when a worker result contains the loaders.gl Arrow IPC transport format. */
+function isSerializedArrowTable(value: unknown): value is SerializedArrowTable {
+  const table = value as SerializedArrowTable;
+  return Boolean(
+    table &&
+      typeof table === 'object' &&
+      table.shape === 'arrow-table' &&
+      table.data &&
+      table.data.transport === 'arrow-ipc'
+  );
+}

package/src/lib/parsers/convert-parquet-tables.ts

@@ -0,0 +1,39 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import {convertArrowToTable} from '@loaders.gl/schema-utils';
+import type {
+  ArrowTable,
+  ArrowTableBatch,
+  ObjectRowTable,
+  ObjectRowTableBatch
+} from '@loaders.gl/schema';
+
+/**
+ * Converts an Arrow-backed table into object rows.
+ *
+ * @param arrowTable - Arrow table wrapper.
+ * @returns Object-row table.
+ */
+export function convertArrowTableToObjectRows(arrowTable: ArrowTable): ObjectRowTable {
+  return convertArrowToTable(arrowTable.data, 'object-row-table') as ObjectRowTable;
+}
+
+/**
+ * Converts an Arrow batch into object-row output.
+ *
+ * @param batch - Arrow table batch wrapper.
+ * @returns Object-row batch.
+ */
+export function convertArrowBatchToObjectRows(batch: ArrowTableBatch): ObjectRowTableBatch {
+  const objectRowTable = convertArrowToTable(batch.data, 'object-row-table') as ObjectRowTable;
+
+  return {
+    batchType: batch.batchType,
+    shape: objectRowTable.shape,
+    schema: objectRowTable.schema,
+    data: objectRowTable.data,
+    length: batch.length
+  };
+}

package/src/lib/parsers/get-parquet-schema.ts

@@ -4,14 +4,16 @@
 
 // loaders.gl
 import {Schema} from '@loaders.gl/schema';
-import {unpackGeoMetadata, unpackJSONStringMetadata} from '@loaders.gl/geoarrow';
+import {unpackGeoMetadata, unpackJSONStringMetadata} from '@loaders.gl/gis';
 import {ParquetReader} from '../../parquetjs/parser/parquet-reader';
 import {convertParquetSchema} from '../arrow/convert-schema-from-parquet';
+import {applyGeoParquetToFieldMetadata} from '../geo/geospatial-metadata';
 
 export async function getSchemaFromParquetReader(reader: ParquetReader): Promise<Schema> {
   const parquetSchema = await reader.getSchema();
   const parquetMetadata = await reader.getFileMetadata();
   const schema = convertParquetSchema(parquetSchema, parquetMetadata);
+  applyGeoParquetToFieldMetadata(schema);
   unpackGeoMetadata(schema.metadata);
   unpackJSONStringMetadata(schema.metadata, 'pandas');
   return schema;

package/src/lib/parsers/parse-geoparquet-to-geojson.ts

@@ -6,31 +6,44 @@ import type {ReadableFile} from '@loaders.gl/loader-utils';
 import type {
   GeoJSONTable,
   GeoJSONTableBatch,
-  ObjectRowTable,
-  ObjectRowTableBatch
+  ArrowTableBatch,
+  ObjectRowTable
 } from '@loaders.gl/schema';
 import {convertWKBTableToGeoJSON} from '@loaders.gl/gis';
+import {convertArrowToTable} from '@loaders.gl/schema-utils';
 
-import type {ParquetJSONLoaderOptions} from '../../parquet-json-loader';
-
-import {parseParquetFile, parseParquetFileInBatches} from './parse-parquet-to-json';
+import type {ParquetLoaderOptions} from '../../parquet-loader-options';
+import {parseParquetArrowTable, parseParquetArrowTableInBatches} from './parse-parquet-tables';
 
 export async function parseGeoParquetFile(
   file: ReadableFile,
-  options?: ParquetJSONLoaderOptions
-): Promise<ObjectRowTable | GeoJSONTable> {
-  const table = await parseParquetFile(file, {...options, shape: 'object-row-table'});
-  return convertWKBTableToGeoJSON(table, table.schema!);
+  options?: ParquetLoaderOptions
+): Promise<GeoJSONTable> {
+  const arrowTable = await parseParquetArrowTable(file, options || {});
+  const objectRowTable = convertArrowToTable(arrowTable.data, 'object-row-table') as ObjectRowTable;
+  return convertWKBTableToGeoJSON(objectRowTable, objectRowTable.schema!);
 }
 
 export async function* parseGeoParquetFileInBatches(
   file: ReadableFile,
-  options?: ParquetJSONLoaderOptions
-): AsyncIterable<ObjectRowTableBatch | GeoJSONTableBatch> {
-  const tableBatches = parseParquetFileInBatches(file, {...options, shape: 'object-row-table'});
+  options?: ParquetLoaderOptions
+): AsyncIterable<GeoJSONTableBatch> {
+  const tableBatches = parseParquetArrowTableInBatches(file, options || {});
 
   for await (const batch of tableBatches) {
-    const geojson = convertWKBTableToGeoJSON(batch as ObjectRowTable, batch.schema!);
-    yield {...batch, ...geojson};
+    yield convertGeoParquetArrowBatch(batch);
   }
 }
+
+function convertGeoParquetArrowBatch(batch: ArrowTableBatch): GeoJSONTableBatch {
+  const objectRowTable = convertArrowToTable(batch.data, 'object-row-table') as ObjectRowTable;
+  const geojsonTable = convertWKBTableToGeoJSON(objectRowTable, objectRowTable.schema!);
+  return {
+    batchType: batch.batchType,
+    shape: geojsonTable.shape,
+    type: geojsonTable.type,
+    schema: geojsonTable.schema,
+    features: geojsonTable.features,
+    length: batch.length
+  };
+}

package/src/lib/parsers/parse-parquet-tables.ts

@@ -0,0 +1,82 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {ReadableFile} from '@loaders.gl/loader-utils';
+import type {
+  ArrowTable,
+  ArrowTableBatch,
+  ObjectRowTable,
+  ObjectRowTableBatch
+} from '@loaders.gl/schema';
+
+import type {ParquetLoaderOptions} from '../../parquet-loader-options';
+import {parseParquetFileToArrow, parseParquetFileToArrowInBatches} from './parse-parquet-to-arrow';
+import {
+  convertArrowBatchToObjectRows,
+  convertArrowTableToObjectRows
+} from './convert-parquet-tables';
+
+export {
+  convertArrowBatchToObjectRows,
+  convertArrowTableToObjectRows
+} from './convert-parquet-tables';
+
+/**
+ * Parses a parquet file into an Arrow-backed table.
+ *
+ * @param file - Parquet file abstraction.
+ * @param options - Normalized parquet options.
+ * @returns Arrow table output.
+ */
+export function parseParquetArrowTable(
+  file: ReadableFile,
+  options: ParquetLoaderOptions
+): Promise<ArrowTable> {
+  return parseParquetFileToArrow(file, options.parquet);
+}
+
+/**
+ * Parses a parquet file into Arrow-backed table batches.
+ *
+ * @param file - Parquet file abstraction.
+ * @param options - Normalized parquet options.
+ * @returns Async iterable of Arrow table batches.
+ */
+export function parseParquetArrowTableInBatches(
+  file: ReadableFile,
+  options: ParquetLoaderOptions
+): AsyncIterable<ArrowTableBatch> {
+  return parseParquetFileToArrowInBatches(file, options.parquet);
+}
+
+/**
+ * Parses a parquet file into plain object rows.
+ *
+ * @param file - Parquet file abstraction.
+ * @param options - Normalized parquet options.
+ * @returns Object-row table output.
+ */
+export async function parseParquetObjectRowTable(
+  file: ReadableFile,
+  options: ParquetLoaderOptions
+): Promise<ObjectRowTable> {
+  const arrowTable = await parseParquetArrowTable(file, options);
+  return convertArrowTableToObjectRows(arrowTable);
+}
+
+/**
+ * Parses a parquet file into object-row batches.
+ *
+ * @param file - Parquet file abstraction.
+ * @param options - Normalized parquet options.
+ * @returns Async iterable of object-row table batches.
+ */
+export async function* parseParquetObjectRowTableInBatches(
+  file: ReadableFile,
+  options: ParquetLoaderOptions
+): AsyncIterable<ObjectRowTableBatch> {
+  for await (const batch of parseParquetArrowTableInBatches(file, options)) {
+    yield convertArrowBatchToObjectRows(batch);
+  }
+}