@loaders.gl/parquet 4.4.2 → 5.0.0-alpha.1

package/src/parquet-format.ts

@@ -7,6 +7,8 @@ import type {Format} from '@loaders.gl/loader-utils';
 export const ParquetFormat = {
   name: 'Apache Parquet',
   category: 'table',
+  encoding: 'parquet',
+  format: 'parquet',
   extensions: ['parquet'],
   mimeTypes: ['application/octet-stream'],
   binary: true,

package/src/parquet-js-loader.ts

@@ -0,0 +1,43 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {Loader} from '@loaders.gl/loader-utils';
+import type {ObjectRowTable, ObjectRowTableBatch} from '@loaders.gl/schema';
+
+import {ParquetFormat} from './parquet-format';
+import type {ParquetJSLoaderOptions} from './parquet-loader-options';
+
+// __VERSION__ is injected by babel-plugin-version-inline
+// @ts-ignore TS2304: Cannot find name '__VERSION__'.
+const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
+
+/** Default option bag for the experimental parquetjs plain-row loader. */
+const DEFAULT_PARQUET_JS_OPTIONS = {
+  backend: 'typescript' as const,
+  columns: undefined,
+  preserveBinary: false
+};
+
+/** Preloads the parser-bearing parquetjs loader implementation. */
+async function preload() {
+  const {ParquetLoaderWithParser} = await import('./parquet-loader-with-parser');
+  return ParquetLoaderWithParser;
+}
+
+/** Metadata-only plain-row Parquet loader backed by the experimental parquetjs implementation. */
+export const ParquetJSLoader = {
+  ...ParquetFormat,
+
+  dataType: null as unknown as ObjectRowTable,
+  batchType: null as unknown as ObjectRowTableBatch,
+
+  id: 'parquet-js',
+  module: 'parquet',
+  version: VERSION,
+  worker: false,
+  options: {
+    parquet: DEFAULT_PARQUET_JS_OPTIONS
+  },
+  preload
+} as const satisfies Loader<ObjectRowTable, ObjectRowTableBatch, ParquetJSLoaderOptions>;

package/src/parquet-js-writer.ts

@@ -0,0 +1,52 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {WriterOptions, WriterWithEncoder} from '@loaders.gl/loader-utils';
+import type {ObjectRowTable, Table, TableBatch} from '@loaders.gl/schema';
+import {convertTable, deduceTableSchema} from '@loaders.gl/schema-utils';
+
+import {normalizeParquetOptions} from './lib/utils/normalize-parquet-options';
+import {encodeTableToParquetJs} from './lib/encoders/encode-table-to-parquet-js';
+import {ParquetFormat} from './parquet-format';
+
+// __VERSION__ is injected by babel-plugin-version-inline
+// @ts-ignore TS2304: Cannot find name '__VERSION__'.
+const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
+
+/** Encoder-specific options for the experimental parquetjs writer. */
+type ParquetJSWriterEncoderOptions = {
+  rowGroupSize?: number;
+  pageSize?: number;
+  useDataPageV2?: boolean;
+};
+
+/** Public options for the parquetjs-backed plain-table writer. */
+export type ParquetJSWriterOptions = WriterOptions & {
+  parquet?: {
+    [Key in keyof ParquetJSWriterEncoderOptions]?: ParquetJSWriterEncoderOptions[Key];
+  };
+};
+
+/** Default option bag for the experimental parquetjs writer. */
+const DEFAULT_PARQUET_JS_OPTIONS = {};
+
+/** Plain-row Parquet writer backed by the experimental parquetjs implementation. */
+export const ParquetJSWriter = {
+  ...ParquetFormat,
+  id: 'parquet-js',
+  module: 'parquet',
+  version: VERSION,
+  options: {
+    parquet: DEFAULT_PARQUET_JS_OPTIONS
+  },
+  async encode(table: Table, options?: ParquetJSWriterOptions) {
+    const schema = table.schema || deduceTableSchema(table);
+    const objectRowTable = convertTable({...table, schema}, 'object-row-table') as ObjectRowTable;
+    return await encodeTableToParquetJs(
+      {...table, schema},
+      objectRowTable,
+      normalizeParquetOptions(options, DEFAULT_PARQUET_JS_OPTIONS)
+    );
+  }
+} as const satisfies WriterWithEncoder<Table, TableBatch, ParquetJSWriterOptions>;

package/src/parquet-loader-options.ts

@@ -0,0 +1,63 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {LoaderOptions} from '@loaders.gl/loader-utils';
+
+/** Shared row-level options supported by both Parquet backends. */
+type ParquetCommonLoaderOptions = {
+  limit?: number;
+  offset?: number;
+  batchSize?: number;
+  columns?: string[];
+  preserveBinary?: boolean;
+};
+
+/** Additional options supported by the wasm-backed Parquet reader. */
+type ParquetWasmLoaderOptions = ParquetCommonLoaderOptions & {
+  rowGroups?: number[];
+  concurrency?: number;
+  wasmUrl?: string;
+};
+
+/** Internal superset of Parquet loader options that retains backend selection. */
+type ParquetInternalLoaderOptions = ParquetWasmLoaderOptions & {
+  shape?: 'object-row-table' | 'arrow-table';
+  backend?: 'wasm' | 'typescript';
+  /** @deprecated Use `backend` instead. */
+  implementation?: 'wasm' | 'js';
+};
+
+/** Public options for the wasm-backed `ParquetLoader` and `GeoParquetLoader`. */
+export type ParquetLoaderOptions = LoaderOptions & {
+  parquet?: {
+    shape?: 'object-row-table' | 'arrow-table';
+    backend?: 'wasm' | 'typescript';
+    /** @deprecated Use `backend` instead. */
+    implementation?: 'wasm' | 'js';
+  } & {
+    [Key in keyof ParquetWasmLoaderOptions]?: ParquetWasmLoaderOptions[Key];
+  };
+};
+
+/** Public options for the experimental parquetjs-backed `ParquetJSLoader`. */
+export type ParquetJSLoaderOptions = LoaderOptions & {
+  parquet?: {backend?: 'typescript'} & {
+    [Key in keyof ParquetCommonLoaderOptions]?: ParquetCommonLoaderOptions[Key];
+  };
+};
+
+/** Internal options used by shared Parquet helpers that need backend selection. */
+export type ParquetLoaderImplementationOptions = LoaderOptions & {
+  parquet?: {
+    [Key in keyof ParquetInternalLoaderOptions]?: ParquetInternalLoaderOptions[Key];
+  };
+};
+
+/** Shared default option bag for the wasm-backed Parquet loaders. */
+export const PARQUET_LOADER_DEFAULT_OPTIONS = {
+  backend: 'wasm',
+  columns: undefined,
+  preserveBinary: false,
+  shape: 'object-row-table'
+} as const;

package/src/parquet-loader-with-parser.ts

@@ -0,0 +1,68 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {LoaderWithParser} from '@loaders.gl/loader-utils';
+import {BlobFile, concatenateArrayBuffersAsync} from '@loaders.gl/loader-utils';
+import type {
+  ArrowTable,
+  ArrowTableBatch,
+  ObjectRowTable,
+  ObjectRowTableBatch
+} from '@loaders.gl/schema';
+import type {ReadableFile} from '@loaders.gl/loader-utils';
+
+import {parseParquetFile, parseParquetFileInBatches} from './lib/parsers/parse-parquet-to-json';
+import {normalizeParquetOptions} from './lib/utils/normalize-parquet-options';
+import type {
+  ParquetLoaderImplementationOptions,
+  ParquetLoaderOptions
+} from './parquet-loader-options';
+import {ParquetLoader as ParquetLoaderMetadata} from './parquet-loader';
+
+const {preload: _ParquetLoaderPreload, ...ParquetLoaderMetadataWithoutPreload} =
+  ParquetLoaderMetadata;
+
+/** Default option bag for the experimental parquetjs plain-row loader. */
+const DEFAULT_PARQUET_JS_OPTIONS = {
+  backend: 'typescript' as const,
+  columns: undefined,
+  preserveBinary: false
+};
+
+/** Parser-bearing TypeScript-only Parquet loader implementation. */
+export const ParquetLoaderWithParser = {
+  ...ParquetLoaderMetadataWithoutPreload,
+  parse(arrayBuffer: ArrayBuffer, options?: ParquetLoaderOptions) {
+    return parseParquetFile(new BlobFile(arrayBuffer), getParquetOptions(options));
+  },
+  parseFile(file: ReadableFile, options?: ParquetLoaderOptions) {
+    return parseParquetFile(file, getParquetOptions(options));
+  },
+  parseFileInBatches(file: ReadableFile, options?: ParquetLoaderOptions) {
+    return parseParquetFileInBatches(file, getParquetOptions(options));
+  },
+  async *parseInBatches(
+    asyncIterator:
+      | AsyncIterable<ArrayBufferLike | ArrayBufferView>
+      | Iterable<ArrayBufferLike | ArrayBufferView>,
+    options?: ParquetLoaderOptions,
+    _context?: unknown
+  ) {
+    const arrayBuffer = await concatenateArrayBuffersAsync(asyncIterator);
+    yield* parseParquetFileInBatches(new BlobFile(arrayBuffer), getParquetOptions(options));
+  }
+} as const satisfies LoaderWithParser<
+  ObjectRowTable | ArrowTable,
+  ObjectRowTableBatch | ArrowTableBatch,
+  ParquetLoaderOptions
+>;
+
+/**
+ * Normalizes caller options for the parquetjs-backed loader.
+ * @param options caller-supplied loader options
+ * @returns normalized options with parquetjs defaults applied
+ */
+function getParquetOptions(options?: ParquetLoaderOptions): ParquetLoaderImplementationOptions {
+  return normalizeParquetOptions(options, DEFAULT_PARQUET_JS_OPTIONS);
+}

package/src/parquet-loader.ts

@@ -0,0 +1,79 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {Loader, LoaderOptions} from '@loaders.gl/loader-utils';
+import type {
+  ObjectRowTable,
+  ObjectRowTableBatch,
+  ArrowTable,
+  ArrowTableBatch
+} from '@loaders.gl/schema';
+
+import {ParquetFormat} from './parquet-format';
+import {
+  PARQUET_LOADER_DEFAULT_OPTIONS,
+  type ParquetLoaderOptions as SharedParquetLoaderOptions
+} from './parquet-loader-options';
+
+// __VERSION__ is injected by babel-plugin-version-inline
+// @ts-ignore TS2304: Cannot find name '__VERSION__'.
+const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
+
+/** Options for the parquet loader */
+export type ParquetLoaderOptions = SharedParquetLoaderOptions;
+
+/** Preloads the parser-bearing Parquet loader implementation selected by `parquet.backend`. */
+async function preloadParquetLoader(_url: string, options?: LoaderOptions) {
+  const parquetOptions = options as ParquetLoaderOptions | undefined;
+  switch (getParquetBackend(parquetOptions)) {
+    case 'wasm': {
+      const {ParquetWASMLoaderWithParser} = await import('./parquet-wasm-loader-with-parser');
+      return ParquetWASMLoaderWithParser;
+    }
+
+    case 'typescript': {
+      if (parquetOptions?.parquet?.shape === 'arrow-table') {
+        throw new Error('ParquetLoader: backend "typescript" does not support shape "arrow-table"');
+      }
+      const {ParquetLoaderWithParser} = await import('./parquet-loader-with-parser');
+      return ParquetLoaderWithParser;
+    }
+
+    default:
+      throw new Error(`ParquetLoader: unsupported backend "${parquetOptions?.parquet?.backend}"`);
+  }
+}
+
+/** Metadata-only Parquet table loader supporting object-row and Arrow table output. */
+export const ParquetLoader = {
+  ...ParquetFormat,
+
+  dataType: null as unknown as ObjectRowTable | ArrowTable,
+  batchType: null as unknown as ObjectRowTableBatch | ArrowTableBatch,
+
+  id: 'parquet',
+  module: 'parquet',
+  version: VERSION,
+  worker: false,
+  options: {
+    parquet: {
+      ...PARQUET_LOADER_DEFAULT_OPTIONS
+    }
+  },
+  preload: preloadParquetLoader
+} as const satisfies Loader<
+  ObjectRowTable | ArrowTable,
+  ObjectRowTableBatch | ArrowTableBatch,
+  ParquetLoaderOptions
+>;
+
+function getParquetBackend(options?: ParquetLoaderOptions): 'wasm' | 'typescript' {
+  if (options?.parquet?.backend) {
+    return options.parquet.backend;
+  }
+  if (options?.parquet?.implementation === 'js') {
+    return 'typescript';
+  }
+  return ParquetLoader.options.parquet.backend;
+}

package/src/parquet-wasm-loader-with-parser.ts

@@ -0,0 +1,115 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {LoaderWithParser} from '@loaders.gl/loader-utils';
+import {concatenateArrayBuffersAsync} from '@loaders.gl/loader-utils';
+import type {
+  ObjectRowTable,
+  ObjectRowTableBatch,
+  ArrowTable,
+  ArrowTableBatch
+  // ColumnarTable,
+  // ColumnarTableBatch
+} from '@loaders.gl/schema';
+import {BlobFile} from '@loaders.gl/loader-utils';
+import type {ReadableFile} from '@loaders.gl/loader-utils';
+
+import {
+  parseParquetArrowTable,
+  parseParquetArrowTableInBatches,
+  parseParquetObjectRowTable,
+  parseParquetObjectRowTableInBatches
+} from './lib/parsers/parse-parquet-tables';
+import {normalizeParquetOptions} from './lib/utils/normalize-parquet-options';
+import {ParquetLoader as ParquetLoaderMetadata, type ParquetLoaderOptions} from './parquet-loader';
+
+const {preload: _ParquetLoaderPreload, ...ParquetLoaderMetadataWithoutPreload} =
+  ParquetLoaderMetadata;
+
+export type {ParquetLoaderOptions} from './parquet-loader';
+
+/** WASM-backed Parquet table loader supporting object-row and Arrow table output. */
+export const ParquetWASMLoaderWithParser = {
+  ...ParquetLoaderMetadataWithoutPreload,
+  parse(arrayBuffer: ArrayBuffer, options?: ParquetLoaderOptions) {
+    return parseParquetTable(new BlobFile(arrayBuffer), options);
+  },
+  parseFile(file, options?: ParquetLoaderOptions) {
+    return parseParquetTable(file, options);
+  },
+  parseFileInBatches(file, options?: ParquetLoaderOptions) {
+    return parseParquetTableInBatches(file, options);
+  },
+  async *parseInBatches(
+    asyncIterator:
+      | AsyncIterable<ArrayBufferLike | ArrayBufferView>
+      | Iterable<ArrayBufferLike | ArrayBufferView>,
+    options?: ParquetLoaderOptions,
+    _context?: unknown
+  ) {
+    const arrayBuffer = await concatenateArrayBuffersAsync(asyncIterator);
+    yield* parseParquetTableInBatches(new BlobFile(arrayBuffer), options);
+  }
+} as const satisfies LoaderWithParser<
+  ObjectRowTable | ArrowTable,
+  ObjectRowTableBatch | ArrowTableBatch,
+  ParquetLoaderOptions
+>;
+
+/**
+ * Parses a Parquet file using the canonical wasm-backed table loader.
+ * @param file readable file abstraction
+ * @param options optional loader options
+ * @returns object-row or Arrow table output depending on `parquet.shape`
+ */
+async function parseParquetTable(
+  file: BlobFile | ReadableFile,
+  options?: ParquetLoaderOptions
+): Promise<ObjectRowTable | ArrowTable> {
+  const parquetOptions = getParquetOptions(options);
+
+  if (parquetOptions.parquet?.shape === 'arrow-table') {
+    return await parseParquetArrowTable(file, parquetOptions);
+  }
+
+  return await parseParquetObjectRowTable(file, parquetOptions);
+}
+
+/**
+ * Parses a Parquet file into streamed table batches using the canonical wasm-backed loader.
+ * @param file readable file abstraction
+ * @param options optional loader options
+ * @returns async iterable of object-row or Arrow batches
+ */
+async function* parseParquetTableInBatches(
+  file: BlobFile | ReadableFile,
+  options?: ParquetLoaderOptions
+): AsyncIterable<ObjectRowTableBatch | ArrowTableBatch> {
+  const parquetOptions = getParquetOptions(options);
+
+  if (parquetOptions.parquet?.shape === 'arrow-table') {
+    yield* parseParquetArrowTableInBatches(file, parquetOptions);
+    return;
+  }
+
+  yield* parseParquetObjectRowTableInBatches(file, parquetOptions);
+}
+
+/**
+ * Normalizes caller options for the canonical wasm-backed Parquet loaders.
+ * @param options caller-supplied loader options
+ * @returns normalized loader options
+ */
+export function getParquetOptions(options?: ParquetLoaderOptions): ParquetLoaderOptions {
+  return normalizeParquetOptions(
+    {
+      ...options,
+      parquet: {
+        ...(options?.parquet || {}),
+        backend: 'wasm'
+      }
+    },
+    ParquetWASMLoaderWithParser.options.parquet
+  );
+}

package/src/parquet-writer.ts

@@ -0,0 +1,44 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {WriterOptions, WriterWithEncoder} from '@loaders.gl/loader-utils';
+import type {ArrowTable, Table, TableBatch} from '@loaders.gl/schema';
+import {convertTable, deduceTableSchema} from '@loaders.gl/schema-utils';
+
+import {encodeArrowToParquet} from './lib/encoders/encode-arrow-to-parquet';
+import {ensureGeoParquetMetadataOnArrowTable} from './lib/geo/geospatial-metadata';
+import {normalizeParquetOptions} from './lib/utils/normalize-parquet-options';
+import {ParquetFormat} from './parquet-format';
+import {VERSION, PARQUET_WASM_URL} from './lib/constants';
+
+/** Public options for the wasm-backed Parquet writer. */
+export type ParquetWriterOptions = WriterOptions & {
+  parquet?: {
+    wasmUrl?: string;
+  };
+};
+
+/** Plain-row Parquet writer that converts tables to Arrow before wasm encoding. */
+export const ParquetWriter = {
+  ...ParquetFormat,
+  id: 'parquet',
+  module: 'parquet',
+  version: VERSION,
+  options: {
+    parquet: {
+      wasmUrl: PARQUET_WASM_URL
+    }
+  },
+  encode(table: Table, options?: ParquetWriterOptions) {
+    const arrowTable =
+      table.shape === 'arrow-table'
+        ? (table as ArrowTable)
+        : (convertTable(
+            {...table, schema: table.schema || deduceTableSchema(table)},
+            'arrow-table'
+          ) as ArrowTable);
+    const parquetOptions = normalizeParquetOptions(options, ParquetWriter.options.parquet);
+    return encodeArrowToParquet(ensureGeoParquetMetadataOnArrowTable(arrowTable), parquetOptions);
+  }
+} as const satisfies WriterWithEncoder<Table, TableBatch, ParquetWriterOptions>;

package/src/parquetjs/codecs/declare.ts

@@ -7,7 +7,7 @@
 import {PrimitiveType} from '../schema/declare';
 
 export interface CursorBuffer {
-  buffer: Buffer;
+  buffer: Uint8Array;
   offset: number;
   size?: number;
 }
@@ -19,7 +19,7 @@ export interface ParquetCodecOptions {
 }
 
 export interface ParquetCodecKit {
-  encodeValues(type: PrimitiveType, values: any[], opts?: ParquetCodecOptions): Buffer;
+  encodeValues(type: PrimitiveType, values: any[], opts?: ParquetCodecOptions): Uint8Array;
   decodeValues(
     type: PrimitiveType,
     cursor: CursorBuffer,

package/src/parquetjs/codecs/dictionary.ts

@@ -7,7 +7,7 @@
 import {decodeValues as decodeRleValues} from './rle';
 
 export function decodeValues(type, cursor, count, opts) {
-  opts.bitWidth = cursor.buffer.slice(cursor.offset, cursor.offset + 1).readInt8(0);
+  opts.bitWidth = cursor.buffer[cursor.offset];
   cursor.offset += 1;
   return decodeRleValues(type, cursor, count, {...opts, disableEnvelope: true});
 }