@loaders.gl/parquet 4.5.3 → 5.0.0-alpha.10

package/src/parquet-source-capabilities.ts

@@ -0,0 +1,73 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {TableQueryCapabilities} from '@loaders.gl/loader-utils';
+
+/** Portable table-query capabilities of Parquet-backed scans. */
+export const PARQUET_TABLE_QUERY_CAPABILITIES: TableQueryCapabilities = Object.freeze({
+  projection: 'pushdown',
+  predicate: 'pushdown+residual',
+  limit: 'pushdown',
+  streaming: true,
+  cancellation: true
+});
+
+/** Feature support advertised by a {@link ParquetSource}. */
+export type ParquetSourceCapabilities = Readonly<{
+  /** Common portable query contract supported by this source. */
+  tableQuery: TableQueryCapabilities;
+  /** Schema and footer metadata are cached for the lifetime of the source. */
+  supportsCachedMetadata: boolean;
+  /** Reads can select explicit Parquet row groups. */
+  supportsRowGroupSelection: boolean;
+  /** Reads can project explicit Parquet column paths. */
+  supportsColumnProjection: boolean;
+  /** Every emitted batch identifies its source row group and row offsets. */
+  supportsBatchProvenance: boolean;
+  /** Read cancellation is propagated cooperatively through the active stream. */
+  supportsCooperativeReadCancellation: boolean;
+  /** The package includes a local parquet-wasm binary for self-hosted delivery. */
+  supportsLocalWasmAsset: boolean;
+  /** Column min/max/null statistics are exposed in public metadata. */
+  supportsColumnStatistics: boolean;
+  /** Serializable predicates can prune impossible row groups using column statistics. */
+  supportsPredicatePushdown: boolean;
+  /** Column and offset indexes can avoid irrelevant data-page reads for non-repeated columns. */
+  supportsPageIndexPruning: boolean;
+  /** Native geospatial statistics and GeoParquet 1.1 coverings drive spatial pruning. */
+  supportsGeoParquetSpatialPruning: boolean;
+  /** Surviving decoded rows are filtered exactly on the caller thread or worker. */
+  supportsExactPredicateFiltering: boolean;
+  /** Callers can supply the random-access transport used by the source. */
+  supportsCustomRangeTransport: boolean;
+  /** Range requests validate that the source object version remains unchanged. */
+  supportsObjectVersionValidation: boolean;
+  /** Requested bytes, downloaded bytes, request counts, and network time are reported. */
+  supportsNetworkTelemetry: boolean;
+  /** Decoder time is reported separately from network and Arrow conversion time. */
+  supportsDecodeTelemetry: boolean;
+  /** Parquet decoding can run in a worker and transfer Arrow output. */
+  supportsWorkerDecoding: boolean;
+}>;
+
+/** Capabilities of the current range-backed TypeScript {@link ParquetSource}. */
+export const PARQUET_SOURCE_CAPABILITIES: ParquetSourceCapabilities = Object.freeze({
+  tableQuery: PARQUET_TABLE_QUERY_CAPABILITIES,
+  supportsCachedMetadata: true,
+  supportsRowGroupSelection: true,
+  supportsColumnProjection: true,
+  supportsBatchProvenance: true,
+  supportsCooperativeReadCancellation: true,
+  supportsLocalWasmAsset: true,
+  supportsColumnStatistics: true,
+  supportsPredicatePushdown: true,
+  supportsPageIndexPruning: true,
+  supportsGeoParquetSpatialPruning: true,
+  supportsExactPredicateFiltering: true,
+  supportsCustomRangeTransport: true,
+  supportsObjectVersionValidation: true,
+  supportsNetworkTelemetry: true,
+  supportsDecodeTelemetry: true,
+  supportsWorkerDecoding: true
+});

package/src/parquet-source-loader-types.ts

@@ -0,0 +1,97 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {CoreAPI, SourceLoader} from '@loaders.gl/loader-utils';
+
+import {ParquetFormat} from './parquet-format';
+import type {ParquetSource} from './parquet-source-loader';
+import type {ParquetSourceLoaderOptions} from './parquet-source-types';
+
+// __VERSION__ is injected by babel-plugin-version-inline
+// @ts-ignore TS2304: Cannot find name '__VERSION__'.
+const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
+
+/** Loads the runtime Parquet source implementation from its explicit package subpath. */
+async function preloadParquetSourceLoader(): Promise<SourceLoader<ParquetSource>> {
+  const {ParquetSourceLoaderWithParser} = await import('@loaders.gl/parquet/parquet-source-loader');
+  return ParquetSourceLoaderWithParser;
+}
+
+/**
+ * Lightweight Parquet source metadata.
+ *
+ * Use with the async `load()` API, which preloads the implementation, or import the runtime source
+ * loader from `@loaders.gl/parquet/parquet-source-loader` for synchronous `createDataSource()`.
+ */
+export const ParquetSourceLoader = {
+  ...ParquetFormat,
+  dataType: null as unknown as ParquetSource,
+  batchType: null as never,
+  name: 'ParquetSourceLoader',
+  id: 'parquet-source',
+  module: 'parquet',
+  version: VERSION,
+  type: 'parquet',
+  fromUrl: true,
+  fromBlob: true,
+  options: {
+    parquet: {
+      rowGroups: undefined,
+      columns: undefined,
+      batchSize: undefined,
+      concurrency: undefined,
+      rowGroupFilter: undefined,
+      predicate: undefined,
+      bbox: undefined,
+      geometryColumn: undefined,
+      headers: undefined,
+      preserveBinary: false,
+      verifyPageChecksums: false,
+      verifyFooterSignature: true,
+      onTelemetry: undefined,
+      workerUrl: undefined,
+      wasmUrl: undefined
+    },
+    rangeRequests: {
+      batchDelayMs: 0,
+      maxGapBytes: 0,
+      rangeExpansionBytes: 0
+    }
+  },
+  defaultOptions: {
+    parquet: {
+      rowGroups: undefined!,
+      columns: undefined!,
+      batchSize: undefined!,
+      concurrency: undefined!,
+      rowGroupFilter: undefined!,
+      predicate: undefined!,
+      bbox: undefined!,
+      geometryColumn: undefined!,
+      headers: undefined!,
+      preserveBinary: false,
+      verifyPageChecksums: false,
+      verifyFooterSignature: true,
+      onTelemetry: undefined!,
+      workerUrl: undefined!,
+      wasmUrl: undefined!
+    },
+    rangeRequests: {
+      batchDelayMs: 0,
+      maxGapBytes: 0,
+      rangeExpansionBytes: 0
+    }
+  },
+  testURL: (url: string): boolean => /\.parquet(?:$|[?#])/i.test(url),
+  preload: preloadParquetSourceLoader,
+  createDataSource(
+    _data: string | Blob,
+    _options: ParquetSourceLoaderOptions,
+    _coreApi?: CoreAPI
+  ): ParquetSource {
+    throw new Error(
+      'ParquetSourceLoader requires async load() or an explicit @loaders.gl/parquet/parquet-source-loader import'
+    );
+  }
+} as const satisfies SourceLoader<ParquetSource>;