@loaders.gl/parquet 4.4.5 → 5.0.0-alpha.2

package/src/lib/utils/make-stream-iterator.ts

@@ -7,6 +7,8 @@ import {isBrowser} from '@loaders.gl/loader-utils';
 
 export type StreamIteratorOptions = {
   _streamReadAhead?: boolean;
+  /** Cancels the active stream read and rejects iteration with the signal reason. */
+  signal?: AbortSignal;
 };
 
 /**
@@ -17,11 +19,16 @@ export function makeStreamIterator<T>(
   stream: ReadableStream<T> | Readable,
   options?: StreamIteratorOptions
 ): AsyncIterable<T> {
-  return isBrowser
+  return isBrowser || isReadableStream(stream)
     ? makeBrowserStreamIterator(stream as ReadableStream<T>, options)
     : makeNodeStreamIterator(stream as Readable, options);
 }
 
+/** Detects WHATWG streams in Node.js as well as in browsers. */
+function isReadableStream<T>(stream: ReadableStream<T> | Readable): stream is ReadableStream<T> {
+  return typeof (stream as ReadableStream<T>).getReader === 'function';
+}
+
 /**
  * Returns an async iterable that reads from a DOM (browser) stream
  * @param stream stream to iterate from
@@ -41,34 +48,76 @@ async function* makeBrowserStreamIterator<T>(
 
   // In the browser, we first need to get a lock on the stream
   const reader = stream.getReader();
+  const signal = options?.signal;
+  const abortListener = () => {
+    // Canceling the reader settles a pending read. The loop then throws the signal reason.
+    void reader.cancel(signal?.reason).catch(() => {});
+  };
 
   let nextBatchPromise: Promise<{done?: boolean; value?: T}> | undefined;
+  let streamFinished = false;
+  let streamReadFailed = false;
+  let iteratorFailed = false;
 
   try {
+    throwIfAborted(signal);
+    signal?.addEventListener('abort', abortListener, {once: true});
+
     // eslint-disable-next-line no-constant-condition
     while (true) {
       const currentBatchPromise = nextBatchPromise || reader.read();
-      // Issue a read for an additional batch, while we await the next batch
-      // Idea is to make fetching happen in parallel with processing / parsing
-      if (options?._streamReadAhead) {
-        nextBatchPromise = reader.read();
-      }
+      nextBatchPromise = undefined;
       // Read from the stream
-      // value is a Uint8Array
-      const {done, value} = await currentBatchPromise;
+      let batch;
+      try {
+        batch = await currentBatchPromise;
+      } catch (error) {
+        streamReadFailed = true;
+        throw error;
+      }
+      throwIfAborted(signal);
+      const {done, value} = batch;
       // Exit if we're done
       if (done) {
+        streamFinished = true;
         return;
       }
+      // Issue a read for an additional batch while the current batch is processed
+      if (options?._streamReadAhead) {
+        nextBatchPromise = reader.read();
+        // Observe a prefetched rejection even if the consumer stops before requesting it
+        void nextBatchPromise.catch(() => {});
+      }
       // Else yield the chunk
       if (value) {
         yield value;
       }
     }
   } catch (error) {
-    // TODO - examples makes it look like this should always be called,
-    // but that generates exceptions so only call it if we do not reach the end
-    reader.releaseLock();
+    iteratorFailed = true;
+    throw error;
+  } finally {
+    signal?.removeEventListener('abort', abortListener);
+    let cleanupError: unknown;
+
+    if (!streamFinished && !streamReadFailed) {
+      try {
+        await reader.cancel();
+      } catch (error) {
+        cleanupError = error;
+      }
+    }
+
+    try {
+      reader.releaseLock();
+    } catch (error) {
+      cleanupError ||= error;
+    }
+
+    // Preserve iteration errors instead of replacing them with cleanup errors
+    if (!iteratorFailed && cleanupError) {
+      throw cleanupError;
+    }
   }
 }
 
@@ -81,7 +130,70 @@ async function* makeNodeStreamIterator<T>(
   stream: Readable,
   options?: StreamIteratorOptions
 ): AsyncIterable<T> {
-  // Hacky test for node version to ensure we don't call bad polyfills
-  // NODE 10+: stream is an asyncIterator
-  yield* stream;
+  // Node streams and modern Node.js Web streams both expose an async iterator.
+  const iterator = stream[Symbol.asyncIterator]();
+  const signal = options?.signal;
+  let abortListener: (() => void) | undefined;
+  const abortPromise = signal
+    ? new Promise<never>((_resolve, reject) => {
+        abortListener = () => reject(getAbortReason(signal));
+        signal.addEventListener('abort', abortListener, {once: true});
+      })
+    : null;
+  let iteratorFailed = false;
+  let streamFinished = false;
+
+  try {
+    throwIfAborted(signal);
+    // eslint-disable-next-line no-constant-condition
+    while (true) {
+      const nextPromise = iterator.next();
+      if (abortPromise) {
+        // Observe a late failure if cancellation wins the race.
+        void nextPromise.catch(() => {});
+      }
+      const result = abortPromise
+        ? await Promise.race([nextPromise, abortPromise])
+        : await nextPromise;
+      throwIfAborted(signal);
+      if (result.done) {
+        streamFinished = true;
+        return;
+      }
+      yield result.value;
+    }
+  } catch (error) {
+    iteratorFailed = true;
+    throw error;
+  } finally {
+    if (abortListener) {
+      signal?.removeEventListener('abort', abortListener);
+    }
+    if (!streamFinished) {
+      try {
+        await iterator.return?.();
+      } catch (error) {
+        if (!iteratorFailed) {
+          throw error;
+        }
+      }
+    }
+  }
+}
+
+/** Returns the standard AbortSignal reason, including a fallback for older runtimes. */
+function getAbortReason(signal: AbortSignal): unknown {
+  if (signal.reason !== undefined) {
+    return signal.reason;
+  }
+  const error = new Error('The operation was aborted');
+  error.name = 'AbortError';
+  return error;
+}
+
+/** Throws at a cancellation checkpoint. */
+function throwIfAborted(signal?: AbortSignal): void {
+  if (signal?.aborted) {
+    throw getAbortReason(signal);
+  }
 }

package/src/lib/utils/normalize-parquet-options.ts

@@ -0,0 +1,14 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+/** Merge caller options with default parquet options while preserving top-level option bags. */
+export function normalizeParquetOptions<OptionsT extends {parquet?: unknown}>(
+  options: OptionsT | undefined,
+  parquetDefaults: Record<string, unknown>
+): OptionsT {
+  return {
+    ...options,
+    parquet: {...parquetDefaults, ...(options?.parquet as Record<string, unknown> | undefined)}
+  } as unknown as OptionsT;
+}

package/src/parquet-dataset-source.ts

@@ -0,0 +1,488 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {CoreAPI} from '@loaders.gl/loader-utils';
+import type {Schema} from '@loaders.gl/schema';
+
+import {SingleBatchQueue} from './lib/sources/single-batch-queue';
+import {ParquetSource} from './parquet-source-loader';
+import type {
+  ParquetDatasetBatch,
+  ParquetDatasetBatchProvenance,
+  ParquetDatasetBoundingBox,
+  ParquetDatasetFile,
+  ParquetDatasetFileCollection,
+  ParquetDatasetFileQuery,
+  ParquetDatasetFiles,
+  ParquetDatasetPartitionValue,
+  ParquetDatasetReadOptions,
+  ParquetDatasetSourceOptions,
+  ParquetDatasetTelemetry,
+  ParquetSourceBatch,
+  ParquetTelemetry
+} from './parquet-source-types';
+
+export type {
+  ParquetDatasetBatch,
+  ParquetDatasetBatchProvenance,
+  ParquetDatasetBoundingBox,
+  ParquetDatasetFile,
+  ParquetDatasetFileCollection,
+  ParquetDatasetFileProvider,
+  ParquetDatasetFileQuery,
+  ParquetDatasetFiles,
+  ParquetDatasetPartitionValue,
+  ParquetDatasetReadOptions,
+  ParquetDatasetSourceOptions,
+  ParquetDatasetTelemetry
+} from './parquet-source-types';
+
+const DEFAULT_FILE_CONCURRENCY = 4;
+
+type IndexedParquetDatasetFile = {
+  /** Descriptor returned by the provider. */
+  file: ParquetDatasetFile;
+  /** Position in provider output before local pruning. */
+  index: number;
+};
+
+type ScheduledFileRead = {
+  /** Bounded batch queue for this file. */
+  queue: SingleBatchQueue<ParquetDatasetBatch>;
+  /** File task used for deterministic cleanup. */
+  task: Promise<void>;
+};
+
+/**
+ * Multi-file Parquet source that composes catalog discovery with selective `ParquetSource` reads.
+ *
+ * The source accepts static descriptors or a lazy provider, allowing STAC and other catalogs to be
+ * injected without coupling `@loaders.gl/parquet` to a catalog protocol.
+ */
+export class ParquetDatasetSource {
+  /** Static descriptors or lazy catalog-backed file provider. */
+  readonly files: ParquetDatasetFiles;
+  /** Child-source and dataset orchestration options. */
+  readonly options: ParquetDatasetSourceOptions;
+  /** Runtime hooks forwarded to child Parquet sources. */
+  readonly coreApi?: CoreAPI;
+
+  /** Abort controllers for active dataset reads. */
+  private readonly activeReadControllers = new Set<AbortController>();
+  /** Whether the dataset source has been permanently closed. */
+  private closed = false;
+  /** Cumulative dataset and child-source telemetry. */
+  private readonly telemetry = createParquetDatasetTelemetry();
+
+  /** Creates a logical dataset over static or lazily discovered Parquet files. */
+  constructor(
+    files: ParquetDatasetFiles,
+    options: ParquetDatasetSourceOptions = {},
+    coreApi?: CoreAPI
+  ) {
+    if (!Array.isArray(files) && typeof files !== 'function') {
+      throw new Error(
+        'ParquetDatasetSource files must be a reusable descriptor array or a provider function'
+      );
+    }
+    this.files = files;
+    this.options = options;
+    this.coreApi = coreApi;
+  }
+
+  /** Returns the schema of the first selected file, closing its temporary child source afterward. */
+  async getSchema(query: ParquetDatasetFileQuery = {}): Promise<Schema> {
+    this.assertOpen();
+    const iterator = this.getSelectedFiles(query)[Symbol.asyncIterator]();
+    try {
+      const first = await iterator.next();
+      if (first.done) {
+        throw new Error('ParquetDatasetSource query selected no files');
+      }
+      const source = this.createSource(first.value.file);
+      this.telemetry.filesOpened++;
+      try {
+        return await source.getSchema({signal: query.signal});
+      } finally {
+        await source.close();
+        this.addParquetTelemetry(source.getTelemetry());
+      }
+    } finally {
+      await iterator.return?.();
+    }
+  }
+
+  /** Returns an immutable snapshot of cumulative dataset and child-source telemetry. */
+  getTelemetry(): ParquetDatasetTelemetry {
+    return Object.freeze({
+      ...this.telemetry,
+      parquet: Object.freeze({...this.telemetry.parquet})
+    });
+  }
+
+  /**
+   * Reads selected files as deterministic, file-ordered Arrow batches.
+   *
+   * Files are decoded concurrently. Each later file is limited to one buffered batch until all
+   * earlier files have been yielded, bounding memory while retaining stable provider order.
+   */
+  async *read(options: ParquetDatasetReadOptions = {}): AsyncIterable<ParquetDatasetBatch> {
+    this.assertOpen();
+    const readContext = createDatasetAbortContext(options.signal);
+    this.activeReadControllers.add(readContext.abortController);
+    const selectedFiles = this.getSelectedFiles({
+      ...options,
+      signal: readContext.abortController.signal
+    });
+    const iterator = selectedFiles[Symbol.asyncIterator]();
+    const scheduledReads = new Map<number, ScheduledFileRead>();
+    let nextSchedulePosition = 0;
+    let nextYieldPosition = 0;
+    let providerComplete = false;
+    let discoveryError: unknown;
+    const fileConcurrency = normalizeFileConcurrency(
+      options.fileConcurrency ?? this.options.parquetDataset?.fileConcurrency
+    );
+
+    const scheduleNext = async (): Promise<void> => {
+      if (providerComplete) {
+        return;
+      }
+      const next = await iterator.next();
+      if (next.done) {
+        providerComplete = true;
+        return;
+      }
+      const position = nextSchedulePosition++;
+      const queue = new SingleBatchQueue<ParquetDatasetBatch>();
+      const task = this.readFile(next.value, options, queue, readContext.abortController.signal);
+      scheduledReads.set(position, {queue, task});
+    };
+
+    const fillAvailableSlots = async (): Promise<void> => {
+      try {
+        while (scheduledReads.size < fileConcurrency && !providerComplete) {
+          await scheduleNext();
+        }
+      } catch (error) {
+        discoveryError = error;
+        providerComplete = true;
+      }
+    };
+
+    try {
+      await scheduleNext();
+      let discoveryPromise = fillAvailableSlots();
+
+      while (scheduledReads.size > 0) {
+        const scheduledRead = scheduledReads.get(nextYieldPosition);
+        if (!scheduledRead) {
+          throw new Error('ParquetDatasetSource internal file ordering error');
+        }
+        for await (const batch of scheduledRead.queue) {
+          throwIfAborted(readContext.abortController.signal);
+          this.telemetry.batchesEmitted++;
+          this.telemetry.rowsEmitted += batch.length;
+          yield batch;
+        }
+        await scheduledRead.task;
+        scheduledReads.delete(nextYieldPosition++);
+        await discoveryPromise;
+        if (discoveryError !== undefined) {
+          throw discoveryError;
+        }
+        discoveryPromise = fillAvailableSlots();
+      }
+      await discoveryPromise;
+      if (discoveryError !== undefined) {
+        throw discoveryError;
+      }
+    } finally {
+      readContext.abortController.abort();
+      readContext.removeSignalListener();
+      this.activeReadControllers.delete(readContext.abortController);
+      await iterator.return?.();
+      await Promise.allSettled([...scheduledReads.values()].map(read => read.task));
+    }
+  }
+
+  /** Permanently closes this dataset source and aborts every active read. */
+  async close(): Promise<void> {
+    if (this.closed) {
+      return;
+    }
+    this.closed = true;
+    for (const abortController of this.activeReadControllers) {
+      abortController.abort();
+    }
+  }
+
+  /** Reads one child source into its bounded file-order queue. */
+  private async readFile(
+    indexedFile: IndexedParquetDatasetFile,
+    options: ParquetDatasetReadOptions,
+    queue: SingleBatchQueue<ParquetDatasetBatch>,
+    signal: AbortSignal
+  ): Promise<void> {
+    const source = this.createSource(indexedFile.file);
+    this.telemetry.filesOpened++;
+    try {
+      if (this.options.parquetDataset?.validateSchema !== false) {
+        await this.validateFileSchema(source, indexedFile, signal);
+      }
+      for await (const batch of source.read({
+        columns: options.columns,
+        batchSize: options.batchSize,
+        concurrency: options.concurrency,
+        rowGroupFilter: options.rowGroupFilter,
+        predicate: options.predicate,
+        signal
+      })) {
+        await queue.push(createDatasetBatch(batch, indexedFile), signal);
+      }
+      queue.finish();
+    } catch (error) {
+      queue.fail(error);
+    } finally {
+      await source.close();
+      this.addParquetTelemetry(source.getTelemetry());
+    }
+  }
+
+  /** Establishes and validates a stable field schema for concurrently opened files. */
+  private schemaFingerprintPromise: Promise<string> | null = null;
+
+  /** Validates one child schema against the first file schema observed by this source. */
+  private async validateFileSchema(
+    source: ParquetSource,
+    indexedFile: IndexedParquetDatasetFile,
+    signal: AbortSignal
+  ): Promise<void> {
+    const schema = await source.getSchema({signal});
+    const fingerprint = getSchemaFingerprint(schema);
+    if (!this.schemaFingerprintPromise) {
+      this.schemaFingerprintPromise = Promise.resolve(fingerprint);
+      return;
+    }
+    const expectedFingerprint = await this.schemaFingerprintPromise;
+    if (fingerprint !== expectedFingerprint) {
+      throw new Error(
+        `Parquet dataset file ${getDatasetFileId(indexedFile.file, indexedFile.index)} has an incompatible schema`
+      );
+    }
+  }
+
+  /** Creates a child source while preserving caller fetch, worker, range, and decode options. */
+  private createSource(file: ParquetDatasetFile): ParquetSource {
+    return new ParquetSource(file.data, this.options, this.coreApi);
+  }
+
+  /** Lazily discovers descriptors and applies conservative local pruning. */
+  private async *getSelectedFiles(
+    query: ParquetDatasetFileQuery
+  ): AsyncIterable<IndexedParquetDatasetFile> {
+    throwIfAborted(query.signal);
+    const collection = await getFileCollection(this.files, query);
+    let index = 0;
+    for await (const file of collection) {
+      throwIfAborted(query.signal);
+      validateDatasetFile(file, index);
+      this.telemetry.filesDiscovered++;
+      if (query.bbox && file.bbox && !doBoundingBoxesIntersect(query.bbox, file.bbox)) {
+        this.telemetry.filesPrunedByBoundingBox++;
+        index++;
+        continue;
+      }
+      if (query.partitions && !matchesPartitions(file.partitions, query.partitions)) {
+        this.telemetry.filesPrunedByPartitions++;
+        index++;
+        continue;
+      }
+      this.telemetry.filesSelected++;
+      yield {file, index: index++};
+    }
+  }
+
+  /** Adds a final child-source telemetry snapshot exactly once. */
+  private addParquetTelemetry(telemetry: ParquetTelemetry): void {
+    for (const key of Object.keys(telemetry) as Array<keyof ParquetTelemetry>) {
+      this.telemetry.parquet[key] += telemetry[key];
+    }
+  }
+
+  /** Throws when an operation is attempted after `close()`. */
+  private assertOpen(): void {
+    if (this.closed) {
+      throw new Error('ParquetDatasetSource is closed');
+    }
+  }
+}
+
+/** Normalizes static descriptors and lazy providers into one file collection. */
+async function getFileCollection(
+  files: ParquetDatasetFiles,
+  query: ParquetDatasetFileQuery
+): Promise<ParquetDatasetFileCollection> {
+  return typeof files === 'function' ? await files(query) : files;
+}
+
+/** Attaches dataset-level descriptor provenance without copying Arrow buffers. */
+function createDatasetBatch(
+  batch: ParquetSourceBatch,
+  indexedFile: IndexedParquetDatasetFile
+): ParquetDatasetBatch {
+  const datasetProvenance = {
+    datasetFileIndex: indexedFile.index,
+    datasetFileId: getDatasetFileId(indexedFile.file, indexedFile.index),
+    datasetPartitions: indexedFile.file.partitions,
+    datasetFileMetadata: indexedFile.file.metadata
+  };
+  const provenance: ParquetDatasetBatchProvenance = Object.freeze({
+    sourceId: batch.sourceId,
+    sourceUrl: batch.sourceUrl,
+    source: batch.source,
+    rowGroupIndex: batch.rowGroupIndex,
+    rowOffset: batch.rowOffset,
+    rowGroupRowOffset: batch.rowGroupRowOffset,
+    rowCount: batch.rowCount,
+    rowGroupRowIndices: batch.rowGroupRowIndices,
+    rowIndices: batch.rowIndices,
+    ...datasetProvenance
+  });
+  return {...batch, metadata: provenance, ...datasetProvenance};
+}
+
+/** Returns a stable descriptor identity for batch provenance and diagnostics. */
+function getDatasetFileId(file: ParquetDatasetFile, index: number): string {
+  if (file.id) {
+    return file.id;
+  }
+  if (typeof file.data === 'string') {
+    return file.data;
+  }
+  return typeof File !== 'undefined' && file.data instanceof File && file.data.name
+    ? file.data.name
+    : `blob-${index}`;
+}
+
+/** Validates a provider descriptor before opening a child source. */
+function validateDatasetFile(file: ParquetDatasetFile, index: number): void {
+  if (!file || (typeof file.data !== 'string' && !(file.data instanceof Blob))) {
+    throw new Error(`Invalid Parquet dataset file descriptor at index ${index}`);
+  }
+}
+
+/** Tests 2D intersection while conservatively accepting additional dimensions. */
+function doBoundingBoxesIntersect(
+  left: ParquetDatasetBoundingBox,
+  right: ParquetDatasetBoundingBox
+): boolean {
+  const leftMaximumX = left.length === 6 ? left[3] : left[2];
+  const leftMaximumY = left.length === 6 ? left[4] : left[3];
+  const rightMaximumX = right.length === 6 ? right[3] : right[2];
+  const rightMaximumY = right.length === 6 ? right[4] : right[3];
+  return (
+    left[0] <= rightMaximumX &&
+    leftMaximumX >= right[0] &&
+    left[1] <= rightMaximumY &&
+    leftMaximumY >= right[1]
+  );
+}
+
+/** Tests exact partition constraints without rejecting files that omit a requested key. */
+function matchesPartitions(
+  partitions: Readonly<Record<string, ParquetDatasetPartitionValue>> | undefined,
+  query: NonNullable<ParquetDatasetFileQuery['partitions']>
+): boolean {
+  for (const [key, requested] of Object.entries(query)) {
+    const actual = partitions?.[key];
+    if (actual === undefined) {
+      continue;
+    }
+    const accepted = Array.isArray(requested) ? requested : [requested];
+    if (!accepted.includes(actual)) {
+      return false;
+    }
+  }
+  return true;
+}
+
+/** Creates a deterministic field-only schema fingerprint for cross-file validation. */
+function getSchemaFingerprint(schema: Schema): string {
+  return JSON.stringify(schema.fields);
+}
+
+/** Validates and normalizes dataset-level file concurrency. */
+function normalizeFileConcurrency(value: number | undefined): number {
+  const concurrency = value ?? DEFAULT_FILE_CONCURRENCY;
+  if (!Number.isInteger(concurrency) || concurrency < 1) {
+    throw new Error(`parquetDataset.fileConcurrency must be a positive integer, received ${value}`);
+  }
+  return concurrency;
+}
+
+/** Creates linked cancellation for one dataset read. */
+function createDatasetAbortContext(signal?: AbortSignal): {
+  abortController: AbortController;
+  removeSignalListener: () => void;
+} {
+  const abortController = new AbortController();
+  const abort = (): void => abortController.abort(signal?.reason);
+  if (signal?.aborted) {
+    abort();
+  } else {
+    signal?.addEventListener('abort', abort, {once: true});
+  }
+  return {
+    abortController,
+    removeSignalListener: () => signal?.removeEventListener('abort', abort)
+  };
+}
+
+/** Throws a standard abort failure when a dataset read has been cancelled. */
+function throwIfAborted(signal?: AbortSignal): void {
+  if (signal?.aborted) {
+    throw signal.reason || new DOMException('Aborted', 'AbortError');
+  }
+}
+
+/** Creates zeroed cumulative dataset and child-source telemetry. */
+function createParquetDatasetTelemetry(): ParquetDatasetTelemetry {
+  return {
+    filesDiscovered: 0,
+    filesSelected: 0,
+    filesPrunedByBoundingBox: 0,
+    filesPrunedByPartitions: 0,
+    filesOpened: 0,
+    batchesEmitted: 0,
+    rowsEmitted: 0,
+    parquet: {
+      rangeRequestCount: 0,
+      requestedBytes: 0,
+      downloadedBytes: 0,
+      cacheHits: 0,
+      networkDurationMs: 0,
+      failedRangeRequestCount: 0,
+      abortedRangeRequestCount: 0,
+      retryCount: 0,
+      decodeDurationMs: 0,
+      arrowConversionDurationMs: 0,
+      rowGroupsRequested: 0,
+      rowGroupsPruned: 0,
+      rowGroupsPrunedByStatistics: 0,
+      rowGroupsPrunedByPageIndex: 0,
+      pageIndexesRead: 0,
+      pagesRead: 0,
+      pagesPruned: 0,
+      rowsPrunedByPageIndex: 0,
+      rowGroupsDecoded: 0,
+      batchesEmitted: 0,
+      rowsEmitted: 0,
+      predicateRowsTested: 0,
+      predicateRowsMatched: 0,
+      cancellationCount: 0,
+      failedReadCount: 0
+    }
+  };
+}

package/src/parquet-format.ts

@@ -7,6 +7,8 @@ import type {Format} from '@loaders.gl/loader-utils';
 export const ParquetFormat = {
   name: 'Apache Parquet',
   category: 'table',
+  encoding: 'parquet',
+  format: 'parquet',
   extensions: ['parquet'],
   mimeTypes: ['application/octet-stream'],
   binary: true,