@loaders.gl/parquet 4.4.5 → 5.0.0-alpha.2

package/src/lib/sources/parquet-range-file.ts

@@ -0,0 +1,359 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {ReadableFile, Stat} from '@loaders.gl/loader-utils';
+import {RangeRequestScheduler} from '@loaders.gl/loader-utils';
+
+import type {ParquetObjectVersion, ParquetRangeRequestOptions} from '../../parquet-source-types';
+
+type FetchLike = (url: string, options?: RequestInit) => Promise<Response>;
+
+type ParquetRangeFileOptions = {
+  /** Fetch implementation used for every HTTP range. */
+  fetch: FetchLike;
+  /** Headers forwarded to every HTTP range. */
+  headers?: HeadersInit;
+  /** Range coalescing and diagnostics configuration. */
+  rangeRequests?: ParquetRangeRequestOptions;
+  /** Receives source-local transport telemetry deltas. */
+  onTelemetry?: (event: ParquetRangeTelemetryEvent) => void;
+};
+
+type ParquetRangeTelemetryEvent = {
+  /** Transport operation that produced this event. */
+  type: 'range-request' | 'cache-hit';
+  /** Number of transport requests contributed by this event. */
+  rangeRequestCount?: number;
+  /** Number of transport bytes requested by this event. */
+  requestedBytes?: number;
+  /** Number of response bytes downloaded by this event. */
+  downloadedBytes?: number;
+  /** Number of cache hits contributed by this event. */
+  cacheHits?: number;
+  /** Network duration contributed by this event. */
+  networkDurationMs?: number;
+  /** Number of failed requests contributed by this event. */
+  failedRangeRequestCount?: number;
+  /** Number of aborted requests contributed by this event. */
+  abortedRangeRequestCount?: number;
+  /** Error associated with a failed range request. */
+  error?: unknown;
+};
+
+type ContentRange = {
+  start: number;
+  end: number;
+  total: number;
+};
+
+/** Strict HTTP range-backed file with Parquet object-version validation. */
+export class ParquetRangeFile implements ReadableFile {
+  /** URL used as the file handle. */
+  readonly handle: string;
+  /** URL of the remote Parquet object. */
+  readonly url: string;
+
+  /** Fetch implementation inherited from source loader options. */
+  private readonly fetch: FetchLike;
+  /** Headers inherited from source loader options. */
+  private readonly headers?: HeadersInit;
+  /** Source telemetry callback invoked after transport and cache operations. */
+  private readonly onTelemetry?: (event: ParquetRangeTelemetryEvent) => void;
+  /** Scheduler used to coalesce later row-group reads. */
+  private readonly scheduler: RangeRequestScheduler;
+  /** Aborts all active requests when the file is closed. */
+  private readonly closeController = new AbortController();
+  /** Exact ranges already loaded during initialization. */
+  private readonly cache = new Map<string, ArrayBuffer>();
+  /** Remote object byte length discovered from `Content-Range`. */
+  private fileByteLength = 0;
+  /** Object validators captured by the opening range request. */
+  private version: ParquetObjectVersion = {};
+  /** Whether `close()` has been called. */
+  private closed = false;
+
+  /** Creates an unopened remote Parquet file. Use `open()` before reading it. */
+  constructor(url: string, options: ParquetRangeFileOptions) {
+    this.handle = url;
+    this.url = url;
+    this.fetch = options.fetch;
+    this.headers = options.headers;
+    this.onTelemetry = options.onTelemetry;
+    this.scheduler =
+      options.rangeRequests?.scheduler ||
+      new RangeRequestScheduler({
+        batchDelayMs: options.rangeRequests?.batchDelayMs ?? 0,
+        maxGapBytes: options.rangeRequests?.maxGapBytes,
+        rangeExpansionBytes: options.rangeRequests?.rangeExpansionBytes,
+        maxMergedBytes: options.rangeRequests?.maxMergedBytes,
+        stats: options.rangeRequests?.stats,
+        onEvent: options.rangeRequests?.onEvent
+      });
+  }
+
+  /** Total remote object byte length after `open()`. */
+  get size(): number {
+    return this.fileByteLength;
+  }
+
+  /** Total remote object byte length as a bigint after `open()`. */
+  get bigsize(): bigint {
+    return BigInt(this.fileByteLength);
+  }
+
+  /** Immutable copy of the HTTP validators captured when the object was opened. */
+  get objectVersion(): ParquetObjectVersion {
+    return {...this.version};
+  }
+
+  /** Opens the object with a four-byte range probe and caches the Parquet header. */
+  async open(signal?: AbortSignal): Promise<this> {
+    this.assertOpen();
+    const result = await this.fetchExactRange(0, 4, signal, false);
+    this.fileByteLength = result.contentRange.total;
+    this.version = getObjectVersion(result.response.headers);
+    this.cache.set(getRangeKey(0, 4), result.arrayBuffer);
+    return this;
+  }
+
+  /** Reads an exact byte range without downloading the complete object. */
+  async read(
+    start: number | bigint = 0,
+    length: number = this.size - Number(start),
+    signal?: AbortSignal
+  ): Promise<ArrayBuffer> {
+    this.assertOpen();
+    const offset = Number(start);
+    assertValidRange(offset, length, this.size);
+    if (length === 0) {
+      return new ArrayBuffer(0);
+    }
+
+    const cached = this.cache.get(getRangeKey(offset, length));
+    if (cached) {
+      this.onTelemetry?.({type: 'cache-hit', cacheHits: 1});
+      return cached.slice(0);
+    }
+
+    const abortContext = createCombinedAbortSignal(signal, this.closeController.signal);
+    try {
+      return await this.scheduler.scheduleRequest({
+        sourceId: getVersionedSourceId(this.url, this.version),
+        offset,
+        length,
+        signal: abortContext.signal,
+        fetchRange: async (transportOffset, transportLength, transportSignal) => {
+          const result = await this.fetchExactRange(
+            transportOffset,
+            transportLength,
+            transportSignal,
+            true
+          );
+          return {
+            arrayBuffer: result.arrayBuffer,
+            status: result.response.status,
+            transportBytes: result.arrayBuffer.byteLength
+          };
+        }
+      });
+    } finally {
+      abortContext.dispose();
+    }
+  }
+
+  /** Returns file length information without another HTTP request. */
+  async stat(): Promise<Stat> {
+    this.assertOpen();
+    return {size: this.size, bigsize: this.bigsize, isDirectory: false};
+  }
+
+  /** Aborts active requests and prevents future reads. */
+  async close(): Promise<void> {
+    if (!this.closed) {
+      this.closed = true;
+      this.closeController.abort();
+      this.cache.clear();
+    }
+  }
+
+  /** Fetches and validates one exact HTTP byte range. */
+  private async fetchExactRange(
+    offset: number,
+    length: number,
+    signal: AbortSignal | undefined,
+    validateVersion: boolean
+  ): Promise<{response: Response; arrayBuffer: ArrayBuffer; contentRange: ContentRange}> {
+    const abortContext = createCombinedAbortSignal(signal, this.closeController.signal);
+    const startTime = getCurrentTime();
+    try {
+      const response = await this.fetch(this.url, {
+        headers: createRangeHeaders(this.headers, offset, length, validateVersion && this.version),
+        signal: abortContext.signal
+      });
+
+      if (response.status === 412) {
+        throw new Error('Parquet object changed while it was being read');
+      }
+      if (response.status !== 206) {
+        await response.body?.cancel().catch(() => {});
+        throw new Error(
+          `ParquetSource requires HTTP byte ranges (expected 206, received ${response.status})`
+        );
+      }
+
+      const contentRange = parseContentRange(response.headers.get('Content-Range'));
+      if (
+        contentRange.start !== offset ||
+        contentRange.end !== offset + length - 1 ||
+        (this.fileByteLength > 0 && contentRange.total !== this.fileByteLength)
+      ) {
+        await response.body?.cancel().catch(() => {});
+        throw new Error('Parquet range response does not match the requested object range');
+      }
+
+      if (validateVersion) {
+        assertObjectVersion(this.version, getObjectVersion(response.headers));
+      }
+
+      const arrayBuffer = await response.arrayBuffer();
+      if (arrayBuffer.byteLength !== length) {
+        throw new Error(
+          `Parquet range response returned ${arrayBuffer.byteLength} bytes; expected ${length}`
+        );
+      }
+      this.onTelemetry?.({
+        type: 'range-request',
+        rangeRequestCount: 1,
+        requestedBytes: length,
+        downloadedBytes: arrayBuffer.byteLength,
+        networkDurationMs: getCurrentTime() - startTime
+      });
+      return {response, arrayBuffer, contentRange};
+    } catch (error) {
+      this.onTelemetry?.({
+        type: 'range-request',
+        rangeRequestCount: 1,
+        requestedBytes: length,
+        networkDurationMs: getCurrentTime() - startTime,
+        failedRangeRequestCount: 1,
+        abortedRangeRequestCount:
+          abortContext.signal.aborted || isAbortError(error) ? 1 : undefined,
+        error
+      });
+      throw error;
+    } finally {
+      abortContext.dispose();
+    }
+  }
+
+  /** Throws when the file has already been closed. */
+  private assertOpen(): void {
+    if (this.closed) {
+      throw new Error('Parquet source is closed');
+    }
+  }
+}
+
+/** Returns a monotonic timestamp when available and falls back to wall-clock time. */
+function getCurrentTime(): number {
+  return globalThis.performance?.now() ?? Date.now();
+}
+
+/** Returns true for DOM and cross-runtime abort errors. */
+function isAbortError(error: unknown): boolean {
+  return error instanceof Error && error.name === 'AbortError';
+}
+
+/** Validates a requested range against the known object byte length. */
+function assertValidRange(offset: number, length: number, fileByteLength: number): void {
+  if (!Number.isSafeInteger(offset) || !Number.isSafeInteger(length) || offset < 0 || length < 0) {
+    throw new Error('Parquet byte range must use non-negative safe integers');
+  }
+  if (offset + length > fileByteLength) {
+    throw new Error('Parquet byte range exceeds the object length');
+  }
+}
+
+/** Creates request headers for one byte range and optional object precondition. */
+function createRangeHeaders(
+  defaultHeaders: HeadersInit | undefined,
+  offset: number,
+  length: number,
+  version: ParquetObjectVersion | false
+): Headers {
+  const headers = new Headers(defaultHeaders);
+  headers.set('Range', `bytes=${offset}-${offset + length - 1}`);
+  if (version && version.etag && !version.etag.startsWith('W/')) {
+    headers.set('If-Match', version.etag);
+  } else if (version && version.lastModified) {
+    headers.set('If-Unmodified-Since', version.lastModified);
+  }
+  return headers;
+}
+
+/** Parses a satisfied single-range `Content-Range` header. */
+function parseContentRange(value: string | null): ContentRange {
+  const match = value?.match(/^bytes (\d+)-(\d+)\/(\d+)$/);
+  if (!match) {
+    throw new Error(`Invalid Parquet Content-Range header: ${value}`);
+  }
+  return {start: Number(match[1]), end: Number(match[2]), total: Number(match[3])};
+}
+
+/** Reads supported object validators from response headers. */
+function getObjectVersion(headers: Headers): ParquetObjectVersion {
+  const etag = headers.get('ETag') || undefined;
+  const lastModified = headers.get('Last-Modified') || undefined;
+  return {etag, lastModified};
+}
+
+/** Rejects a response that identifies a different remote object version. */
+function assertObjectVersion(expected: ParquetObjectVersion, actual: ParquetObjectVersion): void {
+  if (expected.etag && actual.etag && expected.etag !== actual.etag) {
+    throw new Error('Parquet object ETag changed while it was being read');
+  }
+  if (
+    !expected.etag &&
+    expected.lastModified &&
+    actual.lastModified &&
+    expected.lastModified !== actual.lastModified
+  ) {
+    throw new Error('Parquet object Last-Modified changed while it was being read');
+  }
+}
+
+/** Creates a cache key for one exact range. */
+function getRangeKey(offset: number, length: number): string {
+  return `${offset}:${length}`;
+}
+
+/** Creates a scheduler identity that isolates object versions. */
+function getVersionedSourceId(url: string, version: ParquetObjectVersion): string {
+  return `${url}#${version.etag || version.lastModified || 'unversioned'}`;
+}
+
+/** Combines two abort signals and returns cleanup for installed listeners. */
+function createCombinedAbortSignal(
+  firstSignal?: AbortSignal,
+  secondSignal?: AbortSignal
+): {signal: AbortSignal; dispose: () => void} {
+  const controller = new AbortController();
+  const abort = () => controller.abort();
+  const signals = [firstSignal, secondSignal].filter(Boolean) as AbortSignal[];
+  for (const signal of signals) {
+    if (signal.aborted) {
+      controller.abort();
+      break;
+    }
+    signal.addEventListener('abort', abort, {once: true});
+  }
+  return {
+    signal: controller.signal,
+    dispose: () => {
+      for (const signal of signals) {
+        signal.removeEventListener('abort', abort);
+      }
+    }
+  };
+}

package/src/lib/sources/single-batch-queue.ts

@@ -0,0 +1,95 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+/** One-slot asynchronous queue that applies backpressure between a producer and ordered output. */
+export class SingleBatchQueue<Value> implements AsyncIterable<Value> {
+  /** Buffered batch, limited to one value. */
+  private value: Value | undefined;
+  /** Whether the producer completed successfully. */
+  private finished = false;
+  /** Producer failure rethrown by the consumer. */
+  private error: unknown;
+  /** Wakes a consumer waiting for a value. */
+  private wakeConsumer: (() => void) | null = null;
+  /** Wakes a producer after its buffered value is consumed. */
+  private wakeProducer: (() => void) | null = null;
+
+  /** Buffers one value and waits until it has been consumed. */
+  async push(value: Value, signal: AbortSignal): Promise<void> {
+    throwIfAborted(signal);
+    while (this.value !== undefined) {
+      await this.waitForWakeup('producer', signal);
+    }
+    this.value = value;
+    this.wakeConsumer?.();
+    this.wakeConsumer = null;
+    while (this.value !== undefined) {
+      await this.waitForWakeup('producer', signal);
+    }
+  }
+
+  /** Marks the queue complete and wakes its consumer. */
+  finish(): void {
+    this.finished = true;
+    this.wakeConsumer?.();
+    this.wakeConsumer = null;
+  }
+
+  /** Stores a producer error and wakes its consumer. */
+  fail(error: unknown): void {
+    this.error = error;
+    this.finished = true;
+    this.wakeConsumer?.();
+    this.wakeConsumer = null;
+  }
+
+  /** Returns the ordered asynchronous batch iterator. */
+  async *[Symbol.asyncIterator](): AsyncIterator<Value> {
+    while (true) {
+      if (this.value !== undefined) {
+        const value = this.value;
+        this.value = undefined;
+        this.wakeProducer?.();
+        this.wakeProducer = null;
+        yield value;
+        continue;
+      }
+      if (this.error !== undefined) {
+        throw this.error;
+      }
+      if (this.finished) {
+        return;
+      }
+      await this.waitForWakeup('consumer');
+    }
+  }
+
+  /** Waits for a producer or consumer queue transition and supports read cancellation. */
+  private waitForWakeup(type: 'producer' | 'consumer', signal?: AbortSignal): Promise<void> {
+    return new Promise((resolve, reject) => {
+      const abort = (): void => reject(signal?.reason || new DOMException('Aborted', 'AbortError'));
+      const wakeup = (): void => {
+        signal?.removeEventListener('abort', abort);
+        resolve();
+      };
+      if (type === 'producer') {
+        this.wakeProducer = wakeup;
+      } else {
+        this.wakeConsumer = wakeup;
+      }
+      if (signal?.aborted) {
+        abort();
+      } else {
+        signal?.addEventListener('abort', abort, {once: true});
+      }
+    });
+  }
+}
+
+/** Throws a standard abort failure when a queued operation has been cancelled. */
+function throwIfAborted(signal?: AbortSignal): void {
+  if (signal?.aborted) {
+    throw signal.reason || new DOMException('Aborted', 'AbortError');
+  }
+}

package/src/lib/utils/load-wasm-node.ts

@@ -0,0 +1,25 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+/** Loads the installed parquet-wasm binary in Node.js or defers to browser asset resolution. */
+export async function loadNodeWasmInput(): Promise<Uint8Array | undefined> {
+  if (!isNodeRuntime()) {
+    return undefined;
+  }
+  // Keep these specifiers dynamic so browser bundles do not include Node.js built-ins.
+  const nodeProtocol = 'node:';
+  const {readFile} = await import(/* @vite-ignore */ `${nodeProtocol}fs/promises`);
+  const {createRequire} = await import(/* @vite-ignore */ `${nodeProtocol}module`);
+  const {pathToFileURL} = await import(/* @vite-ignore */ `${nodeProtocol}url`);
+  const moduleUrl =
+    typeof __filename === 'string' ? pathToFileURL(__filename).toString() : import.meta.url;
+  const require = createRequire(moduleUrl);
+  const wasmPath = require.resolve('parquet-wasm/esm/parquet_wasm_bg.wasm');
+  return await readFile(wasmPath);
+}
+
+/** Returns true when Node.js built-ins are available in the current runtime. */
+function isNodeRuntime(): boolean {
+  return Boolean((globalThis as {process?: {versions?: {node?: string}}}).process?.versions?.node);
+}

package/src/lib/utils/load-wasm.ts

@@ -2,21 +2,44 @@
 // SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
-// eslint-disable-next-line import/default
-import initWasm from 'parquet-wasm';
-import * as parquetWasm from 'parquet-wasm';
-import {PARQUET_WASM_URL} from '../constants';
+import type * as ParquetWasm from 'parquet-wasm/esm/parquet_wasm.js';
+import {loadNodeWasmInput} from './load-wasm-node';
 
-let initializePromise: any;
+let initializePromise: Promise<typeof ParquetWasm> | undefined;
 
-export async function loadWasm(wasmUrl: string = PARQUET_WASM_URL) {
-  if (!initializePromise && typeof initWasm === 'function') {
-    if (!wasmUrl) {
-      throw new Error('ParquetLoader: No wasmUrl provided');
-    }
-    // @ts-ignore
-    initializePromise = initWasm(wasmUrl);
+export async function loadWasm(
+  wasmUrl?: ParquetWasm.InitInput | Promise<ParquetWasm.InitInput>
+): Promise<typeof ParquetWasm> {
+  if (!initializePromise) {
+    const nextInitializePromise = loadAndInitializeWasm(wasmUrl);
+    const cachedInitializePromise = nextInitializePromise.catch(error => {
+      if (initializePromise === cachedInitializePromise) {
+        initializePromise = undefined;
+      }
+      throw error;
+    });
+    initializePromise = cachedInitializePromise;
+  }
+
+  return await initializePromise;
+}
+
+async function loadAndInitializeWasm(
+  wasmUrl?: ParquetWasm.InitInput | Promise<ParquetWasm.InitInput>
+): Promise<typeof ParquetWasm> {
+  const parquetWasm = await import('parquet-wasm/esm/parquet_wasm.js');
+  const moduleOrPath = wasmUrl ? await wasmUrl : await loadDefaultWasmInput();
+  if (moduleOrPath) {
+    await parquetWasm.default({module_or_path: moduleOrPath});
+  } else {
+    // parquet-wasm resolves its sibling asset with new URL(..., import.meta.url), which allows
+    // browser bundlers to copy and fingerprint the WASM file without a network CDN default.
+    await parquetWasm.default();
   }
-  await initializePromise;
   return parquetWasm;
 }
+
+/** Loads a local Node asset or defers to parquet-wasm's browser import.meta resolver. */
+async function loadDefaultWasmInput(): Promise<Uint8Array | undefined> {
+  return await loadNodeWasmInput();
+}