@loaders.gl/parquet 4.4.5 → 5.0.0-alpha.2

package/dist/lib/utils/make-stream-iterator.js

@@ -7,10 +7,14 @@ import { isBrowser } from '@loaders.gl/loader-utils';
  * @param stream stream to iterator over
  */
 export function makeStreamIterator(stream, options) {
-    return isBrowser
+    return isBrowser || isReadableStream(stream)
         ? makeBrowserStreamIterator(stream, options)
         : makeNodeStreamIterator(stream, options);
 }
+/** Detects WHATWG streams in Node.js as well as in browsers. */
+function isReadableStream(stream) {
+    return typeof stream.getReader === 'function';
+}
 /**
  * Returns an async iterable that reads from a DOM (browser) stream
  * @param stream stream to iterate from
@@ -26,23 +30,44 @@ async function* makeBrowserStreamIterator(stream, options) {
     // }
     // In the browser, we first need to get a lock on the stream
     const reader = stream.getReader();
+    const signal = options?.signal;
+    const abortListener = () => {
+        // Canceling the reader settles a pending read. The loop then throws the signal reason.
+        void reader.cancel(signal?.reason).catch(() => { });
+    };
     let nextBatchPromise;
+    let streamFinished = false;
+    let streamReadFailed = false;
+    let iteratorFailed = false;
     try {
+        throwIfAborted(signal);
+        signal?.addEventListener('abort', abortListener, { once: true });
         // eslint-disable-next-line no-constant-condition
         while (true) {
             const currentBatchPromise = nextBatchPromise || reader.read();
-            // Issue a read for an additional batch, while we await the next batch
-            // Idea is to make fetching happen in parallel with processing / parsing
-            if (options?._streamReadAhead) {
-                nextBatchPromise = reader.read();
-            }
+            nextBatchPromise = undefined;
             // Read from the stream
-            // value is a Uint8Array
-            const { done, value } = await currentBatchPromise;
+            let batch;
+            try {
+                batch = await currentBatchPromise;
+            }
+            catch (error) {
+                streamReadFailed = true;
+                throw error;
+            }
+            throwIfAborted(signal);
+            const { done, value } = batch;
             // Exit if we're done
             if (done) {
+                streamFinished = true;
                 return;
             }
+            // Issue a read for an additional batch while the current batch is processed
+            if (options?._streamReadAhead) {
+                nextBatchPromise = reader.read();
+                // Observe a prefetched rejection even if the consumer stops before requesting it
+                void nextBatchPromise.catch(() => { });
+            }
             // Else yield the chunk
             if (value) {
                 yield value;
@@ -50,9 +75,30 @@ async function* makeBrowserStreamIterator(stream, options) {
         }
     }
     catch (error) {
-        // TODO - examples makes it look like this should always be called,
-        // but that generates exceptions so only call it if we do not reach the end
-        reader.releaseLock();
+        iteratorFailed = true;
+        throw error;
+    }
+    finally {
+        signal?.removeEventListener('abort', abortListener);
+        let cleanupError;
+        if (!streamFinished && !streamReadFailed) {
+            try {
+                await reader.cancel();
+            }
+            catch (error) {
+                cleanupError = error;
+            }
+        }
+        try {
+            reader.releaseLock();
+        }
+        catch (error) {
+            cleanupError ||= error;
+        }
+        // Preserve iteration errors instead of replacing them with cleanup errors
+        if (!iteratorFailed && cleanupError) {
+            throw cleanupError;
+        }
     }
 }
 /**
@@ -61,8 +107,71 @@ async function* makeBrowserStreamIterator(stream, options) {
  * @note Requires Node.js >= 10
  */
 async function* makeNodeStreamIterator(stream, options) {
-    // Hacky test for node version to ensure we don't call bad polyfills
-    // NODE 10+: stream is an asyncIterator
-    yield* stream;
+    // Node streams and modern Node.js Web streams both expose an async iterator.
+    const iterator = stream[Symbol.asyncIterator]();
+    const signal = options?.signal;
+    let abortListener;
+    const abortPromise = signal
+        ? new Promise((_resolve, reject) => {
+            abortListener = () => reject(getAbortReason(signal));
+            signal.addEventListener('abort', abortListener, { once: true });
+        })
+        : null;
+    let iteratorFailed = false;
+    let streamFinished = false;
+    try {
+        throwIfAborted(signal);
+        // eslint-disable-next-line no-constant-condition
+        while (true) {
+            const nextPromise = iterator.next();
+            if (abortPromise) {
+                // Observe a late failure if cancellation wins the race.
+                void nextPromise.catch(() => { });
+            }
+            const result = abortPromise
+                ? await Promise.race([nextPromise, abortPromise])
+                : await nextPromise;
+            throwIfAborted(signal);
+            if (result.done) {
+                streamFinished = true;
+                return;
+            }
+            yield result.value;
+        }
+    }
+    catch (error) {
+        iteratorFailed = true;
+        throw error;
+    }
+    finally {
+        if (abortListener) {
+            signal?.removeEventListener('abort', abortListener);
+        }
+        if (!streamFinished) {
+            try {
+                await iterator.return?.();
+            }
+            catch (error) {
+                if (!iteratorFailed) {
+                    throw error;
+                }
+            }
+        }
+    }
+}
+/** Returns the standard AbortSignal reason, including a fallback for older runtimes. */
+function getAbortReason(signal) {
+    if (signal.reason !== undefined) {
+        return signal.reason;
+    }
+    const error = new Error('The operation was aborted');
+    error.name = 'AbortError';
+    return error;
+}
+/** Throws at a cancellation checkpoint. */
+function throwIfAborted(signal) {
+    if (signal?.aborted) {
+        throw getAbortReason(signal);
+    }
 }
 //# sourceMappingURL=make-stream-iterator.js.map

package/dist/lib/utils/make-stream-iterator.js.map

@@ -1 +1 @@
-{"version":3,"file":"make-stream-iterator.js","sourceRoot":"","sources":["../../../src/lib/utils/make-stream-iterator.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;AAGpC,OAAO,EAAC,SAAS,EAAC,MAAM,0BAA0B,CAAC;AAMnD;;;GAGG;AACH,MAAM,UAAU,kBAAkB,CAChC,MAAoC,EACpC,OAA+B;IAE/B,OAAO,SAAS;QACd,CAAC,CAAC,yBAAyB,CAAC,MAA2B,EAAE,OAAO,CAAC;QACjE,CAAC,CAAC,sBAAsB,CAAC,MAAkB,EAAE,OAAO,CAAC,CAAC;AAC1D,CAAC;AAED;;;;GAIG;AACH,KAAK,SAAS,CAAC,CAAC,yBAAyB,CACvC,MAAyB,EACzB,OAA+B;IAE/B,4DAA4D;IAC5D,kDAAkD;IAClD,iCAAiC;IACjC,IAAI;IACJ,4DAA4D;IAC5D,8CAA8C;IAC9C,IAAI;IAEJ,4DAA4D;IAC5D,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;IAElC,IAAI,gBAAkE,CAAC;IAEvE,IAAI,CAAC;QACH,iDAAiD;QACjD,OAAO,IAAI,EAAE,CAAC;YACZ,MAAM,mBAAmB,GAAG,gBAAgB,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;YAC9D,sEAAsE;YACtE,wEAAwE;YACxE,IAAI,OAAO,EAAE,gBAAgB,EAAE,CAAC;gBAC9B,gBAAgB,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC;YACnC,CAAC;YACD,uBAAuB;YACvB,wBAAwB;YACxB,MAAM,EAAC,IAAI,EAAE,KAAK,EAAC,GAAG,MAAM,mBAAmB,CAAC;YAChD,qBAAqB;YACrB,IAAI,IAAI,EAAE,CAAC;gBACT,OAAO;YACT,CAAC;YACD,uBAAuB;YACvB,IAAI,KAAK,EAAE,CAAC;gBACV,MAAM,KAAK,CAAC;YACd,CAAC;QACH,CAAC;IACH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,mEAAmE;QACnE,2EAA2E;QAC3E,MAAM,CAAC,WAAW,EAAE,CAAC;IACvB,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,KAAK,SAAS,CAAC,CAAC,sBAAsB,CACpC,MAAgB,EAChB,OAA+B;IAE/B,oEAAoE;IACpE,uCAAuC;IACvC,KAAK,CAAC,CAAC,MAAM,CAAC;AAChB,CAAC"}
+{"version":3,"file":"make-stream-iterator.js","sourceRoot":"","sources":["../../../src/lib/utils/make-stream-iterator.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;AAGpC,OAAO,EAAC,SAAS,EAAC,MAAM,0BAA0B,CAAC;AAQnD;;;GAGG;AACH,MAAM,UAAU,kBAAkB,CAChC,MAAoC,EACpC,OAA+B;IAE/B,OAAO,SAAS,IAAI,gBAAgB,CAAC,MAAM,CAAC;QAC1C,CAAC,CAAC,yBAAyB,CAAC,MAA2B,EAAE,OAAO,CAAC;QACjE,CAAC,CAAC,sBAAsB,CAAC,MAAkB,EAAE,OAAO,CAAC,CAAC;AAC1D,CAAC;AAED,gEAAgE;AAChE,SAAS,gBAAgB,CAAI,MAAoC;IAC/D,OAAO,OAAQ,MAA4B,CAAC,SAAS,KAAK,UAAU,CAAC;AACvE,CAAC;AAED;;;;GAIG;AACH,KAAK,SAAS,CAAC,CAAC,yBAAyB,CACvC,MAAyB,EACzB,OAA+B;IAE/B,4DAA4D;IAC5D,kDAAkD;IAClD,iCAAiC;IACjC,IAAI;IACJ,4DAA4D;IAC5D,8CAA8C;IAC9C,IAAI;IAEJ,4DAA4D;IAC5D,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;IAClC,MAAM,MAAM,GAAG,OAAO,EAAE,MAAM,CAAC;IAC/B,MAAM,aAAa,GAAG,GAAG,EAAE;QACzB,uFAAuF;QACvF,KAAK,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IACrD,CAAC,CAAC;IAEF,IAAI,gBAAkE,CAAC;IACvE,IAAI,cAAc,GAAG,KAAK,CAAC;IAC3B,IAAI,gBAAgB,GAAG,KAAK,CAAC;IAC7B,IAAI,cAAc,GAAG,KAAK,CAAC;IAE3B,IAAI,CAAC;QACH,cAAc,CAAC,MAAM,CAAC,CAAC;QACvB,MAAM,EAAE,gBAAgB,CAAC,OAAO,EAAE,aAAa,EAAE,EAAC,IAAI,EAAE,IAAI,EAAC,CAAC,CAAC;QAE/D,iDAAiD;QACjD,OAAO,IAAI,EAAE,CAAC;YACZ,MAAM,mBAAmB,GAAG,gBAAgB,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;YAC9D,gBAAgB,GAAG,SAAS,CAAC;YAC7B,uBAAuB;YACvB,IAAI,KAAK,CAAC;YACV,IAAI,CAAC;gBACH,KAAK,GAAG,MAAM,mBAAmB,CAAC;YACpC,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,gBAAgB,GAAG,IAAI,CAAC;gBACxB,MAAM,KAAK,CAAC;YACd,CAAC;YACD,cAAc,CAAC,MAAM,CAAC,CAAC;YACvB,MAAM,EAAC,IAAI,EAAE,KAAK,EAAC,GAAG,KAAK,CAAC;YAC5B,qBAAqB;YACrB,IAAI,IAAI,EAAE,CAAC;gBACT,cAAc,GAAG,IAAI,CAAC;gBACtB,OAAO;YACT,CAAC;YACD,4EAA4E;YAC5E,IAAI,OAAO,EAAE,gBAAgB,EAAE,CAAC;gBAC9B,gBAAgB,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC;gBACjC,iFAAiF;gBACjF,KAAK,gBAAgB,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;YACxC,CAAC;YACD,uBAAuB;YACvB,IAAI,KAAK,EAAE,CAAC;gBACV,MAAM,KAAK,CAAC;YACd,CAAC;QACH,CAAC;IACH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,cAAc,GAAG,IAAI,CAAC;QACtB,MAAM,KAAK,CAAC;IACd,CAAC;YAAS,CAAC;QACT,MAAM,EAAE,mBAAmB,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC;QACpD,IAAI,YAAqB,CAAC;QAE1B,IAAI,CAAC,cAAc,IAAI,CAAC,gBAAgB,EAAE,CAAC;YACzC,IAAI,CAAC;gBACH,MAAM,MAAM,CAAC,MAAM,EAAE,CAAC;YACxB,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,YAAY,GAAG,KAAK,CAAC;YACvB,CAAC;QACH,CAAC;QAED,IAAI,CAAC;YACH,MAAM,CAAC,WAAW,EAAE,CAAC;QACvB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,YAAY,KAAK,KAAK,CAAC;QACzB,CAAC;QAED,0EAA0E;QAC1E,IAAI,CAAC,cAAc,IAAI,YAAY,EAAE,CAAC;YACpC,MAAM,YAAY,CAAC;QACrB,CAAC;IACH,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,KAAK,SAAS,CAAC,CAAC,sBAAsB,CACpC,MAAgB,EAChB,OAA+B;IAE/B,6EAA6E;IAC7E,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,CAAC;IAChD,MAAM,MAAM,GAAG,OAAO,EAAE,MAAM,CAAC;IAC/B,IAAI,aAAuC,CAAC;IAC5C,MAAM,YAAY,GAAG,MAAM;QACzB,CAAC,CAAC,IAAI,OAAO,CAAQ,CAAC,QAAQ,EAAE,MAAM,EAAE,EAAE;YACtC,aAAa,GAAG,GAAG,EAAE,CAAC,MAAM,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC;YACrD,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,aAAa,EAAE,EAAC,IAAI,EAAE,IAAI,EAAC,CAAC,CAAC;QAChE,CAAC,CAAC;QACJ,CAAC,CAAC,IAAI,CAAC;IACT,IAAI,cAAc,GAAG,KAAK,CAAC;IAC3B,IAAI,cAAc,GAAG,KAAK,CAAC;IAE3B,IAAI,CAAC;QACH,cAAc,CAAC,MAAM,CAAC,CAAC;QACvB,iDAAiD;QACjD,OAAO,IAAI,EAAE,CAAC;YACZ,MAAM,WAAW,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC;YACpC,IAAI,YAAY,EAAE,CAAC;gBACjB,wDAAwD;gBACxD,KAAK,WAAW,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;YACnC,CAAC;YACD,MAAM,MAAM,GAAG,YAAY;gBACzB,CAAC,CAAC,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,YAAY,CAAC,CAAC;gBACjD,CAAC,CAAC,MAAM,WAAW,CAAC;YACtB,cAAc,CAAC,MAAM,CAAC,CAAC;YACvB,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;gBAChB,cAAc,GAAG,IAAI,CAAC;gBACtB,OAAO;YACT,CAAC;YACD,MAAM,MAAM,CAAC,KAAK,CAAC;QACrB,CAAC;IACH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,cAAc,GAAG,IAAI,CAAC;QACtB,MAAM,KAAK,CAAC;IACd,CAAC;YAAS,CAAC;QACT,IAAI,aAAa,EAAE,CAAC;YAClB,MAAM,EAAE,mBAAmB,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC;QACtD,CAAC;QACD,IAAI,CAAC,cAAc,EAAE,CAAC;YACpB,IAAI,CAAC;gBACH,MAAM,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC;YAC5B,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,CAAC,cAAc,EAAE,CAAC;oBACpB,MAAM,KAAK,CAAC;gBACd,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;AACH,CAAC;AAED,wFAAwF;AACxF,SAAS,cAAc,CAAC,MAAmB;IACzC,IAAI,MAAM,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;QAChC,OAAO,MAAM,CAAC,MAAM,CAAC;IACvB,CAAC;IACD,MAAM,KAAK,GAAG,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;IACrD,KAAK,CAAC,IAAI,GAAG,YAAY,CAAC;IAC1B,OAAO,KAAK,CAAC;AACf,CAAC;AAED,2CAA2C;AAC3C,SAAS,cAAc,CAAC,MAAoB;IAC1C,IAAI,MAAM,EAAE,OAAO,EAAE,CAAC;QACpB,MAAM,cAAc,CAAC,MAAM,CAAC,CAAC;IAC/B,CAAC;AACH,CAAC"}

package/dist/lib/utils/normalize-parquet-options.d.ts

@@ -0,0 +1,5 @@
+/** Merge caller options with default parquet options while preserving top-level option bags. */
+export declare function normalizeParquetOptions<OptionsT extends {
+    parquet?: unknown;
+}>(options: OptionsT | undefined, parquetDefaults: Record<string, unknown>): OptionsT;
+//# sourceMappingURL=normalize-parquet-options.d.ts.map

package/dist/lib/utils/normalize-parquet-options.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"normalize-parquet-options.d.ts","sourceRoot":"","sources":["../../../src/lib/utils/normalize-parquet-options.ts"],"names":[],"mappings":"AAIA,gGAAgG;AAChG,wBAAgB,uBAAuB,CAAC,QAAQ,SAAS;IAAC,OAAO,CAAC,EAAE,OAAO,CAAA;CAAC,EAC1E,OAAO,EAAE,QAAQ,GAAG,SAAS,EAC7B,eAAe,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GACvC,QAAQ,CAKV"}

package/dist/lib/utils/normalize-parquet-options.js

@@ -0,0 +1,11 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+/** Merge caller options with default parquet options while preserving top-level option bags. */
+export function normalizeParquetOptions(options, parquetDefaults) {
+    return {
+        ...options,
+        parquet: { ...parquetDefaults, ...options?.parquet }
+    };
+}
+//# sourceMappingURL=normalize-parquet-options.js.map

package/dist/lib/utils/normalize-parquet-options.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"normalize-parquet-options.js","sourceRoot":"","sources":["../../../src/lib/utils/normalize-parquet-options.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;AAEpC,gGAAgG;AAChG,MAAM,UAAU,uBAAuB,CACrC,OAA6B,EAC7B,eAAwC;IAExC,OAAO;QACL,GAAG,OAAO;QACV,OAAO,EAAE,EAAC,GAAG,eAAe,EAAE,GAAI,OAAO,EAAE,OAA+C,EAAC;KACrE,CAAC;AAC3B,CAAC"}