@loaders.gl/core 5.0.0-alpha.1 → 5.0.0-alpha.11

package/dist/null-worker.js

@@ -71,18 +71,18 @@
     static async addEventListener(onMessage) {
       let onMessageWrapper = onMessageWrapperMap.get(onMessage);
       if (!onMessageWrapper) {
-        onMessageWrapper = async (message) => {
-          if (!isKnownMessage(message)) {
+        onMessageWrapper = (message) => {
+          const messageData = getWorkerMessageData(message);
+          if (!messageData) {
             return;
           }
-          const parentPort3 = await getParentPort();
-          const { type, payload } = parentPort3 ? message : message.data;
-          onMessage(type, payload);
+          onMessage(messageData.type, messageData.payload);
         };
+        onMessageWrapperMap.set(onMessage, onMessageWrapper);
       }
       const parentPort2 = await getParentPort();
       if (parentPort2) {
-        console.error("not implemented");
+        parentPort2.on("message", onMessageWrapper);
       } else {
         globalThis.addEventListener("message", onMessageWrapper);
       }
@@ -90,9 +90,12 @@
     static async removeEventListener(onMessage) {
       const onMessageWrapper = onMessageWrapperMap.get(onMessage);
       onMessageWrapperMap.delete(onMessage);
+      if (!onMessageWrapper) {
+        return;
+      }
       const parentPort2 = await getParentPort();
       if (parentPort2) {
-        console.error("not implemented");
+        parentPort2.off("message", onMessageWrapper);
       } else {
         globalThis.removeEventListener("message", onMessageWrapper);
       }
@@ -113,9 +116,12 @@
       }
     }
   };
-  function isKnownMessage(message) {
-    const { type, data } = message;
-    return type === "message" && data && typeof data.source === "string" && data.source.startsWith("loaders.gl");
+  function getWorkerMessageData(message) {
+    if (!message || typeof message !== "object") {
+      return null;
+    }
+    const messageData = "data" in message && message.type === "message" ? message.data : message;
+    return messageData && typeof messageData.source === "string" && messageData.source.startsWith("loaders.gl") ? messageData : null;
   }
 
   // ../worker-utils/src/lib/async-queue/async-queue.ts
@@ -190,8 +196,8 @@
 
   // ../worker-utils/src/lib/worker-api/create-worker.ts
   var requestId = 0;
-  var inputBatches;
-  var options;
+  var activeInputBatches = null;
+  var activeOutputAcknowledgements = null;
   async function createWorker(process, processInBatches) {
     if (!await WorkerBody.inWorkerThread()) {
       return;
@@ -221,24 +227,44 @@
             if (!processInBatches) {
               throw new Error("Worker does not support batched processing");
             }
-            inputBatches = new AsyncQueue();
-            options = payload.options || {};
-            const resultIterator = processInBatches(
-              inputBatches,
-              options,
-              context,
-              payload.context || {}
-            );
-            for await (const batch of resultIterator) {
-              WorkerBody.postMessage("output-batch", { result: batch });
+            const inputBatches = new AsyncQueue();
+            const outputAcknowledgements = new AsyncQueue();
+            activeInputBatches = inputBatches;
+            activeOutputAcknowledgements = outputAcknowledgements;
+            try {
+              const resultIterator = processInBatches(
+                createDemandDrivenIterator(inputBatches),
+                payload.options || {},
+                context,
+                payload.context || {}
+              );
+              for await (const batch of resultIterator) {
+                await WorkerBody.postMessage("output-batch", { result: batch });
+                await outputAcknowledgements.next();
+              }
+              await WorkerBody.postMessage("done", {});
+            } finally {
+              activeInputBatches = null;
+              activeOutputAcknowledgements = null;
             }
-            WorkerBody.postMessage("done", {});
             break;
           case "input-batch":
-            inputBatches.push(payload.input);
+            if (!activeInputBatches) {
+              throw new Error("Worker has no active batched processing session");
+            }
+            activeInputBatches.push(payload.input);
             break;
           case "input-done":
-            inputBatches.close();
+            if (!activeInputBatches) {
+              throw new Error("Worker has no active batched processing session");
+            }
+            activeInputBatches.close();
+            break;
+          case "output-ack":
+            if (!activeOutputAcknowledgements) {
+              throw new Error("Worker has no active batched processing session");
+            }
+            activeOutputAcknowledgements.push(void 0);
             break;
           default:
         }
@@ -248,7 +274,17 @@
       }
     };
   }
-  function processOnMainThread(arrayBuffer, options2 = {}, jobContext = {}) {
+  async function* createDemandDrivenIterator(inputBatches) {
+    while (true) {
+      await WorkerBody.postMessage("input-request", {});
+      const nextBatch = await inputBatches.next();
+      if (nextBatch.done) {
+        return;
+      }
+      yield nextBatch.value;
+    }
+  }
+  function processOnMainThread(arrayBuffer, options = {}, jobContext = {}) {
     return new Promise((resolve, reject) => {
       const id = requestId++;
       const onMessage = (type, payload2) => {
@@ -268,29 +304,56 @@
         }
       };
       WorkerBody.addEventListener(onMessage);
-      const payload = { id, input: arrayBuffer, options: options2, context: jobContext };
+      const payload = { id, input: arrayBuffer, options, context: jobContext };
       WorkerBody.postMessage("process", payload);
     });
   }
 
   // ../loader-utils/src/lib/worker-loader-utils/create-loader-worker.ts
-  async function createLoaderWorker(loader) {
+  async function createLoaderWorker(loader, selectLoader = () => loader) {
     await createWorker(
-      async (input, options2 = {}, workerContext, loaderContext = {}) => {
-        const result = await parseData({
-          loader,
-          arrayBuffer: input,
-          options: options2,
-          context: {
-            ...loaderContext,
-            coreApi: createWorkerCoreApi(),
-            _parse: createParseOnMainThread(workerContext?.process)
-          }
-        });
-        return loader.serializeWorkerResult ? loader.serializeWorkerResult(result, options2, loaderContext) : result;
-      }
+      (input, options, workerContext, loaderContext) => processLoaderWorkerData(
+        selectLoader(options || {}),
+        input,
+        options,
+        workerContext,
+        loaderContext
+      ),
+      (inputIterator, options, workerContext, loaderContext) => processLoaderWorkerBatches(
+        selectLoader(options || {}),
+        inputIterator,
+        options,
+        workerContext,
+        loaderContext
+      )
     );
   }
+  async function processLoaderWorkerData(loader, input, options = {}, workerContext, loaderContext = {}) {
+    const result = await parseData({
+      loader,
+      arrayBuffer: input,
+      options,
+      context: {
+        ...loaderContext,
+        coreApi: createWorkerCoreApi(),
+        _parse: createParseOnMainThread(workerContext?.process)
+      }
+    });
+    return loader.serializeWorkerResult ? loader.serializeWorkerResult(result, options, loaderContext) : result;
+  }
+  async function* processLoaderWorkerBatches(loader, inputIterator, options = {}, workerContext, loaderContext = {}) {
+    if (!loader.parseInBatches) {
+      throw new Error(`${loader.id} loader does not support batched parsing`);
+    }
+    const resultIterator = loader.parseInBatches(inputIterator, options, {
+      ...loaderContext,
+      coreApi: createWorkerCoreApi(),
+      _parse: createParseOnMainThread(workerContext?.process)
+    });
+    for await (const batch of resultIterator) {
+      yield loader.serializeWorkerBatch ? loader.serializeWorkerBatch(batch, options, loaderContext) : batch;
+    }
+  }
   function createWorkerCoreApi() {
     const unavailable = (methodName) => () => {
       throw new Error(`context.coreApi.${methodName} is unavailable inside worker loaders.`);
@@ -306,17 +369,23 @@
     };
   }
   function createParseOnMainThread(processOnMainThread2) {
-    return (arrayBuffer, loaders, options2, context) => {
+    return (arrayBuffer, loaders, options, context) => {
       if (!processOnMainThread2) {
         throw new Error("Worker not set up to parse on main thread");
       }
-      const parseArguments = getMainThreadParseArguments(loaders, options2, context);
-      return processOnMainThread2(arrayBuffer, parseArguments.options, parseArguments.context);
+      const parseArguments = getMainThreadParseArguments(loaders, options, context);
+      return processOnMainThread2(arrayBuffer, parseArguments.options, {
+        ...parseArguments.context,
+        ...Array.isArray(loaders) || loaders && isLoaderObject(loaders) ? {
+          _loaderIsArray: Array.isArray(loaders),
+          _loaderIds: (Array.isArray(loaders) ? loaders : [loaders]).map((loader) => loader.id)
+        } : {}
+      });
     };
   }
-  function getMainThreadParseArguments(loaders, options2, context) {
-    if (options2) {
-      return { options: options2, context: getSerializableLoaderContext(context) };
+  function getMainThreadParseArguments(loaders, options, context) {
+    if (options) {
+      return { options, context: getSerializableLoaderContext(context) };
     }
     if (Array.isArray(loaders) || loaders && isLoaderObject(loaders)) {
       return { options: void 0, context: getSerializableLoaderContext(context) };
@@ -339,7 +408,7 @@
   async function parseData({
     loader,
     arrayBuffer,
-    options: options2,
+    options,
     context
   }) {
     let data;
@@ -354,19 +423,22 @@
     } else {
       throw new Error(`Could not load data with ${loader.name} loader`);
     }
-    options2 = {
-      ...options2,
-      modules: loader && loader.options && loader.options.modules || {},
+    options = {
+      ...options,
+      modules: {
+        ...loader && loader.options && loader.options.modules || {},
+        ...options.modules || {}
+      },
       core: {
-        ...options2.core,
+        ...options.core,
         worker: false
       }
     };
-    return await parser(data, { ...options2 }, context, loader);
+    return await parser(data, { ...options }, context, loader);
   }
 
   // src/null-loader.ts
-  var VERSION = true ? "5.0.0-alpha.1" : "latest";
+  var VERSION = true ? "5.0.0-alpha.11" : "latest";
   var NullLoader = {
     dataType: null,
     batchType: null,
@@ -376,11 +448,11 @@
     version: VERSION,
     mimeTypes: ["application/x.empty"],
     extensions: ["null"],
-    parse: async (arrayBuffer, options2, context) => parseSync(arrayBuffer, options2 || {}, context),
+    parse: async (arrayBuffer, options, context) => parseSync(arrayBuffer, options || {}, context),
     parseSync,
-    parseInBatches: async function* generator(asyncIterator, options2, context) {
+    parseInBatches: async function* generator(asyncIterator, options, context) {
       for await (const batch of asyncIterator) {
-        yield parseSync(batch, options2, context);
+        yield parseSync(batch, options, context);
       }
     },
     tests: [() => false],
@@ -388,7 +460,7 @@
       null: {}
     }
   };
-  function parseSync(arrayBuffer, options2, context) {
+  function parseSync(arrayBuffer, options, context) {
     return null;
   }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@loaders.gl/core",
-  "version": "5.0.0-alpha.1",
+  "version": "5.0.0-alpha.11",
   "description": "The core API for working with loaders.gl loaders and writers",
   "license": "MIT",
   "type": "module",
@@ -58,11 +58,11 @@
     "build-worker-node": "esbuild src/workers/null-worker.ts --outfile=dist/null-worker-node.js --bundle --platform=node --target=node16 --define:__VERSION__=\\\"$npm_package_version\\\""
   },
   "dependencies": {
-    "@loaders.gl/loader-utils": "5.0.0-alpha.1",
-    "@loaders.gl/schema": "5.0.0-alpha.1",
-    "@loaders.gl/schema-utils": "5.0.0-alpha.1",
-    "@loaders.gl/worker-utils": "5.0.0-alpha.1",
+    "@loaders.gl/loader-utils": "5.0.0-alpha.11",
+    "@loaders.gl/schema": "5.0.0-alpha.11",
+    "@loaders.gl/schema-utils": "5.0.0-alpha.11",
+    "@loaders.gl/worker-utils": "5.0.0-alpha.11",
     "@probe.gl/log": "^4.1.1"
   },
-  "gitHead": "77a55028e8204e09df1258f10cb61ed35353b258"
+  "gitHead": "31449661e98c1c3f97bb4f8a7bcba7f19596ac00"
 }

package/src/iterators/make-iterator/make-stream-iterator.ts

@@ -43,30 +43,68 @@ async function* makeBrowserStreamIterator(
   const reader = stream.getReader();
 
   let nextBatchPromise: Promise<{done?: boolean; value?: Uint8Array}> | undefined;
+  let streamFinished = false;
+  let streamReadFailed = false;
+  let iteratorFailed = false;
 
   try {
     // eslint-disable-next-line no-constant-condition
     while (true) {
       const currentBatchPromise = nextBatchPromise || reader.read();
-      // Issue a read for an additional batch, while we await the next batch
-      // Idea is to make fetching happen in parallel with processing / parsing
-      if (options?._streamReadAhead) {
-        nextBatchPromise = reader.read();
-      }
+      nextBatchPromise = undefined;
       // Read from the stream
       // value is a Uint8Array
-      const {done, value} = await currentBatchPromise;
+      let batch;
+      try {
+        batch = await currentBatchPromise;
+      } catch (error) {
+        streamReadFailed = true;
+        throw error;
+      }
+      const {done, value} = batch;
       // Exit if we're done
       if (done) {
+        streamFinished = true;
         return;
       }
+      // Issue a read for an additional batch while the current batch is processed
+      if (options?._streamReadAhead) {
+        nextBatchPromise = reader.read();
+        // Observe a prefetched rejection even if the consumer stops before requesting it
+        void nextBatchPromise.catch(() => {});
+      }
       // Else yield the chunk
       yield toArrayBuffer(value);
     }
-  } catch (_error) {
-    // TODO - examples makes it look like this should always be called,
-    // but that generates exceptions so only call it if we do not reach the end
-    reader.releaseLock();
+  } catch (error) {
+    iteratorFailed = true;
+    throw error;
+  } finally {
+    let cleanupFailed = false;
+    let cleanupError: unknown;
+
+    if (!streamFinished && !streamReadFailed) {
+      try {
+        await reader.cancel();
+      } catch (error) {
+        cleanupFailed = true;
+        cleanupError = error;
+      }
+    }
+
+    try {
+      reader.releaseLock();
+    } catch (error) {
+      if (!cleanupFailed) {
+        cleanupFailed = true;
+        cleanupError = error;
+      }
+    }
+
+    // Preserve errors from iteration instead of replacing them with cleanup errors
+    if (!iteratorFailed && cleanupFailed) {
+      throw cleanupError;
+    }
   }
 }
 

package/src/lib/api/create-data-source.ts

@@ -7,6 +7,7 @@ import {
   SourceArrayOptionsType,
   SourceArrayDataSourceType
 } from '@loaders.gl/loader-utils';
+import {resolveAuthenticationOptions} from '@loaders.gl/loader-utils';
 import {coreApi} from './core-api';
 import {selectLoaderSync} from './select-loader';
 
@@ -22,7 +23,9 @@ import {selectLoaderSync} from './select-loader';
 export function createDataSource<SourceArrayT extends SourceLoader[]>(
   data: unknown,
   sources: Readonly<SourceArrayT>,
-  options: Readonly<SourceArrayOptionsType<SourceArrayT>>
+  options: Readonly<
+    SourceArrayOptionsType<SourceArrayT>
+  > = {} as SourceArrayOptionsType<SourceArrayT>
 ): SourceArrayDataSourceType<SourceArrayT> {
   const resolvedOptions = (options || {}) as SourceArrayOptionsType<SourceArrayT>;
   const type = resolvedOptions?.core?.type || (resolvedOptions.type as unknown as string) || 'auto';
@@ -41,7 +44,33 @@ export function createDataSource<SourceArrayT extends SourceLoader[]>(
   if (!source) {
     throw new Error('Not a valid source type');
   }
-  return source.createDataSource(data as string | Blob, resolvedOptions, coreApi);
+  const contributions = resolvedOptions.core?.credentials?.length
+    ? source.getAuthentications?.(typeof data === 'string' ? data : '', resolvedOptions)
+    : undefined;
+  if (contributions && 'then' in contributions) {
+    // Avoid an unhandled rejection from a hook that cannot be awaited by this sync API.
+    void contributions.catch(() => {});
+    throw new Error('Use load() with an asynchronous getAuthentications() hook.');
+  }
+  const authenticatedOptions = resolveAuthenticationOptions(
+    contributions?.length
+      ? {
+          ...resolvedOptions,
+          core: {
+            ...resolvedOptions.core,
+            authentications: [
+              ...(resolvedOptions.core?.authentications || []),
+              ...(contributions || [])
+            ]
+          }
+        }
+      : resolvedOptions
+  );
+  return source.createDataSource(
+    data as string | Blob,
+    authenticatedOptions,
+    coreApi
+  ) as SourceArrayDataSourceType<SourceArrayT>;
 }
 
 /** Guess service type from URL */

package/src/lib/api/load-in-batches.ts

@@ -6,17 +6,20 @@ import type {
   Loader,
   LoaderOptions,
   LoaderContext,
-  FetchLike,
   BatchableDataType,
   LoaderBatchType,
   LoaderOptionsType,
   LoaderOptionsWithShape,
   LoaderShapeType
 } from '@loaders.gl/loader-utils';
+import {resolveLoaderAuthenticationOptions} from '@loaders.gl/loader-utils';
 import {isLoaderObject} from '../loader-utils/normalize-loader';
 import {getFetchFunction} from '../loader-utils/get-fetch-function';
+import {applySearchParamsToUrl} from '../loader-utils/option-utils';
 
 import {parseInBatches} from './parse-in-batches';
+import {selectLoader} from './select-loader';
+import {getLoaderImplementation} from './load-loader';
 
 type FileType = string | File | Blob | Response | (string | File | Blob | Response)[] | FileList;
 
@@ -73,18 +76,13 @@ export function loadInBatches(
     loadersArray = loaders as Loader | Loader[] | undefined;
   }
 
-  // Select fetch function
-  const fetch = getFetchFunction(options || {});
-
   // Single url/file
   if (!Array.isArray(files)) {
-    return loadOneFileInBatches(files, loadersArray!, options || {}, fetch);
+    return loadOneFileInBatches(files, loadersArray!, options || {});
   }
 
   // Multiple URLs / files
-  const promises = files.map(file =>
-    loadOneFileInBatches(file, loadersArray!, options || {}, fetch)
-  );
+  const promises = files.map(file => loadOneFileInBatches(file, loadersArray!, options || {}));
 
   // No point in waiting here for all responses before starting to stream individual streams?
   return promises;
@@ -93,12 +91,21 @@ export function loadInBatches(
 async function loadOneFileInBatches(
   file: FileType,
   loaders: Loader | Loader[],
-  options: LoaderOptions,
-  fetch: FetchLike
+  options: LoaderOptions
 ): Promise<AsyncIterable<unknown>> {
   if (typeof file === 'string') {
     const url = file;
-    const response = await fetch(url);
+    const loader = await selectLoader(url, loaders, {
+      ...options,
+      core: {...options.core, nothrow: true}
+    });
+    const authenticationLoader =
+      loader && !loader.getAuthentications && options.core?.credentials?.length
+        ? await getLoaderImplementation(loader, options, url)
+        : loader;
+    options = await resolveLoaderAuthenticationOptions(authenticationLoader, url, options);
+    const fetch = getFetchFunction(options);
+    const response = await fetch(applySearchParamsToUrl(url, options));
     // pick right overload
     return Array.isArray(loaders)
       ? await parseInBatches(response, loaders, options)