@loaders.gl/core 4.5.3 → 5.0.0-alpha.10

package/dist/null-worker.js

@@ -71,18 +71,18 @@
     static async addEventListener(onMessage) {
       let onMessageWrapper = onMessageWrapperMap.get(onMessage);
       if (!onMessageWrapper) {
-        onMessageWrapper = async (message) => {
-          if (!isKnownMessage(message)) {
+        onMessageWrapper = (message) => {
+          const messageData = getWorkerMessageData(message);
+          if (!messageData) {
             return;
           }
-          const parentPort3 = await getParentPort();
-          const { type, payload } = parentPort3 ? message : message.data;
-          onMessage(type, payload);
+          onMessage(messageData.type, messageData.payload);
         };
+        onMessageWrapperMap.set(onMessage, onMessageWrapper);
       }
       const parentPort2 = await getParentPort();
       if (parentPort2) {
-        console.error("not implemented");
+        parentPort2.on("message", onMessageWrapper);
       } else {
         globalThis.addEventListener("message", onMessageWrapper);
       }
@@ -90,9 +90,12 @@
     static async removeEventListener(onMessage) {
       const onMessageWrapper = onMessageWrapperMap.get(onMessage);
       onMessageWrapperMap.delete(onMessage);
+      if (!onMessageWrapper) {
+        return;
+      }
       const parentPort2 = await getParentPort();
       if (parentPort2) {
-        console.error("not implemented");
+        parentPort2.off("message", onMessageWrapper);
       } else {
         globalThis.removeEventListener("message", onMessageWrapper);
       }
@@ -113,44 +116,175 @@
       }
     }
   };
-  function isKnownMessage(message) {
-    const { type, data } = message;
-    return type === "message" && data && typeof data.source === "string" && data.source.startsWith("loaders.gl");
+  function getWorkerMessageData(message) {
+    if (!message || typeof message !== "object") {
+      return null;
+    }
+    const messageData = "data" in message && message.type === "message" ? message.data : message;
+    return messageData && typeof messageData.source === "string" && messageData.source.startsWith("loaders.gl") ? messageData : null;
   }
 
-  // ../loader-utils/src/lib/worker-loader-utils/create-loader-worker.ts
+  // ../worker-utils/src/lib/async-queue/async-queue.ts
+  var AsyncQueue = class {
+    _values;
+    _settlers;
+    _closed;
+    constructor() {
+      this._values = [];
+      this._settlers = [];
+      this._closed = false;
+    }
+    /** Return an async iterator for this queue */
+    [Symbol.asyncIterator]() {
+      return this;
+    }
+    /** Push a new value - the async iterator will yield a promise resolved to this value */
+    push(value) {
+      return this.enqueue(value);
+    }
+    /**
+     * Push a new value - the async iterator will yield a promise resolved to this value
+     * Add an error - the async iterator will yield a promise rejected with this value
+     */
+    enqueue(value) {
+      if (this._closed) {
+        throw new Error("Closed");
+      }
+      if (this._settlers.length > 0) {
+        if (this._values.length > 0) {
+          throw new Error("Illegal internal state");
+        }
+        const settler = this._settlers.shift();
+        if (value instanceof Error) {
+          settler.reject(value);
+        } else {
+          settler.resolve({ value });
+        }
+      } else {
+        this._values.push(value);
+      }
+    }
+    /** Indicate that we not waiting for more values - The async iterator will be done */
+    close() {
+      while (this._settlers.length > 0) {
+        const settler = this._settlers.shift();
+        settler.resolve({ done: true });
+      }
+      this._closed = true;
+    }
+    // ITERATOR IMPLEMENTATION
+    /** @returns a Promise for an IteratorResult */
+    next() {
+      if (this._values.length > 0) {
+        const value = this._values.shift();
+        if (value instanceof Error) {
+          return Promise.reject(value);
+        }
+        return Promise.resolve({ done: false, value });
+      }
+      if (this._closed) {
+        if (this._settlers.length > 0) {
+          throw new Error("Illegal internal state");
+        }
+        return Promise.resolve({ done: true, value: void 0 });
+      }
+      return new Promise((resolve, reject) => {
+        this._settlers.push({ resolve, reject });
+      });
+    }
+  };
+
+  // ../worker-utils/src/lib/worker-api/create-worker.ts
   var requestId = 0;
-  async function createLoaderWorker(loader, selectLoader = () => loader) {
+  var activeInputBatches = null;
+  var activeOutputAcknowledgements = null;
+  async function createWorker(process, processInBatches) {
     if (!await WorkerBody.inWorkerThread()) {
       return;
     }
+    const context = {
+      process: processOnMainThread
+    };
     WorkerBody.onmessage = async (type, payload) => {
-      switch (type) {
-        case "process":
-          try {
-            const { input, options = {}, context = {} } = payload;
-            const selectedLoader = selectLoader(options);
-            const result = await parseData({
-              loader: selectedLoader,
-              arrayBuffer: input,
-              options,
-              // @ts-expect-error fetch missing
-              context: {
-                ...context,
-                _parse: parseOnMainThread
-              }
-            });
+      try {
+        switch (type) {
+          case "preload":
+            WorkerBody.postMessage("done", {});
+            break;
+          case "process":
+            if (!process) {
+              throw new Error("Worker does not support atomic processing");
+            }
+            const result = await process(
+              payload.input,
+              payload.options || {},
+              context,
+              payload.context || {}
+            );
             WorkerBody.postMessage("done", { result });
-          } catch (error) {
-            const message = error instanceof Error ? error.message : "";
-            WorkerBody.postMessage("error", { error: message });
-          }
-          break;
-        default:
+            break;
+          case "process-in-batches":
+            if (!processInBatches) {
+              throw new Error("Worker does not support batched processing");
+            }
+            const inputBatches = new AsyncQueue();
+            const outputAcknowledgements = new AsyncQueue();
+            activeInputBatches = inputBatches;
+            activeOutputAcknowledgements = outputAcknowledgements;
+            try {
+              const resultIterator = processInBatches(
+                createDemandDrivenIterator(inputBatches),
+                payload.options || {},
+                context,
+                payload.context || {}
+              );
+              for await (const batch of resultIterator) {
+                await WorkerBody.postMessage("output-batch", { result: batch });
+                await outputAcknowledgements.next();
+              }
+              await WorkerBody.postMessage("done", {});
+            } finally {
+              activeInputBatches = null;
+              activeOutputAcknowledgements = null;
+            }
+            break;
+          case "input-batch":
+            if (!activeInputBatches) {
+              throw new Error("Worker has no active batched processing session");
+            }
+            activeInputBatches.push(payload.input);
+            break;
+          case "input-done":
+            if (!activeInputBatches) {
+              throw new Error("Worker has no active batched processing session");
+            }
+            activeInputBatches.close();
+            break;
+          case "output-ack":
+            if (!activeOutputAcknowledgements) {
+              throw new Error("Worker has no active batched processing session");
+            }
+            activeOutputAcknowledgements.push(void 0);
+            break;
+          default:
+        }
+      } catch (error) {
+        const message = error instanceof Error ? error.message : "";
+        WorkerBody.postMessage("error", { error: message });
       }
     };
   }
-  function parseOnMainThread(arrayBuffer, loader, options, context) {
+  async function* createDemandDrivenIterator(inputBatches) {
+    while (true) {
+      await WorkerBody.postMessage("input-request", {});
+      const nextBatch = await inputBatches.next();
+      if (nextBatch.done) {
+        return;
+      }
+      yield nextBatch.value;
+    }
+  }
+  function processOnMainThread(arrayBuffer, options = {}, jobContext = {}) {
     return new Promise((resolve, reject) => {
       const id = requestId++;
       const onMessage = (type, payload2) => {
@@ -170,10 +304,107 @@
         }
       };
       WorkerBody.addEventListener(onMessage);
-      const payload = { id, input: arrayBuffer, options };
+      const payload = { id, input: arrayBuffer, options, context: jobContext };
       WorkerBody.postMessage("process", payload);
     });
   }
+
+  // ../loader-utils/src/lib/worker-loader-utils/create-loader-worker.ts
+  async function createLoaderWorker(loader, selectLoader = () => loader) {
+    await createWorker(
+      (input, options, workerContext, loaderContext) => processLoaderWorkerData(
+        selectLoader(options || {}),
+        input,
+        options,
+        workerContext,
+        loaderContext
+      ),
+      (inputIterator, options, workerContext, loaderContext) => processLoaderWorkerBatches(
+        selectLoader(options || {}),
+        inputIterator,
+        options,
+        workerContext,
+        loaderContext
+      )
+    );
+  }
+  async function processLoaderWorkerData(loader, input, options = {}, workerContext, loaderContext = {}) {
+    const result = await parseData({
+      loader,
+      arrayBuffer: input,
+      options,
+      context: {
+        ...loaderContext,
+        coreApi: createWorkerCoreApi(),
+        _parse: createParseOnMainThread(workerContext?.process)
+      }
+    });
+    return loader.serializeWorkerResult ? loader.serializeWorkerResult(result, options, loaderContext) : result;
+  }
+  async function* processLoaderWorkerBatches(loader, inputIterator, options = {}, workerContext, loaderContext = {}) {
+    if (!loader.parseInBatches) {
+      throw new Error(`${loader.id} loader does not support batched parsing`);
+    }
+    const resultIterator = loader.parseInBatches(inputIterator, options, {
+      ...loaderContext,
+      coreApi: createWorkerCoreApi(),
+      _parse: createParseOnMainThread(workerContext?.process)
+    });
+    for await (const batch of resultIterator) {
+      yield loader.serializeWorkerBatch ? loader.serializeWorkerBatch(batch, options, loaderContext) : batch;
+    }
+  }
+  function createWorkerCoreApi() {
+    const unavailable = (methodName) => () => {
+      throw new Error(`context.coreApi.${methodName} is unavailable inside worker loaders.`);
+    };
+    return {
+      fetchFile: async (urlOrData, fetchOptions) => await fetch(urlOrData, fetchOptions),
+      parseSync: unavailable("parseSync"),
+      parse: unavailable("parse"),
+      parseFile: unavailable("parseFile"),
+      parseInBatches: unavailable("parseInBatches"),
+      load: unavailable("load"),
+      loadInBatches: unavailable("loadInBatches")
+    };
+  }
+  function createParseOnMainThread(processOnMainThread2) {
+    return (arrayBuffer, loaders, options, context) => {
+      if (!processOnMainThread2) {
+        throw new Error("Worker not set up to parse on main thread");
+      }
+      const parseArguments = getMainThreadParseArguments(loaders, options, context);
+      return processOnMainThread2(arrayBuffer, parseArguments.options, {
+        ...parseArguments.context,
+        ...Array.isArray(loaders) || loaders && isLoaderObject(loaders) ? {
+          _loaderIsArray: Array.isArray(loaders),
+          _loaderIds: (Array.isArray(loaders) ? loaders : [loaders]).map((loader) => loader.id)
+        } : {}
+      });
+    };
+  }
+  function getMainThreadParseArguments(loaders, options, context) {
+    if (options) {
+      return { options, context: getSerializableLoaderContext(context) };
+    }
+    if (Array.isArray(loaders) || loaders && isLoaderObject(loaders)) {
+      return { options: void 0, context: getSerializableLoaderContext(context) };
+    }
+    if (loaders && !Array.isArray(loaders)) {
+      return { options: loaders };
+    }
+    return { options: void 0, context: getSerializableLoaderContext(context) };
+  }
+  function isLoaderObject(value) {
+    return "id" in value && "extensions" in value;
+  }
+  function getSerializableLoaderContext(context) {
+    if (!context) {
+      return void 0;
+    }
+    const { fetch: fetch2, loaders, coreApi, _parse, _parseSync, _parseInBatches, ...serializableContext } = context;
+    return JSON.parse(JSON.stringify(serializableContext));
+  }
   async function parseData({
     loader,
     arrayBuffer,
@@ -182,9 +413,9 @@
   }) {
     let data;
     let parser;
-    if (loader.parseSync || loader.parse) {
+    if (loader.parse || loader.parseSync) {
       data = arrayBuffer;
-      parser = loader.parseSync || loader.parse;
+      parser = loader.parse || loader.parseSync;
     } else if (loader.parseTextSync) {
       const textDecoder = new TextDecoder();
       data = textDecoder.decode(arrayBuffer);
@@ -192,14 +423,11 @@
     } else {
       throw new Error(`Could not load data with ${loader.name} loader`);
     }
-    const workerModules = Object.fromEntries(
-      Object.entries(options.modules || {}).filter(([, value]) => typeof value === "string")
-    );
     options = {
       ...options,
       modules: {
         ...loader && loader.options && loader.options.modules || {},
-        ...workerModules
+        ...options.modules || {}
       },
       core: {
         ...options.core,
@@ -210,7 +438,7 @@
   }
 
   // src/null-loader.ts
-  var VERSION = true ? "4.5.3" : "latest";
+  var VERSION = true ? "5.0.0-alpha.10" : "latest";
   var NullLoader = {
     dataType: null,
     batchType: null,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@loaders.gl/core",
-  "version": "4.5.3",
+  "version": "5.0.0-alpha.10",
   "description": "The core API for working with loaders.gl loaders and writers",
   "license": "MIT",
   "type": "module",
@@ -58,11 +58,11 @@
     "build-worker-node": "esbuild src/workers/null-worker.ts --outfile=dist/null-worker-node.js --bundle --platform=node --target=node16 --define:__VERSION__=\\\"$npm_package_version\\\""
   },
   "dependencies": {
-    "@loaders.gl/loader-utils": "4.5.3",
-    "@loaders.gl/schema": "4.5.3",
-    "@loaders.gl/schema-utils": "4.5.3",
-    "@loaders.gl/worker-utils": "4.5.3",
+    "@loaders.gl/loader-utils": "5.0.0-alpha.10",
+    "@loaders.gl/schema": "5.0.0-alpha.10",
+    "@loaders.gl/schema-utils": "5.0.0-alpha.10",
+    "@loaders.gl/worker-utils": "5.0.0-alpha.10",
     "@probe.gl/log": "^4.1.1"
   },
-  "gitHead": "3ad4ab002b4a53ac5d88feb856117a8a1e8c5bb0"
+  "gitHead": "7cbfaef633e86356c946541dab4977b522745b6c"
 }

package/src/index.ts

@@ -7,6 +7,7 @@ export type {
   Loader,
   LoaderWithParser,
   LoaderContext,
+  CoreAPI,
   LoaderOptions,
   Writer,
   WriterOptions,
@@ -19,6 +20,7 @@ export type {
   FileSystem,
   RandomAccessFileSystem
 } from '@loaders.gl/loader-utils';
+export type {SourceLoader} from '@loaders.gl/loader-utils';
 
 // FILE READING AND WRITING
 export {fetchFile} from './lib/fetch/fetch-file';
@@ -35,8 +37,11 @@ export {selectLoader, selectLoaderSync} from './lib/api/select-loader';
 
 // LOADING (READING + PARSING)
 export {parse} from './lib/api/parse';
+export {parseFile} from './lib/api/parse-file';
 export {parseSync} from './lib/api/parse-sync';
 export {parseInBatches} from './lib/api/parse-in-batches';
+export {preload, preloadSync} from './lib/api/preload';
+export {coreApi} from './lib/api/core-api';
 
 export {load} from './lib/api/load';
 export {loadInBatches} from './lib/api/load-in-batches';
@@ -59,7 +64,7 @@ export {makeIterator} from './iterators/make-iterator/make-iterator';
 export {makeStream} from './iterators/make-stream/make-stream';
 
 // CORE LOADERS
-export {NullWorkerLoader, NullLoader} from './null-loader';
+export {NullLoader} from './null-loader';
 export {JSONLoader} from '@loaders.gl/loader-utils';
 
 // EXPERIMENTAL
@@ -95,3 +100,7 @@ export {
   isReadableStream,
   isWritableStream
 } from '@loaders.gl/loader-utils';
+
+// DEPRECATED EXPORTS
+/** @deprecated Use NullLoader. */
+export {NullWorkerLoader} from './null-loader';

package/src/iterators/make-iterator/make-stream-iterator.ts

@@ -43,30 +43,68 @@ async function* makeBrowserStreamIterator(
   const reader = stream.getReader();
 
   let nextBatchPromise: Promise<{done?: boolean; value?: Uint8Array}> | undefined;
+  let streamFinished = false;
+  let streamReadFailed = false;
+  let iteratorFailed = false;
 
   try {
     // eslint-disable-next-line no-constant-condition
     while (true) {
       const currentBatchPromise = nextBatchPromise || reader.read();
-      // Issue a read for an additional batch, while we await the next batch
-      // Idea is to make fetching happen in parallel with processing / parsing
-      if (options?._streamReadAhead) {
-        nextBatchPromise = reader.read();
-      }
+      nextBatchPromise = undefined;
       // Read from the stream
       // value is a Uint8Array
-      const {done, value} = await currentBatchPromise;
+      let batch;
+      try {
+        batch = await currentBatchPromise;
+      } catch (error) {
+        streamReadFailed = true;
+        throw error;
+      }
+      const {done, value} = batch;
       // Exit if we're done
       if (done) {
+        streamFinished = true;
         return;
       }
+      // Issue a read for an additional batch while the current batch is processed
+      if (options?._streamReadAhead) {
+        nextBatchPromise = reader.read();
+        // Observe a prefetched rejection even if the consumer stops before requesting it
+        void nextBatchPromise.catch(() => {});
+      }
       // Else yield the chunk
       yield toArrayBuffer(value);
     }
   } catch (error) {
-    // TODO - examples makes it look like this should always be called,
-    // but that generates exceptions so only call it if we do not reach the end
-    reader.releaseLock();
+    iteratorFailed = true;
+    throw error;
+  } finally {
+    let cleanupFailed = false;
+    let cleanupError: unknown;
+
+    if (!streamFinished && !streamReadFailed) {
+      try {
+        await reader.cancel();
+      } catch (error) {
+        cleanupFailed = true;
+        cleanupError = error;
+      }
+    }
+
+    try {
+      reader.releaseLock();
+    } catch (error) {
+      if (!cleanupFailed) {
+        cleanupFailed = true;
+        cleanupError = error;
+      }
+    }
+
+    // Preserve errors from iteration instead of replacing them with cleanup errors
+    if (!iteratorFailed && cleanupFailed) {
+      throw cleanupError;
+    }
   }
 }
 

package/src/lib/api/core-api.ts

@@ -0,0 +1,25 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {CoreAPI} from '@loaders.gl/loader-utils';
+import {fetchFile} from '../fetch/fetch-file';
+import {parse} from './parse';
+import {parseFile} from './parse-file';
+import {parseSync} from './parse-sync';
+import {parseInBatches} from './parse-in-batches';
+import {load} from './load';
+import {loadInBatches} from './load-in-batches';
+
+/**
+ * Shared Core API surface that can be injected into contexts and sources.
+ */
+export const coreApi: CoreAPI = Object.freeze({
+  fetchFile,
+  parse,
+  parseFile,
+  parseSync,
+  parseInBatches,
+  load,
+  loadInBatches
+});

package/src/lib/api/create-data-source.ts

@@ -2,7 +2,14 @@
 // SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
-import {Source, SourceArrayOptionsType, SourceArrayDataSourceType} from '@loaders.gl/loader-utils';
+import {
+  SourceLoader,
+  SourceArrayOptionsType,
+  SourceArrayDataSourceType
+} from '@loaders.gl/loader-utils';
+import {resolveAuthenticationOptions} from '@loaders.gl/loader-utils';
+import {coreApi} from './core-api';
+import {selectLoaderSync} from './select-loader';
 
 /**
  * Creates a source from a service
@@ -13,39 +20,61 @@ import {Source, SourceArrayOptionsType, SourceArrayDataSourceType} from '@loader
  */
 // DataSourceOptionsT extends DataSourceOptions = DataSourceOptions,
 // DataSourceT extends DataSource = DataSource
-export function createDataSource<SourceArrayT extends Source[]>(
-  data: string | Blob,
+export function createDataSource<SourceArrayT extends SourceLoader[]>(
+  data: unknown,
   sources: Readonly<SourceArrayT>,
-  options: Readonly<SourceArrayOptionsType<SourceArrayT>>
+  options: Readonly<
+    SourceArrayOptionsType<SourceArrayT>
+  > = {} as SourceArrayOptionsType<SourceArrayT>
 ): SourceArrayDataSourceType<SourceArrayT> {
-  const type = options?.core?.type || (options.type as unknown as string) || 'auto';
-  const source = type === 'auto' ? selectSource(data, sources) : getSourceOfType(type, sources);
+  const resolvedOptions = (options || {}) as SourceArrayOptionsType<SourceArrayT>;
+  const type = resolvedOptions?.core?.type || (resolvedOptions.type as unknown as string) || 'auto';
+  const source =
+    sources.length === 1
+      ? sources[0]
+      : type === 'auto'
+        ? (selectLoaderSync(data as never, sources as unknown as SourceLoader[], {
+            // `createDataSource` supports non-fetch inputs such as in-memory tables.
+            // Selection remains best-effort for those cases.
+            ...resolvedOptions,
+            core: {...resolvedOptions?.core, type: 'auto'}
+          }) as SourceArrayT[number] | null)
+        : getSourceOfType(type, sources);
 
   if (!source) {
     throw new Error('Not a valid source type');
   }
-  return source.createDataSource(data, options);
-}
-
-// TODO - use selectSource...
-
-/** Guess service type from URL */
-function selectSource<SourceArrayT extends Source[]>(
-  url: string | Blob,
-  sources: Readonly<SourceArrayT>
-): SourceArrayT[number] | null {
-  for (const service of sources) {
-    // @ts-expect-error
-    if (service.testURL && service.testURL(url)) {
-      return service;
-    }
+  const contributions = resolvedOptions.core?.credentials?.length
+    ? source.getAuthentications?.(typeof data === 'string' ? data : '', resolvedOptions)
+    : undefined;
+  if (contributions && 'then' in contributions) {
+    // Avoid an unhandled rejection from a hook that cannot be awaited by this sync API.
+    void contributions.catch(() => {});
+    throw new Error('Use load() with an asynchronous getAuthentications() hook.');
   }
-
-  return null;
+  const authenticatedOptions = resolveAuthenticationOptions(
+    contributions?.length
+      ? {
+          ...resolvedOptions,
+          core: {
+            ...resolvedOptions.core,
+            authentications: [
+              ...(resolvedOptions.core?.authentications || []),
+              ...(contributions || [])
+            ]
+          }
+        }
+      : resolvedOptions
+  );
+  return source.createDataSource(
+    data as string | Blob,
+    authenticatedOptions,
+    coreApi
+  ) as SourceArrayDataSourceType<SourceArrayT>;
 }
 
 /** Guess service type from URL */
-function getSourceOfType(type: string, sources: Readonly<Source[]>): Source | null {
+function getSourceOfType(type: string, sources: Readonly<SourceLoader[]>): SourceLoader | null {
   for (const service of sources) {
     if (service.type === type) {
       return service;