@loaders.gl/core 4.4.2 → 5.0.0-alpha.1

package/dist/null-worker-node.js

@@ -153,38 +153,137 @@ function isKnownMessage(message) {
   return type === "message" && data && typeof data.source === "string" && data.source.startsWith("loaders.gl");
 }
 
-// ../loader-utils/src/lib/worker-loader-utils/create-loader-worker.ts
+// ../worker-utils/src/lib/async-queue/async-queue.ts
+var AsyncQueue = class {
+  _values;
+  _settlers;
+  _closed;
+  constructor() {
+    this._values = [];
+    this._settlers = [];
+    this._closed = false;
+  }
+  /** Return an async iterator for this queue */
+  [Symbol.asyncIterator]() {
+    return this;
+  }
+  /** Push a new value - the async iterator will yield a promise resolved to this value */
+  push(value) {
+    return this.enqueue(value);
+  }
+  /**
+   * Push a new value - the async iterator will yield a promise resolved to this value
+   * Add an error - the async iterator will yield a promise rejected with this value
+   */
+  enqueue(value) {
+    if (this._closed) {
+      throw new Error("Closed");
+    }
+    if (this._settlers.length > 0) {
+      if (this._values.length > 0) {
+        throw new Error("Illegal internal state");
+      }
+      const settler = this._settlers.shift();
+      if (value instanceof Error) {
+        settler.reject(value);
+      } else {
+        settler.resolve({ value });
+      }
+    } else {
+      this._values.push(value);
+    }
+  }
+  /** Indicate that we not waiting for more values - The async iterator will be done */
+  close() {
+    while (this._settlers.length > 0) {
+      const settler = this._settlers.shift();
+      settler.resolve({ done: true });
+    }
+    this._closed = true;
+  }
+  // ITERATOR IMPLEMENTATION
+  /** @returns a Promise for an IteratorResult */
+  next() {
+    if (this._values.length > 0) {
+      const value = this._values.shift();
+      if (value instanceof Error) {
+        return Promise.reject(value);
+      }
+      return Promise.resolve({ done: false, value });
+    }
+    if (this._closed) {
+      if (this._settlers.length > 0) {
+        throw new Error("Illegal internal state");
+      }
+      return Promise.resolve({ done: true, value: void 0 });
+    }
+    return new Promise((resolve, reject) => {
+      this._settlers.push({ resolve, reject });
+    });
+  }
+};
+
+// ../worker-utils/src/lib/worker-api/create-worker.ts
 var requestId = 0;
-async function createLoaderWorker(loader) {
+var inputBatches;
+var options;
+async function createWorker(process, processInBatches) {
   if (!await WorkerBody.inWorkerThread()) {
     return;
   }
+  const context = {
+    process: processOnMainThread
+  };
   WorkerBody.onmessage = async (type, payload) => {
-    switch (type) {
-      case "process":
-        try {
-          const { input, options = {}, context = {} } = payload;
-          const result = await parseData({
-            loader,
-            arrayBuffer: input,
-            options,
-            // @ts-expect-error fetch missing
-            context: {
-              ...context,
-              _parse: parseOnMainThread
-            }
-          });
+    try {
+      switch (type) {
+        case "preload":
+          WorkerBody.postMessage("done", {});
+          break;
+        case "process":
+          if (!process) {
+            throw new Error("Worker does not support atomic processing");
+          }
+          const result = await process(
+            payload.input,
+            payload.options || {},
+            context,
+            payload.context || {}
+          );
           WorkerBody.postMessage("done", { result });
-        } catch (error) {
-          const message = error instanceof Error ? error.message : "";
-          WorkerBody.postMessage("error", { error: message });
-        }
-        break;
-      default:
+          break;
+        case "process-in-batches":
+          if (!processInBatches) {
+            throw new Error("Worker does not support batched processing");
+          }
+          inputBatches = new AsyncQueue();
+          options = payload.options || {};
+          const resultIterator = processInBatches(
+            inputBatches,
+            options,
+            context,
+            payload.context || {}
+          );
+          for await (const batch of resultIterator) {
+            WorkerBody.postMessage("output-batch", { result: batch });
+          }
+          WorkerBody.postMessage("done", {});
+          break;
+        case "input-batch":
+          inputBatches.push(payload.input);
+          break;
+        case "input-done":
+          inputBatches.close();
+          break;
+        default:
+      }
+    } catch (error) {
+      const message = error instanceof Error ? error.message : "";
+      WorkerBody.postMessage("error", { error: message });
     }
   };
 }
-function parseOnMainThread(arrayBuffer, loader, options, context) {
+function processOnMainThread(arrayBuffer, options2 = {}, jobContext = {}) {
   return new Promise((resolve, reject) => {
     const id = requestId++;
     const onMessage = (type, payload2) => {
@@ -204,21 +303,85 @@ function parseOnMainThread(arrayBuffer, loader, options, context) {
       }
     };
     WorkerBody.addEventListener(onMessage);
-    const payload = { id, input: arrayBuffer, options };
+    const payload = { id, input: arrayBuffer, options: options2, context: jobContext };
     WorkerBody.postMessage("process", payload);
   });
 }
+
+// ../loader-utils/src/lib/worker-loader-utils/create-loader-worker.ts
+async function createLoaderWorker(loader) {
+  await createWorker(
+    async (input, options2 = {}, workerContext, loaderContext = {}) => {
+      const result = await parseData({
+        loader,
+        arrayBuffer: input,
+        options: options2,
+        context: {
+          ...loaderContext,
+          coreApi: createWorkerCoreApi(),
+          _parse: createParseOnMainThread(workerContext == null ? void 0 : workerContext.process)
+        }
+      });
+      return loader.serializeWorkerResult ? loader.serializeWorkerResult(result, options2, loaderContext) : result;
+    }
+  );
+}
+function createWorkerCoreApi() {
+  const unavailable = (methodName) => () => {
+    throw new Error(`context.coreApi.${methodName} is unavailable inside worker loaders.`);
+  };
+  return {
+    fetchFile: async (urlOrData, fetchOptions) => await fetch(urlOrData, fetchOptions),
+    parseSync: unavailable("parseSync"),
+    parse: unavailable("parse"),
+    parseFile: unavailable("parseFile"),
+    parseInBatches: unavailable("parseInBatches"),
+    load: unavailable("load"),
+    loadInBatches: unavailable("loadInBatches")
+  };
+}
+function createParseOnMainThread(processOnMainThread2) {
+  return (arrayBuffer, loaders, options2, context) => {
+    if (!processOnMainThread2) {
+      throw new Error("Worker not set up to parse on main thread");
+    }
+    const parseArguments = getMainThreadParseArguments(loaders, options2, context);
+    return processOnMainThread2(arrayBuffer, parseArguments.options, parseArguments.context);
+  };
+}
+function getMainThreadParseArguments(loaders, options2, context) {
+  if (options2) {
+    return { options: options2, context: getSerializableLoaderContext(context) };
+  }
+  if (Array.isArray(loaders) || loaders && isLoaderObject(loaders)) {
+    return { options: void 0, context: getSerializableLoaderContext(context) };
+  }
+  if (loaders && !Array.isArray(loaders)) {
+    return { options: loaders };
+  }
+  return { options: void 0, context: getSerializableLoaderContext(context) };
+}
+function isLoaderObject(value) {
+  return "id" in value && "extensions" in value;
+}
+function getSerializableLoaderContext(context) {
+  if (!context) {
+    return void 0;
+  }
+  const { fetch: fetch2, loaders, coreApi, _parse, _parseSync, _parseInBatches, ...serializableContext } = context;
+  return JSON.parse(JSON.stringify(serializableContext));
+}
 async function parseData({
   loader,
   arrayBuffer,
-  options,
+  options: options2,
   context
 }) {
   let data;
   let parser;
-  if (loader.parseSync || loader.parse) {
+  if (loader.parse || loader.parseSync) {
     data = arrayBuffer;
-    parser = loader.parseSync || loader.parse;
+    parser = loader.parse || loader.parseSync;
   } else if (loader.parseTextSync) {
     const textDecoder = new TextDecoder();
     data = textDecoder.decode(arrayBuffer);
@@ -226,19 +389,19 @@ async function parseData({
   } else {
     throw new Error(`Could not load data with ${loader.name} loader`);
   }
-  options = {
-    ...options,
+  options2 = {
+    ...options2,
     modules: loader && loader.options && loader.options.modules || {},
     core: {
-      ...options.core,
+      ...options2.core,
       worker: false
     }
   };
-  return await parser(data, { ...options }, context, loader);
+  return await parser(data, { ...options2 }, context, loader);
 }
 
 // src/null-loader.ts
-var VERSION = true ? "4.4.2" : "latest";
+var VERSION = true ? "5.0.0-alpha.1" : "latest";
 var NullLoader = {
   dataType: null,
   batchType: null,
@@ -248,11 +411,11 @@ var NullLoader = {
   version: VERSION,
   mimeTypes: ["application/x.empty"],
   extensions: ["null"],
-  parse: async (arrayBuffer, options, context) => parseSync(arrayBuffer, options || {}, context),
+  parse: async (arrayBuffer, options2, context) => parseSync(arrayBuffer, options2 || {}, context),
   parseSync,
-  parseInBatches: async function* generator(asyncIterator, options, context) {
+  parseInBatches: async function* generator(asyncIterator, options2, context) {
     for await (const batch of asyncIterator) {
-      yield parseSync(batch, options, context);
+      yield parseSync(batch, options2, context);
     }
   },
   tests: [() => false],
@@ -260,7 +423,7 @@ var NullLoader = {
     null: {}
   }
 };
-function parseSync(arrayBuffer, options, context) {
+function parseSync(arrayBuffer, options2, context) {
   return null;
 }
 

package/dist/null-worker.js

@@ -118,38 +118,137 @@
     return type === "message" && data && typeof data.source === "string" && data.source.startsWith("loaders.gl");
   }
 
-  // ../loader-utils/src/lib/worker-loader-utils/create-loader-worker.ts
+  // ../worker-utils/src/lib/async-queue/async-queue.ts
+  var AsyncQueue = class {
+    _values;
+    _settlers;
+    _closed;
+    constructor() {
+      this._values = [];
+      this._settlers = [];
+      this._closed = false;
+    }
+    /** Return an async iterator for this queue */
+    [Symbol.asyncIterator]() {
+      return this;
+    }
+    /** Push a new value - the async iterator will yield a promise resolved to this value */
+    push(value) {
+      return this.enqueue(value);
+    }
+    /**
+     * Push a new value - the async iterator will yield a promise resolved to this value
+     * Add an error - the async iterator will yield a promise rejected with this value
+     */
+    enqueue(value) {
+      if (this._closed) {
+        throw new Error("Closed");
+      }
+      if (this._settlers.length > 0) {
+        if (this._values.length > 0) {
+          throw new Error("Illegal internal state");
+        }
+        const settler = this._settlers.shift();
+        if (value instanceof Error) {
+          settler.reject(value);
+        } else {
+          settler.resolve({ value });
+        }
+      } else {
+        this._values.push(value);
+      }
+    }
+    /** Indicate that we not waiting for more values - The async iterator will be done */
+    close() {
+      while (this._settlers.length > 0) {
+        const settler = this._settlers.shift();
+        settler.resolve({ done: true });
+      }
+      this._closed = true;
+    }
+    // ITERATOR IMPLEMENTATION
+    /** @returns a Promise for an IteratorResult */
+    next() {
+      if (this._values.length > 0) {
+        const value = this._values.shift();
+        if (value instanceof Error) {
+          return Promise.reject(value);
+        }
+        return Promise.resolve({ done: false, value });
+      }
+      if (this._closed) {
+        if (this._settlers.length > 0) {
+          throw new Error("Illegal internal state");
+        }
+        return Promise.resolve({ done: true, value: void 0 });
+      }
+      return new Promise((resolve, reject) => {
+        this._settlers.push({ resolve, reject });
+      });
+    }
+  };
+
+  // ../worker-utils/src/lib/worker-api/create-worker.ts
   var requestId = 0;
-  async function createLoaderWorker(loader) {
+  var inputBatches;
+  var options;
+  async function createWorker(process, processInBatches) {
     if (!await WorkerBody.inWorkerThread()) {
       return;
     }
+    const context = {
+      process: processOnMainThread
+    };
     WorkerBody.onmessage = async (type, payload) => {
-      switch (type) {
-        case "process":
-          try {
-            const { input, options = {}, context = {} } = payload;
-            const result = await parseData({
-              loader,
-              arrayBuffer: input,
-              options,
-              // @ts-expect-error fetch missing
-              context: {
-                ...context,
-                _parse: parseOnMainThread
-              }
-            });
+      try {
+        switch (type) {
+          case "preload":
+            WorkerBody.postMessage("done", {});
+            break;
+          case "process":
+            if (!process) {
+              throw new Error("Worker does not support atomic processing");
+            }
+            const result = await process(
+              payload.input,
+              payload.options || {},
+              context,
+              payload.context || {}
+            );
             WorkerBody.postMessage("done", { result });
-          } catch (error) {
-            const message = error instanceof Error ? error.message : "";
-            WorkerBody.postMessage("error", { error: message });
-          }
-          break;
-        default:
+            break;
+          case "process-in-batches":
+            if (!processInBatches) {
+              throw new Error("Worker does not support batched processing");
+            }
+            inputBatches = new AsyncQueue();
+            options = payload.options || {};
+            const resultIterator = processInBatches(
+              inputBatches,
+              options,
+              context,
+              payload.context || {}
+            );
+            for await (const batch of resultIterator) {
+              WorkerBody.postMessage("output-batch", { result: batch });
+            }
+            WorkerBody.postMessage("done", {});
+            break;
+          case "input-batch":
+            inputBatches.push(payload.input);
+            break;
+          case "input-done":
+            inputBatches.close();
+            break;
+          default:
+        }
+      } catch (error) {
+        const message = error instanceof Error ? error.message : "";
+        WorkerBody.postMessage("error", { error: message });
       }
     };
   }
-  function parseOnMainThread(arrayBuffer, loader, options, context) {
+  function processOnMainThread(arrayBuffer, options2 = {}, jobContext = {}) {
     return new Promise((resolve, reject) => {
       const id = requestId++;
       const onMessage = (type, payload2) => {
@@ -169,21 +268,85 @@
         }
       };
       WorkerBody.addEventListener(onMessage);
-      const payload = { id, input: arrayBuffer, options };
+      const payload = { id, input: arrayBuffer, options: options2, context: jobContext };
       WorkerBody.postMessage("process", payload);
     });
   }
+
+  // ../loader-utils/src/lib/worker-loader-utils/create-loader-worker.ts
+  async function createLoaderWorker(loader) {
+    await createWorker(
+      async (input, options2 = {}, workerContext, loaderContext = {}) => {
+        const result = await parseData({
+          loader,
+          arrayBuffer: input,
+          options: options2,
+          context: {
+            ...loaderContext,
+            coreApi: createWorkerCoreApi(),
+            _parse: createParseOnMainThread(workerContext?.process)
+          }
+        });
+        return loader.serializeWorkerResult ? loader.serializeWorkerResult(result, options2, loaderContext) : result;
+      }
+    );
+  }
+  function createWorkerCoreApi() {
+    const unavailable = (methodName) => () => {
+      throw new Error(`context.coreApi.${methodName} is unavailable inside worker loaders.`);
+    };
+    return {
+      fetchFile: async (urlOrData, fetchOptions) => await fetch(urlOrData, fetchOptions),
+      parseSync: unavailable("parseSync"),
+      parse: unavailable("parse"),
+      parseFile: unavailable("parseFile"),
+      parseInBatches: unavailable("parseInBatches"),
+      load: unavailable("load"),
+      loadInBatches: unavailable("loadInBatches")
+    };
+  }
+  function createParseOnMainThread(processOnMainThread2) {
+    return (arrayBuffer, loaders, options2, context) => {
+      if (!processOnMainThread2) {
+        throw new Error("Worker not set up to parse on main thread");
+      }
+      const parseArguments = getMainThreadParseArguments(loaders, options2, context);
+      return processOnMainThread2(arrayBuffer, parseArguments.options, parseArguments.context);
+    };
+  }
+  function getMainThreadParseArguments(loaders, options2, context) {
+    if (options2) {
+      return { options: options2, context: getSerializableLoaderContext(context) };
+    }
+    if (Array.isArray(loaders) || loaders && isLoaderObject(loaders)) {
+      return { options: void 0, context: getSerializableLoaderContext(context) };
+    }
+    if (loaders && !Array.isArray(loaders)) {
+      return { options: loaders };
+    }
+    return { options: void 0, context: getSerializableLoaderContext(context) };
+  }
+  function isLoaderObject(value) {
+    return "id" in value && "extensions" in value;
+  }
+  function getSerializableLoaderContext(context) {
+    if (!context) {
+      return void 0;
+    }
+    const { fetch: fetch2, loaders, coreApi, _parse, _parseSync, _parseInBatches, ...serializableContext } = context;
+    return JSON.parse(JSON.stringify(serializableContext));
+  }
   async function parseData({
     loader,
     arrayBuffer,
-    options,
+    options: options2,
     context
   }) {
     let data;
     let parser;
-    if (loader.parseSync || loader.parse) {
+    if (loader.parse || loader.parseSync) {
       data = arrayBuffer;
-      parser = loader.parseSync || loader.parse;
+      parser = loader.parse || loader.parseSync;
     } else if (loader.parseTextSync) {
       const textDecoder = new TextDecoder();
       data = textDecoder.decode(arrayBuffer);
@@ -191,19 +354,19 @@
     } else {
       throw new Error(`Could not load data with ${loader.name} loader`);
     }
-    options = {
-      ...options,
+    options2 = {
+      ...options2,
       modules: loader && loader.options && loader.options.modules || {},
       core: {
-        ...options.core,
+        ...options2.core,
         worker: false
       }
     };
-    return await parser(data, { ...options }, context, loader);
+    return await parser(data, { ...options2 }, context, loader);
   }
 
   // src/null-loader.ts
-  var VERSION = true ? "4.4.2" : "latest";
+  var VERSION = true ? "5.0.0-alpha.1" : "latest";
   var NullLoader = {
     dataType: null,
     batchType: null,
@@ -213,11 +376,11 @@
     version: VERSION,
     mimeTypes: ["application/x.empty"],
     extensions: ["null"],
-    parse: async (arrayBuffer, options, context) => parseSync(arrayBuffer, options || {}, context),
+    parse: async (arrayBuffer, options2, context) => parseSync(arrayBuffer, options2 || {}, context),
     parseSync,
-    parseInBatches: async function* generator(asyncIterator, options, context) {
+    parseInBatches: async function* generator(asyncIterator, options2, context) {
       for await (const batch of asyncIterator) {
-        yield parseSync(batch, options, context);
+        yield parseSync(batch, options2, context);
       }
     },
     tests: [() => false],
@@ -225,7 +388,7 @@
       null: {}
     }
   };
-  function parseSync(arrayBuffer, options, context) {
+  function parseSync(arrayBuffer, options2, context) {
     return null;
   }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@loaders.gl/core",
-  "version": "4.4.2",
+  "version": "5.0.0-alpha.1",
   "description": "The core API for working with loaders.gl loaders and writers",
   "license": "MIT",
   "type": "module",
@@ -58,11 +58,11 @@
     "build-worker-node": "esbuild src/workers/null-worker.ts --outfile=dist/null-worker-node.js --bundle --platform=node --target=node16 --define:__VERSION__=\\\"$npm_package_version\\\""
   },
   "dependencies": {
-    "@loaders.gl/loader-utils": "4.4.2",
-    "@loaders.gl/schema": "4.4.2",
-    "@loaders.gl/schema-utils": "4.4.2",
-    "@loaders.gl/worker-utils": "4.4.2",
+    "@loaders.gl/loader-utils": "5.0.0-alpha.1",
+    "@loaders.gl/schema": "5.0.0-alpha.1",
+    "@loaders.gl/schema-utils": "5.0.0-alpha.1",
+    "@loaders.gl/worker-utils": "5.0.0-alpha.1",
     "@probe.gl/log": "^4.1.1"
   },
-  "gitHead": "1c9649781db0403dbf7c3f1224bbeec77bd812eb"
+  "gitHead": "77a55028e8204e09df1258f10cb61ed35353b258"
 }

package/src/index.ts

@@ -7,6 +7,7 @@ export type {
   Loader,
   LoaderWithParser,
   LoaderContext,
+  CoreAPI,
   LoaderOptions,
   Writer,
   WriterOptions,
@@ -19,6 +20,7 @@ export type {
   FileSystem,
   RandomAccessFileSystem
 } from '@loaders.gl/loader-utils';
+export type {SourceLoader} from '@loaders.gl/loader-utils';
 
 // FILE READING AND WRITING
 export {fetchFile} from './lib/fetch/fetch-file';
@@ -35,8 +37,11 @@ export {selectLoader, selectLoaderSync} from './lib/api/select-loader';
 
 // LOADING (READING + PARSING)
 export {parse} from './lib/api/parse';
+export {parseFile} from './lib/api/parse-file';
 export {parseSync} from './lib/api/parse-sync';
 export {parseInBatches} from './lib/api/parse-in-batches';
+export {preload, preloadSync} from './lib/api/preload';
+export {coreApi} from './lib/api/core-api';
 
 export {load} from './lib/api/load';
 export {loadInBatches} from './lib/api/load-in-batches';
@@ -59,7 +64,7 @@ export {makeIterator} from './iterators/make-iterator/make-iterator';
 export {makeStream} from './iterators/make-stream/make-stream';
 
 // CORE LOADERS
-export {NullWorkerLoader, NullLoader} from './null-loader';
+export {NullLoader} from './null-loader';
 export {JSONLoader} from '@loaders.gl/loader-utils';
 
 // EXPERIMENTAL
@@ -95,3 +100,7 @@ export {
   isReadableStream,
   isWritableStream
 } from '@loaders.gl/loader-utils';
+
+// DEPRECATED EXPORTS
+/** @deprecated Use NullLoader. */
+export {NullWorkerLoader} from './null-loader';

package/src/iterators/make-iterator/make-stream-iterator.ts

@@ -63,7 +63,7 @@ async function* makeBrowserStreamIterator(
       // Else yield the chunk
       yield toArrayBuffer(value);
     }
-  } catch (error) {
+  } catch (_error) {
     // TODO - examples makes it look like this should always be called,
     // but that generates exceptions so only call it if we do not reach the end
     reader.releaseLock();