@loaders.gl/core 5.0.0-alpha.4 → 5.0.0-alpha.6

package/dist/dist.dev.js

@@ -925,6 +925,10 @@ var __exports__ = (() => {
     name;
     source;
     url;
+    /** Lazily resolves the classic worker URL used as a fallback. */
+    getUrl;
+    /** Creates a browser Worker directly. */
+    loadWorker;
     terminated = false;
     worker;
     onMessage;
@@ -935,11 +939,13 @@ var __exports__ = (() => {
       return typeof Worker !== "undefined" && isBrowser3 || typeof NodeWorker !== "undefined" && !isBrowser3;
     }
     constructor(props) {
-      const { name, source, url } = props;
-      assert3(source || url);
+      const { name, source, url, getUrl, loadWorker } = props;
+      assert3(source || url || getUrl || loadWorker);
       this.name = name;
       this.source = source;
       this.url = url;
+      this.getUrl = getUrl;
+      this.loadWorker = loadWorker;
       this.onMessage = NOOP;
       this.onError = (error) => console.log(error);
       this.worker = isBrowser3 ? this._createBrowserWorker() : this._createNodeWorker();
@@ -985,7 +991,7 @@ var __exports__ = (() => {
      */
     _getErrorFromErrorEvent(event) {
       let message = "Failed to load ";
-      message += `worker ${this.name} from ${this.url}. `;
+      message += `worker ${this.name} from ${this.url || "built-in worker factory"}. `;
       if (event.message) {
         message += `${event.message} in `;
       }
@@ -998,8 +1004,25 @@ var __exports__ = (() => {
      * Creates a worker thread on the browser
      */
     _createBrowserWorker() {
-      this._loadableURL = getLoadableWorkerURL({ source: this.source, url: this.url });
+      if (this.loadWorker) {
+        try {
+          const worker2 = this.loadWorker();
+          if (worker2) {
+            return this._initializeBrowserWorker(worker2);
+          }
+        } catch (error) {
+          if (!this.source && !this.url && !this.getUrl) {
+            throw error;
+          }
+        }
+      }
+      const url = this.url || this.getUrl?.();
+      this._loadableURL = getLoadableWorkerURL({ source: this.source, url });
       const worker = new Worker(this._loadableURL, { name: this.name });
+      return this._initializeBrowserWorker(worker);
+    }
+    /** Attaches protocol handlers to a browser worker. */
+    _initializeBrowserWorker(worker) {
       worker.onmessage = (event) => {
         if (!event.data) {
           this.onError(new Error("No data received"));
@@ -1049,6 +1072,10 @@ var __exports__ = (() => {
     source;
     // | Function;
     url;
+    /** Lazily resolves a classic worker fallback URL. */
+    getUrl;
+    /** Creates a browser Worker directly. */
+    loadWorker;
     maxConcurrency = 1;
     maxMobileConcurrency = 1;
     onDebug = () => {
@@ -1070,6 +1097,8 @@ var __exports__ = (() => {
     constructor(props) {
       this.source = props.source;
       this.url = props.url;
+      this.getUrl = props.getUrl;
+      this.loadWorker = props.loadWorker;
       this.setProps(props);
     }
     /**
@@ -1179,7 +1208,13 @@ var __exports__ = (() => {
       if (this.count < this._getMaxConcurrency()) {
         this.count++;
         const name = `${this.name.toLowerCase()} (#${this.count} of ${this.maxConcurrency})`;
-        return new WorkerThread({ name, source: this.source, url: this.url });
+        return new WorkerThread({
+          name,
+          source: this.source,
+          url: this.url,
+          getUrl: this.getUrl,
+          loadWorker: this.loadWorker
+        });
       }
       return null;
     }
@@ -1196,6 +1231,8 @@ var __exports__ = (() => {
     onDebug: () => {
     }
   };
+  var loadWorkerIds = /* @__PURE__ */ new WeakMap();
+  var nextLoadWorkerId = 1;
   var WorkerFarm = class _WorkerFarm {
     props;
     workerPools = /* @__PURE__ */ new Map();
@@ -1239,24 +1276,28 @@ var __exports__ = (() => {
     }
     /**
      * Returns a worker pool for the specified worker
-     * @param options - only used first time for a specific worker name
-     * @param options.name - the name of the worker - used to identify worker pool
-     * @param options.url -
-     * @param options.source -
+     * @param options Worker target used to identify and initialize the pool.
+     * @param options.name Worker name used as the base pool identity.
+     * @param options.url Worker script URL.
+     * @param options.source Inline worker source.
+     * @param options.loadWorker Built-in browser worker factory.
      * @example
      *   const job = WorkerFarm.getWorkerFarm().getWorkerPool({name, url}).startJob(...);
      */
     getWorkerPool(options) {
-      const { name, source, url } = options;
-      let workerPool = this.workerPools.get(name);
+      const { name, source, url, getUrl, loadWorker } = options;
+      const workerPoolKey = getWorkerPoolKey(options);
+      let workerPool = this.workerPools.get(workerPoolKey);
       if (!workerPool) {
         workerPool = new WorkerPool({
           name,
           source,
-          url
+          url,
+          getUrl,
+          loadWorker
         });
         workerPool.setProps(this._getWorkerPoolProps());
-        this.workerPools.set(name, workerPool);
+        this.workerPools.set(workerPoolKey, workerPool);
       }
       return workerPool;
     }
@@ -1269,6 +1310,91 @@ var __exports__ = (() => {
       };
     }
   };
+  function getWorkerPoolKey(options) {
+    const { name, source, url, urlKey, loadWorker } = options;
+    const sourceKey = source ? `source:${source}` : "";
+    const urlKeyPart = url || urlKey ? `url:${url || urlKey}` : "";
+    const loadWorkerKey = loadWorker ? `loadWorker:${getLoadWorkerId(loadWorker)}` : "";
+    return [name, sourceKey, urlKeyPart, loadWorkerKey].filter(Boolean).join("|");
+  }
+  function getLoadWorkerId(loadWorker) {
+    let loadWorkerId = loadWorkerIds.get(loadWorker);
+    if (!loadWorkerId) {
+      loadWorkerId = nextLoadWorkerId++;
+      loadWorkerIds.set(loadWorker, loadWorkerId);
+    }
+    return loadWorkerId;
+  }
+
+  // ../worker-utils/src/lib/async-queue/async-queue.ts
+  var AsyncQueue = class {
+    _values;
+    _settlers;
+    _closed;
+    constructor() {
+      this._values = [];
+      this._settlers = [];
+      this._closed = false;
+    }
+    /** Return an async iterator for this queue */
+    [Symbol.asyncIterator]() {
+      return this;
+    }
+    /** Push a new value - the async iterator will yield a promise resolved to this value */
+    push(value) {
+      return this.enqueue(value);
+    }
+    /**
+     * Push a new value - the async iterator will yield a promise resolved to this value
+     * Add an error - the async iterator will yield a promise rejected with this value
+     */
+    enqueue(value) {
+      if (this._closed) {
+        throw new Error("Closed");
+      }
+      if (this._settlers.length > 0) {
+        if (this._values.length > 0) {
+          throw new Error("Illegal internal state");
+        }
+        const settler = this._settlers.shift();
+        if (value instanceof Error) {
+          settler.reject(value);
+        } else {
+          settler.resolve({ value });
+        }
+      } else {
+        this._values.push(value);
+      }
+    }
+    /** Indicate that we not waiting for more values - The async iterator will be done */
+    close() {
+      while (this._settlers.length > 0) {
+        const settler = this._settlers.shift();
+        settler.resolve({ done: true });
+      }
+      this._closed = true;
+    }
+    // ITERATOR IMPLEMENTATION
+    /** @returns a Promise for an IteratorResult */
+    next() {
+      if (this._values.length > 0) {
+        const value = this._values.shift();
+        if (value instanceof Error) {
+          return Promise.reject(value);
+        }
+        return Promise.resolve({ done: false, value });
+      }
+      if (this._closed) {
+        if (this._settlers.length > 0) {
+          throw new Error("Illegal internal state");
+        }
+        return Promise.resolve({ done: true, value: void 0 });
+      }
+      return new Promise((resolve2, reject) => {
+        this._settlers.push({ resolve: resolve2, reject });
+      });
+    }
+  };
 
   // ../worker-utils/src/lib/worker-api/get-worker-url.ts
   var warnedWorkerVersionFallbacks = /* @__PURE__ */ new Set();
@@ -1276,34 +1402,41 @@ var __exports__ = (() => {
     const warning = worker.version !== VERSION3 ? ` (worker-utils@${VERSION3})` : "";
     return `${worker.name}@${worker.version}${warning}`;
   }
-  function getWorkerURL(worker, options = {}) {
+  function getCustomWorkerURL(worker, options = {}) {
     const workerOptions = options[worker.id] || {};
-    const workerFile = isBrowser3 ? `${worker.id}-worker.js` : worker.workerNode || `${worker.id}-worker-node.js`;
+    const workerFile = isBrowser3 ? worker.workerFile || `${worker.id}-worker.js` : worker.workerNode || `${worker.id}-worker-node.js`;
     let url = workerOptions.workerUrl;
-    if (!url && typeof worker.worker === "string") {
-      url = worker.worker;
-    }
     if (!url && worker.id === "compression") {
       url = options.workerUrl;
     }
     const workerType = options._workerType || options?.core?._workerType;
-    if (workerType === "test") {
+    if (!url && workerType === "test") {
       if (isBrowser3) {
         url = `modules/${worker.module}/dist/${workerFile}`;
       } else {
         url = `modules/${worker.module}/src/workers/${worker.id}-worker-node.ts`;
       }
     }
-    if (!url) {
-      let version2 = worker.version;
-      if (version2 === "latest") {
-        version2 = NPM_TAG;
-      }
-      const versionTag = version2 ? `@${version2}` : "";
-      url = `https://unpkg.com/@loaders.gl/${worker.module}${versionTag}/dist/${workerFile}`;
+    return url || null;
+  }
+  function getWorkerURL(worker, options = {}) {
+    const customWorkerUrl = getCustomWorkerURL(worker, options);
+    const descriptorWorkerUrl = typeof worker.worker === "string" ? worker.worker : null;
+    const url = customWorkerUrl || descriptorWorkerUrl || getDefaultWorkerURL(worker, true);
+    assert3(url);
+    return url;
+  }
+  function getDefaultWorkerURL(worker, warn = false) {
+    const workerFile = isBrowser3 ? worker.workerFile || `${worker.id}-worker.js` : worker.workerNode || `${worker.id}-worker-node.js`;
+    let version2 = worker.version;
+    if (version2 === "latest") {
+      version2 = NPM_TAG;
+    }
+    const versionTag = version2 ? `@${version2}` : "";
+    const url = `https://unpkg.com/@loaders.gl/${worker.module}${versionTag}/dist/${workerFile}`;
+    if (warn) {
       warnIfUsingNpmTagFallback(worker, url);
     }
-    assert3(url);
     return url;
   }
   function warnIfUsingNpmTagFallback(worker, url) {
@@ -1326,18 +1459,15 @@ var __exports__ = (() => {
       return false;
     }
     const workerOptions = options?.[worker.id];
-    return Boolean((worker.worker || workerOptions?.workerUrl) && options?.worker);
+    return Boolean(
+      (worker.worker || worker.loadWorker || workerOptions?.workerUrl) && options?.worker
+    );
   }
   async function processOnWorker(worker, data, options = {}, context = {}, jobContext = {}) {
     throwIfAborted(options.signal);
     const name = getWorkerName(worker);
     const workerFarm = WorkerFarm.getWorkerFarm(options);
-    const { source } = options;
-    const workerPoolProps = { name, source };
-    if (!source) {
-      workerPoolProps.url = getWorkerURL(worker, options);
-    }
-    const workerPool = workerFarm.getWorkerPool(workerPoolProps);
+    const workerPool = workerFarm.getWorkerPool(getWorkerPoolTarget(worker, options, name));
     const jobName = options.jobName || worker.name;
     const job = await workerPool.startJob(
       jobName,
@@ -1366,6 +1496,114 @@ var __exports__ = (() => {
       options.signal?.removeEventListener("abort", abortJob);
     }
   }
+  function processOnWorkerInBatches(worker, input, options = {}, context = {}, jobContext = {}) {
+    return processOnWorkerInBatchesIterator(worker, input, options, context, jobContext);
+  }
+  async function* processOnWorkerInBatchesIterator(worker, input, options, context, jobContext) {
+    throwIfAborted(options.signal);
+    const name = getWorkerName(worker);
+    const workerFarm = WorkerFarm.getWorkerFarm(options);
+    const workerPool = workerFarm.getWorkerPool(getWorkerPoolTarget(worker, options, name));
+    const outputBatches = new AsyncQueue();
+    const inputIterator = getAsyncIterator(input);
+    let outputFinished = false;
+    let inputFinished = false;
+    let inputRequest = Promise.resolve();
+    const batchJobFailed = new Error("Worker batch job failed");
+    const finishOutput = () => {
+      if (!outputFinished) {
+        outputFinished = true;
+        outputBatches.close();
+      }
+    };
+    const failOutput = (_error) => {
+      if (!outputFinished) {
+        outputFinished = true;
+        outputBatches.enqueue(batchJobFailed);
+        outputBatches.close();
+      }
+    };
+    const job = await workerPool.startJob(
+      options.jobName || worker.name,
+      (activeJob, type, payload) => {
+        switch (type) {
+          case "input-request":
+            inputRequest = inputRequest.then(async () => {
+              const nextBatch = await inputIterator.next();
+              if (!activeJob.isRunning) {
+                return;
+              }
+              if (nextBatch.done) {
+                inputFinished = true;
+                activeJob.postMessage("input-done", {});
+              } else {
+                activeJob.postMessage("input-batch", { input: nextBatch.value });
+              }
+            }).catch((error) => activeJob.abort(error));
+            break;
+          case "output-batch":
+            outputBatches.push(payload.result);
+            break;
+          case "done":
+            activeJob.done(payload);
+            finishOutput();
+            break;
+          case "error": {
+            const error = new Error(payload.error || "Worker batch processing failed");
+            activeJob.error(error);
+            failOutput(error);
+            break;
+          }
+          case "process":
+            void onMessage(context, activeJob, type, payload);
+            break;
+          default: {
+            const error = new Error(`Unexpected worker batch message: ${type}`);
+            activeJob.error(error);
+            failOutput(error);
+            break;
+          }
+        }
+      }
+    );
+    const abortJob = () => job.abort(getAbortReason(options.signal));
+    options.signal?.addEventListener("abort", abortJob, { once: true });
+    if (options.signal?.aborted) {
+      abortJob();
+    }
+    void job.result.catch(failOutput);
+    try {
+      const { signal: _signal, ...workerOptions } = options;
+      if (job.isRunning) {
+        job.postMessage("process-in-batches", {
+          options: getTransferListForWriter(workerOptions),
+          context: getTransferListForWriter(jobContext)
+        });
+      }
+      try {
+        for await (const outputBatch of outputBatches) {
+          yield outputBatch;
+          if (job.isRunning) {
+            job.postMessage("output-ack", {});
+          }
+        }
+      } catch (error) {
+        if (error === batchJobFailed) {
+          await job.result;
+        }
+        throw error;
+      }
+      await job.result;
+    } finally {
+      options.signal?.removeEventListener("abort", abortJob);
+      if (job.isRunning) {
+        job.abort(createAbortError2("Worker batch iterator was closed"));
+      }
+      if (!inputFinished) {
+        closeInputIterator(inputIterator);
+      }
+    }
+  }
   function throwIfAborted(signal) {
     if (!signal?.aborted) {
       return;
@@ -1380,6 +1618,38 @@ var __exports__ = (() => {
     error.name = "AbortError";
     return error;
   }
+  function getAsyncIterator(input) {
+    if (Symbol.asyncIterator in input) {
+      return input[Symbol.asyncIterator]();
+    }
+    return (async function* iterateSynchronously() {
+      yield* input;
+    })();
+  }
+  function closeInputIterator(inputIterator) {
+    try {
+      void Promise.resolve(inputIterator.return?.()).catch(() => void 0);
+    } catch {
+    }
+  }
+  function getWorkerPoolTarget(worker, options, name) {
+    if (options.source) {
+      return { name, source: options.source };
+    }
+    const customWorkerUrl = getCustomWorkerURL(worker, options);
+    if (customWorkerUrl) {
+      return { name, url: customWorkerUrl };
+    }
+    if (isBrowser3 && worker.loadWorker) {
+      return {
+        name,
+        loadWorker: worker.loadWorker,
+        getUrl: () => getWorkerURL(worker, options),
+        urlKey: getDefaultWorkerURL(worker)
+      };
+    }
+    return { name, url: getWorkerURL(worker, options) };
+  }
   async function onMessage(context, job, type, payload) {
     switch (type) {
       case "done":
@@ -1432,12 +1702,29 @@ var __exports__ = (() => {
     }
     return Boolean(canProcessOnWorker(loader, workerOptions));
   }
+  function shouldParseWithWorker(loader, data, options, context, estimateLoader = loader) {
+    if (!canParseWithWorker(loader, options)) {
+      return false;
+    }
+    if (options?.core?.worker !== "auto" || !estimateLoader.getWorkerEstimate) {
+      return true;
+    }
+    try {
+      const estimate = estimateLoader.getWorkerEstimate(data, options, context);
+      if (estimate === void 0 || !Number.isFinite(estimate) || estimate < 0 || estimate > 1) {
+        return true;
+      }
+      return estimate >= (options.core?.workerThreshold ?? 0.1);
+    } catch {
+      return true;
+    }
+  }
   async function parseWithWorker(loader, data, options, context, parseOnMainThread, signal) {
     const workerSignal = signal || getWorkerAbortSignal(options);
     const result = await processOnWorker(
       loader,
       data,
-      { ...getWorkerOptions(options), signal: workerSignal },
+      { ...getWorkerOptions(options, loader), signal: workerSignal },
       {
         process: async (input, processOptions, _workerContext, parseContext) => {
           if (!parseOnMainThread) {
@@ -1456,6 +1743,32 @@ var __exports__ = (() => {
     );
     return isLoaderWithWorkerResultDeserializer(loader) ? loader.deserializeWorkerResult(result, options, context) : result;
   }
+  async function* parseWithWorkerInBatches(loader, inputIterator, options, context, parseOnMainThread, signal) {
+    const workerSignal = signal || getWorkerAbortSignal(options);
+    const outputIterator = processOnWorkerInBatches(
+      loader,
+      inputIterator,
+      { ...getWorkerOptions(options), signal: workerSignal },
+      {
+        process: async (input, processOptions, _workerContext, parseContext) => {
+          if (!parseOnMainThread) {
+            throw new Error("Worker not set up to parse on main thread");
+          }
+          const mainThreadContext = context ? { ...context, ...parseContext || {} } : void 0;
+          return await callParseOnMainThread(
+            parseOnMainThread,
+            input,
+            processOptions,
+            mainThreadContext
+          );
+        }
+      },
+      getSerializableLoaderContext(context)
+    );
+    for await (const batch of outputIterator) {
+      yield loader.deserializeWorkerBatch ? loader.deserializeWorkerBatch(batch, options, context) : batch;
+    }
+  }
   function getWorkerAbortSignal(options) {
     const parquetSignal = options?.parquet?.signal;
     if (typeof AbortSignal !== "undefined" && parquetSignal instanceof AbortSignal) {
@@ -1469,12 +1782,19 @@ var __exports__ = (() => {
     }
     return parseOnMainThread(input, void 0, options, context);
   }
-  function getWorkerOptions(options = {}) {
+  function getWorkerOptions(options = {}, loader) {
     const serializedOptions = JSON.parse(JSON.stringify(options));
-    return {
+    const workerOptions = {
       ...serializedOptions.core,
       ...serializedOptions
     };
+    if (workerOptions.worker === "auto") {
+      workerOptions.worker = true;
+    }
+    if (loader) {
+      workerOptions._workerLoaderId = loader.id;
+    }
+    return workerOptions;
   }
   function getSerializableLoaderContext(context) {
     if (!context) {
@@ -2728,6 +3048,7 @@ var __exports__ = (() => {
       CDN: "https://unpkg.com/@loaders.gl",
       worker: true,
       // By default, use worker if provided by loader.
+      workerThreshold: 0.1,
       maxConcurrency: 3,
       // How many worker instances should be created for each loader.
       maxMobileConcurrency: 1,
@@ -2803,6 +3124,7 @@ var __exports__ = (() => {
     "useLocalLibraries",
     "CDN",
     "worker",
+    "workerThreshold",
     "maxConcurrency",
     "maxMobileConcurrency",
     "reuseWorkers",
@@ -2843,7 +3165,10 @@ var __exports__ = (() => {
     loaders = loaders || [];
     loaders = Array.isArray(loaders) ? loaders : [loaders];
     validateOptions(options, loaders);
-    return normalizeLoaderOptions(normalizeOptionsInternal(loader, options, url));
+    validateWorkerThreshold(options);
+    const normalizedOptions = normalizeLoaderOptions(normalizeOptionsInternal(loader, options, url));
+    validateWorkerThreshold(normalizedOptions);
+    return normalizedOptions;
   }
   function normalizeLoaderOptions(options) {
     const normalized = cloneLoaderOptions(options);
@@ -2867,6 +3192,12 @@ var __exports__ = (() => {
       validateOptionsObject(idOptions, loader.id, loaderOptions, deprecatedOptions, loaders);
     }
   }
+  function validateWorkerThreshold(options) {
+    const threshold = options.core?.workerThreshold;
+    if (threshold !== void 0 && (!Number.isFinite(threshold) || threshold < 0 || threshold > 1)) {
+      throw new Error("core.workerThreshold must be a finite number between 0 and 1");
+    }
+  }
   function validateOptionsObject(options, id, defaultOptions, deprecatedOptions, loaders) {
     const loaderName = id || "Top level";
     const prefix = id ? `${id}.` : "";
@@ -3580,7 +3911,8 @@ var __exports__ = (() => {
     if (typeof fetchOption === "function") {
       fetchFunction = fetchOption;
     } else if (isObject(fetchOption)) {
-      fetchFunction = (url, requestOptions) => fetchFile(url, mergeFetchOptions(fetchOption, requestOptions));
+      const baseFetch = context?.fetch || fetchFile;
+      fetchFunction = (url, requestOptions) => baseFetch(url, mergeFetchOptions(fetchOption, requestOptions));
     } else if (context?.fetch) {
       fetchFunction = context.fetch;
     } else {
@@ -3898,6 +4230,9 @@ var __exports__ = (() => {
       options?.core?.transforms || []
     );
     if (loader.parseInBatches) {
+      if (loader.id === "csv" && isBrowser && canParseWithWorker(loader, options)) {
+        return parseWithWorkerInBatches(loader, transformedIterator, options, context, parse);
+      }
       return loader.parseInBatches(transformedIterator, options, context);
     }
     return parseChunkInBatches(transformedIterator, loader, options, context);
@@ -4084,7 +4419,10 @@ var __exports__ = (() => {
   // src/lib/loader-utils/loader-context.ts
   function getLoaderContext(context, options, parentContext) {
     if (parentContext) {
-      return parentContext;
+      return {
+        ...parentContext,
+        fetch: getFetchFunction(options, parentContext)
+      };
     }
     const newContext = {
       ...context,
@@ -4157,7 +4495,7 @@ var __exports__ = (() => {
       context.response = { headers, ok, redirected, status, statusText, type, url };
     }
     const loaderWithParser = await getLoaderImplementation(loader, options, context.url);
-    if (canParseWithWorker(loaderWithParser, options)) {
+    if (shouldParseWithWorker(loaderWithParser, data, options, context, loader)) {
       data = await getArrayBufferFromData(data, options);
       return await parseWithWorker(loaderWithParser, data, options, context, parse);
     }