@loaders.gl/core 5.0.0-alpha.13 → 5.0.0-alpha.14

package/dist/dist.dev.js

@@ -792,6 +792,7 @@ var __exports__ = (() => {
      * @param data any data structure, ideally consisting mostly of transferrable objects
      */
     postMessage(type, payload) {
+      if (!this.isRunning) return;
       this.workerThread.postMessage({
         source: "loaders.gl",
         // Lets worker ignore unrelated messages
@@ -827,9 +828,10 @@ var __exports__ = (() => {
       if (!this.isRunning) {
         return;
       }
-      const error = reason ?? createAbortError(`Worker job "${this.name}" was aborted`);
       this.workerThread.destroy();
-      this.error(error);
+      this.error(
+        arguments.length ? reason : createAbortError(`Worker job "${this.name}" was aborted`)
+      );
     }
   };
   function createAbortError(message) {
@@ -1132,9 +1134,13 @@ var __exports__ = (() => {
      * @note Can free up significant memory
      */
     destroy(reason = new Error("Worker pool was destroyed")) {
+      this.isDestroyed = true;
       this.activeJobs.forEach((job) => job.abort(reason));
       const queuedJobs = this.jobQueue.splice(0);
-      queuedJobs.forEach((queuedJob) => queuedJob.onReject(reason));
+      queuedJobs.forEach((queuedJob) => {
+        queuedJob.removeAbortListener();
+        queuedJob.onReject(reason);
+      });
       this.idleQueue.forEach((worker) => worker.destroy());
       this.isDestroyed = true;
     }
@@ -1156,12 +1162,33 @@ var __exports__ = (() => {
         this.onDebug = props.onDebug;
       }
     }
-    async startJob(name, onMessage2 = (job, type, data) => job.done(data), onError = (job, error) => job.error(error)) {
+    async startJob(name, onMessage2 = (job, type, data) => job.done(data), onError = (job, error) => job.error(error), signal) {
+      if (signal?.aborted) {
+        throw signal.reason;
+      }
       if (this.isDestroyed) {
         return Promise.reject(new Error("Worker pool was destroyed"));
       }
       const startPromise = new Promise((onStart, onReject) => {
-        this.jobQueue.push({ name, onMessage: onMessage2, onError, onStart, onReject });
+        const abortQueuedJob = () => {
+          const index = this.jobQueue.indexOf(queuedJob);
+          if (index >= 0) {
+            this.jobQueue.splice(index, 1);
+            queuedJob.removeAbortListener();
+            onReject(signal?.reason);
+          }
+        };
+        const queuedJob = {
+          name,
+          onMessage: onMessage2,
+          onError,
+          onStart,
+          onReject,
+          signal,
+          removeAbortListener: () => signal?.removeEventListener("abort", abortQueuedJob)
+        };
+        this.jobQueue.push(queuedJob);
+        signal?.addEventListener("abort", abortQueuedJob, { once: true });
         return this;
       });
       this._startQueuedJob();
@@ -1176,7 +1203,16 @@ var __exports__ = (() => {
       if (!this.jobQueue.length) {
         return;
       }
-      const workerThread = this._getAvailableWorker();
+      let workerThread;
+      try {
+        workerThread = this._getAvailableWorker();
+      } catch (error) {
+        const queuedJob2 = this.jobQueue.shift();
+        queuedJob2?.removeAbortListener();
+        queuedJob2?.onReject(error);
+        void this._startQueuedJob();
+        return;
+      }
       if (!workerThread) {
         return;
       }
@@ -1189,18 +1225,27 @@ var __exports__ = (() => {
           backlog: this.jobQueue.length
         });
         const job = new WorkerJob(queuedJob.name, workerThread);
+        queuedJob.removeAbortListener();
+        const abortJob = () => job.abort(queuedJob.signal?.reason);
+        queuedJob.signal?.addEventListener("abort", abortJob, { once: true });
+        if (queuedJob.signal?.aborted) abortJob();
         this.activeJobs.add(job);
         workerThread.ref();
-        workerThread.onMessage = (data) => queuedJob.onMessage(job, data.type, data.payload);
+        workerThread.onMessage = (data) => {
+          if (job.isRunning) queuedJob.onMessage(job, data.type, data.payload);
+        };
         workerThread.onError = (error) => queuedJob.onError(job, error);
         queuedJob.onStart(job);
         try {
           await job.result;
         } catch {
         } finally {
+          queuedJob.signal?.removeEventListener("abort", abortJob);
           this.activeJobs.delete(job);
           this.returnWorkerToQueue(workerThread);
         }
+      } else {
+        this.returnWorkerToQueue(workerThread);
       }
     }
     /**
@@ -1242,13 +1287,18 @@ var __exports__ = (() => {
       if (this.count < this._getMaxConcurrency()) {
         this.count++;
         const name = `${this.name.toLowerCase()} (#${this.count} of ${this.maxConcurrency})`;
-        return new WorkerThread({
-          name,
-          source: this.source,
-          url: this.url,
-          getUrl: this.getUrl,
-          loadWorker: this.loadWorker
-        });
+        try {
+          return new WorkerThread({
+            name,
+            source: this.source,
+            url: this.url,
+            getUrl: this.getUrl,
+            loadWorker: this.loadWorker
+          });
+        } catch (error) {
+          this.count--;
+          throw error;
+        }
       }
       return null;
     }
@@ -1438,12 +1488,8 @@ var __exports__ = (() => {
     return `${worker.name}@${worker.version}${warning}`;
   }
   function getCustomWorkerURL(worker, options = {}) {
-    const workerOptions = options[worker.id] || {};
     const workerFile = isBrowser3 ? worker.workerFile || `${worker.id}-worker.js` : worker.workerNode || `${worker.id}-worker-node.js`;
-    let url = workerOptions.workerUrl;
-    if (!url && worker.id === "compression") {
-      url = options.workerUrl;
-    }
+    let url = getConfiguredWorkerURL(worker, options);
     const workerType = options._workerType || options?.core?._workerType;
     if (!url && workerType === "test") {
       if (isBrowser3) {
@@ -1456,6 +1502,10 @@ var __exports__ = (() => {
     }
     return url || null;
   }
+  function getConfiguredWorkerURL(worker, options = {}) {
+    const workerOptions = options[worker.id] || {};
+    return workerOptions.workerUrl || (worker.id === "compression" ? options.workerUrl : null) || null;
+  }
   function getWorkerURL(worker, options = {}) {
     const customWorkerUrl = getCustomWorkerURL(worker, options);
     const descriptorWorkerUrl = typeof worker.worker === "string" ? worker.worker : null;
@@ -1500,6 +1550,13 @@ var __exports__ = (() => {
       (worker.worker || worker.loadWorker || workerOptions?.workerUrl) && options?.worker
     );
   }
+  function canProcessOnWorkerInBatches(worker, options = {}) {
+    if (!canProcessOnWorker(worker, options)) return false;
+    if (options.workerInBatches !== void 0) return options.workerInBatches;
+    return Boolean(
+      worker.workerInBatches && !options.source && !getConfiguredWorkerURL(worker, options)
+    );
+  }
   async function processOnWorker(worker, data, options = {}, context = {}, jobContext = {}) {
     throwIfAborted(options.signal);
     const name = getWorkerName(worker);
@@ -1509,15 +1566,13 @@ var __exports__ = (() => {
     const job = await workerPool.startJob(
       jobName,
       // eslint-disable-next-line
-      onMessage.bind(null, context)
+      onMessage.bind(null, context, options),
+      void 0,
+      options.signal
     );
-    const abortJob = () => job.abort(getAbortReason(options.signal));
-    options.signal?.addEventListener("abort", abortJob, { once: true });
-    if (options.signal?.aborted) {
-      abortJob();
-    }
     try {
-      const { signal: _signal, ...workerOptions } = options;
+      if (options.signal?.aborted) job.abort(getAbortReason(options.signal));
+      const { signal: _signal, onProgress: _onProgress, ...workerOptions } = options;
       const transferableOptions = getTransferListForWriter(workerOptions);
       const transferableContext = getTransferListForWriter(jobContext);
       if (job.isRunning) {
@@ -1529,8 +1584,9 @@ var __exports__ = (() => {
       }
       const result = await job.result;
       return result.result;
-    } finally {
-      options.signal?.removeEventListener("abort", abortJob);
+    } catch (error) {
+      job.abort(error);
+      throw error;
     }
   }
   function processOnWorkerInBatches(worker, input, options = {}, context = {}, jobContext = {}) {
@@ -1545,8 +1601,15 @@ var __exports__ = (() => {
     const inputIterator = getAsyncIterator(input);
     let outputFinished = false;
     let inputFinished = false;
+    let inputCleanupRequested = false;
     let inputRequest = Promise.resolve();
     const batchJobFailed = new Error("Worker batch job failed");
+    const closeJobInput = () => {
+      if (!inputFinished && !inputCleanupRequested) {
+        inputCleanupRequested = true;
+        closeInputIterator(inputIterator);
+      }
+    };
     const finishOutput = () => {
       if (!outputFinished) {
         outputFinished = true;
@@ -1564,8 +1627,12 @@ var __exports__ = (() => {
       options.jobName || worker.name,
       (activeJob, type, payload) => {
         switch (type) {
+          case "progress":
+            reportProgress(activeJob, options, payload);
+            break;
           case "input-request":
             inputRequest = inputRequest.then(async () => {
+              if (!activeJob.isRunning) return;
               const nextBatch = await inputIterator.next();
               if (!activeJob.isRunning) {
                 return;
@@ -1592,7 +1659,7 @@ var __exports__ = (() => {
             break;
           }
           case "process":
-            void onMessage(context, activeJob, type, payload);
+            void onMessage(context, options, activeJob, type, payload);
             break;
           default: {
             const error = new Error(`Unexpected worker batch message: ${type}`);
@@ -1601,16 +1668,19 @@ var __exports__ = (() => {
             break;
           }
         }
-      }
-    );
-    const abortJob = () => job.abort(getAbortReason(options.signal));
-    options.signal?.addEventListener("abort", abortJob, { once: true });
-    if (options.signal?.aborted) {
-      abortJob();
-    }
-    void job.result.catch(failOutput);
+      },
+      void 0,
+      options.signal
+    ).catch((error) => {
+      closeJobInput();
+      throw error;
+    });
+    void job.result.catch((error) => {
+      failOutput(error);
+      closeJobInput();
+    });
     try {
-      const { signal: _signal, ...workerOptions } = options;
+      const { signal: _signal, onProgress: _onProgress, ...workerOptions } = options;
       if (job.isRunning) {
         job.postMessage("process-in-batches", {
           options: getTransferListForWriter(workerOptions),
@@ -1632,13 +1702,10 @@ var __exports__ = (() => {
       }
       await job.result;
     } finally {
-      options.signal?.removeEventListener("abort", abortJob);
       if (job.isRunning) {
         job.abort(createAbortError2("Worker batch iterator was closed"));
       }
-      if (!inputFinished) {
-        closeInputIterator(inputIterator);
-      }
+      closeJobInput();
     }
   }
   function throwIfAborted(signal) {
@@ -1648,7 +1715,7 @@ var __exports__ = (() => {
     throw getAbortReason(signal);
   }
   function getAbortReason(signal) {
-    return signal?.reason ?? createAbortError2("Worker job was aborted");
+    return signal ? signal.reason : createAbortError2("Worker job was aborted");
   }
   function createAbortError2(message) {
     const error = new Error(message);
@@ -1687,8 +1754,11 @@ var __exports__ = (() => {
     }
     return { name, url: getWorkerURL(worker, options) };
   }
-  async function onMessage(context, job, type, payload) {
+  async function onMessage(context, options, job, type, payload) {
     switch (type) {
+      case "progress":
+        reportProgress(job, options, payload);
+        break;
       case "done":
         job.done(payload);
         break;
@@ -1696,13 +1766,18 @@ var __exports__ = (() => {
         job.error(new Error(payload.error));
         break;
       case "process":
-        const { id, input, options } = payload;
+        const { id, input, options: processOptions } = payload;
         try {
           if (!context.process) {
             job.postMessage("error", { id, error: "Worker not set up to process on main thread" });
             return;
           }
-          const result = await context.process(input, options, void 0, payload.context || {});
+          const result = await context.process(
+            input,
+            { ...processOptions, signal: options.signal, onProgress: options.onProgress },
+            void 0,
+            payload.context || {}
+          );
           job.postMessage("done", { id, result });
         } catch (error) {
           const message = error instanceof Error ? error.message : "unknown error";
@@ -1713,6 +1788,14 @@ var __exports__ = (() => {
         console.warn(`process-on-worker: unknown message ${type}`);
     }
   }
+  function reportProgress(job, options, payload) {
+    if (!job.isRunning || !payload.progress) return;
+    try {
+      options.onProgress?.(payload.progress);
+    } catch (error) {
+      job.abort(error);
+    }
+  }
 
   // src/worker-utils/lib/worker-api/validate-worker-version.ts
   function validateWorkerVersion(worker, coreVersion = VERSION3) {
@@ -1746,6 +1829,15 @@ var __exports__ = (() => {
     }
     return Boolean(canProcessOnWorker(loader, workerOptions));
   }
+  function canParseWithWorkerInBatches(loader, options) {
+    const parquetOptions = options?.parquet;
+    if (loader.id === "parquet" && typeof parquetOptions?.keyRetriever === "function") {
+      return false;
+    }
+    return Boolean(
+      loader.workerInBatches && canParseWithWorker(loader, options) && canProcessOnWorkerInBatches(loader, getWorkerOptions(options))
+    );
+  }
   function shouldParseWithWorker(loader, data, options, context, estimateLoader = loader) {
     if (!canParseWithWorker(loader, options)) {
       return false;
@@ -1764,11 +1856,15 @@ var __exports__ = (() => {
     }
   }
   async function parseWithWorker(loader, data, options, context, parseOnMainThread, signal) {
-    const workerSignal = signal || getWorkerAbortSignal(options);
+    const workerSignal = signal || getWorkerAbortSignal(options) || context?.signal;
     const result = await processOnWorker(
       loader,
       data,
-      { ...getWorkerOptions(options, loader), signal: workerSignal },
+      {
+        ...getWorkerOptions(options, loader),
+        signal: workerSignal,
+        onProgress: options?.onProgress || context?.reportProgress
+      },
       {
         process: async (input, processOptions, _workerContext, parseContext) => {
           if (!parseOnMainThread) {
@@ -1790,11 +1886,15 @@ var __exports__ = (() => {
     return isLoaderWithWorkerResultDeserializer(loader) ? loader.deserializeWorkerResult(result, options, context) : result;
   }
   async function* parseWithWorkerInBatches(loader, inputIterator, options, context, parseOnMainThread, signal) {
-    const workerSignal = signal || getWorkerAbortSignal(options);
+    const workerSignal = signal || getWorkerAbortSignal(options) || context?.signal;
     const outputIterator = processOnWorkerInBatches(
       loader,
       inputIterator,
-      { ...getWorkerOptions(options), signal: workerSignal },
+      {
+        ...getWorkerOptions(options, loader),
+        signal: workerSignal,
+        onProgress: options?.onProgress || context?.reportProgress
+      },
       {
         process: async (input, processOptions, _workerContext, parseContext) => {
           if (!parseOnMainThread) {
@@ -1818,6 +1918,7 @@ var __exports__ = (() => {
     }
   }
   function getWorkerAbortSignal(options) {
+    if (options?.signal) return options.signal;
     const parquetSignal = options?.parquet?.signal;
     if (typeof AbortSignal !== "undefined" && parquetSignal instanceof AbortSignal) {
       return parquetSignal;
@@ -1861,10 +1962,15 @@ var __exports__ = (() => {
   function getWorkerOptions(options = {}, loader) {
     const optionsToSerialize = {
       ...options,
+      signal: void 0,
+      onProgress: void 0,
       ...options.core ? { core: { ...options.core, authentications: void 0 } } : {}
     };
     const serializedOptions = JSON.parse(
-      JSON.stringify(optionsToSerialize, (_key, value) => typeof value === "bigint" ? null : value)
+      JSON.stringify(optionsToSerialize, (_key, value) => {
+        if (typeof AbortSignal !== "undefined" && value instanceof AbortSignal) return void 0;
+        return typeof value === "bigint" ? null : value;
+      })
     );
     restoreBigIntOptions(optionsToSerialize, serializedOptions);
     const workerOptions = {
@@ -1891,7 +1997,17 @@ var __exports__ = (() => {
     if (!context) {
       return {};
     }
-    const { fetch: fetch2, loaders, coreApi: coreApi2, _parse, _parseSync, _parseInBatches, ...serializableContext } = context;
+    const {
+      fetch: fetch2,
+      loaders,
+      coreApi: coreApi2,
+      signal,
+      reportProgress: reportProgress2,
+      _parse,
+      _parseSync,
+      _parseInBatches,
+      ...serializableContext
+    } = context;
     return JSON.parse(JSON.stringify(serializableContext));
   }
   function shouldParseCSVWithWorker(options) {
@@ -2499,7 +2615,7 @@ var __exports__ = (() => {
     for (const credential of requestCredentials) normalizeRequestOrigins(credential.origins);
     const authenticatedFetch = async (url, requestOptions = {}) => {
       requestOptions = mergeFetchOptions(fetchOptions, requestOptions);
-      if (requestOptions.signal?.aborted) throw createAbortError3();
+      requestOptions.signal?.throwIfAborted();
       if (!credentials.length) return baseFetch(url, requestOptions);
       const resolvedURL = parseHTTPURL(url);
       const authorization = resolvedURL ? await authorizeRequest(resolvedURL, requestOptions, tokenCredentials) : { url, options: requestOptions, credentials: [] };
@@ -2515,7 +2631,7 @@ var __exports__ = (() => {
         (credential) => credential.refreshStatusCodes?.includes(response.status)
       );
       if (!refreshableCredentials.length && !shouldRetrySignature) return response;
-      if (requestOptions.signal?.aborted) throw createAbortError3();
+      requestOptions.signal?.throwIfAborted();
       const refreshedTokens = /* @__PURE__ */ new Map();
       await Promise.all(
         refreshableCredentials.map(async ({ credential }) => {
@@ -2526,7 +2642,7 @@ var __exports__ = (() => {
       if (!refreshedTokens.size && !shouldRetrySignature) return response;
       await response.body?.cancel().catch(() => {
       });
-      if (requestOptions.signal?.aborted) throw createAbortError3();
+      requestOptions.signal?.throwIfAborted();
       const replayAuthorization = resolvedURL ? await authorizeRequest(
         resolvedURL,
         requestOptions,
@@ -2567,7 +2683,7 @@ var __exports__ = (() => {
     for (const credential of credentials) {
       const origin = getRequestOrigin(request.url);
       if (!origin || !normalizeRequestOrigins(credential.origins).includes(origin)) continue;
-      if (signal?.aborted) throw createAbortError3();
+      signal?.throwIfAborted();
       const authenticatedRequest = await credential.authenticate({
         url: request.url,
         options: { ...request.options, headers: new Headers(request.options.headers) },
@@ -2583,7 +2699,7 @@ var __exports__ = (() => {
       };
       appliedCredentials.push(credential);
     }
-    if (signal?.aborted) throw createAbortError3();
+    signal?.throwIfAborted();
     return { ...request, credentials: appliedCredentials };
   }
   function getRequestOrigin(url) {
@@ -2673,9 +2789,6 @@ var __exports__ = (() => {
     if (!["GET", "HEAD", "OPTIONS", "PUT", "DELETE"].includes(method)) return false;
     return !(options.body && typeof ReadableStream !== "undefined" && options.body instanceof ReadableStream);
   }
-  function createAbortError3() {
-    return new DOMException("The operation was aborted.", "AbortError");
-  }
 
   // src/loader-utils/lib/path-utils/file-aliases.ts
   var pathPrefix = "";
@@ -3310,6 +3423,7 @@ var __exports__ = (() => {
       worker: true,
       // By default, use worker if provided by loader.
       workerThreshold: 0.1,
+      workerInBatches: void 0,
       maxConcurrency: 3,
       // How many worker instances should be created for each loader.
       maxMobileConcurrency: 1,
@@ -3485,7 +3599,7 @@ var __exports__ = (() => {
     const prefix = id ? `${id}.` : "";
     for (const key in options) {
       const isSubOptions = !id && isObject(options[key]);
-      const isBaseUriOption = key === "baseUri" && !id;
+      const isBaseUriOption = !id && ["baseUri", "signal", "onProgress"].includes(key);
       const isWorkerUrlOption = key === "workerUrl" && id;
       if (!(key in defaultOptions) && !isBaseUriOption && !isWorkerUrlOption) {
         if (key in deprecatedOptions) {
@@ -4199,11 +4313,13 @@ var __exports__ = (() => {
     } else {
       fetchFunction = fetchFile;
     }
+    const signal = loaderOptions.signal || context?.signal;
+    const requestOptions = isObject(fetchOption) ? fetchOption : void 0;
     return createAuthenticatedFetch({
       fetch: fetchFunction,
       credentials: loaderOptions.core?.credentials || [],
       authentications: loaderOptions.core?.authentications,
-      fetchOptions: isObject(fetchOption) ? fetchOption : void 0
+      fetchOptions: signal ? { ...requestOptions, signal } : requestOptions
     });
   }
 
@@ -4426,6 +4542,7 @@ var __exports__ = (() => {
       loaders = void 0;
     }
     options = options || {};
+    (options.signal || context?.signal)?.throwIfAborted();
     const url = file.url || "";
     const typedLoaders = loaders;
     const candidateLoaders = getLoadersFromContext(typedLoaders, context);
@@ -4454,6 +4571,7 @@ var __exports__ = (() => {
     return await parseFileWithLoader(loader, file, strictOptions, context);
   }
   async function parseFileWithLoader(loader, file, options, context) {
+    (options.signal || context.signal)?.throwIfAborted();
     validateWorkerVersion(loader);
     options = mergeOptions(loader.options, options);
     const loaderWithParser = await getLoaderImplementation(loader, options, context.url);
@@ -4535,8 +4653,10 @@ var __exports__ = (() => {
       options = loaders;
       loaders = void 0;
     }
+    (options?.signal || context?.signal)?.throwIfAborted();
     data = await data;
     options = options || {};
+    (options.signal || context?.signal)?.throwIfAborted();
     const url = getResourceUrl(data);
     const loader = await selectLoader(data, loaders, options);
     if (!loader) {
@@ -4586,7 +4706,7 @@ var __exports__ = (() => {
       options?.core?.transforms || []
     );
     if (loader.parseInBatches) {
-      if (loader.id === "csv" && isBrowser && canParseWithWorker(loader, options)) {
+      if (canParseWithWorkerInBatches(loader, options)) {
         return parseWithWorkerInBatches(loader, transformedIterator, options, context, parse);
       }
       return loader.parseInBatches(transformedIterator, options, context);
@@ -4841,6 +4961,8 @@ var __exports__ = (() => {
     if (parentContext) {
       return {
         ...parentContext,
+        signal: options.signal || parentContext.signal,
+        reportProgress: options.onProgress || parentContext.reportProgress,
         coreApi: parentContext.coreApi || context.coreApi || coreApi,
         _parse: parentContext._parse || context._parse,
         fetch: getFetchFunction(options, parentContext)
@@ -4848,6 +4970,8 @@ var __exports__ = (() => {
     }
     const newContext = {
       ...context,
+      signal: options.signal || context.signal,
+      reportProgress: options.onProgress || context.reportProgress,
       fetch: getFetchFunction(options, context),
       coreApi: context.coreApi || coreApi
     };
@@ -4885,8 +5009,10 @@ var __exports__ = (() => {
       options = loaders;
       loaders = void 0;
     }
+    (options?.signal || context?.signal)?.throwIfAborted();
     data = await data;
     options = options || {};
+    (options.signal || context?.signal)?.throwIfAborted();
     const url = getResourceUrl(data);
     const typedLoaders = loaders;
     const candidateLoaders = getLoadersFromContext(typedLoaders, context);
@@ -4915,6 +5041,7 @@ var __exports__ = (() => {
     return await parseWithLoader(loader, data, strictOptions, context);
   }
   async function parseWithLoader(loader, data, options, context) {
+    (options.signal || context.signal)?.throwIfAborted();
     validateWorkerVersion(loader);
     options = mergeOptions(loader.options, options);
     if (isResponse(data)) {