@loaders.gl/parquet 5.0.0-alpha.5 → 5.0.0-alpha.6

package/dist/delta-source-loader-types.js

@@ -4,7 +4,7 @@
 import { DeltaFormat } from "./delta-format.js";
 // __VERSION__ is injected by babel-plugin-version-inline
 // @ts-ignore TS2304: Cannot find name '__VERSION__'.
-const VERSION = typeof "5.0.0-alpha.5" !== 'undefined' ? "5.0.0-alpha.5" : 'latest';
+const VERSION = typeof "5.0.0-alpha.6" !== 'undefined' ? "5.0.0-alpha.6" : 'latest';
 /** Loads the parser-bearing Delta source implementation on demand. */
 async function preloadDeltaSourceLoader() {
     const { DeltaSourceLoaderWithParser } = await import('@loaders.gl/parquet/delta-source');

package/dist/delta-source.js

@@ -7,7 +7,7 @@ import { PARQUET_TABLE_QUERY_CAPABILITIES } from "./parquet-source-capabilities.
 import { DeltaFormat } from "./delta-format.js";
 // __VERSION__ is injected by babel-plugin-version-inline
 // @ts-ignore TS2304: Cannot find name '__VERSION__'.
-const VERSION = typeof "5.0.0-alpha.5" !== 'undefined' ? "5.0.0-alpha.5" : 'latest';
+const VERSION = typeof "5.0.0-alpha.6" !== 'undefined' ? "5.0.0-alpha.6" : 'latest';
 /** Lightweight Delta Lake snapshot source backed by a newline-delimited commit log. */
 export class DeltaTableSource extends DataSource {
     actionsPromises = new Map();

package/dist/geoparquet-loader.js

@@ -5,7 +5,7 @@ import { ParquetFormat } from "./parquet-format.js";
 import { PARQUET_LOADER_DEFAULT_OPTIONS } from "./parquet-loader-options.js";
 // __VERSION__ is injected by babel-plugin-version-inline
 // @ts-ignore TS2304: Cannot find name '__VERSION__'.
-const VERSION = typeof "5.0.0-alpha.5" !== 'undefined' ? "5.0.0-alpha.5" : 'latest';
+const VERSION = typeof "5.0.0-alpha.6" !== 'undefined' ? "5.0.0-alpha.6" : 'latest';
 /** Preloads the parser-bearing GeoParquet loader implementation. */
 async function preloadGeoParquetLoader() {
     const { GeoParquetLoaderWithParser } = await import("./geoparquet-loader-with-parser.js");

package/dist/index.cjs

@@ -73,7 +73,7 @@ var init_parquet_loader_base = __esm({
     "use strict";
     init_parquet_format();
     init_parquet_loader_options();
-    VERSION = true ? "5.0.0-alpha.5" : "latest";
+    VERSION = true ? "5.0.0-alpha.6" : "latest";
     PARQUET_LOADER_BASE = {
       ...ParquetFormat,
       dataType: null,
@@ -497,6 +497,10 @@ var init_worker_thread = __esm({
       name;
       source;
       url;
+      /** Lazily resolves the classic worker URL used as a fallback. */
+      getUrl;
+      /** Creates a browser Worker directly. */
+      loadWorker;
       terminated = false;
       worker;
       onMessage;
@@ -507,11 +511,13 @@ var init_worker_thread = __esm({
         return typeof Worker !== "undefined" && isBrowser2 || typeof NodeWorker !== "undefined" && !isBrowser2;
       }
       constructor(props) {
-        const { name, source, url } = props;
-        assert(source || url);
+        const { name, source, url, getUrl, loadWorker } = props;
+        assert(source || url || getUrl || loadWorker);
         this.name = name;
         this.source = source;
         this.url = url;
+        this.getUrl = getUrl;
+        this.loadWorker = loadWorker;
         this.onMessage = NOOP;
         this.onError = (error) => console.log(error);
         this.worker = isBrowser2 ? this._createBrowserWorker() : this._createNodeWorker();
@@ -557,7 +563,7 @@ var init_worker_thread = __esm({
        */
       _getErrorFromErrorEvent(event) {
         let message = "Failed to load ";
-        message += `worker ${this.name} from ${this.url}. `;
+        message += `worker ${this.name} from ${this.url || "built-in worker factory"}. `;
         if (event.message) {
           message += `${event.message} in `;
         }
@@ -570,8 +576,26 @@ var init_worker_thread = __esm({
        * Creates a worker thread on the browser
        */
       _createBrowserWorker() {
-        this._loadableURL = getLoadableWorkerURL({ source: this.source, url: this.url });
+        var _a;
+        if (this.loadWorker) {
+          try {
+            const worker2 = this.loadWorker();
+            if (worker2) {
+              return this._initializeBrowserWorker(worker2);
+            }
+          } catch (error) {
+            if (!this.source && !this.url && !this.getUrl) {
+              throw error;
+            }
+          }
+        }
+        const url = this.url || ((_a = this.getUrl) == null ? void 0 : _a.call(this));
+        this._loadableURL = getLoadableWorkerURL({ source: this.source, url });
         const worker = new Worker(this._loadableURL, { name: this.name });
+        return this._initializeBrowserWorker(worker);
+      }
+      /** Attaches protocol handlers to a browser worker. */
+      _initializeBrowserWorker(worker) {
         worker.onmessage = (event) => {
           if (!event.data) {
             this.onError(new Error("No data received"));
@@ -630,6 +654,10 @@ var init_worker_pool = __esm({
       source;
       // | Function;
       url;
+      /** Lazily resolves a classic worker fallback URL. */
+      getUrl;
+      /** Creates a browser Worker directly. */
+      loadWorker;
       maxConcurrency = 1;
       maxMobileConcurrency = 1;
       onDebug = () => {
@@ -651,6 +679,8 @@ var init_worker_pool = __esm({
       constructor(props) {
         this.source = props.source;
         this.url = props.url;
+        this.getUrl = props.getUrl;
+        this.loadWorker = props.loadWorker;
         this.setProps(props);
       }
       /**
@@ -760,7 +790,13 @@ var init_worker_pool = __esm({
         if (this.count < this._getMaxConcurrency()) {
           this.count++;
           const name = `${this.name.toLowerCase()} (#${this.count} of ${this.maxConcurrency})`;
-          return new WorkerThread({ name, source: this.source, url: this.url });
+          return new WorkerThread({
+            name,
+            source: this.source,
+            url: this.url,
+            getUrl: this.getUrl,
+            loadWorker: this.loadWorker
+          });
         }
         return null;
       }
@@ -772,7 +808,22 @@ var init_worker_pool = __esm({
 });
 
 // ../worker-utils/src/lib/worker-farm/worker-farm.ts
-var DEFAULT_PROPS, WorkerFarm;
+function getWorkerPoolKey(options) {
+  const { name, source, url, urlKey, loadWorker } = options;
+  const sourceKey = source ? `source:${source}` : "";
+  const urlKeyPart = url || urlKey ? `url:${url || urlKey}` : "";
+  const loadWorkerKey = loadWorker ? `loadWorker:${getLoadWorkerId(loadWorker)}` : "";
+  return [name, sourceKey, urlKeyPart, loadWorkerKey].filter(Boolean).join("|");
+}
+function getLoadWorkerId(loadWorker) {
+  let loadWorkerId = loadWorkerIds.get(loadWorker);
+  if (!loadWorkerId) {
+    loadWorkerId = nextLoadWorkerId++;
+    loadWorkerIds.set(loadWorker, loadWorkerId);
+  }
+  return loadWorkerId;
+}
+var DEFAULT_PROPS, loadWorkerIds, nextLoadWorkerId, WorkerFarm;
 var init_worker_farm = __esm({
   "../worker-utils/src/lib/worker-farm/worker-farm.ts"() {
     "use strict";
@@ -785,6 +836,8 @@ var init_worker_farm = __esm({
       onDebug: () => {
       }
     };
+    loadWorkerIds = /* @__PURE__ */ new WeakMap();
+    nextLoadWorkerId = 1;
     WorkerFarm = class _WorkerFarm {
       props;
       workerPools = /* @__PURE__ */ new Map();
@@ -828,24 +881,28 @@ var init_worker_farm = __esm({
       }
       /**
        * Returns a worker pool for the specified worker
-       * @param options - only used first time for a specific worker name
-       * @param options.name - the name of the worker - used to identify worker pool
-       * @param options.url -
-       * @param options.source -
+       * @param options Worker target used to identify and initialize the pool.
+       * @param options.name Worker name used as the base pool identity.
+       * @param options.url Worker script URL.
+       * @param options.source Inline worker source.
+       * @param options.loadWorker Built-in browser worker factory.
        * @example
        *   const job = WorkerFarm.getWorkerFarm().getWorkerPool({name, url}).startJob(...);
        */
       getWorkerPool(options) {
-        const { name, source, url } = options;
-        let workerPool = this.workerPools.get(name);
+        const { name, source, url, getUrl, loadWorker } = options;
+        const workerPoolKey = getWorkerPoolKey(options);
+        let workerPool = this.workerPools.get(workerPoolKey);
         if (!workerPool) {
           workerPool = new WorkerPool({
             name,
             source,
-            url
+            url,
+            getUrl,
+            loadWorker
           });
           workerPool.setProps(this._getWorkerPoolProps());
-          this.workerPools.set(name, workerPool);
+          this.workerPools.set(workerPoolKey, workerPool);
         }
         return workerPool;
       }
@@ -866,35 +923,42 @@ function getWorkerName(worker) {
   const warning = worker.version !== VERSION3 ? ` (worker-utils@${VERSION3})` : "";
   return `${worker.name}@${worker.version}${warning}`;
 }
-function getWorkerURL(worker, options = {}) {
+function getCustomWorkerURL(worker, options = {}) {
   var _a;
   const workerOptions = options[worker.id] || {};
-  const workerFile = isBrowser2 ? `${worker.id}-worker.js` : worker.workerNode || `${worker.id}-worker-node.js`;
+  const workerFile = isBrowser2 ? worker.workerFile || `${worker.id}-worker.js` : worker.workerNode || `${worker.id}-worker-node.js`;
   let url = workerOptions.workerUrl;
-  if (!url && typeof worker.worker === "string") {
-    url = worker.worker;
-  }
   if (!url && worker.id === "compression") {
     url = options.workerUrl;
   }
   const workerType = options._workerType || ((_a = options == null ? void 0 : options.core) == null ? void 0 : _a._workerType);
-  if (workerType === "test") {
+  if (!url && workerType === "test") {
     if (isBrowser2) {
       url = `modules/${worker.module}/dist/${workerFile}`;
     } else {
       url = `modules/${worker.module}/src/workers/${worker.id}-worker-node.ts`;
     }
   }
-  if (!url) {
-    let version2 = worker.version;
-    if (version2 === "latest") {
-      version2 = NPM_TAG;
-    }
-    const versionTag = version2 ? `@${version2}` : "";
-    url = `https://unpkg.com/@loaders.gl/${worker.module}${versionTag}/dist/${workerFile}`;
+  return url || null;
+}
+function getWorkerURL(worker, options = {}) {
+  const customWorkerUrl = getCustomWorkerURL(worker, options);
+  const descriptorWorkerUrl = typeof worker.worker === "string" ? worker.worker : null;
+  const url = customWorkerUrl || descriptorWorkerUrl || getDefaultWorkerURL(worker, true);
+  assert(url);
+  return url;
+}
+function getDefaultWorkerURL(worker, warn = false) {
+  const workerFile = isBrowser2 ? worker.workerFile || `${worker.id}-worker.js` : worker.workerNode || `${worker.id}-worker-node.js`;
+  let version2 = worker.version;
+  if (version2 === "latest") {
+    version2 = NPM_TAG;
+  }
+  const versionTag = version2 ? `@${version2}` : "";
+  const url = `https://unpkg.com/@loaders.gl/${worker.module}${versionTag}/dist/${workerFile}`;
+  if (warn) {
     warnIfUsingNpmTagFallback(worker, url);
   }
-  assert(url);
   return url;
 }
 function warnIfUsingNpmTagFallback(worker, url) {
@@ -928,19 +992,16 @@ function canProcessOnWorker(worker, options) {
     return false;
   }
   const workerOptions = options == null ? void 0 : options[worker.id];
-  return Boolean((worker.worker || (workerOptions == null ? void 0 : workerOptions.workerUrl)) && (options == null ? void 0 : options.worker));
+  return Boolean(
+    (worker.worker || worker.loadWorker || (workerOptions == null ? void 0 : workerOptions.workerUrl)) && (options == null ? void 0 : options.worker)
+  );
 }
 async function processOnWorker(worker, data, options = {}, context = {}, jobContext = {}) {
   var _a, _b, _c;
   throwIfAborted(options.signal);
   const name = getWorkerName(worker);
   const workerFarm = WorkerFarm.getWorkerFarm(options);
-  const { source } = options;
-  const workerPoolProps = { name, source };
-  if (!source) {
-    workerPoolProps.url = getWorkerURL(worker, options);
-  }
-  const workerPool = workerFarm.getWorkerPool(workerPoolProps);
+  const workerPool = workerFarm.getWorkerPool(getWorkerPoolTarget(worker, options, name));
   const jobName = options.jobName || worker.name;
   const job = await workerPool.startJob(
     jobName,
@@ -983,6 +1044,24 @@ function createAbortError2(message) {
   error.name = "AbortError";
   return error;
 }
+function getWorkerPoolTarget(worker, options, name) {
+  if (options.source) {
+    return { name, source: options.source };
+  }
+  const customWorkerUrl = getCustomWorkerURL(worker, options);
+  if (customWorkerUrl) {
+    return { name, url: customWorkerUrl };
+  }
+  if (isBrowser2 && worker.loadWorker) {
+    return {
+      name,
+      loadWorker: worker.loadWorker,
+      getUrl: () => getWorkerURL(worker, options),
+      urlKey: getDefaultWorkerURL(worker)
+    };
+  }
+  return { name, url: getWorkerURL(worker, options) };
+}
 async function onMessage(context, job, type, payload) {
   switch (type) {
     case "done":
@@ -1015,6 +1094,7 @@ var init_process_on_worker = __esm({
     init_worker_farm();
     init_get_worker_url();
     init_get_transfer_list();
+    init_globals2();
   }
 });
 
@@ -1048,7 +1128,7 @@ async function parseWithWorker(loader, data, options, context, parseOnMainThread
   const result = await processOnWorker(
     loader,
     data,
-    { ...getWorkerOptions(options), signal: workerSignal },
+    { ...getWorkerOptions(options, loader), signal: workerSignal },
     {
       process: async (input, processOptions, _workerContext, parseContext) => {
         if (!parseOnMainThread) {
@@ -1081,12 +1161,19 @@ function callParseOnMainThread(parseOnMainThread, input, options, context) {
   }
   return parseOnMainThread(input, void 0, options, context);
 }
-function getWorkerOptions(options = {}) {
+function getWorkerOptions(options = {}, loader) {
   const serializedOptions = JSON.parse(JSON.stringify(options));
-  return {
+  const workerOptions = {
     ...serializedOptions.core,
     ...serializedOptions
   };
+  if (workerOptions.worker === "auto") {
+    workerOptions.worker = true;
+  }
+  if (loader) {
+    workerOptions._workerLoaderId = loader.id;
+  }
+  return workerOptions;
 }
 function getSerializableLoaderContext(context) {
   if (!context) {
@@ -7191,7 +7278,7 @@ var init_geoparquet_loader = __esm({
     "use strict";
     init_parquet_format();
     init_parquet_loader_options();
-    VERSION5 = true ? "5.0.0-alpha.5" : "latest";
+    VERSION5 = true ? "5.0.0-alpha.6" : "latest";
     GeoParquetLoader = {
       ...ParquetFormat,
       dataType: null,
@@ -25899,7 +25986,7 @@ function resolveParquetSourceWorkerUrl() {
   }
 }
 function getParquetSourceWorkerFile() {
-  return "./parquet-source-worker.js";
+  return "../dist/parquet-source-worker.js";
 }
 var import_meta5, PARQUET_SOURCE_WORKER_URL;
 var init_parquet_source_worker_url = __esm({
@@ -28013,7 +28100,7 @@ init_geoparquet_loader();
 
 // dist/parquet-js-loader-types.js
 init_parquet_format();
-var VERSION8 = true ? "5.0.0-alpha.5" : "latest";
+var VERSION8 = true ? "5.0.0-alpha.6" : "latest";
 var DEFAULT_PARQUET_JS_OPTIONS3 = {
   columns: void 0,
   preserveBinary: false,
@@ -28040,7 +28127,7 @@ var ParquetJSLoader2 = {
 
 // dist/parquet-source-loader-types.js
 init_parquet_format();
-var VERSION10 = true ? "5.0.0-alpha.5" : "latest";
+var VERSION10 = true ? "5.0.0-alpha.6" : "latest";
 async function preloadParquetSourceLoader2() {
   const { ParquetSourceLoaderWithParser: ParquetSourceLoaderWithParser2 } = await Promise.resolve().then(() => (init_parquet_source_loader(), parquet_source_loader_exports));
   return ParquetSourceLoaderWithParser2;
@@ -28164,7 +28251,7 @@ init_normalize_parquet_options2();
 init_parquet_format();
 
 // dist/lib/constants.js
-var VERSION11 = true ? "5.0.0-alpha.5" : "latest";
+var VERSION11 = true ? "5.0.0-alpha.6" : "latest";
 var PARQUET_MAGIC2 = "PAR1";
 var PARQUET_MAGIC_ENCRYPTED2 = "PARE";
 var PARQUET_RDLVL_TYPE2 = "INT32";
@@ -39027,7 +39114,7 @@ function formatDataType(dataType) {
 
 // dist/parquet-js-writer.js
 init_parquet_format();
-var VERSION12 = true ? "5.0.0-alpha.5" : "latest";
+var VERSION12 = true ? "5.0.0-alpha.6" : "latest";
 var DEFAULT_PARQUET_JS_OPTIONS4 = {};
 var ParquetJSWriter = {
   ...ParquetFormat,