@loaders.gl/scan 5.0.0-alpha.5 → 5.0.0-alpha.7

package/dist/iceberg.cjs

@@ -424,6 +424,10 @@ var init_worker_thread = __esm({
       name;
       source;
       url;
+      /** Lazily resolves the classic worker URL used as a fallback. */
+      getUrl;
+      /** Creates a browser Worker directly. */
+      loadWorker;
       terminated = false;
       worker;
       onMessage;
@@ -434,11 +438,13 @@ var init_worker_thread = __esm({
         return typeof Worker !== "undefined" && isBrowser2 || typeof NodeWorker !== "undefined" && !isBrowser2;
       }
       constructor(props) {
-        const { name, source, url } = props;
-        assert(source || url);
+        const { name, source, url, getUrl, loadWorker } = props;
+        assert(source || url || getUrl || loadWorker);
         this.name = name;
         this.source = source;
         this.url = url;
+        this.getUrl = getUrl;
+        this.loadWorker = loadWorker;
         this.onMessage = NOOP;
         this.onError = (error) => console.log(error);
         this.worker = isBrowser2 ? this._createBrowserWorker() : this._createNodeWorker();
@@ -484,7 +490,7 @@ var init_worker_thread = __esm({
        */
       _getErrorFromErrorEvent(event) {
         let message = "Failed to load ";
-        message += `worker ${this.name} from ${this.url}. `;
+        message += `worker ${this.name} from ${this.url || "built-in worker factory"}. `;
         if (event.message) {
           message += `${event.message} in `;
         }
@@ -497,8 +503,26 @@ var init_worker_thread = __esm({
        * Creates a worker thread on the browser
        */
       _createBrowserWorker() {
-        this._loadableURL = getLoadableWorkerURL({ source: this.source, url: this.url });
+        var _a;
+        if (this.loadWorker) {
+          try {
+            const worker2 = this.loadWorker();
+            if (worker2) {
+              return this._initializeBrowserWorker(worker2);
+            }
+          } catch (error) {
+            if (!this.source && !this.url && !this.getUrl) {
+              throw error;
+            }
+          }
+        }
+        const url = this.url || ((_a = this.getUrl) == null ? void 0 : _a.call(this));
+        this._loadableURL = getLoadableWorkerURL({ source: this.source, url });
         const worker = new Worker(this._loadableURL, { name: this.name });
+        return this._initializeBrowserWorker(worker);
+      }
+      /** Attaches protocol handlers to a browser worker. */
+      _initializeBrowserWorker(worker) {
         worker.onmessage = (event) => {
           if (!event.data) {
             this.onError(new Error("No data received"));
@@ -557,6 +581,10 @@ var init_worker_pool = __esm({
       source;
       // | Function;
       url;
+      /** Lazily resolves a classic worker fallback URL. */
+      getUrl;
+      /** Creates a browser Worker directly. */
+      loadWorker;
       maxConcurrency = 1;
       maxMobileConcurrency = 1;
       onDebug = () => {
@@ -565,6 +593,7 @@ var init_worker_pool = __esm({
       props = {};
       jobQueue = [];
       idleQueue = [];
+      activeJobs = /* @__PURE__ */ new Set();
       count = 0;
       isDestroyed = false;
       /** Checks if workers are supported on this platform */
@@ -578,13 +607,19 @@ var init_worker_pool = __esm({
       constructor(props) {
         this.source = props.source;
         this.url = props.url;
+        this.getUrl = props.getUrl;
+        this.loadWorker = props.loadWorker;
         this.setProps(props);
       }
       /**
-       * Terminates all workers in the pool
+       * Terminates all workers in the pool and aborts active jobs.
+       * @param reason Optional error delivered to active jobs.
        * @note Can free up significant memory
        */
-      destroy() {
+      destroy(reason = new Error("Worker pool was destroyed")) {
+        this.activeJobs.forEach((job) => job.abort(reason));
+        const queuedJobs = this.jobQueue.splice(0);
+        queuedJobs.forEach((queuedJob) => queuedJob.onReject(reason));
         this.idleQueue.forEach((worker) => worker.destroy());
         this.isDestroyed = true;
       }
@@ -607,8 +642,11 @@ var init_worker_pool = __esm({
         }
       }
       async startJob(name, onMessage2 = (job, type, data) => job.done(data), onError = (job, error) => job.error(error)) {
-        const startPromise = new Promise((onStart) => {
-          this.jobQueue.push({ name, onMessage: onMessage2, onError, onStart });
+        if (this.isDestroyed) {
+          return Promise.reject(new Error("Worker pool was destroyed"));
+        }
+        const startPromise = new Promise((onStart, onReject) => {
+          this.jobQueue.push({ name, onMessage: onMessage2, onError, onStart, onReject });
           return this;
         });
         this._startQueuedJob();
@@ -636,6 +674,7 @@ var init_worker_pool = __esm({
             backlog: this.jobQueue.length
           });
           const job = new WorkerJob(queuedJob.name, workerThread);
+          this.activeJobs.add(job);
           workerThread.ref();
           workerThread.onMessage = (data) => queuedJob.onMessage(job, data.type, data.payload);
           workerThread.onError = (error) => queuedJob.onError(job, error);
@@ -644,6 +683,7 @@ var init_worker_pool = __esm({
             await job.result;
           } catch {
           } finally {
+            this.activeJobs.delete(job);
             this.returnWorkerToQueue(workerThread);
           }
         }
@@ -687,7 +727,13 @@ var init_worker_pool = __esm({
         if (this.count < this._getMaxConcurrency()) {
           this.count++;
           const name = `${this.name.toLowerCase()} (#${this.count} of ${this.maxConcurrency})`;
-          return new WorkerThread({ name, source: this.source, url: this.url });
+          return new WorkerThread({
+            name,
+            source: this.source,
+            url: this.url,
+            getUrl: this.getUrl,
+            loadWorker: this.loadWorker
+          });
         }
         return null;
       }
@@ -699,7 +745,22 @@ var init_worker_pool = __esm({
 });
 
 // ../worker-utils/src/lib/worker-farm/worker-farm.ts
-var DEFAULT_PROPS, WorkerFarm;
+function getWorkerPoolKey(options) {
+  const { name, source, url, urlKey, loadWorker } = options;
+  const sourceKey = source ? `source:${source}` : "";
+  const urlKeyPart = url || urlKey ? `url:${url || urlKey}` : "";
+  const loadWorkerKey = loadWorker ? `loadWorker:${getLoadWorkerId(loadWorker)}` : "";
+  return [name, sourceKey, urlKeyPart, loadWorkerKey].filter(Boolean).join("|");
+}
+function getLoadWorkerId(loadWorker) {
+  let loadWorkerId = loadWorkerIds.get(loadWorker);
+  if (!loadWorkerId) {
+    loadWorkerId = nextLoadWorkerId++;
+    loadWorkerIds.set(loadWorker, loadWorkerId);
+  }
+  return loadWorkerId;
+}
+var DEFAULT_PROPS, loadWorkerIds, nextLoadWorkerId, WorkerFarm;
 var init_worker_farm = __esm({
   "../worker-utils/src/lib/worker-farm/worker-farm.ts"() {
     "use strict";
@@ -712,6 +773,8 @@ var init_worker_farm = __esm({
       onDebug: () => {
       }
     };
+    loadWorkerIds = /* @__PURE__ */ new WeakMap();
+    nextLoadWorkerId = 1;
     WorkerFarm = class _WorkerFarm {
       props;
       workerPools = /* @__PURE__ */ new Map();
@@ -734,12 +797,13 @@ var init_worker_farm = __esm({
         this.workerPools = /* @__PURE__ */ new Map();
       }
       /**
-       * Terminate all workers in the farm
+       * Terminate all workers in the farm and abort active jobs.
+       * @param reason Optional error delivered to active jobs.
        * @note Can free up significant memory
        */
-      destroy() {
+      destroy(reason) {
         for (const workerPool of this.workerPools.values()) {
-          workerPool.destroy();
+          workerPool.destroy(reason);
         }
         this.workerPools = /* @__PURE__ */ new Map();
       }
@@ -755,24 +819,28 @@ var init_worker_farm = __esm({
       }
       /**
        * Returns a worker pool for the specified worker
-       * @param options - only used first time for a specific worker name
-       * @param options.name - the name of the worker - used to identify worker pool
-       * @param options.url -
-       * @param options.source -
+       * @param options Worker target used to identify and initialize the pool.
+       * @param options.name Worker name used as the base pool identity.
+       * @param options.url Worker script URL.
+       * @param options.source Inline worker source.
+       * @param options.loadWorker Built-in browser worker factory.
        * @example
        *   const job = WorkerFarm.getWorkerFarm().getWorkerPool({name, url}).startJob(...);
        */
       getWorkerPool(options) {
-        const { name, source, url } = options;
-        let workerPool = this.workerPools.get(name);
+        const { name, source, url, getUrl, loadWorker } = options;
+        const workerPoolKey = getWorkerPoolKey(options);
+        let workerPool = this.workerPools.get(workerPoolKey);
         if (!workerPool) {
           workerPool = new WorkerPool({
             name,
             source,
-            url
+            url,
+            getUrl,
+            loadWorker
           });
           workerPool.setProps(this._getWorkerPoolProps());
-          this.workerPools.set(name, workerPool);
+          this.workerPools.set(workerPoolKey, workerPool);
         }
         return workerPool;
       }
@@ -793,35 +861,42 @@ function getWorkerName(worker) {
   const warning = worker.version !== VERSION2 ? ` (worker-utils@${VERSION2})` : "";
   return `${worker.name}@${worker.version}${warning}`;
 }
-function getWorkerURL(worker, options = {}) {
+function getCustomWorkerURL(worker, options = {}) {
   var _a;
   const workerOptions = options[worker.id] || {};
-  const workerFile = isBrowser2 ? `${worker.id}-worker.js` : worker.workerNode || `${worker.id}-worker-node.js`;
+  const workerFile = isBrowser2 ? worker.workerFile || `${worker.id}-worker.js` : worker.workerNode || `${worker.id}-worker-node.js`;
   let url = workerOptions.workerUrl;
-  if (!url && typeof worker.worker === "string") {
-    url = worker.worker;
-  }
   if (!url && worker.id === "compression") {
     url = options.workerUrl;
   }
   const workerType = options._workerType || ((_a = options == null ? void 0 : options.core) == null ? void 0 : _a._workerType);
-  if (workerType === "test") {
+  if (!url && workerType === "test") {
     if (isBrowser2) {
       url = `modules/${worker.module}/dist/${workerFile}`;
     } else {
       url = `modules/${worker.module}/src/workers/${worker.id}-worker-node.ts`;
     }
   }
-  if (!url) {
-    let version2 = worker.version;
-    if (version2 === "latest") {
-      version2 = NPM_TAG;
-    }
-    const versionTag = version2 ? `@${version2}` : "";
-    url = `https://unpkg.com/@loaders.gl/${worker.module}${versionTag}/dist/${workerFile}`;
+  return url || null;
+}
+function getWorkerURL(worker, options = {}) {
+  const customWorkerUrl = getCustomWorkerURL(worker, options);
+  const descriptorWorkerUrl = typeof worker.worker === "string" ? worker.worker : null;
+  const url = customWorkerUrl || descriptorWorkerUrl || getDefaultWorkerURL(worker, true);
+  assert(url);
+  return url;
+}
+function getDefaultWorkerURL(worker, warn = false) {
+  const workerFile = isBrowser2 ? worker.workerFile || `${worker.id}-worker.js` : worker.workerNode || `${worker.id}-worker-node.js`;
+  let version2 = worker.version;
+  if (version2 === "latest") {
+    version2 = NPM_TAG;
+  }
+  const versionTag = version2 ? `@${version2}` : "";
+  const url = `https://unpkg.com/@loaders.gl/${worker.module}${versionTag}/dist/${workerFile}`;
+  if (warn) {
     warnIfUsingNpmTagFallback(worker, url);
   }
-  assert(url);
   return url;
 }
 function warnIfUsingNpmTagFallback(worker, url) {
@@ -855,19 +930,16 @@ function canProcessOnWorker(worker, options) {
     return false;
   }
   const workerOptions = options == null ? void 0 : options[worker.id];
-  return Boolean((worker.worker || (workerOptions == null ? void 0 : workerOptions.workerUrl)) && (options == null ? void 0 : options.worker));
+  return Boolean(
+    (worker.worker || worker.loadWorker || (workerOptions == null ? void 0 : workerOptions.workerUrl)) && (options == null ? void 0 : options.worker)
+  );
 }
 async function processOnWorker(worker, data, options = {}, context = {}, jobContext = {}) {
   var _a, _b, _c;
   throwIfAborted(options.signal);
   const name = getWorkerName(worker);
   const workerFarm = WorkerFarm.getWorkerFarm(options);
-  const { source } = options;
-  const workerPoolProps = { name, source };
-  if (!source) {
-    workerPoolProps.url = getWorkerURL(worker, options);
-  }
-  const workerPool = workerFarm.getWorkerPool(workerPoolProps);
+  const workerPool = workerFarm.getWorkerPool(getWorkerPoolTarget(worker, options, name));
   const jobName = options.jobName || worker.name;
   const job = await workerPool.startJob(
     jobName,
@@ -910,6 +982,24 @@ function createAbortError2(message) {
   error.name = "AbortError";
   return error;
 }
+function getWorkerPoolTarget(worker, options, name) {
+  if (options.source) {
+    return { name, source: options.source };
+  }
+  const customWorkerUrl = getCustomWorkerURL(worker, options);
+  if (customWorkerUrl) {
+    return { name, url: customWorkerUrl };
+  }
+  if (isBrowser2 && worker.loadWorker) {
+    return {
+      name,
+      loadWorker: worker.loadWorker,
+      getUrl: () => getWorkerURL(worker, options),
+      urlKey: getDefaultWorkerURL(worker)
+    };
+  }
+  return { name, url: getWorkerURL(worker, options) };
+}
 async function onMessage(context, job, type, payload) {
   switch (type) {
     case "done":
@@ -942,6 +1032,7 @@ var init_process_on_worker = __esm({
     init_worker_farm();
     init_get_worker_url();
     init_get_transfer_list();
+    init_globals2();
   }
 });
 
@@ -956,13 +1047,13 @@ var init_src = __esm({
 
 // ../loader-utils/src/lib/worker-loader-utils/parse-with-worker.ts
 function canParseWithWorker(loader, options) {
-  var _a, _b;
+  var _a;
   const workerOptions = getWorkerOptions(options);
   const nodeWorkers = workerOptions._nodeWorkers;
   if (!isBrowser2 && !nodeWorkers) {
     return false;
   }
-  if (loader.id === "excel" && ((_a = options == null ? void 0 : options.excel) == null ? void 0 : _a.shape) === "arrow-table" || loader.id === "ply" && ((_b = options == null ? void 0 : options.ply) == null ? void 0 : _b.shape) === "arrow-table") {
+  if (loader.id === "las" && (((_a = options == null ? void 0 : options.las) == null ? void 0 : _a.shape) ?? "arrow-table") === "arrow-table") {
     return false;
   }
   if (loader.id === "csv" && !shouldParseCSVWithWorker(options)) {
@@ -975,7 +1066,7 @@ async function parseWithWorker(loader, data, options, context, parseOnMainThread
   const result = await processOnWorker(
     loader,
     data,
-    { ...getWorkerOptions(options), signal: workerSignal },
+    { ...getWorkerOptions(options, loader), signal: workerSignal },
     {
       process: async (input, processOptions, _workerContext, parseContext) => {
         if (!parseOnMainThread) {
@@ -1008,12 +1099,19 @@ function callParseOnMainThread(parseOnMainThread, input, options, context) {
   }
   return parseOnMainThread(input, void 0, options, context);
 }
-function getWorkerOptions(options = {}) {
+function getWorkerOptions(options = {}, loader) {
   const serializedOptions = JSON.parse(JSON.stringify(options));
-  return {
+  const workerOptions = {
     ...serializedOptions.core,
     ...serializedOptions
   };
+  if (workerOptions.worker === "auto") {
+    workerOptions.worker = true;
+  }
+  if (loader) {
+    workerOptions._workerLoaderId = loader.id;
+  }
+  return workerOptions;
 }
 function getSerializableLoaderContext(context) {
   if (!context) {
@@ -6593,6 +6691,9 @@ function getDataTypeFromArray(array) {
   return { type: "null", nullable: true };
 }
 function getDataTypeFromTypedArray(array) {
+  if (typeof Float16Array !== "undefined" && array instanceof Float16Array) {
+    return "float16";
+  }
   switch (array.constructor) {
     case Int8Array:
       return "int8";
@@ -6634,15 +6735,19 @@ function getArrayTypeFromDataType(type, nullable) {
         return Float32Array;
       case "float64":
         return Float64Array;
+      case "float16":
+        return (0, import_types.getFloat16ArrayConstructor)();
       default:
         break;
     }
   }
   return Array;
 }
+var import_types;
 var init_data_type = __esm({
   "../schema-utils/src/lib/schema/data-type.ts"() {
     "use strict";
+    import_types = require("@math.gl/types");
   }
 });
 
@@ -6691,6 +6796,13 @@ function serializeArrowType(arrowType) {
   if (isArrowViewType(arrowType, "Utf8View")) {
     return "utf8-view";
   }
+  if (isOptionalArrowType(arrowType, "LargeList")) {
+    const largeListType = arrowType;
+    return {
+      type: "large-list",
+      children: [serializeArrowField(largeListType.valueField)]
+    };
+  }
   switch (arrowType.constructor) {
     case arrow2.Null:
       return "null";
@@ -6829,12 +6941,6 @@ function serializeArrowType(arrowType) {
         type: "list",
         children: [serializeArrowField(listField)]
       };
-    case arrow2.LargeList:
-      const largeListType = arrowType;
-      return {
-        type: "large-list",
-        children: [serializeArrowField(largeListType.valueField)]
-      };
     case arrow2.FixedSizeList:
       const fixedSizeList = arrowType;
       return {
@@ -6887,7 +6993,13 @@ function deserializeArrowType(dataType, options) {
       }
       case "large-list": {
         const field = deserializeArrowField(dataType.children[0], options);
-        return new arrow2.LargeList(field);
+        const LargeList = getOptionalArrowConstructor("LargeList");
+        if (!LargeList) {
+          throw new Error(
+            "LargeList requires an Apache Arrow runtime that exports the LargeList constructor"
+          );
+        }
+        return new LargeList(field);
       }
       case "fixed-size-list": {
         const child = deserializeArrowField(dataType.children[0], options);
@@ -6983,11 +7095,11 @@ function deserializeArrowType(dataType, options) {
       throw new Error("array type not supported");
   }
 }
-function getArrowViewConstructors() {
-  return arrow2;
+function getOptionalArrowConstructor(typeName) {
+  return arrow2[typeName];
 }
 function isArrowViewType(arrowType, viewTypeName) {
-  const constructor = getArrowViewConstructors()[viewTypeName];
+  const constructor = getOptionalArrowConstructor(viewTypeName);
   if (constructor && arrowType instanceof constructor) {
     return true;
   }
@@ -6995,12 +7107,16 @@ function isArrowViewType(arrowType, viewTypeName) {
   const guard = viewTypeName === "BinaryView" ? guards.isBinaryView : guards.isUtf8View;
   return Boolean(guard == null ? void 0 : guard(arrowType));
 }
+function isOptionalArrowType(arrowType, typeName) {
+  const constructor = getOptionalArrowConstructor(typeName);
+  return Boolean(constructor && arrowType instanceof constructor);
+}
 function makeVariableWidthArrowType(viewTypeName, fallbackType, options) {
   const viewTypeMode = (options == null ? void 0 : options.viewTypes) || "never";
   if (viewTypeMode === "never") {
     return fallbackType;
   }
-  const constructor = getArrowViewConstructors()[viewTypeName];
+  const constructor = getOptionalArrowConstructor(viewTypeName);
   if (constructor) {
     return new constructor();
   }
@@ -7012,7 +7128,7 @@ function makeVariableWidthArrowType(viewTypeName, fallbackType, options) {
   return fallbackType;
 }
 function makeRequiredArrowViewType(viewTypeName) {
-  const constructor = getArrowViewConstructors()[viewTypeName];
+  const constructor = getOptionalArrowConstructor(viewTypeName);
   if (!constructor) {
     throw new Error(
       `${viewTypeName} requires apache-arrow 21.2.0 or later; the installed runtime does not support it`
@@ -16509,7 +16625,7 @@ var init_parquet_loader_options = __esm({
     PARQUET_LOADER_DEFAULT_OPTIONS = {
       columns: void 0,
       preserveBinary: false,
-      shape: "object-row-table"
+      shape: "arrow-table"
     };
   }
 });
@@ -16538,6 +16654,15 @@ var init_parquet_loader_base = __esm({
   }
 });
 
+// ../parquet/src/parquet-source-worker-factory.ts
+var PARQUET_SOURCE_WORKER_LOAD_WORKER;
+var init_parquet_source_worker_factory = __esm({
+  "../parquet/src/parquet-source-worker-factory.ts"() {
+    "use strict";
+    PARQUET_SOURCE_WORKER_LOAD_WORKER = void 0;
+  }
+});
+
 // ../parquet/src/parquet-source-worker-url.ts
 function resolveParquetSourceWorkerUrl() {
   if (!import_meta.url) {
@@ -16550,7 +16675,7 @@ function resolveParquetSourceWorkerUrl() {
   }
 }
 function getParquetSourceWorkerFile() {
-  return "./parquet-source-worker.js";
+  return "../dist/parquet-source-worker.js";
 }
 var import_meta, PARQUET_SOURCE_WORKER_URL;
 var init_parquet_source_worker_url = __esm({
@@ -16578,12 +16703,14 @@ var init_parquet_source_worker_client = __esm({
     "use strict";
     init_src2();
     init_parquet_loader_base();
+    init_parquet_source_worker_factory();
     init_parquet_source_worker_url();
     PARQUET_SOURCE_WORKER = {
       ...PARQUET_LOADER_BASE,
       id: "parquet-source",
       name: "ParquetSource",
-      worker: PARQUET_SOURCE_WORKER_URL
+      worker: PARQUET_SOURCE_WORKER_URL,
+      loadWorker: PARQUET_SOURCE_WORKER_LOAD_WORKER
     };
   }
 });