@loaders.gl/parquet 5.0.0-alpha.6 → 5.0.0-alpha.7

package/dist/delta-source-loader-types.js

@@ -4,7 +4,7 @@
 import { DeltaFormat } from "./delta-format.js";
 // __VERSION__ is injected by babel-plugin-version-inline
 // @ts-ignore TS2304: Cannot find name '__VERSION__'.
-const VERSION = typeof "5.0.0-alpha.6" !== 'undefined' ? "5.0.0-alpha.6" : 'latest';
+const VERSION = typeof "5.0.0-alpha.7" !== 'undefined' ? "5.0.0-alpha.7" : 'latest';
 /** Loads the parser-bearing Delta source implementation on demand. */
 async function preloadDeltaSourceLoader() {
     const { DeltaSourceLoaderWithParser } = await import('@loaders.gl/parquet/delta-source');

package/dist/delta-source.js

@@ -7,7 +7,7 @@ import { PARQUET_TABLE_QUERY_CAPABILITIES } from "./parquet-source-capabilities.
 import { DeltaFormat } from "./delta-format.js";
 // __VERSION__ is injected by babel-plugin-version-inline
 // @ts-ignore TS2304: Cannot find name '__VERSION__'.
-const VERSION = typeof "5.0.0-alpha.6" !== 'undefined' ? "5.0.0-alpha.6" : 'latest';
+const VERSION = typeof "5.0.0-alpha.7" !== 'undefined' ? "5.0.0-alpha.7" : 'latest';
 /** Lightweight Delta Lake snapshot source backed by a newline-delimited commit log. */
 export class DeltaTableSource extends DataSource {
     actionsPromises = new Map();

package/dist/geoparquet-loader-with-parser.d.ts

@@ -18,7 +18,7 @@ export declare const GeoParquetLoaderWithParser: {
         readonly parquet: {
             readonly columns: undefined;
             readonly preserveBinary: false;
-            readonly shape: "object-row-table";
+            readonly shape: "arrow-table";
         };
     };
     readonly name: "Apache Parquet";

package/dist/geoparquet-loader.d.ts

@@ -18,7 +18,7 @@ declare function preloadGeoParquetLoader(): Promise<{
         readonly parquet: {
             readonly columns: undefined;
             readonly preserveBinary: false;
-            readonly shape: "object-row-table";
+            readonly shape: "arrow-table";
         };
     };
     readonly name: "Apache Parquet";
@@ -42,7 +42,7 @@ export declare const GeoParquetLoader: {
         readonly parquet: {
             readonly columns: undefined;
             readonly preserveBinary: false;
-            readonly shape: "object-row-table";
+            readonly shape: "arrow-table";
         };
     };
     readonly preload: typeof preloadGeoParquetLoader;

package/dist/geoparquet-loader.js

@@ -5,7 +5,7 @@ import { ParquetFormat } from "./parquet-format.js";
 import { PARQUET_LOADER_DEFAULT_OPTIONS } from "./parquet-loader-options.js";
 // __VERSION__ is injected by babel-plugin-version-inline
 // @ts-ignore TS2304: Cannot find name '__VERSION__'.
-const VERSION = typeof "5.0.0-alpha.6" !== 'undefined' ? "5.0.0-alpha.6" : 'latest';
+const VERSION = typeof "5.0.0-alpha.7" !== 'undefined' ? "5.0.0-alpha.7" : 'latest';
 /** Preloads the parser-bearing GeoParquet loader implementation. */
 async function preloadGeoParquetLoader() {
     const { GeoParquetLoaderWithParser } = await import("./geoparquet-loader-with-parser.js");

package/dist/index.cjs

@@ -61,7 +61,7 @@ var init_parquet_loader_options = __esm({
     PARQUET_LOADER_DEFAULT_OPTIONS = {
       columns: void 0,
       preserveBinary: false,
-      shape: "object-row-table"
+      shape: "arrow-table"
     };
   }
 });
@@ -73,7 +73,7 @@ var init_parquet_loader_base = __esm({
     "use strict";
     init_parquet_format();
     init_parquet_loader_options();
-    VERSION = true ? "5.0.0-alpha.6" : "latest";
+    VERSION = true ? "5.0.0-alpha.7" : "latest";
     PARQUET_LOADER_BASE = {
       ...ParquetFormat,
       dataType: null,
@@ -666,6 +666,7 @@ var init_worker_pool = __esm({
       props = {};
       jobQueue = [];
       idleQueue = [];
+      activeJobs = /* @__PURE__ */ new Set();
       count = 0;
       isDestroyed = false;
       /** Checks if workers are supported on this platform */
@@ -684,10 +685,14 @@ var init_worker_pool = __esm({
         this.setProps(props);
       }
       /**
-       * Terminates all workers in the pool
+       * Terminates all workers in the pool and aborts active jobs.
+       * @param reason Optional error delivered to active jobs.
        * @note Can free up significant memory
        */
-      destroy() {
+      destroy(reason = new Error("Worker pool was destroyed")) {
+        this.activeJobs.forEach((job) => job.abort(reason));
+        const queuedJobs = this.jobQueue.splice(0);
+        queuedJobs.forEach((queuedJob) => queuedJob.onReject(reason));
         this.idleQueue.forEach((worker) => worker.destroy());
         this.isDestroyed = true;
       }
@@ -710,8 +715,11 @@ var init_worker_pool = __esm({
         }
       }
       async startJob(name, onMessage2 = (job, type, data) => job.done(data), onError = (job, error) => job.error(error)) {
-        const startPromise = new Promise((onStart) => {
-          this.jobQueue.push({ name, onMessage: onMessage2, onError, onStart });
+        if (this.isDestroyed) {
+          return Promise.reject(new Error("Worker pool was destroyed"));
+        }
+        const startPromise = new Promise((onStart, onReject) => {
+          this.jobQueue.push({ name, onMessage: onMessage2, onError, onStart, onReject });
           return this;
         });
         this._startQueuedJob();
@@ -739,6 +747,7 @@ var init_worker_pool = __esm({
             backlog: this.jobQueue.length
           });
           const job = new WorkerJob(queuedJob.name, workerThread);
+          this.activeJobs.add(job);
           workerThread.ref();
           workerThread.onMessage = (data) => queuedJob.onMessage(job, data.type, data.payload);
           workerThread.onError = (error) => queuedJob.onError(job, error);
@@ -747,6 +756,7 @@ var init_worker_pool = __esm({
             await job.result;
           } catch {
           } finally {
+            this.activeJobs.delete(job);
             this.returnWorkerToQueue(workerThread);
           }
         }
@@ -860,12 +870,13 @@ var init_worker_farm = __esm({
         this.workerPools = /* @__PURE__ */ new Map();
       }
       /**
-       * Terminate all workers in the farm
+       * Terminate all workers in the farm and abort active jobs.
+       * @param reason Optional error delivered to active jobs.
        * @note Can free up significant memory
        */
-      destroy() {
+      destroy(reason) {
         for (const workerPool of this.workerPools.values()) {
-          workerPool.destroy();
+          workerPool.destroy(reason);
         }
         this.workerPools = /* @__PURE__ */ new Map();
       }
@@ -1109,13 +1120,13 @@ var init_src = __esm({
 
 // ../loader-utils/src/lib/worker-loader-utils/parse-with-worker.ts
 function canParseWithWorker(loader, options) {
-  var _a, _b;
+  var _a;
   const workerOptions = getWorkerOptions(options);
   const nodeWorkers = workerOptions._nodeWorkers;
   if (!isBrowser2 && !nodeWorkers) {
     return false;
   }
-  if (loader.id === "excel" && ((_a = options == null ? void 0 : options.excel) == null ? void 0 : _a.shape) === "arrow-table" || loader.id === "ply" && ((_b = options == null ? void 0 : options.ply) == null ? void 0 : _b.shape) === "arrow-table") {
+  if (loader.id === "las" && (((_a = options == null ? void 0 : options.las) == null ? void 0 : _a.shape) ?? "arrow-table") === "arrow-table") {
     return false;
   }
   if (loader.id === "csv" && !shouldParseCSVWithWorker(options)) {
@@ -2937,6 +2948,9 @@ function getDataTypeFromArray(array) {
   return { type: "null", nullable: true };
 }
 function getDataTypeFromTypedArray(array) {
+  if (typeof Float16Array !== "undefined" && array instanceof Float16Array) {
+    return "float16";
+  }
   switch (array.constructor) {
     case Int8Array:
       return "int8";
@@ -2978,15 +2992,19 @@ function getArrayTypeFromDataType(type, nullable) {
         return Float32Array;
       case "float64":
         return Float64Array;
+      case "float16":
+        return (0, import_types.getFloat16ArrayConstructor)();
       default:
         break;
     }
   }
   return Array;
 }
+var import_types;
 var init_data_type = __esm({
   "../schema-utils/src/lib/schema/data-type.ts"() {
     "use strict";
+    import_types = require("@math.gl/types");
   }
 });
 
@@ -3038,6 +3056,13 @@ function serializeArrowType(arrowType) {
   if (isArrowViewType(arrowType, "Utf8View")) {
     return "utf8-view";
   }
+  if (isOptionalArrowType(arrowType, "LargeList")) {
+    const largeListType = arrowType;
+    return {
+      type: "large-list",
+      children: [serializeArrowField(largeListType.valueField)]
+    };
+  }
   switch (arrowType.constructor) {
     case arrow.Null:
       return "null";
@@ -3176,12 +3201,6 @@ function serializeArrowType(arrowType) {
         type: "list",
         children: [serializeArrowField(listField)]
       };
-    case arrow.LargeList:
-      const largeListType = arrowType;
-      return {
-        type: "large-list",
-        children: [serializeArrowField(largeListType.valueField)]
-      };
     case arrow.FixedSizeList:
       const fixedSizeList = arrowType;
       return {
@@ -3234,7 +3253,13 @@ function deserializeArrowType(dataType, options) {
       }
       case "large-list": {
         const field = deserializeArrowField(dataType.children[0], options);
-        return new arrow.LargeList(field);
+        const LargeList = getOptionalArrowConstructor("LargeList");
+        if (!LargeList) {
+          throw new Error(
+            "LargeList requires an Apache Arrow runtime that exports the LargeList constructor"
+          );
+        }
+        return new LargeList(field);
       }
       case "fixed-size-list": {
         const child = deserializeArrowField(dataType.children[0], options);
@@ -3330,11 +3355,11 @@ function deserializeArrowType(dataType, options) {
       throw new Error("array type not supported");
   }
 }
-function getArrowViewConstructors() {
-  return arrow;
+function getOptionalArrowConstructor(typeName) {
+  return arrow[typeName];
 }
 function isArrowViewType(arrowType, viewTypeName) {
-  const constructor = getArrowViewConstructors()[viewTypeName];
+  const constructor = getOptionalArrowConstructor(viewTypeName);
   if (constructor && arrowType instanceof constructor) {
     return true;
   }
@@ -3342,12 +3367,16 @@ function isArrowViewType(arrowType, viewTypeName) {
   const guard = viewTypeName === "BinaryView" ? guards.isBinaryView : guards.isUtf8View;
   return Boolean(guard == null ? void 0 : guard(arrowType));
 }
+function isOptionalArrowType(arrowType, typeName) {
+  const constructor = getOptionalArrowConstructor(typeName);
+  return Boolean(constructor && arrowType instanceof constructor);
+}
 function makeVariableWidthArrowType(viewTypeName, fallbackType, options) {
   const viewTypeMode = (options == null ? void 0 : options.viewTypes) || "never";
   if (viewTypeMode === "never") {
     return fallbackType;
   }
-  const constructor = getArrowViewConstructors()[viewTypeName];
+  const constructor = getOptionalArrowConstructor(viewTypeName);
   if (constructor) {
     return new constructor();
   }
@@ -3359,7 +3388,7 @@ function makeVariableWidthArrowType(viewTypeName, fallbackType, options) {
   return fallbackType;
 }
 function makeRequiredArrowViewType(viewTypeName) {
-  const constructor = getArrowViewConstructors()[viewTypeName];
+  const constructor = getOptionalArrowConstructor(viewTypeName);
   if (!constructor) {
     throw new Error(
       `${viewTypeName} requires apache-arrow 21.2.0 or later; the installed runtime does not support it`
@@ -6265,7 +6294,7 @@ var init_parquet_loader_options2 = __esm({
     PARQUET_LOADER_DEFAULT_OPTIONS2 = {
       columns: void 0,
       preserveBinary: false,
-      shape: "object-row-table"
+      shape: "arrow-table"
     };
   }
 });
@@ -7278,7 +7307,7 @@ var init_geoparquet_loader = __esm({
     "use strict";
     init_parquet_format();
     init_parquet_loader_options();
-    VERSION5 = true ? "5.0.0-alpha.6" : "latest";
+    VERSION5 = true ? "5.0.0-alpha.7" : "latest";
     GeoParquetLoader = {
       ...ParquetFormat,
       dataType: null,
@@ -24792,7 +24821,7 @@ var init_parquet_js_loader_types = __esm({
       columns: void 0,
       preserveBinary: false,
       verifyFooterSignature: true,
-      shape: "object-row-table"
+      shape: "arrow-table"
     };
     ParquetJSLoader = {
       ...ParquetFormat2,
@@ -24831,7 +24860,8 @@ var init_parquet_js_loader = __esm({
     DEFAULT_PARQUET_JS_OPTIONS2 = {
       columns: void 0,
       preserveBinary: false,
-      verifyFooterSignature: true
+      verifyFooterSignature: true,
+      shape: "arrow-table"
     };
     ParquetJSLoaderWithParser = {
       ...ParquetJSLoaderMetadataWithoutPreload,
@@ -25974,6 +26004,15 @@ var init_parquet_page_index = __esm({
   }
 });
 
+// src/parquet-source-worker-factory.ts
+var PARQUET_SOURCE_WORKER_LOAD_WORKER;
+var init_parquet_source_worker_factory = __esm({
+  "src/parquet-source-worker-factory.ts"() {
+    "use strict";
+    PARQUET_SOURCE_WORKER_LOAD_WORKER = void 0;
+  }
+});
+
 // src/parquet-source-worker-url.ts
 function resolveParquetSourceWorkerUrl() {
   if (!import_meta5.url) {
@@ -26014,12 +26053,14 @@ var init_parquet_source_worker_client = __esm({
     "use strict";
     init_src2();
     init_parquet_loader_base2();
+    init_parquet_source_worker_factory();
     init_parquet_source_worker_url();
     PARQUET_SOURCE_WORKER = {
       ...PARQUET_LOADER_BASE2,
       id: "parquet-source",
       name: "ParquetSource",
-      worker: PARQUET_SOURCE_WORKER_URL
+      worker: PARQUET_SOURCE_WORKER_URL,
+      loadWorker: PARQUET_SOURCE_WORKER_LOAD_WORKER
     };
   }
 });
@@ -28100,12 +28141,12 @@ init_geoparquet_loader();
 
 // dist/parquet-js-loader-types.js
 init_parquet_format();
-var VERSION8 = true ? "5.0.0-alpha.6" : "latest";
+var VERSION8 = true ? "5.0.0-alpha.7" : "latest";
 var DEFAULT_PARQUET_JS_OPTIONS3 = {
   columns: void 0,
   preserveBinary: false,
   verifyFooterSignature: true,
-  shape: "object-row-table"
+  shape: "arrow-table"
 };
 async function preloadParquetJSLoader2() {
   const { ParquetJSLoaderWithParser: ParquetJSLoaderWithParser2 } = await Promise.resolve().then(() => (init_parquet_js_loader(), parquet_js_loader_exports));
@@ -28127,7 +28168,7 @@ var ParquetJSLoader2 = {
 
 // dist/parquet-source-loader-types.js
 init_parquet_format();
-var VERSION10 = true ? "5.0.0-alpha.6" : "latest";
+var VERSION10 = true ? "5.0.0-alpha.7" : "latest";
 async function preloadParquetSourceLoader2() {
   const { ParquetSourceLoaderWithParser: ParquetSourceLoaderWithParser2 } = await Promise.resolve().then(() => (init_parquet_source_loader(), parquet_source_loader_exports));
   return ParquetSourceLoaderWithParser2;
@@ -28251,7 +28292,7 @@ init_normalize_parquet_options2();
 init_parquet_format();
 
 // dist/lib/constants.js
-var VERSION11 = true ? "5.0.0-alpha.6" : "latest";
+var VERSION11 = true ? "5.0.0-alpha.7" : "latest";
 var PARQUET_MAGIC2 = "PAR1";
 var PARQUET_MAGIC_ENCRYPTED2 = "PARE";
 var PARQUET_RDLVL_TYPE2 = "INT32";
@@ -39114,7 +39155,7 @@ function formatDataType(dataType) {
 
 // dist/parquet-js-writer.js
 init_parquet_format();
-var VERSION12 = true ? "5.0.0-alpha.6" : "latest";
+var VERSION12 = true ? "5.0.0-alpha.7" : "latest";
 var DEFAULT_PARQUET_JS_OPTIONS4 = {};
 var ParquetJSWriter = {
   ...ParquetFormat,