@loaders.gl/arrow 4.2.0-alpha.5 → 4.2.0-beta.1

package/dist/arrow-loader.d.ts

@@ -1,14 +1,58 @@
-import type { Loader, LoaderWithParser, LoaderOptions } from '@loaders.gl/loader-utils';
-import type { ArrayRowTable, ArrowTableBatch, ColumnarTable, ObjectRowTable } from '@loaders.gl/schema';
+import type { LoaderOptions } from '@loaders.gl/loader-utils';
+import type { ArrayRowTable, ColumnarTable, ObjectRowTable } from '@loaders.gl/schema';
 import type { ArrowTable } from "./lib/arrow-table.js";
+import { parseArrowInBatches } from "./parsers/parse-arrow-in-batches.js";
+/** ArrowLoader options */
 export type ArrowLoaderOptions = LoaderOptions & {
+    /** ArrowLoader options */
     arrow?: {
+        /** Shape of returned data */
         shape: 'arrow-table' | 'columnar-table' | 'array-row-table' | 'object-row-table';
+        /** Debounce time between batches (prevent excessive numbers of small batches) */
         batchDebounceMs?: number;
+        /** Override the URL to the worker bundle (by default loads from unpkg.com) */
+        workerUrl?: string;
     };
 };
 /** ArrowJS table loader */
-export declare const ArrowWorkerLoader: Loader<ArrowTable, never, ArrowLoaderOptions>;
+export declare const ArrowWorkerLoader: {
+    readonly dataType: ArrowTable;
+    readonly batchType: never;
+    readonly name: "Apache Arrow";
+    readonly id: "arrow";
+    readonly module: "arrow";
+    readonly version: any;
+    readonly category: "table";
+    readonly extensions: ["arrow", "feather"];
+    readonly mimeTypes: ["application/vnd.apache.arrow.file", "application/vnd.apache.arrow.stream", "application/octet-stream"];
+    readonly binary: true;
+    readonly tests: ["ARROW"];
+    readonly options: {
+        readonly arrow: {
+            readonly shape: "columnar-table";
+        };
+    };
+};
 /** ArrowJS table loader */
-export declare const ArrowLoader: LoaderWithParser<ArrowTable | ColumnarTable | ObjectRowTable | ArrayRowTable, ArrowTableBatch, ArrowLoaderOptions>;
+export declare const ArrowLoader: {
+    readonly parse: (arraybuffer: ArrayBuffer, options?: ArrowLoaderOptions) => Promise<ArrowTable | ColumnarTable | ObjectRowTable | ArrayRowTable>;
+    readonly parseSync: (arraybuffer: ArrayBuffer, options?: ArrowLoaderOptions) => ArrowTable | ColumnarTable | ObjectRowTable | ArrayRowTable;
+    readonly parseInBatches: typeof parseArrowInBatches;
+    readonly dataType: ArrowTable;
+    readonly batchType: never;
+    readonly name: "Apache Arrow";
+    readonly id: "arrow";
+    readonly module: "arrow";
+    readonly version: any;
+    readonly category: "table";
+    readonly extensions: ["arrow", "feather"];
+    readonly mimeTypes: ["application/vnd.apache.arrow.file", "application/vnd.apache.arrow.stream", "application/octet-stream"];
+    readonly binary: true;
+    readonly tests: ["ARROW"];
+    readonly options: {
+        readonly arrow: {
+            readonly shape: "columnar-table";
+        };
+    };
+};
 //# sourceMappingURL=arrow-loader.d.ts.map

package/dist/arrow-loader.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"arrow-loader.d.ts","sourceRoot":"","sources":["../src/arrow-loader.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,MAAM,EAAE,gBAAgB,EAAE,aAAa,EAAC,MAAM,0BAA0B,CAAC;AACtF,OAAO,KAAK,EACV,aAAa,EACb,eAAe,EACf,aAAa,EACb,cAAc,EACf,MAAM,oBAAoB,CAAC;AAC5B,OAAO,KAAK,EAAC,UAAU,EAAC,6BAA0B;AAQlD,MAAM,MAAM,kBAAkB,GAAG,aAAa,GAAG;IAC/C,KAAK,CAAC,EAAE;QACN,KAAK,EAAE,aAAa,GAAG,gBAAgB,GAAG,iBAAiB,GAAG,kBAAkB,CAAC;QACjF,eAAe,CAAC,EAAE,MAAM,CAAC;KAC1B,CAAC;CACH,CAAC;AAEF,2BAA2B;AAC3B,eAAO,MAAM,iBAAiB,EAAE,MAAM,CAAC,UAAU,EAAE,KAAK,EAAE,kBAAkB,CAoB3E,CAAC;AAEF,2BAA2B;AAC3B,eAAO,MAAM,WAAW,EAAE,gBAAgB,CACxC,UAAU,GAAG,aAAa,GAAG,cAAc,GAAG,aAAa,EAC3D,eAAe,EACf,kBAAkB,CAQnB,CAAC"}
+{"version":3,"file":"arrow-loader.d.ts","sourceRoot":"","sources":["../src/arrow-loader.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAA2B,aAAa,EAAC,MAAM,0BAA0B,CAAC;AACtF,OAAO,KAAK,EACV,aAAa,EAEb,aAAa,EACb,cAAc,EACf,MAAM,oBAAoB,CAAC;AAC5B,OAAO,KAAK,EAAC,UAAU,EAAC,6BAA0B;AAElD,OAAO,EAAC,mBAAmB,EAAC,4CAAyC;AAMrE,0BAA0B;AAC1B,MAAM,MAAM,kBAAkB,GAAG,aAAa,GAAG;IAC/C,0BAA0B;IAC1B,KAAK,CAAC,EAAE;QACN,6BAA6B;QAC7B,KAAK,EAAE,aAAa,GAAG,gBAAgB,GAAG,iBAAiB,GAAG,kBAAkB,CAAC;QACjF,iFAAiF;QACjF,eAAe,CAAC,EAAE,MAAM,CAAC;QACzB,8EAA8E;QAC9E,SAAS,CAAC,EAAE,MAAM,CAAC;KACpB,CAAC;CACH,CAAC;AAEF,2BAA2B;AAC3B,eAAO,MAAM,iBAAiB;;;;;;;;;;;;;;;;;CAuBoC,CAAC;AAEnE,2BAA2B;AAC3B,eAAO,MAAM,WAAW;kCAEK,WAAW,YAAY,kBAAkB;sCAE3C,WAAW,YAAY,kBAAkB;;;;;;;;;;;;;;;;;;CAOnE,CAAC"}

package/dist/arrow-loader.js

@@ -5,9 +5,11 @@ import { parseArrowSync } from "./parsers/parse-arrow-sync.js";
 import { parseArrowInBatches } from "./parsers/parse-arrow-in-batches.js";
 // __VERSION__ is injected by babel-plugin-version-inline
 // @ts-ignore TS2304: Cannot find name '__VERSION__'.
-const VERSION = typeof "4.2.0-alpha.4" !== 'undefined' ? "4.2.0-alpha.4" : 'latest';
+const VERSION = typeof "4.2.0-alpha.6" !== 'undefined' ? "4.2.0-alpha.6" : 'latest';
 /** ArrowJS table loader */
 export const ArrowWorkerLoader = {
+    dataType: null,
+    batchType: null,
     name: 'Apache Arrow',
     id: 'arrow',
     module: 'arrow',

package/dist/arrow-worker.js

@@ -65,7 +65,9 @@
       }
       getParentPort().then((parentPort2) => {
         if (parentPort2) {
-          parentPort2.on("message", handleMessage);
+          parentPort2.on("message", (message) => {
+            handleMessage(message);
+          });
           parentPort2.on("exit", () => console.debug("Node worker closing"));
         } else {
           globalThis.onmessage = handleMessage;
@@ -12497,8 +12499,10 @@ return true;`);
   }
 
   // src/arrow-loader.ts
-  var VERSION = true ? "4.2.0-alpha.4" : "latest";
+  var VERSION = true ? "4.2.0-alpha.6" : "latest";
   var ArrowWorkerLoader = {
+    dataType: null,
+    batchType: null,
     name: "Apache Arrow",
     id: "arrow",
     module: "arrow",

package/dist/arrow-writer.d.ts

@@ -1,9 +1,20 @@
-import type { WriterWithEncoder, WriterOptions } from '@loaders.gl/loader-utils';
+import type { WriterOptions } from '@loaders.gl/loader-utils';
 import { ColumnarTable } from "./lib/encode-arrow.js";
 type ArrowWriterOptions = WriterOptions & {
     arrow?: {};
 };
 /** Apache Arrow writer */
-export declare const ArrowWriter: WriterWithEncoder<ColumnarTable, never, ArrowWriterOptions>;
+export declare const ArrowWriter: {
+    readonly name: "Apache Arrow";
+    readonly id: "arrow";
+    readonly module: "arrow";
+    readonly version: any;
+    readonly extensions: ["arrow", "feather"];
+    readonly mimeTypes: ["application/vnd.apache.arrow.file", "application/vnd.apache.arrow.stream", "application/octet-stream"];
+    readonly binary: true;
+    readonly options: {};
+    readonly encode: (data: ColumnarTable, options?: ArrowWriterOptions | undefined) => Promise<ArrayBuffer>;
+    readonly encodeSync: (data: ColumnarTable, options?: ArrowWriterOptions | undefined) => ArrayBuffer;
+};
 export {};
 //# sourceMappingURL=arrow-writer.d.ts.map

package/dist/arrow-writer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"arrow-writer.d.ts","sourceRoot":"","sources":["../src/arrow-writer.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAC,iBAAiB,EAAE,aAAa,EAAC,MAAM,0BAA0B,CAAC;AAC/E,OAAO,EAAC,aAAa,EAAC,8BAA2B;AAOjD,KAAK,kBAAkB,GAAG,aAAa,GAAG;IACxC,KAAK,CAAC,EAAE,EAAE,CAAC;CACZ,CAAC;AAEF,0BAA0B;AAC1B,eAAO,MAAM,WAAW,EAAE,iBAAiB,CAAC,aAAa,EAAE,KAAK,EAAE,kBAAkB,CAmBnF,CAAC"}
+{"version":3,"file":"arrow-writer.d.ts","sourceRoot":"","sources":["../src/arrow-writer.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAoB,aAAa,EAAC,MAAM,0BAA0B,CAAC;AAC/E,OAAO,EAAC,aAAa,EAAC,8BAA2B;AAOjD,KAAK,kBAAkB,GAAG,aAAa,GAAG;IACxC,KAAK,CAAC,EAAE,EAAE,CAAC;CACZ,CAAC;AAEF,0BAA0B;AAC1B,eAAO,MAAM,WAAW;;;;;;;;;wFAa8B,QAAQ,WAAW,CAAC;;CAMM,CAAC"}

package/dist/arrow-writer.js

@@ -2,7 +2,7 @@
 import { encodeArrowSync } from "./lib/encode-arrow.js";
 // __VERSION__ is injected by babel-plugin-version-inline
 // @ts-ignore TS2304: Cannot find name '__VERSION__'.
-const VERSION = typeof "4.2.0-alpha.4" !== 'undefined' ? "4.2.0-alpha.4" : 'latest';
+const VERSION = typeof "4.2.0-alpha.6" !== 'undefined' ? "4.2.0-alpha.6" : 'latest';
 /** Apache Arrow writer */
 export const ArrowWriter = {
     name: 'Apache Arrow',

package/dist/dist.dev.js

@@ -12,6 +12,7 @@ var __exports__ = (() => {
   var __getOwnPropNames = Object.getOwnPropertyNames;
   var __getProtoOf = Object.getPrototypeOf;
   var __hasOwnProp = Object.prototype.hasOwnProperty;
+  var __defNormalProp = (obj, key, value) => key in obj ? __defProp(obj, key, { enumerable: true, configurable: true, writable: true, value }) : obj[key] = value;
   var __commonJS = (cb, mod) => function __require() {
     return mod || (0, cb[__getOwnPropNames(cb)[0]])((mod = { exports: {} }).exports, mod), mod.exports;
   };
@@ -37,6 +38,10 @@ var __exports__ = (() => {
     mod
   ));
   var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+  var __publicField = (obj, key, value) => {
+    __defNormalProp(obj, typeof key !== "symbol" ? key + "" : key, value);
+    return value;
+  };
 
   // external-global-plugin:@loaders.gl/core
   var require_core = __commonJS({
@@ -80,11 +85,14 @@ var __exports__ = (() => {
   // ../schema/src/lib/table/batches/base-table-batch-aggregator.ts
   var DEFAULT_ROW_COUNT = 100;
   var BaseTableBatchAggregator = class {
+    schema;
+    options;
+    shape;
+    length = 0;
+    rows = null;
+    cursor = 0;
+    _headers = [];
     constructor(schema, options) {
-      this.length = 0;
-      this.rows = null;
-      this.cursor = 0;
-      this._headers = [];
       this.options = options;
       this.schema = schema;
       if (!Array.isArray(schema)) {
@@ -180,12 +188,14 @@ var __exports__ = (() => {
   // ../schema/src/lib/table/batches/row-table-batch-aggregator.ts
   var DEFAULT_ROW_COUNT2 = 100;
   var RowTableBatchAggregator = class {
+    schema;
+    options;
+    length = 0;
+    objectRows = null;
+    arrayRows = null;
+    cursor = 0;
+    _headers = null;
     constructor(schema, options) {
-      this.length = 0;
-      this.objectRows = null;
-      this.arrayRows = null;
-      this.cursor = 0;
-      this._headers = null;
       this.options = options;
       this.schema = schema;
       if (schema) {
@@ -255,10 +265,11 @@ var __exports__ = (() => {
   // ../schema/src/lib/table/batches/columnar-table-batch-aggregator.ts
   var DEFAULT_ROW_COUNT3 = 100;
   var ColumnarTableBatchAggregator = class {
+    schema;
+    length = 0;
+    allocated = 0;
+    columns = {};
     constructor(schema, options) {
-      this.length = 0;
-      this.allocated = 0;
-      this.columns = {};
       this.schema = schema;
       this._reallocateColumns();
     }
@@ -338,16 +349,18 @@ var __exports__ = (() => {
     _limitMB: 0
   };
   var ERR_MESSAGE = "TableBatchBuilder";
-  var TableBatchBuilder = class {
+  var _TableBatchBuilder = class {
+    schema;
+    options;
+    aggregator = null;
+    batchCount = 0;
+    bytesUsed = 0;
+    isChunkComplete = false;
+    lastBatchEmittedMs = Date.now();
+    totalLength = 0;
+    totalBytes = 0;
+    rowBytes = 0;
     constructor(schema, options) {
-      this.aggregator = null;
-      this.batchCount = 0;
-      this.bytesUsed = 0;
-      this.isChunkComplete = false;
-      this.lastBatchEmittedMs = Date.now();
-      this.totalLength = 0;
-      this.totalBytes = 0;
-      this.rowBytes = 0;
       this.schema = schema;
       this.options = { ...DEFAULT_OPTIONS, ...options };
     }
@@ -454,15 +467,17 @@ var __exports__ = (() => {
         case "columnar-table":
           return ColumnarTableBatchAggregator;
         case "arrow-table":
-          if (!TableBatchBuilder.ArrowBatch) {
+          if (!_TableBatchBuilder.ArrowBatch) {
             throw new Error(ERR_MESSAGE);
           }
-          return TableBatchBuilder.ArrowBatch;
+          return _TableBatchBuilder.ArrowBatch;
         default:
           return BaseTableBatchAggregator;
       }
     }
   };
+  var TableBatchBuilder = _TableBatchBuilder;
+  __publicField(TableBatchBuilder, "ArrowBatch");
 
   // ../schema/src/lib/table/simple-table/table-accessors.ts
   function getTableLength(table) {
@@ -12685,6 +12700,7 @@ return true;`);
 
   // src/lib/arrow-table-batch.ts
   var ArrowTableBatchAggregator = class extends ColumnarTableBatchAggregator {
+    arrowSchema;
     constructor(schema, options) {
       super(schema, options);
       this.arrowSchema = null;
@@ -13106,6 +13122,8 @@ return true;`);
   // src/arrow-loader.ts
   var VERSION = typeof __VERSION__ !== "undefined" ? __VERSION__ : "latest";
   var ArrowWorkerLoader = {
+    dataType: null,
+    batchType: null,
     name: "Apache Arrow",
     id: "arrow",
     module: "arrow",
@@ -13922,12 +13940,7 @@ return true;`);
     }
   };
   var GeoArrowLoader = {
-    ...ArrowWorkerLoader,
-    options: {
-      arrow: {
-        shape: "arrow-table"
-      }
-    },
+    ...GeoArrowWorkerLoader,
     parse: async (arraybuffer, options) => parseGeoArrowSync(arraybuffer, options?.arrow),
     parseSync: (arraybuffer, options) => parseGeoArrowSync(arraybuffer, options?.arrow),
     parseInBatches: parseGeoArrowInBatches
@@ -14494,6 +14507,8 @@ return true;`);
 
   // ../wkt/src/wkt-loader.ts
   var WKTWorkerLoader = {
+    dataType: null,
+    batchType: null,
     name: "WKT (Well-Known Text)",
     id: "wkt",
     module: "wkt",
@@ -14515,7 +14530,7 @@ return true;`);
   var WKTLoader = {
     ...WKTWorkerLoader,
     parse: async (arrayBuffer, options) => parseWKT(new TextDecoder().decode(arrayBuffer), options),
-    parseTextSync: parseWKT
+    parseTextSync: (string, options) => parseWKT(string, options)
   };
 
   // ../wkt/src/lib/parse-wkb-header.ts
@@ -14688,7 +14703,7 @@ return true;`);
       geometry: {
         type: "LineString",
         positions: { value: positions, size: dimension },
-        pathIndices: { value: new Uint16Array(pathIndices), size: 1 }
+        pathIndices: { value: new Uint32Array(pathIndices), size: 1 }
       },
       offset
     };
@@ -14716,10 +14731,10 @@ return true;`);
         type: "Polygon",
         positions: { value: concatenatedPositions, size: dimension },
         polygonIndices: {
-          value: new Uint16Array(polygonIndices),
+          value: new Uint32Array(polygonIndices),
           size: 1
         },
-        primitivePolygonIndices: { value: new Uint16Array(primitivePolygonIndices), size: 1 }
+        primitivePolygonIndices: { value: new Uint32Array(primitivePolygonIndices), size: 1 }
       },
       offset
     };
@@ -14791,7 +14806,7 @@ return true;`);
     return {
       type: "LineString",
       positions: { value: concatenatedPositions, size: dimension },
-      pathIndices: { value: new Uint16Array(pathIndices), size: 1 }
+      pathIndices: { value: new Uint32Array(pathIndices), size: 1 }
     };
   }
   function concatenateBinaryPolygonGeometries(binaryPolygonGeometries, dimension) {
@@ -14814,8 +14829,8 @@ return true;`);
     return {
       type: "Polygon",
       positions: { value: concatenatedPositions, size: dimension },
-      polygonIndices: { value: new Uint16Array(polygonIndices), size: 1 },
-      primitivePolygonIndices: { value: new Uint16Array(primitivePolygonIndices), size: 1 }
+      polygonIndices: { value: new Uint32Array(polygonIndices), size: 1 },
+      primitivePolygonIndices: { value: new Uint32Array(primitivePolygonIndices), size: 1 }
     };
   }
   function concatTypedArrays(arrays) {
@@ -14837,6 +14852,8 @@ return true;`);
 
   // ../wkt/src/wkb-loader.ts
   var WKBWorkerLoader = {
+    dataType: null,
+    batchType: null,
     name: "WKB",
     id: "wkb",
     module: "wkt",
@@ -15097,12 +15114,16 @@ return true;`);
 
   // ../worker-utils/src/lib/worker-farm/worker-job.ts
   var WorkerJob = class {
+    name;
+    workerThread;
+    isRunning = true;
+    /** Promise that resolves when Job is done */
+    result;
+    _resolve = () => {
+    };
+    _reject = () => {
+    };
     constructor(jobName, workerThread) {
-      this.isRunning = true;
-      this._resolve = () => {
-      };
-      this._reject = () => {
-      };
       this.name = jobName;
       this.workerThread = workerThread;
       this.result = new Promise((resolve, reject) => {
@@ -15239,9 +15260,19 @@ return true;`);
   var NOOP = () => {
   };
   var WorkerThread = class {
+    name;
+    source;
+    url;
+    terminated = false;
+    worker;
+    onMessage;
+    onError;
+    _loadableURL = "";
+    /** Checks if workers are supported on this platform */
+    static isSupported() {
+      return typeof Worker !== "undefined" && isBrowser || typeof NodeWorker !== "undefined" && !isBrowser;
+    }
     constructor(props) {
-      this.terminated = false;
-      this._loadableURL = "";
       const { name, source, url } = props;
       assert(source || url);
       this.name = name;
@@ -15251,10 +15282,6 @@ return true;`);
       this.onError = (error) => console.log(error);
       this.worker = isBrowser ? this._createBrowserWorker() : this._createNodeWorker();
     }
-    /** Checks if workers are supported on this platform */
-    static isSupported() {
-      return typeof Worker !== "undefined" && isBrowser || typeof NodeWorker !== "undefined" && !isBrowser;
-    }
     /**
      * Terminate this worker thread
      * @note Can free up significant memory
@@ -15342,30 +15369,33 @@ return true;`);
 
   // ../worker-utils/src/lib/worker-farm/worker-pool.ts
   var WorkerPool = class {
+    name = "unnamed";
+    source;
+    // | Function;
+    url;
+    maxConcurrency = 1;
+    maxMobileConcurrency = 1;
+    onDebug = () => {
+    };
+    reuseWorkers = true;
+    props = {};
+    jobQueue = [];
+    idleQueue = [];
+    count = 0;
+    isDestroyed = false;
+    /** Checks if workers are supported on this platform */
+    static isSupported() {
+      return WorkerThread.isSupported();
+    }
     /**
      * @param processor - worker function
      * @param maxConcurrency - max count of workers
      */
     constructor(props) {
-      this.name = "unnamed";
-      this.maxConcurrency = 1;
-      this.maxMobileConcurrency = 1;
-      this.onDebug = () => {
-      };
-      this.reuseWorkers = true;
-      this.props = {};
-      this.jobQueue = [];
-      this.idleQueue = [];
-      this.count = 0;
-      this.isDestroyed = false;
       this.source = props.source;
       this.url = props.url;
       this.setProps(props);
     }
-    /** Checks if workers are supported on this platform */
-    static isSupported() {
-      return WorkerThread.isSupported();
-    }
     /**
      * Terminates all workers in the pool
      * @note Can free up significant memory
@@ -15488,23 +15518,24 @@ return true;`);
     onDebug: () => {
     }
   };
-  var WorkerFarm = class {
-    /** get global instance with WorkerFarm.getWorkerFarm() */
-    constructor(props) {
-      this.workerPools = /* @__PURE__ */ new Map();
-      this.props = { ...DEFAULT_PROPS };
-      this.setProps(props);
-      this.workerPools = /* @__PURE__ */ new Map();
-    }
+  var _WorkerFarm = class {
+    props;
+    workerPools = /* @__PURE__ */ new Map();
     /** Checks if workers are supported on this platform */
     static isSupported() {
       return WorkerThread.isSupported();
     }
     /** Get the singleton instance of the global worker farm */
     static getWorkerFarm(props = {}) {
-      WorkerFarm._workerFarm = WorkerFarm._workerFarm || new WorkerFarm({});
-      WorkerFarm._workerFarm.setProps(props);
-      return WorkerFarm._workerFarm;
+      _WorkerFarm._workerFarm = _WorkerFarm._workerFarm || new _WorkerFarm({});
+      _WorkerFarm._workerFarm.setProps(props);
+      return _WorkerFarm._workerFarm;
+    }
+    /** get global instance with WorkerFarm.getWorkerFarm() */
+    constructor(props) {
+      this.props = { ...DEFAULT_PROPS };
+      this.setProps(props);
+      this.workerPools = /* @__PURE__ */ new Map();
     }
     /**
      * Terminate all workers in the farm
@@ -15558,6 +15589,9 @@ return true;`);
       };
     }
   };
+  var WorkerFarm = _WorkerFarm;
+  // singleton
+  __publicField(WorkerFarm, "_workerFarm");
 
   // ../worker-utils/src/lib/worker-api/get-worker-url.ts
   function getWorkerName(worker) {