@loaders.gl/arrow 4.1.0-alpha.3 → 4.1.0-alpha.5

package/dist/arrow-loader.d.ts

@@ -4,6 +4,7 @@ import type { ArrowTable } from './lib/arrow-table';
 export type ArrowLoaderOptions = LoaderOptions & {
     arrow?: {
         shape: 'arrow-table' | 'columnar-table' | 'array-row-table' | 'object-row-table';
+        batchDebounceMs?: number;
     };
 };
 /** ArrowJS table loader */

package/dist/arrow-loader.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"arrow-loader.d.ts","sourceRoot":"","sources":["../src/arrow-loader.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,MAAM,EAAE,gBAAgB,EAAE,aAAa,EAAC,MAAM,0BAA0B,CAAC;AACtF,OAAO,KAAK,EACV,aAAa,EACb,eAAe,EACf,aAAa,EACb,cAAc,EACf,MAAM,oBAAoB,CAAC;AAC5B,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,mBAAmB,CAAC;AAQlD,MAAM,MAAM,kBAAkB,GAAG,aAAa,GAAG;IAC/C,KAAK,CAAC,EAAE;QACN,KAAK,EAAE,aAAa,GAAG,gBAAgB,GAAG,iBAAiB,GAAG,kBAAkB,CAAC;KAClF,CAAC;CACH,CAAC;AAEF,2BAA2B;AAC3B,eAAO,MAAM,iBAAiB,EAAE,MAAM,CAAC,UAAU,EAAE,KAAK,EAAE,kBAAkB,CAoB3E,CAAC;AAEF,2BAA2B;AAC3B,eAAO,MAAM,WAAW,EAAE,gBAAgB,CACxC,UAAU,GAAG,aAAa,GAAG,cAAc,GAAG,aAAa,EAC3D,eAAe,EACf,kBAAkB,CAQnB,CAAC"}
+{"version":3,"file":"arrow-loader.d.ts","sourceRoot":"","sources":["../src/arrow-loader.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,MAAM,EAAE,gBAAgB,EAAE,aAAa,EAAC,MAAM,0BAA0B,CAAC;AACtF,OAAO,KAAK,EACV,aAAa,EACb,eAAe,EACf,aAAa,EACb,cAAc,EACf,MAAM,oBAAoB,CAAC;AAC5B,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,mBAAmB,CAAC;AAQlD,MAAM,MAAM,kBAAkB,GAAG,aAAa,GAAG;IAC/C,KAAK,CAAC,EAAE;QACN,KAAK,EAAE,aAAa,GAAG,gBAAgB,GAAG,iBAAiB,GAAG,kBAAkB,CAAC;QACjF,eAAe,CAAC,EAAE,MAAM,CAAC;KAC1B,CAAC;CACH,CAAC;AAEF,2BAA2B;AAC3B,eAAO,MAAM,iBAAiB,EAAE,MAAM,CAAC,UAAU,EAAE,KAAK,EAAE,kBAAkB,CAoB3E,CAAC;AAEF,2BAA2B;AAC3B,eAAO,MAAM,WAAW,EAAE,gBAAgB,CACxC,UAAU,GAAG,aAAa,GAAG,cAAc,GAAG,aAAa,EAC3D,eAAe,EACf,kBAAkB,CAQnB,CAAC"}

package/dist/arrow-loader.js

@@ -1,6 +1,6 @@
 import { parseArrowSync } from "./parsers/parse-arrow-sync.js";
 import { parseArrowInBatches } from "./parsers/parse-arrow-in-batches.js";
-const VERSION = typeof "4.1.0-alpha.3" !== 'undefined' ? "4.1.0-alpha.3" : 'latest';
+const VERSION = typeof "4.1.0-alpha.5" !== 'undefined' ? "4.1.0-alpha.5" : 'latest';
 export const ArrowWorkerLoader = {
   name: 'Apache Arrow',
   id: 'arrow',

package/dist/arrow-loader.js.map

@@ -1 +1 @@
-{"version":3,"file":"arrow-loader.js","names":["parseArrowSync","parseArrowInBatches","VERSION","ArrowWorkerLoader","name","id","module","version","category","extensions","mimeTypes","binary","tests","options","arrow","shape","ArrowLoader","parse","arraybuffer","parseSync","parseInBatches"],"sources":["../src/arrow-loader.ts"],"sourcesContent":["// loaders.gl\n// SPDX-License-Identifier: MIT\n// Copyright (c) vis.gl contributors\n\nimport type {Loader, LoaderWithParser, LoaderOptions} from '@loaders.gl/loader-utils';\nimport type {\n  ArrayRowTable,\n  ArrowTableBatch,\n  ColumnarTable,\n  ObjectRowTable\n} from '@loaders.gl/schema';\nimport type {ArrowTable} from './lib/arrow-table';\nimport {parseArrowSync} from './parsers/parse-arrow-sync';\nimport {parseArrowInBatches} from './parsers/parse-arrow-in-batches';\n\n// __VERSION__ is injected by babel-plugin-version-inline\n// @ts-ignore TS2304: Cannot find name '__VERSION__'.\nconst VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';\n\nexport type ArrowLoaderOptions = LoaderOptions & {\n  arrow?: {\n    shape: 'arrow-table' | 'columnar-table' | 'array-row-table' | 'object-row-table';\n  };\n};\n\n/** ArrowJS table loader */\nexport const ArrowWorkerLoader: Loader<ArrowTable, never, ArrowLoaderOptions> = {\n  name: 'Apache Arrow',\n  id: 'arrow',\n  module: 'arrow',\n  version: VERSION,\n  // worker: true,\n  category: 'table',\n  extensions: ['arrow', 'feather'],\n  mimeTypes: [\n    'application/vnd.apache.arrow.file',\n    'application/vnd.apache.arrow.stream',\n    'application/octet-stream'\n  ],\n  binary: true,\n  tests: ['ARROW'],\n  options: {\n    arrow: {\n      shape: 'columnar-table'\n    }\n  }\n};\n\n/** ArrowJS table loader */\nexport const ArrowLoader: LoaderWithParser<\n  ArrowTable | ColumnarTable | ObjectRowTable | ArrayRowTable,\n  ArrowTableBatch,\n  ArrowLoaderOptions\n> = {\n  ...ArrowWorkerLoader,\n  parse: async (arraybuffer: ArrayBuffer, options?: ArrowLoaderOptions) =>\n    parseArrowSync(arraybuffer, options?.arrow),\n  parseSync: (arraybuffer: ArrayBuffer, options?: ArrowLoaderOptions) =>\n    parseArrowSync(arraybuffer, options?.arrow),\n  parseInBatches: parseArrowInBatches\n};\n"],"mappings":"SAYQA,cAAc;AAAA,SACdC,mBAAmB;AAI3B,MAAMC,OAAO,GAAG,sBAAkB,KAAK,WAAW,qBAAiB,QAAQ;AAS3E,OAAO,MAAMC,iBAAgE,GAAG;EAC9EC,IAAI,EAAE,cAAc;EACpBC,EAAE,EAAE,OAAO;EACXC,MAAM,EAAE,OAAO;EACfC,OAAO,EAAEL,OAAO;EAEhBM,QAAQ,EAAE,OAAO;EACjBC,UAAU,EAAE,CAAC,OAAO,EAAE,SAAS,CAAC;EAChCC,SAAS,EAAE,CACT,mCAAmC,EACnC,qCAAqC,EACrC,0BAA0B,CAC3B;EACDC,MAAM,EAAE,IAAI;EACZC,KAAK,EAAE,CAAC,OAAO,CAAC;EAChBC,OAAO,EAAE;IACPC,KAAK,EAAE;MACLC,KAAK,EAAE;IACT;EACF;AACF,CAAC;AAGD,OAAO,MAAMC,WAIZ,GAAG;EACF,GAAGb,iBAAiB;EACpBc,KAAK,EAAE,MAAAA,CAAOC,WAAwB,EAAEL,OAA4B,KAClEb,cAAc,CAACkB,WAAW,EAAEL,OAAO,aAAPA,OAAO,uBAAPA,OAAO,CAAEC,KAAK,CAAC;EAC7CK,SAAS,EAAEA,CAACD,WAAwB,EAAEL,OAA4B,KAChEb,cAAc,CAACkB,WAAW,EAAEL,OAAO,aAAPA,OAAO,uBAAPA,OAAO,CAAEC,KAAK,CAAC;EAC7CM,cAAc,EAAEnB;AAClB,CAAC"}
+{"version":3,"file":"arrow-loader.js","names":["parseArrowSync","parseArrowInBatches","VERSION","ArrowWorkerLoader","name","id","module","version","category","extensions","mimeTypes","binary","tests","options","arrow","shape","ArrowLoader","parse","arraybuffer","parseSync","parseInBatches"],"sources":["../src/arrow-loader.ts"],"sourcesContent":["// loaders.gl\n// SPDX-License-Identifier: MIT\n// Copyright (c) vis.gl contributors\n\nimport type {Loader, LoaderWithParser, LoaderOptions} from '@loaders.gl/loader-utils';\nimport type {\n  ArrayRowTable,\n  ArrowTableBatch,\n  ColumnarTable,\n  ObjectRowTable\n} from '@loaders.gl/schema';\nimport type {ArrowTable} from './lib/arrow-table';\nimport {parseArrowSync} from './parsers/parse-arrow-sync';\nimport {parseArrowInBatches} from './parsers/parse-arrow-in-batches';\n\n// __VERSION__ is injected by babel-plugin-version-inline\n// @ts-ignore TS2304: Cannot find name '__VERSION__'.\nconst VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';\n\nexport type ArrowLoaderOptions = LoaderOptions & {\n  arrow?: {\n    shape: 'arrow-table' | 'columnar-table' | 'array-row-table' | 'object-row-table';\n    batchDebounceMs?: number;\n  };\n};\n\n/** ArrowJS table loader */\nexport const ArrowWorkerLoader: Loader<ArrowTable, never, ArrowLoaderOptions> = {\n  name: 'Apache Arrow',\n  id: 'arrow',\n  module: 'arrow',\n  version: VERSION,\n  // worker: true,\n  category: 'table',\n  extensions: ['arrow', 'feather'],\n  mimeTypes: [\n    'application/vnd.apache.arrow.file',\n    'application/vnd.apache.arrow.stream',\n    'application/octet-stream'\n  ],\n  binary: true,\n  tests: ['ARROW'],\n  options: {\n    arrow: {\n      shape: 'columnar-table'\n    }\n  }\n};\n\n/** ArrowJS table loader */\nexport const ArrowLoader: LoaderWithParser<\n  ArrowTable | ColumnarTable | ObjectRowTable | ArrayRowTable,\n  ArrowTableBatch,\n  ArrowLoaderOptions\n> = {\n  ...ArrowWorkerLoader,\n  parse: async (arraybuffer: ArrayBuffer, options?: ArrowLoaderOptions) =>\n    parseArrowSync(arraybuffer, options?.arrow),\n  parseSync: (arraybuffer: ArrayBuffer, options?: ArrowLoaderOptions) =>\n    parseArrowSync(arraybuffer, options?.arrow),\n  parseInBatches: parseArrowInBatches\n};\n"],"mappings":"SAYQA,cAAc;AAAA,SACdC,mBAAmB;AAI3B,MAAMC,OAAO,GAAG,sBAAkB,KAAK,WAAW,qBAAiB,QAAQ;AAU3E,OAAO,MAAMC,iBAAgE,GAAG;EAC9EC,IAAI,EAAE,cAAc;EACpBC,EAAE,EAAE,OAAO;EACXC,MAAM,EAAE,OAAO;EACfC,OAAO,EAAEL,OAAO;EAEhBM,QAAQ,EAAE,OAAO;EACjBC,UAAU,EAAE,CAAC,OAAO,EAAE,SAAS,CAAC;EAChCC,SAAS,EAAE,CACT,mCAAmC,EACnC,qCAAqC,EACrC,0BAA0B,CAC3B;EACDC,MAAM,EAAE,IAAI;EACZC,KAAK,EAAE,CAAC,OAAO,CAAC;EAChBC,OAAO,EAAE;IACPC,KAAK,EAAE;MACLC,KAAK,EAAE;IACT;EACF;AACF,CAAC;AAGD,OAAO,MAAMC,WAIZ,GAAG;EACF,GAAGb,iBAAiB;EACpBc,KAAK,EAAE,MAAAA,CAAOC,WAAwB,EAAEL,OAA4B,KAClEb,cAAc,CAACkB,WAAW,EAAEL,OAAO,aAAPA,OAAO,uBAAPA,OAAO,CAAEC,KAAK,CAAC;EAC7CK,SAAS,EAAEA,CAACD,WAAwB,EAAEL,OAA4B,KAChEb,cAAc,CAACkB,WAAW,EAAEL,OAAO,aAAPA,OAAO,uBAAPA,OAAO,CAAEC,KAAK,CAAC;EAC7CM,cAAc,EAAEnB;AAClB,CAAC"}

package/dist/arrow-worker.js

@@ -12154,11 +12154,14 @@ return true;`);
   }
 
   // src/parsers/parse-arrow-in-batches.ts
-  function parseArrowInBatches(asyncIterator) {
+  function parseArrowInBatches(asyncIterator, options) {
     async function* makeArrowAsyncIterator() {
       const readers = RecordBatchReader.readAll(asyncIterator);
       for await (const reader of readers) {
         for await (const recordBatch of reader) {
+          if (options?.arrow?.batchDebounceMs !== void 0 && options?.arrow?.batchDebounceMs > 0) {
+            await new Promise((resolve) => setTimeout(resolve, options.arrow?.batchDebounceMs || 0));
+          }
           const arrowTabledBatch = {
             shape: "arrow-table",
             batchType: "data",
@@ -12174,7 +12177,7 @@ return true;`);
   }
 
   // src/arrow-loader.ts
-  var VERSION = true ? "4.1.0-alpha.3" : "latest";
+  var VERSION = true ? "4.1.0-alpha.5" : "latest";
   var ArrowWorkerLoader = {
     name: "Apache Arrow",
     id: "arrow",

package/dist/arrow-writer.js

@@ -1,5 +1,5 @@
 import { encodeArrowSync } from "./lib/encode-arrow.js";
-const VERSION = typeof "4.1.0-alpha.3" !== 'undefined' ? "4.1.0-alpha.3" : 'latest';
+const VERSION = typeof "4.1.0-alpha.5" !== 'undefined' ? "4.1.0-alpha.5" : 'latest';
 export const ArrowWriter = {
   name: 'Apache Arrow',
   id: 'arrow',

package/dist/dist.dev.js

@@ -12736,11 +12736,14 @@ return true;`);
   }
 
   // src/parsers/parse-arrow-in-batches.ts
-  function parseArrowInBatches(asyncIterator) {
+  function parseArrowInBatches(asyncIterator, options) {
     async function* makeArrowAsyncIterator() {
       const readers = RecordBatchReader.readAll(asyncIterator);
       for await (const reader of readers) {
         for await (const recordBatch of reader) {
+          if (options?.arrow?.batchDebounceMs !== void 0 && options?.arrow?.batchDebounceMs > 0) {
+            await new Promise((resolve) => setTimeout(resolve, options.arrow?.batchDebounceMs || 0));
+          }
           const arrowTabledBatch = {
             shape: "arrow-table",
             batchType: "data",
@@ -12756,7 +12759,7 @@ return true;`);
   }
 
   // src/arrow-loader.ts
-  var VERSION = true ? "4.1.0-alpha.3" : "latest";
+  var VERSION = true ? "4.1.0-alpha.5" : "latest";
   var ArrowWorkerLoader = {
     name: "Apache Arrow",
     id: "arrow",
@@ -12802,7 +12805,7 @@ return true;`);
   }
 
   // src/arrow-writer.ts
-  var VERSION2 = true ? "4.1.0-alpha.3" : "latest";
+  var VERSION2 = true ? "4.1.0-alpha.5" : "latest";
   var ArrowWriter = {
     name: "Apache Arrow",
     id: "arrow",
@@ -13934,7 +13937,7 @@ return true;`);
   }
 
   // ../wkt/src/lib/utils/version.ts
-  var VERSION3 = true ? "4.1.0-alpha.3" : "latest";
+  var VERSION3 = true ? "4.1.0-alpha.5" : "latest";
 
   // ../wkt/src/lib/parse-wkt.ts
   var numberRegexp = /[-+]?([0-9]*\.[0-9]+|[0-9]+)([eE][-+]?[0-9]+)?/;
@@ -14773,7 +14776,7 @@ return true;`);
         console.warn("loaders.gl: The __VERSION__ variable is not injected using babel plugin. Latest unstable workers would be fetched from the CDN.");
         globalThis._loadersgl_.version = NPM_TAG;
       } else {
-        globalThis._loadersgl_.version = "4.1.0-alpha.3";
+        globalThis._loadersgl_.version = "4.1.0-alpha.5";
       }
     }
     return globalThis._loadersgl_.version;
@@ -15315,7 +15318,7 @@ return true;`);
   }
 
   // src/triangulate-on-worker.ts
-  var VERSION5 = true ? "4.1.0-alpha.3" : "latest";
+  var VERSION5 = true ? "4.1.0-alpha.5" : "latest";
   var TriangulationWorker = {
     id: "triangulation",
     name: "Triangulate",

package/dist/geoarrow-writer.js

@@ -1,5 +1,5 @@
 import { encodeGeoArrowSync } from "./lib/encode-geoarrow.js";
-const VERSION = typeof "4.1.0-alpha.3" !== 'undefined' ? "4.1.0-alpha.3" : 'latest';
+const VERSION = typeof "4.1.0-alpha.5" !== 'undefined' ? "4.1.0-alpha.5" : 'latest';
 export const GeoArrowWriter = {
   name: 'Apache Arrow',
   id: 'arrow',

package/dist/index.cjs

@@ -467,11 +467,18 @@ function parseArrowSync(arrayBuffer, options) {
 
 // src/parsers/parse-arrow-in-batches.ts
 var arrow5 = __toESM(require("apache-arrow"), 1);
-function parseArrowInBatches(asyncIterator) {
+function parseArrowInBatches(asyncIterator, options) {
   async function* makeArrowAsyncIterator() {
+    var _a, _b;
     const readers = arrow5.RecordBatchReader.readAll(asyncIterator);
     for await (const reader of readers) {
       for await (const recordBatch of reader) {
+        if (((_a = options == null ? void 0 : options.arrow) == null ? void 0 : _a.batchDebounceMs) !== void 0 && ((_b = options == null ? void 0 : options.arrow) == null ? void 0 : _b.batchDebounceMs) > 0) {
+          await new Promise((resolve) => {
+            var _a2;
+            return setTimeout(resolve, ((_a2 = options.arrow) == null ? void 0 : _a2.batchDebounceMs) || 0);
+          });
+        }
         const arrowTabledBatch = {
           shape: "arrow-table",
           batchType: "data",

package/dist/parsers/parse-arrow-in-batches.d.ts

@@ -1,5 +1,6 @@
 import type { ArrowTableBatch } from '../lib/arrow-table';
+import { ArrowLoaderOptions } from '../arrow-loader';
 /**
  */
-export declare function parseArrowInBatches(asyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>): AsyncIterable<ArrowTableBatch>;
+export declare function parseArrowInBatches(asyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>, options?: ArrowLoaderOptions): AsyncIterable<ArrowTableBatch>;
 //# sourceMappingURL=parse-arrow-in-batches.d.ts.map

package/dist/parsers/parse-arrow-in-batches.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"parse-arrow-in-batches.d.ts","sourceRoot":"","sources":["../../src/parsers/parse-arrow-in-batches.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,oBAAoB,CAAC;AAIxD;GACG;AACH,wBAAgB,mBAAmB,CACjC,aAAa,EAAE,aAAa,CAAC,WAAW,CAAC,GAAG,QAAQ,CAAC,WAAW,CAAC,GAChE,aAAa,CAAC,eAAe,CAAC,CAsChC"}
+{"version":3,"file":"parse-arrow-in-batches.d.ts","sourceRoot":"","sources":["../../src/parsers/parse-arrow-in-batches.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,oBAAoB,CAAC;AAExD,OAAO,EAAC,kBAAkB,EAAC,MAAM,iBAAiB,CAAC;AAGnD;GACG;AACH,wBAAgB,mBAAmB,CACjC,aAAa,EAAE,aAAa,CAAC,WAAW,CAAC,GAAG,QAAQ,CAAC,WAAW,CAAC,EACjE,OAAO,CAAC,EAAE,kBAAkB,GAC3B,aAAa,CAAC,eAAe,CAAC,CA0ChC"}

package/dist/parsers/parse-arrow-in-batches.js

@@ -1,9 +1,16 @@
 import * as arrow from 'apache-arrow';
-export function parseArrowInBatches(asyncIterator) {
+export function parseArrowInBatches(asyncIterator, options) {
   async function* makeArrowAsyncIterator() {
     const readers = arrow.RecordBatchReader.readAll(asyncIterator);
     for await (const reader of readers) {
       for await (const recordBatch of reader) {
+        var _options$arrow, _options$arrow2;
+        if ((options === null || options === void 0 ? void 0 : (_options$arrow = options.arrow) === null || _options$arrow === void 0 ? void 0 : _options$arrow.batchDebounceMs) !== undefined && (options === null || options === void 0 ? void 0 : (_options$arrow2 = options.arrow) === null || _options$arrow2 === void 0 ? void 0 : _options$arrow2.batchDebounceMs) > 0) {
+          await new Promise(resolve => {
+            var _options$arrow3;
+            return setTimeout(resolve, ((_options$arrow3 = options.arrow) === null || _options$arrow3 === void 0 ? void 0 : _options$arrow3.batchDebounceMs) || 0);
+          });
+        }
         const arrowTabledBatch = {
           shape: 'arrow-table',
           batchType: 'data',

package/dist/parsers/parse-arrow-in-batches.js.map

@@ -1 +1 @@
-{"version":3,"file":"parse-arrow-in-batches.js","names":["arrow","parseArrowInBatches","asyncIterator","makeArrowAsyncIterator","readers","RecordBatchReader","readAll","reader","recordBatch","arrowTabledBatch","shape","batchType","data","Table","length"],"sources":["../../src/parsers/parse-arrow-in-batches.ts"],"sourcesContent":["// loaders.gl\n// SPDX-License-Identifier: MIT\n// Copyright (c) vis.gl contributors\n\nimport type {ArrowTableBatch} from '../lib/arrow-table';\nimport * as arrow from 'apache-arrow';\n// import {isIterable} from '@loaders.gl/core';\n\n/**\n */\nexport function parseArrowInBatches(\n  asyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>\n): AsyncIterable<ArrowTableBatch> {\n  // Creates the appropriate arrow.RecordBatchReader subclasses from the input\n  // This will also close the underlying source in case of early termination or errors\n\n  // As an optimization, return a non-async iterator\n  /*\n  if (isIterable(readers)) {\n    function* makeArrowIterator() {\n      for (const reader of readers) {\n        for (const batch of reader) {\n          yield processBatch(batch, reader);\n        }\n        break; // only processing one stream of batches\n      }\n    }\n    const arrowIterator = makeArrowIterator();\n  }\n  */\n\n  async function* makeArrowAsyncIterator(): AsyncIterator<ArrowTableBatch> {\n    // @ts-ignore\n    const readers = arrow.RecordBatchReader.readAll(asyncIterator);\n    for await (const reader of readers) {\n      for await (const recordBatch of reader) {\n        const arrowTabledBatch: ArrowTableBatch = {\n          shape: 'arrow-table',\n          batchType: 'data',\n          data: new arrow.Table([recordBatch]),\n          length: recordBatch.data.length\n        };\n        // processBatch(recordBatch);\n        yield arrowTabledBatch;\n      }\n      break; // only processing one stream of batches\n    }\n  }\n\n  return makeArrowAsyncIterator() as any; // as AsyncIterator<ArrowTableBatch>;\n}\n\n// function processBatch(batch: RecordBatch): ArrowTableBatch {\n//   const values = {};\n//   batch.schema.fields.forEach(({name}, index) => {\n//     values[name] = batch.getChildAt(index)?.toArray();\n//   });\n//   return {\n//   };\n// }\n"],"mappings":"AAKA,OAAO,KAAKA,KAAK,MAAM,cAAc;AAKrC,OAAO,SAASC,mBAAmBA,CACjCC,aAAiE,EACjC;EAmBhC,gBAAgBC,sBAAsBA,CAAA,EAAmC;IAEvE,MAAMC,OAAO,GAAGJ,KAAK,CAACK,iBAAiB,CAACC,OAAO,CAACJ,aAAa,CAAC;IAC9D,WAAW,MAAMK,MAAM,IAAIH,OAAO,EAAE;MAClC,WAAW,MAAMI,WAAW,IAAID,MAAM,EAAE;QACtC,MAAME,gBAAiC,GAAG;UACxCC,KAAK,EAAE,aAAa;UACpBC,SAAS,EAAE,MAAM;UACjBC,IAAI,EAAE,IAAIZ,KAAK,CAACa,KAAK,CAAC,CAACL,WAAW,CAAC,CAAC;UACpCM,MAAM,EAAEN,WAAW,CAACI,IAAI,CAACE;QAC3B,CAAC;QAED,MAAML,gBAAgB;MACxB;MACA;IACF;EACF;EAEA,OAAON,sBAAsB,CAAC,CAAC;AACjC"}
+{"version":3,"file":"parse-arrow-in-batches.js","names":["arrow","parseArrowInBatches","asyncIterator","options","makeArrowAsyncIterator","readers","RecordBatchReader","readAll","reader","recordBatch","_options$arrow","_options$arrow2","batchDebounceMs","undefined","Promise","resolve","_options$arrow3","setTimeout","arrowTabledBatch","shape","batchType","data","Table","length"],"sources":["../../src/parsers/parse-arrow-in-batches.ts"],"sourcesContent":["// loaders.gl\n// SPDX-License-Identifier: MIT\n// Copyright (c) vis.gl contributors\n\nimport type {ArrowTableBatch} from '../lib/arrow-table';\nimport * as arrow from 'apache-arrow';\nimport {ArrowLoaderOptions} from '../arrow-loader';\n// import {isIterable} from '@loaders.gl/core';\n\n/**\n */\nexport function parseArrowInBatches(\n  asyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>,\n  options?: ArrowLoaderOptions\n): AsyncIterable<ArrowTableBatch> {\n  // Creates the appropriate arrow.RecordBatchReader subclasses from the input\n  // This will also close the underlying source in case of early termination or errors\n\n  // As an optimization, return a non-async iterator\n  /*\n  if (isIterable(readers)) {\n    function* makeArrowIterator() {\n      for (const reader of readers) {\n        for (const batch of reader) {\n          yield processBatch(batch, reader);\n        }\n        break; // only processing one stream of batches\n      }\n    }\n    const arrowIterator = makeArrowIterator();\n  }\n  */\n\n  async function* makeArrowAsyncIterator(): AsyncIterator<ArrowTableBatch> {\n    // @ts-ignore\n    const readers = arrow.RecordBatchReader.readAll(asyncIterator);\n    for await (const reader of readers) {\n      for await (const recordBatch of reader) {\n        // use options.batchDebounceMs to add a delay between batches if needed (use case: incremental loading)\n        if (options?.arrow?.batchDebounceMs !== undefined && options?.arrow?.batchDebounceMs > 0) {\n          await new Promise((resolve) => setTimeout(resolve, options.arrow?.batchDebounceMs || 0));\n        }\n        const arrowTabledBatch: ArrowTableBatch = {\n          shape: 'arrow-table',\n          batchType: 'data',\n          data: new arrow.Table([recordBatch]),\n          length: recordBatch.data.length\n        };\n        // processBatch(recordBatch);\n        yield arrowTabledBatch;\n      }\n      break; // only processing one stream of batches\n    }\n  }\n\n  return makeArrowAsyncIterator() as any; // as AsyncIterator<ArrowTableBatch>;\n}\n\n// function processBatch(batch: RecordBatch): ArrowTableBatch {\n//   const values = {};\n//   batch.schema.fields.forEach(({name}, index) => {\n//     values[name] = batch.getChildAt(index)?.toArray();\n//   });\n//   return {\n//   };\n// }\n"],"mappings":"AAKA,OAAO,KAAKA,KAAK,MAAM,cAAc;AAMrC,OAAO,SAASC,mBAAmBA,CACjCC,aAAiE,EACjEC,OAA4B,EACI;EAmBhC,gBAAgBC,sBAAsBA,CAAA,EAAmC;IAEvE,MAAMC,OAAO,GAAGL,KAAK,CAACM,iBAAiB,CAACC,OAAO,CAACL,aAAa,CAAC;IAC9D,WAAW,MAAMM,MAAM,IAAIH,OAAO,EAAE;MAClC,WAAW,MAAMI,WAAW,IAAID,MAAM,EAAE;QAAA,IAAAE,cAAA,EAAAC,eAAA;QAEtC,IAAI,CAAAR,OAAO,aAAPA,OAAO,wBAAAO,cAAA,GAAPP,OAAO,CAAEH,KAAK,cAAAU,cAAA,uBAAdA,cAAA,CAAgBE,eAAe,MAAKC,SAAS,IAAI,CAAAV,OAAO,aAAPA,OAAO,wBAAAQ,eAAA,GAAPR,OAAO,CAAEH,KAAK,cAAAW,eAAA,uBAAdA,eAAA,CAAgBC,eAAe,IAAG,CAAC,EAAE;UACxF,MAAM,IAAIE,OAAO,CAAEC,OAAO;YAAA,IAAAC,eAAA;YAAA,OAAKC,UAAU,CAACF,OAAO,EAAE,EAAAC,eAAA,GAAAb,OAAO,CAACH,KAAK,cAAAgB,eAAA,uBAAbA,eAAA,CAAeJ,eAAe,KAAI,CAAC,CAAC;UAAA,EAAC;QAC1F;QACA,MAAMM,gBAAiC,GAAG;UACxCC,KAAK,EAAE,aAAa;UACpBC,SAAS,EAAE,MAAM;UACjBC,IAAI,EAAE,IAAIrB,KAAK,CAACsB,KAAK,CAAC,CAACb,WAAW,CAAC,CAAC;UACpCc,MAAM,EAAEd,WAAW,CAACY,IAAI,CAACE;QAC3B,CAAC;QAED,MAAML,gBAAgB;MACxB;MACA;IACF;EACF;EAEA,OAAOd,sBAAsB,CAAC,CAAC;AACjC"}

package/dist/triangulate-on-worker.js

@@ -1,5 +1,5 @@
 import { processOnWorker } from '@loaders.gl/worker-utils';
-const VERSION = typeof "4.1.0-alpha.3" !== 'undefined' ? "4.1.0-alpha.3" : 'latest';
+const VERSION = typeof "4.1.0-alpha.5" !== 'undefined' ? "4.1.0-alpha.5" : 'latest';
 export const TriangulationWorker = {
   id: 'triangulation',
   name: 'Triangulate',

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@loaders.gl/arrow",
-  "version": "4.1.0-alpha.3",
+  "version": "4.1.0-alpha.5",
   "description": "Simple columnar table loader for the Apache Arrow format",
   "license": "MIT",
   "type": "module",
@@ -52,12 +52,12 @@
     "build-worker2": "esbuild src/workers/arrow-worker.ts --bundle --outfile=dist/arrow-worker.js --platform=browser --external:{stream}"
   },
   "dependencies": {
-    "@loaders.gl/gis": "4.1.0-alpha.3",
-    "@loaders.gl/loader-utils": "4.1.0-alpha.3",
-    "@loaders.gl/schema": "4.1.0-alpha.3",
-    "@loaders.gl/wkt": "4.1.0-alpha.3",
+    "@loaders.gl/gis": "4.1.0-alpha.5",
+    "@loaders.gl/loader-utils": "4.1.0-alpha.5",
+    "@loaders.gl/schema": "4.1.0-alpha.5",
+    "@loaders.gl/wkt": "4.1.0-alpha.5",
     "@math.gl/polygon": "4.0.0",
     "apache-arrow": "^13.0.0"
   },
-  "gitHead": "b78075a7cb8d4ecd4aac84805ce74b8ceb400cf7"
+  "gitHead": "3250842d8cc0fc8b76a575168c1fd57e5a66b6ba"
 }

package/src/arrow-loader.ts

@@ -20,6 +20,7 @@ const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
 export type ArrowLoaderOptions = LoaderOptions & {
   arrow?: {
     shape: 'arrow-table' | 'columnar-table' | 'array-row-table' | 'object-row-table';
+    batchDebounceMs?: number;
   };
 };
 

package/src/parsers/parse-arrow-in-batches.ts

@@ -4,12 +4,14 @@
 
 import type {ArrowTableBatch} from '../lib/arrow-table';
 import * as arrow from 'apache-arrow';
+import {ArrowLoaderOptions} from '../arrow-loader';
 // import {isIterable} from '@loaders.gl/core';
 
 /**
  */
 export function parseArrowInBatches(
-  asyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>
+  asyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>,
+  options?: ArrowLoaderOptions
 ): AsyncIterable<ArrowTableBatch> {
   // Creates the appropriate arrow.RecordBatchReader subclasses from the input
   // This will also close the underlying source in case of early termination or errors
@@ -34,6 +36,10 @@ export function parseArrowInBatches(
     const readers = arrow.RecordBatchReader.readAll(asyncIterator);
     for await (const reader of readers) {
       for await (const recordBatch of reader) {
+        // use options.batchDebounceMs to add a delay between batches if needed (use case: incremental loading)
+        if (options?.arrow?.batchDebounceMs !== undefined && options?.arrow?.batchDebounceMs > 0) {
+          await new Promise((resolve) => setTimeout(resolve, options.arrow?.batchDebounceMs || 0));
+        }
         const arrowTabledBatch: ArrowTableBatch = {
           shape: 'arrow-table',
           batchType: 'data',