@loaders.gl/arrow 4.1.0-alpha.1 → 4.1.0-alpha.11

package/dist/types.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAGA,KAAK,aAAa,GAAG,SAAS,GAAG,UAAU,GAAG,UAAU,GAAG,WAAW,GAAG,UAAU,GAAG,WAAW,CAAC;AAElG,KAAK,eAAe,GAAG,YAAY,GAAG,YAAY,CAAC;AAEnD,KAAK,UAAU,GAAG,aAAa,GAAG,eAAe,CAAC;AAElD,MAAM,MAAM,YAAY,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,UAAU,CAAC;AAEnD,oBAAY,YAAY;IACtB,KAAK,IAAA;IACL,IAAI,IAAA;CACL"}
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAIA,KAAK,aAAa,GAAG,SAAS,GAAG,UAAU,GAAG,UAAU,GAAG,WAAW,GAAG,UAAU,GAAG,WAAW,CAAC;AAElG,KAAK,eAAe,GAAG,YAAY,GAAG,YAAY,CAAC;AAEnD,KAAK,UAAU,GAAG,aAAa,GAAG,eAAe,CAAC;AAElD,MAAM,MAAM,YAAY,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,UAAU,CAAC;AAEnD,oBAAY,YAAY;IACtB,KAAK,IAAA;IACL,IAAI,IAAA;CACL"}

package/dist/types.js.map

@@ -1 +1 @@
-{"version":3,"file":"types.js","names":["VECTOR_TYPES"],"sources":["../src/types.ts"],"sourcesContent":["// loaders.gl, MIT license\n// Copyright (c) vis.gl contributors\n\ntype TypedIntArray = Int8Array | Uint8Array | Int16Array | Uint16Array | Int32Array | Uint32Array;\n\ntype TypedFloatArray = Float32Array | Float64Array;\n\ntype TypedArray = TypedIntArray | TypedFloatArray;\n\nexport type AnyArrayType = Array<any> | TypedArray;\n\nexport enum VECTOR_TYPES {\n  FLOAT,\n  DATE\n}\n"],"mappings":"AAWA,WAAYA,YAAY,aAAZA,YAAY;EAAZA,YAAY,CAAZA,YAAY;EAAZA,YAAY,CAAZA,YAAY;EAAA,OAAZA,YAAY;AAAA"}
+{"version":3,"file":"types.js","names":["VECTOR_TYPES"],"sources":["../src/types.ts"],"sourcesContent":["// loaders.gl\n// SPDX-License-Identifier: MIT\n// Copyright (c) vis.gl contributors\n\ntype TypedIntArray = Int8Array | Uint8Array | Int16Array | Uint16Array | Int32Array | Uint32Array;\n\ntype TypedFloatArray = Float32Array | Float64Array;\n\ntype TypedArray = TypedIntArray | TypedFloatArray;\n\nexport type AnyArrayType = Array<any> | TypedArray;\n\nexport enum VECTOR_TYPES {\n  FLOAT,\n  DATE\n}\n"],"mappings":"AAYA,WAAYA,YAAY,aAAZA,YAAY;EAAZA,YAAY,CAAZA,YAAY;EAAZA,YAAY,CAAZA,YAAY;EAAA,OAAZA,YAAY;AAAA"}

package/dist/workers/arrow-worker.js.map

@@ -1 +1 @@
-{"version":3,"file":"arrow-worker.js","names":["createLoaderWorker","ArrowLoader"],"sources":["../../src/workers/arrow-worker.ts"],"sourcesContent":["// loaders.gl, MIT license\n// Copyright (c) vis.gl contributors\n\nimport {createLoaderWorker} from '@loaders.gl/loader-utils';\nimport {ArrowLoader} from '../arrow-loader';\n\ncreateLoaderWorker(ArrowLoader);\n"],"mappings":"AAGA,SAAQA,kBAAkB,QAAO,0BAA0B;AAAC,SACpDC,WAAW;AAEnBD,kBAAkB,CAACC,WAAW,CAAC"}
+{"version":3,"file":"arrow-worker.js","names":["createLoaderWorker","ArrowLoader"],"sources":["../../src/workers/arrow-worker.ts"],"sourcesContent":["// loaders.gl\n// SPDX-License-Identifier: MIT\n// Copyright (c) vis.gl contributors\n\nimport {createLoaderWorker} from '@loaders.gl/loader-utils';\nimport {ArrowLoader} from '../arrow-loader';\n\ncreateLoaderWorker(ArrowLoader);\n"],"mappings":"AAIA,SAAQA,kBAAkB,QAAO,0BAA0B;AAAC,SACpDC,WAAW;AAEnBD,kBAAkB,CAACC,WAAW,CAAC"}

package/dist/workers/hard-clone.d.ts

@@ -0,0 +1,23 @@
+import * as arrow from 'apache-arrow';
+/**
+ * Clone an Arrow JS Data or Vector, detaching from an existing ArrayBuffer if
+ * it is shared with other.
+ *
+ * The purpose of this function is to enable transferring a `Data` instance,
+ * e.g. to a web worker, without neutering any other data.
+ *
+ * Any internal buffers that are a slice of a larger `ArrayBuffer` (i.e. where
+ * the typed array's `byteOffset` is not `0` and where its `byteLength` does not
+ * match its `array.buffer.byteLength`) are copied into new `ArrayBuffers`.
+ *
+ * If `force` is `true`, always clone internal buffers, even if not shared. If
+ * the default, `false`, any internal buffers that are **not** a slice of a
+ * larger `ArrayBuffer` will not be copied.
+ */
+export declare function hardClone<T extends arrow.DataType>(input: arrow.Data<T>, force?: boolean): arrow.Data<T>;
+export declare function hardClone<T extends arrow.DataType>(input: arrow.Vector<T>, force?: boolean): arrow.Vector<T>;
+/**
+ * Test whether an arrow.Data instance is a slice of a larger `ArrayBuffer`.
+ */
+export declare function isShared<T extends arrow.DataType>(data: arrow.Data<T> | arrow.Vector<T>): boolean;
+//# sourceMappingURL=hard-clone.d.ts.map

package/dist/workers/hard-clone.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"hard-clone.d.ts","sourceRoot":"","sources":["../../src/workers/hard-clone.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,KAAK,MAAM,cAAc,CAAC;AActC;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,SAAS,CAAC,CAAC,SAAS,KAAK,CAAC,QAAQ,EAChD,KAAK,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EACpB,KAAK,CAAC,EAAE,OAAO,GACd,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AACjB,wBAAgB,SAAS,CAAC,CAAC,SAAS,KAAK,CAAC,QAAQ,EAChD,KAAK,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EACtB,KAAK,CAAC,EAAE,OAAO,GACd,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;AAuDnB;;GAEG;AACH,wBAAgB,QAAQ,CAAC,CAAC,SAAS,KAAK,CAAC,QAAQ,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,OAAO,CAiCjG"}

package/dist/workers/hard-clone.js

@@ -0,0 +1,57 @@
+import * as arrow from 'apache-arrow';
+export function hardClone(data) {
+  let force = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : false;
+  if ('data' in data) {
+    return new arrow.Vector(data.data.map(data => hardClone(data, force)));
+  }
+  const clonedChildren = [];
+  for (const childData of data.children) {
+    clonedChildren.push(hardClone(childData, force));
+  }
+  let clonedDictionary;
+  if (data.dictionary !== undefined) {
+    clonedDictionary = hardClone(data.dictionary, force);
+  }
+  const clonedBuffers = {
+    [arrow.BufferType.OFFSET]: cloneBuffer(data.buffers[arrow.BufferType.OFFSET], force),
+    [arrow.BufferType.DATA]: cloneBuffer(data.buffers[arrow.BufferType.DATA], force),
+    [arrow.BufferType.VALIDITY]: cloneBuffer(data.buffers[arrow.BufferType.VALIDITY], force),
+    [arrow.BufferType.TYPE]: cloneBuffer(data.buffers[arrow.BufferType.TYPE], force)
+  };
+  return new arrow.Data(data.type, data.offset, data.length, data._nullCount, clonedBuffers, clonedChildren, clonedDictionary);
+}
+export function isShared(data) {
+  if ('data' in data) {
+    return data.data.some(data => isShared(data));
+  }
+  for (const childData of data.children) {
+    if (isShared(childData)) {
+      return true;
+    }
+  }
+  if (data.dictionary !== undefined) {
+    if (isShared(data.dictionary)) {
+      return true;
+    }
+  }
+  const bufferTypes = [arrow.BufferType.OFFSET, arrow.BufferType.DATA, arrow.BufferType.VALIDITY, arrow.BufferType.TYPE];
+  for (const bufferType of bufferTypes) {
+    if (data.buffers[bufferType] !== undefined && isTypedArraySliced(data.buffers[bufferType])) {
+      return true;
+    }
+  }
+  return false;
+}
+function isTypedArraySliced(arr) {
+  return !(arr.byteOffset === 0 && arr.byteLength === arr.buffer.byteLength);
+}
+function cloneBuffer(arr, force) {
+  if (arr === undefined) {
+    return arr;
+  }
+  if (!force && !isTypedArraySliced(arr)) {
+    return arr;
+  }
+  return arr.slice();
+}
+//# sourceMappingURL=hard-clone.js.map

package/dist/workers/hard-clone.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"hard-clone.js","names":["arrow","hardClone","data","force","arguments","length","undefined","Vector","map","clonedChildren","childData","children","push","clonedDictionary","dictionary","clonedBuffers","BufferType","OFFSET","cloneBuffer","buffers","DATA","VALIDITY","TYPE","Data","type","offset","_nullCount","isShared","some","bufferTypes","bufferType","isTypedArraySliced","arr","byteOffset","byteLength","buffer","slice"],"sources":["../../src/workers/hard-clone.ts"],"sourcesContent":["import * as arrow from 'apache-arrow';\nimport type {Buffers} from 'apache-arrow/data';\n\ntype TypedArray =\n  | Uint8Array\n  | Uint8ClampedArray\n  | Uint16Array\n  | Uint32Array\n  | Int8Array\n  | Int16Array\n  | Int32Array\n  | Float32Array\n  | Float64Array;\n\n/**\n * Clone an Arrow JS Data or Vector, detaching from an existing ArrayBuffer if\n * it is shared with other.\n *\n * The purpose of this function is to enable transferring a `Data` instance,\n * e.g. to a web worker, without neutering any other data.\n *\n * Any internal buffers that are a slice of a larger `ArrayBuffer` (i.e. where\n * the typed array's `byteOffset` is not `0` and where its `byteLength` does not\n * match its `array.buffer.byteLength`) are copied into new `ArrayBuffers`.\n *\n * If `force` is `true`, always clone internal buffers, even if not shared. If\n * the default, `false`, any internal buffers that are **not** a slice of a\n * larger `ArrayBuffer` will not be copied.\n */\nexport function hardClone<T extends arrow.DataType>(\n  input: arrow.Data<T>,\n  force?: boolean\n): arrow.Data<T>;\nexport function hardClone<T extends arrow.DataType>(\n  input: arrow.Vector<T>,\n  force?: boolean\n): arrow.Vector<T>;\n\nexport function hardClone<T extends arrow.DataType>(\n  data: arrow.Data<T> | arrow.Vector<T>,\n  force: boolean = false\n): arrow.Data<T> | arrow.Vector<T> {\n  // Check if `data` is an arrow.Vector\n  if ('data' in data) {\n    return new arrow.Vector(data.data.map((data) => hardClone(data, force)));\n  }\n\n  // Clone each of the children, recursively\n  const clonedChildren: arrow.Data[] = [];\n  for (const childData of data.children) {\n    clonedChildren.push(hardClone(childData, force));\n  }\n\n  // Clone the dictionary if there is one\n  let clonedDictionary: arrow.Vector | undefined;\n  if (data.dictionary !== undefined) {\n    clonedDictionary = hardClone(data.dictionary, force);\n  }\n\n  // Buffers can have up to four entries. Each of these can be `undefined` for\n  // one or more array types.\n  //\n  // - OFFSET: value offsets for variable size list types\n  // - DATA: the underlying data\n  // - VALIDITY: the null buffer. This may be empty or undefined if all elements\n  //   are non-null/valid.\n  // - TYPE: type ids for a union type.\n  const clonedBuffers: Buffers<T> = {\n    [arrow.BufferType.OFFSET]: cloneBuffer(data.buffers[arrow.BufferType.OFFSET], force),\n    [arrow.BufferType.DATA]: cloneBuffer(data.buffers[arrow.BufferType.DATA], force),\n    [arrow.BufferType.VALIDITY]: cloneBuffer(data.buffers[arrow.BufferType.VALIDITY], force),\n    [arrow.BufferType.TYPE]: cloneBuffer(data.buffers[arrow.BufferType.TYPE], force)\n  };\n\n  // Note: the data.offset is passed on so that a sliced Data instance will not\n  // be \"un-sliced\". However keep in mind that this means we're cloning the\n  // _original backing buffer_, not only the portion of the Data that was\n  // sliced.\n  return new arrow.Data(\n    data.type,\n    data.offset,\n    data.length,\n    // @ts-expect-error _nullCount is protected. We're using it here to mimic\n    // `Data.clone`\n    data._nullCount,\n    clonedBuffers,\n    clonedChildren,\n    clonedDictionary\n  );\n}\n\n/**\n * Test whether an arrow.Data instance is a slice of a larger `ArrayBuffer`.\n */\nexport function isShared<T extends arrow.DataType>(data: arrow.Data<T> | arrow.Vector<T>): boolean {\n  // Loop over arrow.Vector\n  if ('data' in data) {\n    return data.data.some((data) => isShared(data));\n  }\n\n  // Check child data\n  for (const childData of data.children) {\n    if (isShared(childData)) {\n      return true;\n    }\n  }\n\n  // Check dictionary\n  if (data.dictionary !== undefined) {\n    if (isShared(data.dictionary)) {\n      return true;\n    }\n  }\n\n  const bufferTypes = [\n    arrow.BufferType.OFFSET,\n    arrow.BufferType.DATA,\n    arrow.BufferType.VALIDITY,\n    arrow.BufferType.TYPE\n  ];\n  for (const bufferType of bufferTypes) {\n    if (data.buffers[bufferType] !== undefined && isTypedArraySliced(data.buffers[bufferType])) {\n      return true;\n    }\n  }\n\n  return false;\n}\n\n/**\n * Returns true if the current typed array is a partial slice on a larger\n * ArrayBuffer\n */\nfunction isTypedArraySliced(arr: TypedArray): boolean {\n  return !(arr.byteOffset === 0 && arr.byteLength === arr.buffer.byteLength);\n}\n\n/**\n * If a slice of a larger ArrayBuffer, clone to a fresh `ArrayBuffer`.\n *\n * If `force` is `true`, always clone the array, even if not shared.\n */\nfunction cloneBuffer<A extends TypedArray | undefined>(arr: A, force: boolean): A {\n  // Not all buffer types are defined for every type of Arrow array. E.g.\n  // `arrow.BufferType.TYPE` is only defined for the Union type.\n  if (arr === undefined) {\n    return arr;\n  }\n\n  // The current array is not a part of a larger ArrayBuffer, don't clone it\n  if (!force && !isTypedArraySliced(arr)) {\n    return arr;\n  }\n\n  // Note: TypedArray.slice() **copies** into a new ArrayBuffer\n\n  // @ts-expect-error 'Uint8Array' is assignable to the constraint of type 'A',\n  // but 'A' could be instantiated with a different subtype of constraint\n  // 'TypedArray'\n  // We know from arr.slice that it will always return the same\n  return arr.slice();\n}\n"],"mappings":"AAAA,OAAO,KAAKA,KAAK,MAAM,cAAc;AAsCrC,OAAO,SAASC,SAASA,CACvBC,IAAqC,EAEJ;EAAA,IADjCC,KAAc,GAAAC,SAAA,CAAAC,MAAA,QAAAD,SAAA,QAAAE,SAAA,GAAAF,SAAA,MAAG,KAAK;EAGtB,IAAI,MAAM,IAAIF,IAAI,EAAE;IAClB,OAAO,IAAIF,KAAK,CAACO,MAAM,CAACL,IAAI,CAACA,IAAI,CAACM,GAAG,CAAEN,IAAI,IAAKD,SAAS,CAACC,IAAI,EAAEC,KAAK,CAAC,CAAC,CAAC;EAC1E;EAGA,MAAMM,cAA4B,GAAG,EAAE;EACvC,KAAK,MAAMC,SAAS,IAAIR,IAAI,CAACS,QAAQ,EAAE;IACrCF,cAAc,CAACG,IAAI,CAACX,SAAS,CAACS,SAAS,EAAEP,KAAK,CAAC,CAAC;EAClD;EAGA,IAAIU,gBAA0C;EAC9C,IAAIX,IAAI,CAACY,UAAU,KAAKR,SAAS,EAAE;IACjCO,gBAAgB,GAAGZ,SAAS,CAACC,IAAI,CAACY,UAAU,EAAEX,KAAK,CAAC;EACtD;EAUA,MAAMY,aAAyB,GAAG;IAChC,CAACf,KAAK,CAACgB,UAAU,CAACC,MAAM,GAAGC,WAAW,CAAChB,IAAI,CAACiB,OAAO,CAACnB,KAAK,CAACgB,UAAU,CAACC,MAAM,CAAC,EAAEd,KAAK,CAAC;IACpF,CAACH,KAAK,CAACgB,UAAU,CAACI,IAAI,GAAGF,WAAW,CAAChB,IAAI,CAACiB,OAAO,CAACnB,KAAK,CAACgB,UAAU,CAACI,IAAI,CAAC,EAAEjB,KAAK,CAAC;IAChF,CAACH,KAAK,CAACgB,UAAU,CAACK,QAAQ,GAAGH,WAAW,CAAChB,IAAI,CAACiB,OAAO,CAACnB,KAAK,CAACgB,UAAU,CAACK,QAAQ,CAAC,EAAElB,KAAK,CAAC;IACxF,CAACH,KAAK,CAACgB,UAAU,CAACM,IAAI,GAAGJ,WAAW,CAAChB,IAAI,CAACiB,OAAO,CAACnB,KAAK,CAACgB,UAAU,CAACM,IAAI,CAAC,EAAEnB,KAAK;EACjF,CAAC;EAMD,OAAO,IAAIH,KAAK,CAACuB,IAAI,CACnBrB,IAAI,CAACsB,IAAI,EACTtB,IAAI,CAACuB,MAAM,EACXvB,IAAI,CAACG,MAAM,EAGXH,IAAI,CAACwB,UAAU,EACfX,aAAa,EACbN,cAAc,EACdI,gBACF,CAAC;AACH;AAKA,OAAO,SAASc,QAAQA,CAA2BzB,IAAqC,EAAW;EAEjG,IAAI,MAAM,IAAIA,IAAI,EAAE;IAClB,OAAOA,IAAI,CAACA,IAAI,CAAC0B,IAAI,CAAE1B,IAAI,IAAKyB,QAAQ,CAACzB,IAAI,CAAC,CAAC;EACjD;EAGA,KAAK,MAAMQ,SAAS,IAAIR,IAAI,CAACS,QAAQ,EAAE;IACrC,IAAIgB,QAAQ,CAACjB,SAAS,CAAC,EAAE;MACvB,OAAO,IAAI;IACb;EACF;EAGA,IAAIR,IAAI,CAACY,UAAU,KAAKR,SAAS,EAAE;IACjC,IAAIqB,QAAQ,CAACzB,IAAI,CAACY,UAAU,CAAC,EAAE;MAC7B,OAAO,IAAI;IACb;EACF;EAEA,MAAMe,WAAW,GAAG,CAClB7B,KAAK,CAACgB,UAAU,CAACC,MAAM,EACvBjB,KAAK,CAACgB,UAAU,CAACI,IAAI,EACrBpB,KAAK,CAACgB,UAAU,CAACK,QAAQ,EACzBrB,KAAK,CAACgB,UAAU,CAACM,IAAI,CACtB;EACD,KAAK,MAAMQ,UAAU,IAAID,WAAW,EAAE;IACpC,IAAI3B,IAAI,CAACiB,OAAO,CAACW,UAAU,CAAC,KAAKxB,SAAS,IAAIyB,kBAAkB,CAAC7B,IAAI,CAACiB,OAAO,CAACW,UAAU,CAAC,CAAC,EAAE;MAC1F,OAAO,IAAI;IACb;EACF;EAEA,OAAO,KAAK;AACd;AAMA,SAASC,kBAAkBA,CAACC,GAAe,EAAW;EACpD,OAAO,EAAEA,GAAG,CAACC,UAAU,KAAK,CAAC,IAAID,GAAG,CAACE,UAAU,KAAKF,GAAG,CAACG,MAAM,CAACD,UAAU,CAAC;AAC5E;AAOA,SAAShB,WAAWA,CAAmCc,GAAM,EAAE7B,KAAc,EAAK;EAGhF,IAAI6B,GAAG,KAAK1B,SAAS,EAAE;IACrB,OAAO0B,GAAG;EACZ;EAGA,IAAI,CAAC7B,KAAK,IAAI,CAAC4B,kBAAkB,CAACC,GAAG,CAAC,EAAE;IACtC,OAAOA,GAAG;EACZ;EAQA,OAAOA,GAAG,CAACI,KAAK,CAAC,CAAC;AACpB"}

package/dist/workers/triangulation-worker-node.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"triangulation-worker-node.d.ts","sourceRoot":"","sources":["../../src/workers/triangulation-worker-node.ts"],"names":[],"mappings":"AAGA,OAAO,wBAAwB,CAAC"}
+{"version":3,"file":"triangulation-worker-node.d.ts","sourceRoot":"","sources":["../../src/workers/triangulation-worker-node.ts"],"names":[],"mappings":"AAIA,OAAO,wBAAwB,CAAC"}

package/dist/workers/triangulation-worker-node.js.map

@@ -1 +1 @@
-{"version":3,"file":"triangulation-worker-node.js","names":[],"sources":["../../src/workers/triangulation-worker-node.ts"],"sourcesContent":["// loaders.gl, MIT license\n// Copyright (c) vis.gl contributors\n\nimport './triangulation-worker';\n"],"mappings":""}
+{"version":3,"file":"triangulation-worker-node.js","names":[],"sources":["../../src/workers/triangulation-worker-node.ts"],"sourcesContent":["// loaders.gl\n// SPDX-License-Identifier: MIT\n// Copyright (c) vis.gl contributors\n\nimport './triangulation-worker';\n"],"mappings":""}

package/dist/workers/triangulation-worker.js

@@ -1,5 +1,6 @@
+import * as arrow from 'apache-arrow';
 import { createWorker } from '@loaders.gl/worker-utils';
-import { getTriangleIndices } from "../geoarrow/convert-geoarrow-to-binary-geometry.js";
+import { getTriangleIndices, getBinaryGeometriesFromArrow } from "../geoarrow/convert-geoarrow-to-binary-geometry.js";
 createWorker(async function (data) {
   let options = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : {};
   const input = data;
@@ -9,12 +10,13 @@ createWorker(async function (data) {
       return input;
     case 'triangulate':
       return triangulateBatch(data);
+    case 'parse-geoarrow':
+      return parseGeoArrowBatch(data);
     default:
       throw new Error(`TriangulationWorker: Unsupported operation ${operation}. Expected 'triangulate'`);
   }
 });
 function triangulateBatch(data) {
-  console.error('TriangulationWorker: tessellating batch', data);
   const triangleIndices = getTriangleIndices(data.polygonIndices, data.primitivePolygonIndices, data.flatCoordinateArray, data.nDim);
   return {
     ...data,
@@ -23,4 +25,34 @@ function triangulateBatch(data) {
     } : {})
   };
 }
+function parseGeoArrowBatch(data) {
+  let binaryDataFromGeoArrow = null;
+  const {
+    chunkData,
+    chunkIndex,
+    chunkOffset,
+    geometryEncoding,
+    calculateMeanCenters,
+    triangle
+  } = data;
+  const arrowData = new arrow.Data(chunkData.type, chunkData.offset, chunkData.length, chunkData.nullCount, chunkData.buffers, chunkData.children, chunkData.dictionary);
+  const geometryColumn = arrow.makeVector(arrowData);
+  if (geometryColumn) {
+    const options = {
+      calculateMeanCenters,
+      triangle,
+      chunkIndex: 0,
+      chunkOffset
+    };
+    binaryDataFromGeoArrow = getBinaryGeometriesFromArrow(geometryColumn, geometryEncoding, options);
+    return {
+      binaryDataFromGeoArrow,
+      chunkIndex
+    };
+  }
+  return {
+    binaryDataFromGeoArrow,
+    chunkIndex
+  };
+}
 //# sourceMappingURL=triangulation-worker.js.map

package/dist/workers/triangulation-worker.js.map

@@ -1 +1 @@
-{"version":3,"file":"triangulation-worker.js","names":["createWorker","getTriangleIndices","data","options","arguments","length","undefined","input","operation","triangulateBatch","Error","console","error","triangleIndices","polygonIndices","primitivePolygonIndices","flatCoordinateArray","nDim"],"sources":["../../src/workers/triangulation-worker.ts"],"sourcesContent":["// loaders.gl, MIT license\n// Copyright (c) vis.gl contributors\n\nimport {createWorker} from '@loaders.gl/worker-utils';\nimport {getTriangleIndices} from '../geoarrow/convert-geoarrow-to-binary-geometry';\nimport type {\n  TriangulationWorkerInput,\n  TriangulateInput,\n  TriangulateResult\n} from '../triangulate-on-worker';\n\ncreateWorker(async (data, options = {}) => {\n  const input = data as TriangulationWorkerInput;\n  const operation = input?.operation;\n  switch (operation) {\n    case 'test':\n      return input;\n    case 'triangulate':\n      return triangulateBatch(data);\n    default:\n      throw new Error(\n        `TriangulationWorker: Unsupported operation ${operation}. Expected 'triangulate'`\n      );\n  }\n});\n\nfunction triangulateBatch(data: TriangulateInput): TriangulateResult {\n  // Parse any WKT/WKB geometries\n  // Build binary geometries\n  // Call earcut and triangulate\n  console.error('TriangulationWorker: tessellating batch', data);\n  const triangleIndices = getTriangleIndices(\n    data.polygonIndices,\n    data.primitivePolygonIndices,\n    data.flatCoordinateArray,\n    data.nDim\n  );\n  return {...data, ...(triangleIndices ? {triangleIndices} : {})};\n}\n"],"mappings":"AAGA,SAAQA,YAAY,QAAO,0BAA0B;AAAC,SAC9CC,kBAAkB;AAO1BD,YAAY,CAAC,gBAAOE,IAAI,EAAmB;EAAA,IAAjBC,OAAO,GAAAC,SAAA,CAAAC,MAAA,QAAAD,SAAA,QAAAE,SAAA,GAAAF,SAAA,MAAG,CAAC,CAAC;EACpC,MAAMG,KAAK,GAAGL,IAAgC;EAC9C,MAAMM,SAAS,GAAGD,KAAK,aAALA,KAAK,uBAALA,KAAK,CAAEC,SAAS;EAClC,QAAQA,SAAS;IACf,KAAK,MAAM;MACT,OAAOD,KAAK;IACd,KAAK,aAAa;MAChB,OAAOE,gBAAgB,CAACP,IAAI,CAAC;IAC/B;MACE,MAAM,IAAIQ,KAAK,CACZ,8CAA6CF,SAAU,0BAC1D,CAAC;EACL;AACF,CAAC,CAAC;AAEF,SAASC,gBAAgBA,CAACP,IAAsB,EAAqB;EAInES,OAAO,CAACC,KAAK,CAAC,yCAAyC,EAAEV,IAAI,CAAC;EAC9D,MAAMW,eAAe,GAAGZ,kBAAkB,CACxCC,IAAI,CAACY,cAAc,EACnBZ,IAAI,CAACa,uBAAuB,EAC5Bb,IAAI,CAACc,mBAAmB,EACxBd,IAAI,CAACe,IACP,CAAC;EACD,OAAO;IAAC,GAAGf,IAAI;IAAE,IAAIW,eAAe,GAAG;MAACA;IAAe,CAAC,GAAG,CAAC,CAAC;EAAC,CAAC;AACjE"}
+{"version":3,"file":"triangulation-worker.js","names":["arrow","createWorker","getTriangleIndices","getBinaryGeometriesFromArrow","data","options","arguments","length","undefined","input","operation","triangulateBatch","parseGeoArrowBatch","Error","triangleIndices","polygonIndices","primitivePolygonIndices","flatCoordinateArray","nDim","binaryDataFromGeoArrow","chunkData","chunkIndex","chunkOffset","geometryEncoding","calculateMeanCenters","triangle","arrowData","Data","type","offset","nullCount","buffers","children","dictionary","geometryColumn","makeVector"],"sources":["../../src/workers/triangulation-worker.ts"],"sourcesContent":["// loaders.gl\n// SPDX-License-Identifier: MIT\n// Copyright (c) vis.gl contributors\n\nimport * as arrow from 'apache-arrow';\nimport {createWorker} from '@loaders.gl/worker-utils';\nimport {\n  getTriangleIndices,\n  getBinaryGeometriesFromArrow,\n  BinaryDataFromGeoArrow\n} from '../geoarrow/convert-geoarrow-to-binary-geometry';\nimport type {\n  TriangulationWorkerInput,\n  TriangulateInput,\n  TriangulateResult,\n  ParseGeoArrowInput,\n  ParseGeoArrowResult\n} from '../triangulate-on-worker';\n\ncreateWorker(async (data, options = {}) => {\n  const input = data as TriangulationWorkerInput;\n  const operation = input?.operation;\n  switch (operation) {\n    case 'test':\n      return input;\n    case 'triangulate':\n      return triangulateBatch(data);\n    case 'parse-geoarrow':\n      return parseGeoArrowBatch(data);\n    default:\n      throw new Error(\n        `TriangulationWorker: Unsupported operation ${operation}. Expected 'triangulate'`\n      );\n  }\n});\n\nfunction triangulateBatch(data: TriangulateInput): TriangulateResult {\n  // Parse any WKT/WKB geometries\n  // Build binary geometries\n  // Call earcut and triangulate\n  const triangleIndices = getTriangleIndices(\n    data.polygonIndices,\n    data.primitivePolygonIndices,\n    data.flatCoordinateArray,\n    data.nDim\n  );\n  return {...data, ...(triangleIndices ? {triangleIndices} : {})};\n}\n\n/**\n * Reading the arrow file into memory is very fast. Parsing the geoarrow column is slow, and blocking the main thread.\n * To address this issue, we can move the parsing job from main thread to parallel web workers.\n * Each web worker will parse one chunk/batch of geoarrow column, and return binary geometries to main thread.\n * The app on the main thread will render the binary geometries and the parsing will not block the main thread.\n *\n * @param data\n * @returns\n */\nfunction parseGeoArrowBatch(data: ParseGeoArrowInput): ParseGeoArrowResult {\n  let binaryDataFromGeoArrow: BinaryDataFromGeoArrow | null = null;\n  const {chunkData, chunkIndex, chunkOffset, geometryEncoding, calculateMeanCenters, triangle} =\n    data;\n  // rebuild chunkData that is only for geoarrow column\n  const arrowData = new arrow.Data(\n    chunkData.type,\n    chunkData.offset,\n    chunkData.length,\n    chunkData.nullCount,\n    chunkData.buffers,\n    chunkData.children,\n    chunkData.dictionary\n  );\n  // rebuild geometry column with chunkData\n  const geometryColumn = arrow.makeVector(arrowData);\n  if (geometryColumn) {\n    // NOTE: for a rebuild arrow.Vector, there is only one chunk, so chunkIndex is always 0\n    const options = {calculateMeanCenters, triangle, chunkIndex: 0, chunkOffset};\n    binaryDataFromGeoArrow = getBinaryGeometriesFromArrow(\n      geometryColumn,\n      geometryEncoding,\n      options\n    );\n    // NOTE: here binaryGeometry will be copied to main thread\n    return {\n      binaryDataFromGeoArrow,\n      chunkIndex\n    };\n  }\n  return {\n    binaryDataFromGeoArrow,\n    chunkIndex\n  };\n}\n"],"mappings":"AAIA,OAAO,KAAKA,KAAK,MAAM,cAAc;AACrC,SAAQC,YAAY,QAAO,0BAA0B;AAAC,SAEpDC,kBAAkB,EAClBC,4BAA4B;AAW9BF,YAAY,CAAC,gBAAOG,IAAI,EAAmB;EAAA,IAAjBC,OAAO,GAAAC,SAAA,CAAAC,MAAA,QAAAD,SAAA,QAAAE,SAAA,GAAAF,SAAA,MAAG,CAAC,CAAC;EACpC,MAAMG,KAAK,GAAGL,IAAgC;EAC9C,MAAMM,SAAS,GAAGD,KAAK,aAALA,KAAK,uBAALA,KAAK,CAAEC,SAAS;EAClC,QAAQA,SAAS;IACf,KAAK,MAAM;MACT,OAAOD,KAAK;IACd,KAAK,aAAa;MAChB,OAAOE,gBAAgB,CAACP,IAAI,CAAC;IAC/B,KAAK,gBAAgB;MACnB,OAAOQ,kBAAkB,CAACR,IAAI,CAAC;IACjC;MACE,MAAM,IAAIS,KAAK,CACZ,8CAA6CH,SAAU,0BAC1D,CAAC;EACL;AACF,CAAC,CAAC;AAEF,SAASC,gBAAgBA,CAACP,IAAsB,EAAqB;EAInE,MAAMU,eAAe,GAAGZ,kBAAkB,CACxCE,IAAI,CAACW,cAAc,EACnBX,IAAI,CAACY,uBAAuB,EAC5BZ,IAAI,CAACa,mBAAmB,EACxBb,IAAI,CAACc,IACP,CAAC;EACD,OAAO;IAAC,GAAGd,IAAI;IAAE,IAAIU,eAAe,GAAG;MAACA;IAAe,CAAC,GAAG,CAAC,CAAC;EAAC,CAAC;AACjE;AAWA,SAASF,kBAAkBA,CAACR,IAAwB,EAAuB;EACzE,IAAIe,sBAAqD,GAAG,IAAI;EAChE,MAAM;IAACC,SAAS;IAAEC,UAAU;IAAEC,WAAW;IAAEC,gBAAgB;IAAEC,oBAAoB;IAAEC;EAAQ,CAAC,GAC1FrB,IAAI;EAEN,MAAMsB,SAAS,GAAG,IAAI1B,KAAK,CAAC2B,IAAI,CAC9BP,SAAS,CAACQ,IAAI,EACdR,SAAS,CAACS,MAAM,EAChBT,SAAS,CAACb,MAAM,EAChBa,SAAS,CAACU,SAAS,EACnBV,SAAS,CAACW,OAAO,EACjBX,SAAS,CAACY,QAAQ,EAClBZ,SAAS,CAACa,UACZ,CAAC;EAED,MAAMC,cAAc,GAAGlC,KAAK,CAACmC,UAAU,CAACT,SAAS,CAAC;EAClD,IAAIQ,cAAc,EAAE;IAElB,MAAM7B,OAAO,GAAG;MAACmB,oBAAoB;MAAEC,QAAQ;MAAEJ,UAAU,EAAE,CAAC;MAAEC;IAAW,CAAC;IAC5EH,sBAAsB,GAAGhB,4BAA4B,CACnD+B,cAAc,EACdX,gBAAgB,EAChBlB,OACF,CAAC;IAED,OAAO;MACLc,sBAAsB;MACtBE;IACF,CAAC;EACH;EACA,OAAO;IACLF,sBAAsB;IACtBE;EACF,CAAC;AACH"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@loaders.gl/arrow",
-  "version": "4.1.0-alpha.1",
+  "version": "4.1.0-alpha.11",
   "description": "Simple columnar table loader for the Apache Arrow format",
   "license": "MIT",
   "type": "module",
@@ -29,9 +29,9 @@
   "module": "dist/index.js",
   "exports": {
     ".": {
+      "types": "./dist/index.d.ts",
       "import": "./dist/index.js",
-      "require": "./dist/index.cjs",
-      "types": "./dist/index.d.ts"
+      "require": "./dist/index.cjs"
     }
   },
   "sideEffects": false,
@@ -52,12 +52,12 @@
     "build-worker2": "esbuild src/workers/arrow-worker.ts --bundle --outfile=dist/arrow-worker.js --platform=browser --external:{stream}"
   },
   "dependencies": {
-    "@loaders.gl/gis": "4.1.0-alpha.1",
-    "@loaders.gl/loader-utils": "4.1.0-alpha.1",
-    "@loaders.gl/schema": "4.1.0-alpha.1",
-    "@loaders.gl/wkt": "4.1.0-alpha.1",
+    "@loaders.gl/gis": "4.1.0-alpha.11",
+    "@loaders.gl/loader-utils": "4.1.0-alpha.11",
+    "@loaders.gl/schema": "4.1.0-alpha.11",
+    "@loaders.gl/wkt": "4.1.0-alpha.11",
     "@math.gl/polygon": "4.0.0",
-    "apache-arrow": "^13.0.0"
+    "apache-arrow": ">= 15.0.0"
   },
-  "gitHead": "6a4d3da93d45115ad99861474a43c3f4a0b280a7"
+  "gitHead": "5d3e23bf93762b48c8c1d6d926ede7a97fe43ab0"
 }

package/src/arrow-loader.ts

@@ -1,4 +1,5 @@
-// loaders.gl, MIT license
+// loaders.gl
+// SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
 import type {Loader, LoaderWithParser, LoaderOptions} from '@loaders.gl/loader-utils';
@@ -19,6 +20,7 @@ const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
 export type ArrowLoaderOptions = LoaderOptions & {
   arrow?: {
     shape: 'arrow-table' | 'columnar-table' | 'array-row-table' | 'object-row-table';
+    batchDebounceMs?: number;
   };
 };
 

package/src/geoarrow/convert-geoarrow-to-binary-geometry.ts

@@ -1,4 +1,5 @@
-// loaders.gl, MIT license
+// loaders.gl
+// SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
 import * as arrow from 'apache-arrow';
@@ -45,7 +46,7 @@ type BinaryGeometryContent = {
   geomOffset: Int32Array;
   /** Array of geometry indicies: the start index of each geometry */
   geometryIndicies: Uint16Array;
-  /** (Optional) indices of triangels returned from polygon tessellation (Polygon only) */
+  /** (Optional) indices of triangels returned from polygon triangulation (Polygon only) */
   triangles?: Uint32Array;
   /** (Optional) array of mean center of each geometry */
   meanCenters?: Float64Array;
@@ -54,17 +55,21 @@ type BinaryGeometryContent = {
 /**
  * binary geometry template, see deck.gl BinaryGeometry
  */
-export const BINARY_GEOMETRY_TEMPLATE = {
-  globalFeatureIds: {value: new Uint32Array(0), size: 1},
-  positions: {value: new Float32Array(0), size: 2},
-  properties: [],
-  numericProps: {},
-  featureIds: {value: new Uint32Array(0), size: 1}
-};
+export function getBinaryGeometryTemplate() {
+  return {
+    globalFeatureIds: {value: new Uint32Array(0), size: 1},
+    positions: {value: new Float32Array(0), size: 2},
+    properties: [],
+    numericProps: {},
+    featureIds: {value: new Uint32Array(0), size: 1}
+  };
+}
 
 export type BinaryGeometriesFromArrowOptions = {
   /** option to specify which chunk to get binary geometries from, for progressive rendering */
   chunkIndex?: number;
+  /** The offset (beginning index of rows) of input chunk. Used for reconstructing globalFeatureIds in web workers */
+  chunkOffset?: number;
   /** option to get mean centers from geometries, for polygon filtering */
   calculateMeanCenters?: boolean;
   /** option to compute the triangle indices by tesselating polygons */
@@ -90,9 +95,12 @@ export function getBinaryGeometriesFromArrow(
     line: geoEncoding === 'geoarrow.multilinestring' || geoEncoding === 'geoarrow.linestring'
   };
 
-  const chunks = options?.chunkIndex ? [geoColumn.data[options?.chunkIndex]] : geoColumn.data;
+  const chunks =
+    options?.chunkIndex !== undefined && options?.chunkIndex >= 0
+      ? [geoColumn.data[options?.chunkIndex]]
+      : geoColumn.data;
   let bounds: [number, number, number, number] = [Infinity, Infinity, -Infinity, -Infinity];
-  let globalFeatureIdOffset = 0;
+  let globalFeatureIdOffset = options?.chunkOffset || 0;
   const binaryGeometries: BinaryFeatures[] = [];
 
   chunks.forEach((chunk) => {
@@ -111,6 +119,7 @@ export function getBinaryGeometriesFromArrow(
         size: nDim
       },
       featureIds: {value: featureIds, size: 1},
+      // eslint-disable-next-line no-loop-func
       properties: [...Array(chunk.length).keys()].map((i) => ({
         index: i + globalFeatureIdOffset
       }))
@@ -123,18 +132,18 @@ export function getBinaryGeometriesFromArrow(
       shape: 'binary-feature-collection',
       points: {
         type: 'Point',
-        ...BINARY_GEOMETRY_TEMPLATE,
+        ...getBinaryGeometryTemplate(),
         ...(featureTypes.point ? binaryContent : {})
       },
       lines: {
         type: 'LineString',
-        ...BINARY_GEOMETRY_TEMPLATE,
+        ...getBinaryGeometryTemplate(),
         ...(featureTypes.line ? binaryContent : {}),
         pathIndices: {value: featureTypes.line ? geomOffset : new Uint16Array(0), size: 1}
       },
       polygons: {
         type: 'Polygon',
-        ...BINARY_GEOMETRY_TEMPLATE,
+        ...getBinaryGeometryTemplate(),
         ...(featureTypes.polygon ? binaryContent : {}),
         polygonIndices: {
           // use geomOffset as polygonIndices same as primitivePolygonIndices since we are using earcut to get triangule indices
@@ -281,7 +290,7 @@ function getBinaryGeometriesFromChunk(
  * @param primitivePolygonIndices Indices within positions of the start of each primitive Polygon/ring
  * @param flatCoordinateArray Array of x, y or x, y, z positions
  * @param nDim - number of dimensions per position
- * @returns
+ * @returns triangle indices or null if invalid polygon and earcut fails
  */
 export function getTriangleIndices(
   polygonIndices: Uint16Array,
@@ -306,13 +315,14 @@ export function getTriangleIndices(
         }
         primitiveIndex++;
       }
+      // TODO check if each ring is closed
       const triangleIndices = earcut(
         slicedFlatCoords,
         holeIndices.length > 0 ? holeIndices : undefined,
         nDim
       );
       if (triangleIndices.length === 0) {
-        throw Error('can not tesselate invalid polygon');
+        throw Error('earcut failed e.g. invalid polygon');
       }
       for (let j = 0; j < triangleIndices.length; j++) {
         triangles.push(triangleIndices[j] + startIdx);
@@ -325,9 +335,7 @@ export function getTriangleIndices(
     }
     return trianglesUint32;
   } catch (error) {
-    // TODO - add logging
-    // there is an expection when tesselating invalid polygon, e.g. polygon with self-intersection
-    // return null to skip tesselating
+    // if earcut fails, return null
     return null;
   }
 }
@@ -379,8 +387,8 @@ function getBinaryPolygonsFromChunk(
 
   return {
     featureIds,
-    flatCoordinateArray,
     nDim,
+    flatCoordinateArray,
     geomOffset,
     geometryIndicies,
     ...(options?.triangulate && triangles ? {triangles} : {})

package/src/geoarrow/convert-geoarrow-to-geojson-geometry.ts

@@ -1,4 +1,5 @@
-// loaders.gl, MIT license
+// loaders.gl
+// SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
 // import * as arrow from 'apache-arrow';

package/src/geoarrow/get-arrow-bounds.ts

@@ -1,4 +1,5 @@
-// loaders.gl, MIT license
+// loaders.gl
+// SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
 /**

package/src/geoarrow-loader.ts

@@ -1,4 +1,5 @@
-// loaders.gl, MIT license
+// loaders.gl
+// SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
 import type {Loader, LoaderWithParser, LoaderOptions} from '@loaders.gl/loader-utils';

package/src/index.ts

@@ -1,4 +1,5 @@
-// loaders.gl, MIT license
+// loaders.gl
+// SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
 import type {ArrowLoaderOptions} from './arrow-loader';
@@ -52,7 +53,7 @@ export type {
   BinaryGeometriesFromArrowOptions
 } from './geoarrow/convert-geoarrow-to-binary-geometry';
 export {
-  BINARY_GEOMETRY_TEMPLATE,
+  getBinaryGeometryTemplate,
   getBinaryGeometriesFromArrow,
   getTriangleIndices,
   getMeanCentersFromBinaryGeometries
@@ -65,5 +66,11 @@ export {parseGeometryFromArrow} from './geoarrow/convert-geoarrow-to-geojson-geo
 export {convertArrowToGeoJSONTable} from './tables/convert-arrow-to-geojson-table';
 
 // EXPERIMENTAL WORKER
+export {hardClone} from './workers/hard-clone';
 
-export {TriangulationWorker, triangulateOnWorker} from './triangulate-on-worker';
+export type {ParseGeoArrowInput, ParseGeoArrowResult} from './triangulate-on-worker';
+export {
+  TriangulationWorker,
+  triangulateOnWorker,
+  parseGeoArrowOnWorker
+} from './triangulate-on-worker';

package/src/lib/arrow-table-batch.ts

@@ -1,4 +1,5 @@
-// loaders.gl, MIT license
+// loaders.gl
+// SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
 import {ColumnarTableBatchAggregator} from '@loaders.gl/schema';

package/src/lib/arrow-table.ts

@@ -1,4 +1,5 @@
-// loaders.gl, MIT license
+// loaders.gl
+// SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
 import type {Batch, Schema} from '@loaders.gl/schema';

package/src/lib/encode-arrow.ts

@@ -1,4 +1,5 @@
-// loaders.gl, MIT license
+// loaders.gl
+// SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
 import * as arrow from 'apache-arrow';

package/src/lib/encode-geoarrow.ts

@@ -1,4 +1,5 @@
-// loaders.gl, MIT license
+// loaders.gl
+// SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
 import * as arrow from 'apache-arrow';

package/src/parsers/parse-arrow-in-batches.ts

@@ -1,14 +1,17 @@
-// loaders.gl, MIT license
+// loaders.gl
+// SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
 import type {ArrowTableBatch} from '../lib/arrow-table';
 import * as arrow from 'apache-arrow';
+import {ArrowLoaderOptions} from '../arrow-loader';
 // import {isIterable} from '@loaders.gl/core';
 
 /**
  */
 export function parseArrowInBatches(
-  asyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>
+  asyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>,
+  options?: ArrowLoaderOptions
 ): AsyncIterable<ArrowTableBatch> {
   // Creates the appropriate arrow.RecordBatchReader subclasses from the input
   // This will also close the underlying source in case of early termination or errors
@@ -33,6 +36,10 @@ export function parseArrowInBatches(
     const readers = arrow.RecordBatchReader.readAll(asyncIterator);
     for await (const reader of readers) {
       for await (const recordBatch of reader) {
+        // use options.batchDebounceMs to add a delay between batches if needed (use case: incremental loading)
+        if (options?.arrow?.batchDebounceMs !== undefined && options?.arrow?.batchDebounceMs > 0) {
+          await new Promise((resolve) => setTimeout(resolve, options.arrow?.batchDebounceMs || 0));
+        }
         const arrowTabledBatch: ArrowTableBatch = {
           shape: 'arrow-table',
           batchType: 'data',

package/src/parsers/parse-arrow-sync.ts

@@ -1,4 +1,5 @@
-// loaders.gl, MIT license
+// loaders.gl
+// SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
 import type {ArrayRowTable, ColumnarTable, ObjectRowTable} from '@loaders.gl/schema';

package/src/parsers/parse-geoarrow-in-batches.ts

@@ -1,4 +1,5 @@
-// loaders.gl, MIT license
+// loaders.gl
+// SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
 import type {GeoJSONTableBatch} from '@loaders.gl/schema';

package/src/parsers/parse-geoarrow-sync.ts

@@ -1,4 +1,5 @@
-// loaders.gl, MIT license
+// loaders.gl
+// SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
 import type {GeoJSONTable} from '@loaders.gl/schema';

package/src/schema/arrow-type-utils.ts

@@ -1,4 +1,5 @@
-// loaders.gl, MIT license
+// loaders.gl
+// SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
 import type {TypedArray} from '@loaders.gl/schema';

package/src/schema/convert-arrow-schema.ts

@@ -1,4 +1,5 @@
-// loaders.gl, MIT license
+// loaders.gl
+// SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
 import type {DataType, Field, Schema, SchemaMetadata} from '@loaders.gl/schema';

package/src/tables/convert-arrow-to-columnar-table.ts

@@ -1,4 +1,5 @@
-// loaders.gl, MIT license
+// loaders.gl
+// SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
 import type {ColumnarTable} from '@loaders.gl/schema';

package/src/tables/convert-arrow-to-geojson-table.ts

@@ -1,4 +1,5 @@
-// loaders.gl, MIT license
+// loaders.gl
+// SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
 import type {Feature, GeoJSONTable} from '@loaders.gl/schema';

package/src/tables/convert-columnar-to-row-table.ts

@@ -1,4 +1,5 @@
-// loaders.gl, MIT license
+// loaders.gl
+// SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
 import type {ColumnarTable, ObjectRowTable} from '@loaders.gl/schema';

package/src/tables/convert-table-to-arrow.ts

@@ -1,4 +1,5 @@
-// loaders.gl, MIT license
+// loaders.gl
+// SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
 // import {