@loaders.gl/arrow 4.0.3 → 4.1.0-alpha.1

package/src/tables/convert-columnar-to-row-table.ts

@@ -0,0 +1,30 @@
+// loaders.gl, MIT license
+// Copyright (c) vis.gl contributors
+
+import type {ColumnarTable, ObjectRowTable} from '@loaders.gl/schema';
+
+/**
+ *
+ * @note - should be part of schema module
+ */
+export function convertColumnarToRowFormatTable(columnarTable: ColumnarTable): ObjectRowTable {
+  const tableKeys = Object.keys(columnarTable);
+  const tableRowsCount = columnarTable[tableKeys[0]].length;
+
+  const rowFormatTable: {}[] = [];
+
+  for (let index = 0; index < tableRowsCount; index++) {
+    const tableItem = {};
+    for (let keyIndex = 0; keyIndex < tableKeys.length; keyIndex++) {
+      const fieldName = tableKeys[keyIndex];
+      tableItem[fieldName] = columnarTable[fieldName][index];
+    }
+    rowFormatTable.push(tableItem);
+  }
+
+  return {
+    shape: 'object-row-table',
+    schema: columnarTable.schema,
+    data: rowFormatTable
+  };
+}

package/src/triangulate-on-worker.ts

@@ -0,0 +1,47 @@
+// loaders.gl, MIT license
+// Copyright (c) vis.gl contributors
+
+import type {WorkerOptions} from '@loaders.gl/worker-utils';
+import {processOnWorker} from '@loaders.gl/worker-utils';
+
+// __VERSION__ is injected by babel-plugin-version-inline
+// @ts-ignore TS2304: Cannot find name '__VERSION__'.
+const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
+
+export type TriangulationWorkerInput = TriangulateInput | {operation: 'test'; data: any};
+export type TriangulationWorkerOutput = TriangulateResult | {operation: 'test'; data: any};
+
+/** Input data for operation: 'triangulate' */
+export type TriangulateInput = {
+  operation: 'triangulate';
+  polygonIndices: Uint16Array;
+  primitivePolygonIndices: Int32Array;
+  flatCoordinateArray: Float64Array;
+  nDim: number;
+};
+
+/** Result type for operation: 'triangulate' */
+export type TriangulateResult = TriangulateInput & {
+  triangleIndices?: Uint32Array;
+};
+
+/**
+ * Worker for tessellating geometries. Normally called through triangulateOnWorker
+ */
+export const TriangulationWorker = {
+  id: 'triangulation',
+  name: 'Triangulate',
+  module: 'arrow',
+  version: VERSION,
+  options: {}
+};
+
+/**
+ * Provide type safety
+ */
+export function triangulateOnWorker(
+  data: TriangulationWorkerInput,
+  options: WorkerOptions = {}
+): Promise<TriangulationWorkerOutput> {
+  return processOnWorker(TriangulationWorker, data, options);
+}

package/src/workers/arrow-worker.ts

@@ -2,6 +2,6 @@
 // Copyright (c) vis.gl contributors
 
 import {createLoaderWorker} from '@loaders.gl/loader-utils';
-import {ArrowLoader} from '../index';
+import {ArrowLoader} from '../arrow-loader';
 
 createLoaderWorker(ArrowLoader);

package/src/workers/triangulation-worker-node.ts

@@ -0,0 +1,4 @@
+// loaders.gl, MIT license
+// Copyright (c) vis.gl contributors
+
+import './triangulation-worker';

package/src/workers/triangulation-worker.ts

@@ -0,0 +1,39 @@
+// loaders.gl, MIT license
+// Copyright (c) vis.gl contributors
+
+import {createWorker} from '@loaders.gl/worker-utils';
+import {getTriangleIndices} from '../geoarrow/convert-geoarrow-to-binary-geometry';
+import type {
+  TriangulationWorkerInput,
+  TriangulateInput,
+  TriangulateResult
+} from '../triangulate-on-worker';
+
+createWorker(async (data, options = {}) => {
+  const input = data as TriangulationWorkerInput;
+  const operation = input?.operation;
+  switch (operation) {
+    case 'test':
+      return input;
+    case 'triangulate':
+      return triangulateBatch(data);
+    default:
+      throw new Error(
+        `TriangulationWorker: Unsupported operation ${operation}. Expected 'triangulate'`
+      );
+  }
+});
+
+function triangulateBatch(data: TriangulateInput): TriangulateResult {
+  // Parse any WKT/WKB geometries
+  // Build binary geometries
+  // Call earcut and triangulate
+  console.error('TriangulationWorker: tessellating batch', data);
+  const triangleIndices = getTriangleIndices(
+    data.polygonIndices,
+    data.primitivePolygonIndices,
+    data.flatCoordinateArray,
+    data.nDim
+  );
+  return {...data, ...(triangleIndices ? {triangleIndices} : {})};
+}

package/dist/geoarrow/convert-geoarrow-to-geojson.d.ts

@@ -1,19 +0,0 @@
-import { Feature } from '@loaders.gl/schema';
-import type { GeoArrowEncoding } from '@loaders.gl/gis';
-type RawArrowFeature = {
-    encoding?: GeoArrowEncoding;
-    data: any;
-};
-/**
- * parse geometry from arrow data that is returned from processArrowData()
- * NOTE: this function could be duplicated with the binaryToFeature() in deck.gl,
- * it is currently only used for picking because currently deck.gl returns only the index of the feature
- * So the following functions could be deprecated once deck.gl returns the feature directly for binary geojson layer
- *
- * @param rawData the raw geometry data returned from processArrowData, which is an object with two properties: encoding and data
- * @see processArrowData
- * @returns Feature or null
- */
-export declare function parseGeometryFromArrow(rawData: RawArrowFeature): Feature | null;
-export {};
-//# sourceMappingURL=convert-geoarrow-to-geojson.d.ts.map

package/dist/geoarrow/convert-geoarrow-to-geojson.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"convert-geoarrow-to-geojson.d.ts","sourceRoot":"","sources":["../../src/geoarrow/convert-geoarrow-to-geojson.ts"],"names":[],"mappings":"AAIA,OAAO,EACL,OAAO,EAQR,MAAM,oBAAoB,CAAC;AAC5B,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,iBAAiB,CAAC;AAEtD,KAAK,eAAe,GAAG;IACrB,QAAQ,CAAC,EAAE,gBAAgB,CAAC;IAC5B,IAAI,EAAE,GAAG,CAAC;CACX,CAAC;AAEF;;;;;;;;;GASG;AACH,wBAAgB,sBAAsB,CAAC,OAAO,EAAE,eAAe,GAAG,OAAO,GAAG,IAAI,CAqC/E"}

package/dist/geoarrow/convert-geoarrow-to-geojson.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"convert-geoarrow-to-geojson.js","names":["parseGeometryFromArrow","rawData","_rawData$encoding","encoding","toLowerCase","data","geometry","arrowMultiPolygonToFeature","arrowPolygonToFeature","arrowMultiPointToFeature","arrowPointToFeature","arrowMultiLineStringToFeature","arrowLineStringToFeature","Error","type","properties","arrowMultiPolygon","multiPolygon","m","length","arrowPolygon","get","polygon","i","arrowRing","ring","j","arrowCoord","coord","Array","from","push","coordinates","coords","arrowMultiPoint","multiPoint","arrowPoint","point","arrowMultiLineString","multiLineString","arrowLineString","lineString"],"sources":["../../src/geoarrow/convert-geoarrow-to-geojson.ts"],"sourcesContent":["// loaders.gl, MIT license\n// Copyright (c) vis.gl contributors\n\nimport * as arrow from 'apache-arrow';\nimport {\n  Feature,\n  MultiPolygon,\n  Position,\n  Polygon,\n  MultiPoint,\n  Point,\n  MultiLineString,\n  LineString\n} from '@loaders.gl/schema';\nimport type {GeoArrowEncoding} from '@loaders.gl/gis';\n\ntype RawArrowFeature = {\n  encoding?: GeoArrowEncoding;\n  data: any;\n};\n\n/**\n * parse geometry from arrow data that is returned from processArrowData()\n * NOTE: this function could be duplicated with the binaryToFeature() in deck.gl,\n * it is currently only used for picking because currently deck.gl returns only the index of the feature\n * So the following functions could be deprecated once deck.gl returns the feature directly for binary geojson layer\n *\n * @param rawData the raw geometry data returned from processArrowData, which is an object with two properties: encoding and data\n * @see processArrowData\n * @returns Feature or null\n */\nexport function parseGeometryFromArrow(rawData: RawArrowFeature): Feature | null {\n  const encoding = rawData.encoding?.toLowerCase();\n  const data = rawData.data;\n  if (!encoding || !data) {\n    return null;\n  }\n\n  let geometry;\n\n  switch (encoding) {\n    case 'geoarrow.multipolygon':\n      geometry = arrowMultiPolygonToFeature(data);\n      break;\n    case 'geoarrow.polygon':\n      geometry = arrowPolygonToFeature(data);\n      break;\n    case 'geoarrow.multipoint':\n      geometry = arrowMultiPointToFeature(data);\n      break;\n    case 'geoarrow.point':\n      geometry = arrowPointToFeature(data);\n      break;\n    case 'geoarrow.multilinestring':\n      geometry = arrowMultiLineStringToFeature(data);\n      break;\n    case 'geoarrow.linestring':\n      geometry = arrowLineStringToFeature(data);\n      break;\n    default: {\n      throw Error(`GeoArrow encoding not supported ${encoding}`);\n    }\n  }\n  return {\n    type: 'Feature',\n    geometry,\n    properties: {}\n  };\n}\n\n/**\n * convert Arrow MultiPolygon to geojson Feature\n */\nfunction arrowMultiPolygonToFeature(arrowMultiPolygon: arrow.Vector): MultiPolygon {\n  const multiPolygon: Position[][][] = [];\n  for (let m = 0; m < arrowMultiPolygon.length; m++) {\n    const arrowPolygon = arrowMultiPolygon.get(m);\n    const polygon: Position[][] = [];\n    for (let i = 0; arrowPolygon && i < arrowPolygon?.length; i++) {\n      const arrowRing = arrowPolygon?.get(i);\n      const ring: Position[] = [];\n      for (let j = 0; arrowRing && j < arrowRing.length; j++) {\n        const arrowCoord = arrowRing.get(j);\n        const coord: Position = Array.from(arrowCoord);\n        ring.push(coord);\n      }\n      polygon.push(ring);\n    }\n    multiPolygon.push(polygon);\n  }\n  const geometry: MultiPolygon = {\n    type: 'MultiPolygon',\n    coordinates: multiPolygon\n  };\n  return geometry;\n}\n\n/**\n * convert Arrow Polygon to geojson Feature\n */\nfunction arrowPolygonToFeature(arrowPolygon: arrow.Vector): Polygon {\n  const polygon: Position[][] = [];\n  for (let i = 0; arrowPolygon && i < arrowPolygon.length; i++) {\n    const arrowRing = arrowPolygon.get(i);\n    const ring: Position[] = [];\n    for (let j = 0; arrowRing && j < arrowRing.length; j++) {\n      const arrowCoord = arrowRing.get(j);\n      const coords: Position = Array.from(arrowCoord);\n      ring.push(coords);\n    }\n    polygon.push(ring);\n  }\n  const geometry: Polygon = {\n    type: 'Polygon',\n    coordinates: polygon\n  };\n  return geometry;\n}\n\n/**\n * convert Arrow MultiPoint to geojson MultiPoint\n */\nfunction arrowMultiPointToFeature(arrowMultiPoint: arrow.Vector): MultiPoint {\n  const multiPoint: Position[] = [];\n  for (let i = 0; arrowMultiPoint && i < arrowMultiPoint.length; i++) {\n    const arrowPoint = arrowMultiPoint.get(i);\n    if (arrowPoint) {\n      const coord: Position = Array.from(arrowPoint);\n      multiPoint.push(coord);\n    }\n  }\n  const geometry: MultiPoint = {\n    type: 'MultiPoint',\n    coordinates: multiPoint\n  };\n  return geometry;\n}\n\n/**\n * convert Arrow Point to geojson Point\n */\nfunction arrowPointToFeature(arrowPoint: arrow.Vector): Point {\n  const point: Position = Array.from(arrowPoint);\n  const geometry: Point = {\n    type: 'Point',\n    coordinates: point\n  };\n  return geometry;\n}\n\n/**\n * convert Arrow MultiLineString to geojson MultiLineString\n */\nfunction arrowMultiLineStringToFeature(arrowMultiLineString: arrow.Vector): MultiLineString {\n  const multiLineString: Position[][] = [];\n  for (let i = 0; arrowMultiLineString && i < arrowMultiLineString.length; i++) {\n    const arrowLineString = arrowMultiLineString.get(i);\n    const lineString: Position[] = [];\n    for (let j = 0; arrowLineString && j < arrowLineString.length; j++) {\n      const arrowCoord = arrowLineString.get(j);\n      if (arrowCoord) {\n        const coords: Position = Array.from(arrowCoord);\n        lineString.push(coords);\n      }\n    }\n    multiLineString.push(lineString);\n  }\n  const geometry: MultiLineString = {\n    type: 'MultiLineString',\n    coordinates: multiLineString\n  };\n  return geometry;\n}\n\n/**\n * convert Arrow LineString to geojson LineString\n */\nfunction arrowLineStringToFeature(arrowLineString: arrow.Vector): LineString {\n  const lineString: Position[] = [];\n  for (let i = 0; arrowLineString && i < arrowLineString.length; i++) {\n    const arrowCoord = arrowLineString.get(i);\n    if (arrowCoord) {\n      const coords: Position = Array.from(arrowCoord);\n      lineString.push(coords);\n    }\n  }\n  const geometry: LineString = {\n    type: 'LineString',\n    coordinates: lineString\n  };\n  return geometry;\n}\n"],"mappings":"AA+BA,OAAO,SAASA,sBAAsBA,CAACC,OAAwB,EAAkB;EAAA,IAAAC,iBAAA;EAC/E,MAAMC,QAAQ,IAAAD,iBAAA,GAAGD,OAAO,CAACE,QAAQ,cAAAD,iBAAA,uBAAhBA,iBAAA,CAAkBE,WAAW,CAAC,CAAC;EAChD,MAAMC,IAAI,GAAGJ,OAAO,CAACI,IAAI;EACzB,IAAI,CAACF,QAAQ,IAAI,CAACE,IAAI,EAAE;IACtB,OAAO,IAAI;EACb;EAEA,IAAIC,QAAQ;EAEZ,QAAQH,QAAQ;IACd,KAAK,uBAAuB;MAC1BG,QAAQ,GAAGC,0BAA0B,CAACF,IAAI,CAAC;MAC3C;IACF,KAAK,kBAAkB;MACrBC,QAAQ,GAAGE,qBAAqB,CAACH,IAAI,CAAC;MACtC;IACF,KAAK,qBAAqB;MACxBC,QAAQ,GAAGG,wBAAwB,CAACJ,IAAI,CAAC;MACzC;IACF,KAAK,gBAAgB;MACnBC,QAAQ,GAAGI,mBAAmB,CAACL,IAAI,CAAC;MACpC;IACF,KAAK,0BAA0B;MAC7BC,QAAQ,GAAGK,6BAA6B,CAACN,IAAI,CAAC;MAC9C;IACF,KAAK,qBAAqB;MACxBC,QAAQ,GAAGM,wBAAwB,CAACP,IAAI,CAAC;MACzC;IACF;MAAS;QACP,MAAMQ,KAAK,CAAE,mCAAkCV,QAAS,EAAC,CAAC;MAC5D;EACF;EACA,OAAO;IACLW,IAAI,EAAE,SAAS;IACfR,QAAQ;IACRS,UAAU,EAAE,CAAC;EACf,CAAC;AACH;AAKA,SAASR,0BAA0BA,CAACS,iBAA+B,EAAgB;EACjF,MAAMC,YAA4B,GAAG,EAAE;EACvC,KAAK,IAAIC,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGF,iBAAiB,CAACG,MAAM,EAAED,CAAC,EAAE,EAAE;IACjD,MAAME,YAAY,GAAGJ,iBAAiB,CAACK,GAAG,CAACH,CAAC,CAAC;IAC7C,MAAMI,OAAqB,GAAG,EAAE;IAChC,KAAK,IAAIC,CAAC,GAAG,CAAC,EAAEH,YAAY,IAAIG,CAAC,IAAGH,YAAY,aAAZA,YAAY,uBAAZA,YAAY,CAAED,MAAM,GAAEI,CAAC,EAAE,EAAE;MAC7D,MAAMC,SAAS,GAAGJ,YAAY,aAAZA,YAAY,uBAAZA,YAAY,CAAEC,GAAG,CAACE,CAAC,CAAC;MACtC,MAAME,IAAgB,GAAG,EAAE;MAC3B,KAAK,IAAIC,CAAC,GAAG,CAAC,EAAEF,SAAS,IAAIE,CAAC,GAAGF,SAAS,CAACL,MAAM,EAAEO,CAAC,EAAE,EAAE;QACtD,MAAMC,UAAU,GAAGH,SAAS,CAACH,GAAG,CAACK,CAAC,CAAC;QACnC,MAAME,KAAe,GAAGC,KAAK,CAACC,IAAI,CAACH,UAAU,CAAC;QAC9CF,IAAI,CAACM,IAAI,CAACH,KAAK,CAAC;MAClB;MACAN,OAAO,CAACS,IAAI,CAACN,IAAI,CAAC;IACpB;IACAR,YAAY,CAACc,IAAI,CAACT,OAAO,CAAC;EAC5B;EACA,MAAMhB,QAAsB,GAAG;IAC7BQ,IAAI,EAAE,cAAc;IACpBkB,WAAW,EAAEf;EACf,CAAC;EACD,OAAOX,QAAQ;AACjB;AAKA,SAASE,qBAAqBA,CAACY,YAA0B,EAAW;EAClE,MAAME,OAAqB,GAAG,EAAE;EAChC,KAAK,IAAIC,CAAC,GAAG,CAAC,EAAEH,YAAY,IAAIG,CAAC,GAAGH,YAAY,CAACD,MAAM,EAAEI,CAAC,EAAE,EAAE;IAC5D,MAAMC,SAAS,GAAGJ,YAAY,CAACC,GAAG,CAACE,CAAC,CAAC;IACrC,MAAME,IAAgB,GAAG,EAAE;IAC3B,KAAK,IAAIC,CAAC,GAAG,CAAC,EAAEF,SAAS,IAAIE,CAAC,GAAGF,SAAS,CAACL,MAAM,EAAEO,CAAC,EAAE,EAAE;MACtD,MAAMC,UAAU,GAAGH,SAAS,CAACH,GAAG,CAACK,CAAC,CAAC;MACnC,MAAMO,MAAgB,GAAGJ,KAAK,CAACC,IAAI,CAACH,UAAU,CAAC;MAC/CF,IAAI,CAACM,IAAI,CAACE,MAAM,CAAC;IACnB;IACAX,OAAO,CAACS,IAAI,CAACN,IAAI,CAAC;EACpB;EACA,MAAMnB,QAAiB,GAAG;IACxBQ,IAAI,EAAE,SAAS;IACfkB,WAAW,EAAEV;EACf,CAAC;EACD,OAAOhB,QAAQ;AACjB;AAKA,SAASG,wBAAwBA,CAACyB,eAA6B,EAAc;EAC3E,MAAMC,UAAsB,GAAG,EAAE;EACjC,KAAK,IAAIZ,CAAC,GAAG,CAAC,EAAEW,eAAe,IAAIX,CAAC,GAAGW,eAAe,CAACf,MAAM,EAAEI,CAAC,EAAE,EAAE;IAClE,MAAMa,UAAU,GAAGF,eAAe,CAACb,GAAG,CAACE,CAAC,CAAC;IACzC,IAAIa,UAAU,EAAE;MACd,MAAMR,KAAe,GAAGC,KAAK,CAACC,IAAI,CAACM,UAAU,CAAC;MAC9CD,UAAU,CAACJ,IAAI,CAACH,KAAK,CAAC;IACxB;EACF;EACA,MAAMtB,QAAoB,GAAG;IAC3BQ,IAAI,EAAE,YAAY;IAClBkB,WAAW,EAAEG;EACf,CAAC;EACD,OAAO7B,QAAQ;AACjB;AAKA,SAASI,mBAAmBA,CAAC0B,UAAwB,EAAS;EAC5D,MAAMC,KAAe,GAAGR,KAAK,CAACC,IAAI,CAACM,UAAU,CAAC;EAC9C,MAAM9B,QAAe,GAAG;IACtBQ,IAAI,EAAE,OAAO;IACbkB,WAAW,EAAEK;EACf,CAAC;EACD,OAAO/B,QAAQ;AACjB;AAKA,SAASK,6BAA6BA,CAAC2B,oBAAkC,EAAmB;EAC1F,MAAMC,eAA6B,GAAG,EAAE;EACxC,KAAK,IAAIhB,CAAC,GAAG,CAAC,EAAEe,oBAAoB,IAAIf,CAAC,GAAGe,oBAAoB,CAACnB,MAAM,EAAEI,CAAC,EAAE,EAAE;IAC5E,MAAMiB,eAAe,GAAGF,oBAAoB,CAACjB,GAAG,CAACE,CAAC,CAAC;IACnD,MAAMkB,UAAsB,GAAG,EAAE;IACjC,KAAK,IAAIf,CAAC,GAAG,CAAC,EAAEc,eAAe,IAAId,CAAC,GAAGc,eAAe,CAACrB,MAAM,EAAEO,CAAC,EAAE,EAAE;MAClE,MAAMC,UAAU,GAAGa,eAAe,CAACnB,GAAG,CAACK,CAAC,CAAC;MACzC,IAAIC,UAAU,EAAE;QACd,MAAMM,MAAgB,GAAGJ,KAAK,CAACC,IAAI,CAACH,UAAU,CAAC;QAC/Cc,UAAU,CAACV,IAAI,CAACE,MAAM,CAAC;MACzB;IACF;IACAM,eAAe,CAACR,IAAI,CAACU,UAAU,CAAC;EAClC;EACA,MAAMnC,QAAyB,GAAG;IAChCQ,IAAI,EAAE,iBAAiB;IACvBkB,WAAW,EAAEO;EACf,CAAC;EACD,OAAOjC,QAAQ;AACjB;AAKA,SAASM,wBAAwBA,CAAC4B,eAA6B,EAAc;EAC3E,MAAMC,UAAsB,GAAG,EAAE;EACjC,KAAK,IAAIlB,CAAC,GAAG,CAAC,EAAEiB,eAAe,IAAIjB,CAAC,GAAGiB,eAAe,CAACrB,MAAM,EAAEI,CAAC,EAAE,EAAE;IAClE,MAAMI,UAAU,GAAGa,eAAe,CAACnB,GAAG,CAACE,CAAC,CAAC;IACzC,IAAII,UAAU,EAAE;MACd,MAAMM,MAAgB,GAAGJ,KAAK,CAACC,IAAI,CAACH,UAAU,CAAC;MAC/Cc,UAAU,CAACV,IAAI,CAACE,MAAM,CAAC;IACzB;EACF;EACA,MAAM3B,QAAoB,GAAG;IAC3BQ,IAAI,EAAE,YAAY;IAClBkB,WAAW,EAAES;EACf,CAAC;EACD,OAAOnC,QAAQ;AACjB"}

package/dist/lib/parse-arrow-in-batches.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"parse-arrow-in-batches.d.ts","sourceRoot":"","sources":["../../src/lib/parse-arrow-in-batches.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,eAAe,CAAC;AAInD;GACG;AACH,wBAAgB,mBAAmB,CACjC,aAAa,EAAE,aAAa,CAAC,WAAW,CAAC,GAAG,QAAQ,CAAC,WAAW,CAAC,GAChE,aAAa,CAAC,eAAe,CAAC,CAsChC"}

package/dist/lib/parse-arrow-in-batches.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"parse-arrow-in-batches.js","names":["arrow","parseArrowInBatches","asyncIterator","makeArrowAsyncIterator","readers","RecordBatchReader","readAll","reader","recordBatch","arrowTabledBatch","shape","batchType","data","Table","length"],"sources":["../../src/lib/parse-arrow-in-batches.ts"],"sourcesContent":["// TODO - this import defeats the sophisticated typescript checking in ArrowJS\nimport type {ArrowTableBatch} from './arrow-table';\nimport * as arrow from 'apache-arrow';\n// import {isIterable} from '@loaders.gl/core';\n\n/**\n */\nexport function parseArrowInBatches(\n  asyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>\n): AsyncIterable<ArrowTableBatch> {\n  // Creates the appropriate arrow.RecordBatchReader subclasses from the input\n  // This will also close the underlying source in case of early termination or errors\n\n  // As an optimization, return a non-async iterator\n  /*\n  if (isIterable(readers)) {\n    function* makeArrowIterator() {\n      for (const reader of readers) {\n        for (const batch of reader) {\n          yield processBatch(batch, reader);\n        }\n        break; // only processing one stream of batches\n      }\n    }\n    const arrowIterator = makeArrowIterator();\n  }\n  */\n\n  async function* makeArrowAsyncIterator(): AsyncIterator<ArrowTableBatch> {\n    // @ts-ignore\n    const readers = arrow.RecordBatchReader.readAll(asyncIterator);\n    for await (const reader of readers) {\n      for await (const recordBatch of reader) {\n        const arrowTabledBatch: ArrowTableBatch = {\n          shape: 'arrow-table',\n          batchType: 'data',\n          data: new arrow.Table([recordBatch]),\n          length: recordBatch.data.length\n        };\n        // processBatch(recordBatch);\n        yield arrowTabledBatch;\n      }\n      break; // only processing one stream of batches\n    }\n  }\n\n  return makeArrowAsyncIterator() as any; // as AsyncIterator<ArrowTableBatch>;\n}\n\n// function processBatch(batch: RecordBatch): ArrowTableBatch {\n//   const values = {};\n//   batch.schema.fields.forEach(({name}, index) => {\n//     values[name] = batch.getChildAt(index)?.toArray();\n//   });\n//   return {\n//   };\n// }\n"],"mappings":"AAEA,OAAO,KAAKA,KAAK,MAAM,cAAc;AAKrC,OAAO,SAASC,mBAAmBA,CACjCC,aAAiE,EACjC;EAmBhC,gBAAgBC,sBAAsBA,CAAA,EAAmC;IAEvE,MAAMC,OAAO,GAAGJ,KAAK,CAACK,iBAAiB,CAACC,OAAO,CAACJ,aAAa,CAAC;IAC9D,WAAW,MAAMK,MAAM,IAAIH,OAAO,EAAE;MAClC,WAAW,MAAMI,WAAW,IAAID,MAAM,EAAE;QACtC,MAAME,gBAAiC,GAAG;UACxCC,KAAK,EAAE,aAAa;UACpBC,SAAS,EAAE,MAAM;UACjBC,IAAI,EAAE,IAAIZ,KAAK,CAACa,KAAK,CAAC,CAACL,WAAW,CAAC,CAAC;UACpCM,MAAM,EAAEN,WAAW,CAACI,IAAI,CAACE;QAC3B,CAAC;QAED,MAAML,gBAAgB;MACxB;MACA;IACF;EACF;EAEA,OAAON,sBAAsB,CAAC,CAAC;AACjC"}

package/dist/lib/parse-arrow-sync.d.ts

@@ -1,5 +0,0 @@
-import type { ColumnarTable, ObjectRowTable } from '@loaders.gl/schema';
-import type { ArrowTable } from './arrow-table';
-import type { ArrowLoaderOptions } from '../arrow-loader';
-export default function parseArrowSync(arrayBuffer: any, options?: ArrowLoaderOptions): ArrowTable | ColumnarTable | ObjectRowTable;
-//# sourceMappingURL=parse-arrow-sync.d.ts.map

package/dist/lib/parse-arrow-sync.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"parse-arrow-sync.d.ts","sourceRoot":"","sources":["../../src/lib/parse-arrow-sync.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,aAAa,EAAE,cAAc,EAAC,MAAM,oBAAoB,CAAC;AACtE,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,eAAe,CAAC;AAG9C,OAAO,KAAK,EAAC,kBAAkB,EAAC,MAAM,iBAAiB,CAAC;AAOxD,MAAM,CAAC,OAAO,UAAU,cAAc,CACpC,WAAW,KAAA,EACX,OAAO,CAAC,EAAE,kBAAkB,GAC3B,UAAU,GAAG,aAAa,GAAG,cAAc,CAoB7C"}

package/dist/lib/parse-arrow-sync.js

@@ -1,21 +0,0 @@
-import { convertTable } from '@loaders.gl/schema';
-import * as arrow from 'apache-arrow';
-import { convertApacheArrowToArrowTable, convertArrowToColumnarTable } from "../tables/convert-arrow-to-table.js";
-export default function parseArrowSync(arrayBuffer, options) {
-  var _options$arrow;
-  const apacheArrowTable = arrow.tableFromIPC([new Uint8Array(arrayBuffer)]);
-  const arrowTable = convertApacheArrowToArrowTable(apacheArrowTable);
-  const shape = (options === null || options === void 0 ? void 0 : (_options$arrow = options.arrow) === null || _options$arrow === void 0 ? void 0 : _options$arrow.shape) || 'arrow-table';
-  switch (shape) {
-    case 'arrow-table':
-      return arrowTable;
-    case 'columnar-table':
-      return convertArrowToColumnarTable(arrowTable);
-    case 'object-row-table':
-      const columnarTable = convertArrowToColumnarTable(arrowTable);
-      return convertTable(columnarTable, 'object-row-table');
-    default:
-      throw new Error(shape);
-  }
-}
-//# sourceMappingURL=parse-arrow-sync.js.map

package/dist/lib/parse-arrow-sync.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"parse-arrow-sync.js","names":["convertTable","arrow","convertApacheArrowToArrowTable","convertArrowToColumnarTable","parseArrowSync","arrayBuffer","options","_options$arrow","apacheArrowTable","tableFromIPC","Uint8Array","arrowTable","shape","columnarTable","Error"],"sources":["../../src/lib/parse-arrow-sync.ts"],"sourcesContent":["import type {ColumnarTable, ObjectRowTable} from '@loaders.gl/schema';\nimport type {ArrowTable} from './arrow-table';\nimport {convertTable} from '@loaders.gl/schema';\nimport * as arrow from 'apache-arrow';\nimport type {ArrowLoaderOptions} from '../arrow-loader';\nimport {\n  convertApacheArrowToArrowTable,\n  convertArrowToColumnarTable\n} from '../tables/convert-arrow-to-table';\n\n// Parses arrow to a columnar table\nexport default function parseArrowSync(\n  arrayBuffer,\n  options?: ArrowLoaderOptions\n): ArrowTable | ColumnarTable | ObjectRowTable {\n  const apacheArrowTable = arrow.tableFromIPC([new Uint8Array(arrayBuffer)]);\n  const arrowTable = convertApacheArrowToArrowTable(apacheArrowTable);\n\n  const shape = options?.arrow?.shape || 'arrow-table';\n  switch (shape) {\n    case 'arrow-table':\n      return arrowTable;\n\n    case 'columnar-table':\n      return convertArrowToColumnarTable(arrowTable);\n\n    case 'object-row-table':\n      const columnarTable = convertArrowToColumnarTable(arrowTable);\n      return convertTable(columnarTable, 'object-row-table');\n\n    default:\n      // TODO\n      throw new Error(shape);\n  }\n}\n"],"mappings":"AAEA,SAAQA,YAAY,QAAO,oBAAoB;AAC/C,OAAO,KAAKC,KAAK,MAAM,cAAc;AAAC,SAGpCC,8BAA8B,EAC9BC,2BAA2B;AAI7B,eAAe,SAASC,cAAcA,CACpCC,WAAW,EACXC,OAA4B,EACiB;EAAA,IAAAC,cAAA;EAC7C,MAAMC,gBAAgB,GAAGP,KAAK,CAACQ,YAAY,CAAC,CAAC,IAAIC,UAAU,CAACL,WAAW,CAAC,CAAC,CAAC;EAC1E,MAAMM,UAAU,GAAGT,8BAA8B,CAACM,gBAAgB,CAAC;EAEnE,MAAMI,KAAK,GAAG,CAAAN,OAAO,aAAPA,OAAO,wBAAAC,cAAA,GAAPD,OAAO,CAAEL,KAAK,cAAAM,cAAA,uBAAdA,cAAA,CAAgBK,KAAK,KAAI,aAAa;EACpD,QAAQA,KAAK;IACX,KAAK,aAAa;MAChB,OAAOD,UAAU;IAEnB,KAAK,gBAAgB;MACnB,OAAOR,2BAA2B,CAACQ,UAAU,CAAC;IAEhD,KAAK,kBAAkB;MACrB,MAAME,aAAa,GAAGV,2BAA2B,CAACQ,UAAU,CAAC;MAC7D,OAAOX,YAAY,CAACa,aAAa,EAAE,kBAAkB,CAAC;IAExD;MAEE,MAAM,IAAIC,KAAK,CAACF,KAAK,CAAC;EAC1B;AACF"}

package/dist/tables/convert-arrow-to-table.d.ts

@@ -1,21 +0,0 @@
-import type { ColumnarTable, ObjectRowTable } from '@loaders.gl/schema';
-import type * as arrow from 'apache-arrow';
-import type { ArrowTable } from '../lib/arrow-table';
-/**
- * Wrap an apache arrow table in a loaders.gl table wrapper.
- * From this additional conversions are available.
- * @param arrowTable
- * @returns
- */
-export declare function convertApacheArrowToArrowTable(arrowTable: arrow.Table): ArrowTable;
-/**
- * Convert an Apache Arrow table to a ColumnarTable
- * @note Currently does not convert schema
- */
-export declare function convertArrowToColumnarTable(table: ArrowTable): ColumnarTable;
-/**
- *
- * @note - should be part of schema module
- */
-export declare function convertColumnarToRowFormatTable(columnarTable: ColumnarTable): ObjectRowTable;
-//# sourceMappingURL=convert-arrow-to-table.d.ts.map

package/dist/tables/convert-arrow-to-table.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"convert-arrow-to-table.d.ts","sourceRoot":"","sources":["../../src/tables/convert-arrow-to-table.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAC,aAAa,EAAE,cAAc,EAAC,MAAM,oBAAoB,CAAC;AACtE,OAAO,KAAK,KAAK,KAAK,MAAM,cAAc,CAAC;AAC3C,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,oBAAoB,CAAC;AAEnD;;;;;GAKG;AACH,wBAAgB,8BAA8B,CAAC,UAAU,EAAE,KAAK,CAAC,KAAK,GAAG,UAAU,CAKlF;AAED;;;GAGG;AACH,wBAAgB,2BAA2B,CAAC,KAAK,EAAE,UAAU,GAAG,aAAa,CAkB5E;AAED;;;GAGG;AACH,wBAAgB,+BAA+B,CAAC,aAAa,EAAE,aAAa,GAAG,cAAc,CAmB5F"}

package/dist/tables/convert-arrow-to-table.js

@@ -1,37 +0,0 @@
-export function convertApacheArrowToArrowTable(arrowTable) {
-  return {
-    shape: 'arrow-table',
-    data: arrowTable
-  };
-}
-export function convertArrowToColumnarTable(table) {
-  const arrowTable = table.data;
-  const columnarTable = {};
-  for (const field of arrowTable.schema.fields) {
-    const arrowColumn = arrowTable.getChild(field.name);
-    const values = arrowColumn === null || arrowColumn === void 0 ? void 0 : arrowColumn.toArray();
-    columnarTable[field.name] = values;
-  }
-  return {
-    shape: 'columnar-table',
-    data: columnarTable
-  };
-}
-export function convertColumnarToRowFormatTable(columnarTable) {
-  const tableKeys = Object.keys(columnarTable);
-  const tableRowsCount = columnarTable[tableKeys[0]].length;
-  const rowFormatTable = [];
-  for (let index = 0; index < tableRowsCount; index++) {
-    const tableItem = {};
-    for (let keyIndex = 0; keyIndex < tableKeys.length; keyIndex++) {
-      const fieldName = tableKeys[keyIndex];
-      tableItem[fieldName] = columnarTable[fieldName][index];
-    }
-    rowFormatTable.push(tableItem);
-  }
-  return {
-    shape: 'object-row-table',
-    data: rowFormatTable
-  };
-}
-//# sourceMappingURL=convert-arrow-to-table.js.map

package/dist/tables/convert-arrow-to-table.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"convert-arrow-to-table.js","names":["convertApacheArrowToArrowTable","arrowTable","shape","data","convertArrowToColumnarTable","table","columnarTable","field","schema","fields","arrowColumn","getChild","name","values","toArray","convertColumnarToRowFormatTable","tableKeys","Object","keys","tableRowsCount","length","rowFormatTable","index","tableItem","keyIndex","fieldName","push"],"sources":["../../src/tables/convert-arrow-to-table.ts"],"sourcesContent":["// loaders.gl, MIT license\n// Copyright (c) vis.gl contributors\n\nimport type {ColumnarTable, ObjectRowTable} from '@loaders.gl/schema';\nimport type * as arrow from 'apache-arrow';\nimport type {ArrowTable} from '../lib/arrow-table';\n\n/**\n * Wrap an apache arrow table in a loaders.gl table wrapper.\n * From this additional conversions are available.\n * @param arrowTable\n * @returns\n */\nexport function convertApacheArrowToArrowTable(arrowTable: arrow.Table): ArrowTable {\n  return {\n    shape: 'arrow-table',\n    data: arrowTable\n  };\n}\n\n/**\n * Convert an Apache Arrow table to a ColumnarTable\n * @note Currently does not convert schema\n */\nexport function convertArrowToColumnarTable(table: ArrowTable): ColumnarTable {\n  // TODO - avoid calling `getColumn` on columns we are not interested in?\n  // Add options object?\n\n  const arrowTable = table.data;\n  const columnarTable = {};\n\n  for (const field of arrowTable.schema.fields) {\n    // This (is intended to) coalesce all record batches into a single typed array\n    const arrowColumn = arrowTable.getChild(field.name);\n    const values = arrowColumn?.toArray();\n    columnarTable[field.name] = values;\n  }\n\n  return {\n    shape: 'columnar-table',\n    data: columnarTable\n  };\n}\n\n/**\n *\n * @note - should be part of schema module\n */\nexport function convertColumnarToRowFormatTable(columnarTable: ColumnarTable): ObjectRowTable {\n  const tableKeys = Object.keys(columnarTable);\n  const tableRowsCount = columnarTable[tableKeys[0]].length;\n\n  const rowFormatTable: {}[] = [];\n\n  for (let index = 0; index < tableRowsCount; index++) {\n    const tableItem = {};\n    for (let keyIndex = 0; keyIndex < tableKeys.length; keyIndex++) {\n      const fieldName = tableKeys[keyIndex];\n      tableItem[fieldName] = columnarTable[fieldName][index];\n    }\n    rowFormatTable.push(tableItem);\n  }\n\n  return {\n    shape: 'object-row-table',\n    data: rowFormatTable\n  };\n}\n"],"mappings":"AAaA,OAAO,SAASA,8BAA8BA,CAACC,UAAuB,EAAc;EAClF,OAAO;IACLC,KAAK,EAAE,aAAa;IACpBC,IAAI,EAAEF;EACR,CAAC;AACH;AAMA,OAAO,SAASG,2BAA2BA,CAACC,KAAiB,EAAiB;EAI5E,MAAMJ,UAAU,GAAGI,KAAK,CAACF,IAAI;EAC7B,MAAMG,aAAa,GAAG,CAAC,CAAC;EAExB,KAAK,MAAMC,KAAK,IAAIN,UAAU,CAACO,MAAM,CAACC,MAAM,EAAE;IAE5C,MAAMC,WAAW,GAAGT,UAAU,CAACU,QAAQ,CAACJ,KAAK,CAACK,IAAI,CAAC;IACnD,MAAMC,MAAM,GAAGH,WAAW,aAAXA,WAAW,uBAAXA,WAAW,CAAEI,OAAO,CAAC,CAAC;IACrCR,aAAa,CAACC,KAAK,CAACK,IAAI,CAAC,GAAGC,MAAM;EACpC;EAEA,OAAO;IACLX,KAAK,EAAE,gBAAgB;IACvBC,IAAI,EAAEG;EACR,CAAC;AACH;AAMA,OAAO,SAASS,+BAA+BA,CAACT,aAA4B,EAAkB;EAC5F,MAAMU,SAAS,GAAGC,MAAM,CAACC,IAAI,CAACZ,aAAa,CAAC;EAC5C,MAAMa,cAAc,GAAGb,aAAa,CAACU,SAAS,CAAC,CAAC,CAAC,CAAC,CAACI,MAAM;EAEzD,MAAMC,cAAoB,GAAG,EAAE;EAE/B,KAAK,IAAIC,KAAK,GAAG,CAAC,EAAEA,KAAK,GAAGH,cAAc,EAAEG,KAAK,EAAE,EAAE;IACnD,MAAMC,SAAS,GAAG,CAAC,CAAC;IACpB,KAAK,IAAIC,QAAQ,GAAG,CAAC,EAAEA,QAAQ,GAAGR,SAAS,CAACI,MAAM,EAAEI,QAAQ,EAAE,EAAE;MAC9D,MAAMC,SAAS,GAAGT,SAAS,CAACQ,QAAQ,CAAC;MACrCD,SAAS,CAACE,SAAS,CAAC,GAAGnB,aAAa,CAACmB,SAAS,CAAC,CAACH,KAAK,CAAC;IACxD;IACAD,cAAc,CAACK,IAAI,CAACH,SAAS,CAAC;EAChC;EAEA,OAAO;IACLrB,KAAK,EAAE,kBAAkB;IACzBC,IAAI,EAAEkB;EACR,CAAC;AACH"}

package/src/lib/parse-arrow-sync.ts

@@ -1,35 +0,0 @@
-import type {ColumnarTable, ObjectRowTable} from '@loaders.gl/schema';
-import type {ArrowTable} from './arrow-table';
-import {convertTable} from '@loaders.gl/schema';
-import * as arrow from 'apache-arrow';
-import type {ArrowLoaderOptions} from '../arrow-loader';
-import {
-  convertApacheArrowToArrowTable,
-  convertArrowToColumnarTable
-} from '../tables/convert-arrow-to-table';
-
-// Parses arrow to a columnar table
-export default function parseArrowSync(
-  arrayBuffer,
-  options?: ArrowLoaderOptions
-): ArrowTable | ColumnarTable | ObjectRowTable {
-  const apacheArrowTable = arrow.tableFromIPC([new Uint8Array(arrayBuffer)]);
-  const arrowTable = convertApacheArrowToArrowTable(apacheArrowTable);
-
-  const shape = options?.arrow?.shape || 'arrow-table';
-  switch (shape) {
-    case 'arrow-table':
-      return arrowTable;
-
-    case 'columnar-table':
-      return convertArrowToColumnarTable(arrowTable);
-
-    case 'object-row-table':
-      const columnarTable = convertArrowToColumnarTable(arrowTable);
-      return convertTable(columnarTable, 'object-row-table');
-
-    default:
-      // TODO
-      throw new Error(shape);
-  }
-}

package/src/tables/convert-arrow-to-table.ts

@@ -1,68 +0,0 @@
-// loaders.gl, MIT license
-// Copyright (c) vis.gl contributors
-
-import type {ColumnarTable, ObjectRowTable} from '@loaders.gl/schema';
-import type * as arrow from 'apache-arrow';
-import type {ArrowTable} from '../lib/arrow-table';
-
-/**
- * Wrap an apache arrow table in a loaders.gl table wrapper.
- * From this additional conversions are available.
- * @param arrowTable
- * @returns
- */
-export function convertApacheArrowToArrowTable(arrowTable: arrow.Table): ArrowTable {
-  return {
-    shape: 'arrow-table',
-    data: arrowTable
-  };
-}
-
-/**
- * Convert an Apache Arrow table to a ColumnarTable
- * @note Currently does not convert schema
- */
-export function convertArrowToColumnarTable(table: ArrowTable): ColumnarTable {
-  // TODO - avoid calling `getColumn` on columns we are not interested in?
-  // Add options object?
-
-  const arrowTable = table.data;
-  const columnarTable = {};
-
-  for (const field of arrowTable.schema.fields) {
-    // This (is intended to) coalesce all record batches into a single typed array
-    const arrowColumn = arrowTable.getChild(field.name);
-    const values = arrowColumn?.toArray();
-    columnarTable[field.name] = values;
-  }
-
-  return {
-    shape: 'columnar-table',
-    data: columnarTable
-  };
-}
-
-/**
- *
- * @note - should be part of schema module
- */
-export function convertColumnarToRowFormatTable(columnarTable: ColumnarTable): ObjectRowTable {
-  const tableKeys = Object.keys(columnarTable);
-  const tableRowsCount = columnarTable[tableKeys[0]].length;
-
-  const rowFormatTable: {}[] = [];
-
-  for (let index = 0; index < tableRowsCount; index++) {
-    const tableItem = {};
-    for (let keyIndex = 0; keyIndex < tableKeys.length; keyIndex++) {
-      const fieldName = tableKeys[keyIndex];
-      tableItem[fieldName] = columnarTable[fieldName][index];
-    }
-    rowFormatTable.push(tableItem);
-  }
-
-  return {
-    shape: 'object-row-table',
-    data: rowFormatTable
-  };
-}