@loaders.gl/arrow 4.0.2 → 4.0.4

package/src/geoarrow-writer.ts

@@ -0,0 +1,41 @@
+// import type {} from '@loaders.gl/loader-utils';
+
+import type {WriterWithEncoder, WriterOptions} from '@loaders.gl/loader-utils';
+import {GeoJSONTable, BinaryGeometry} from '@loaders.gl/schema';
+import {encodeGeoArrowSync} from './lib/encode-geoarrow';
+
+// __VERSION__ is injected by babel-plugin-version-inline
+// @ts-ignore TS2304: Cannot find name '__VERSION__'.
+const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
+
+type ArrowWriterOptions = WriterOptions & {
+  arrow?: {};
+};
+
+/** Apache Arrow writer */
+export const GeoArrowWriter: WriterWithEncoder<
+  GeoJSONTable | BinaryGeometry,
+  never,
+  ArrowWriterOptions
+> = {
+  name: 'Apache Arrow',
+  id: 'arrow',
+  module: 'arrow',
+  version: VERSION,
+  extensions: ['arrow', 'feather'],
+  mimeTypes: [
+    'application/vnd.apache.arrow.file',
+    'application/vnd.apache.arrow.stream',
+    'application/octet-stream'
+  ],
+  binary: true,
+  options: {},
+  encode: async function encodeArrow(data, options?): Promise<ArrayBuffer> {
+    // @ts-expect-error
+    return encodeGeoArrowSync(data);
+  },
+  encodeSync(data, options?) {
+    // @ts-expect-error
+    return encodeGeoArrowSync(data);
+  }
+};

package/src/index.ts

@@ -1,16 +1,8 @@
 // loaders.gl, MIT license
 // Copyright (c) vis.gl contributors
 
-import type {LoaderWithParser} from '@loaders.gl/loader-utils';
 import type {ArrowLoaderOptions} from './arrow-loader';
-import type {ArrowTableBatch, ColumnarTable, ObjectRowTable} from '@loaders.gl/schema';
-import type {ArrowTable} from './lib/arrow-table';
-
 import {TableBatchBuilder} from '@loaders.gl/schema';
-import {ArrowLoader as ArrowWorkerLoader} from './arrow-loader';
-import parseSync from './lib/parse-arrow-sync';
-import {parseArrowInBatches} from './lib/parse-arrow-in-batches';
-
 import {ArrowTableBatchAggregator} from './lib/arrow-table-batch';
 
 // Make the ArrowBatch type available
@@ -22,6 +14,21 @@ export {getArrowType} from './schema/arrow-type-utils';
 
 // SCHEMA
 
+// Types
+export type {ArrowTable, ArrowTableBatch} from './lib/arrow-table';
+export {VECTOR_TYPES} from './types';
+
+// Arrow loader / Writer
+
+export type {ArrowLoaderOptions};
+export {ArrowLoader, ArrowWorkerLoader} from './arrow-loader';
+
+export {ArrowWriter} from './arrow-writer';
+
+// Geoarrow loader
+export {GeoArrowLoader, GeoArrowWorkerLoader} from './geoarrow-loader';
+
+// Schema utils
 export {
   serializeArrowSchema,
   deserializeArrowSchema,
@@ -33,43 +40,30 @@ export {
   deserializeArrowType
 } from './schema/convert-arrow-schema';
 
-// Types
-export type {ArrowTable, ArrowTableBatch} from './lib/arrow-table';
-export {VECTOR_TYPES} from './types';
-
-// Arrow writer
-
-export {ArrowWriter} from './arrow-writer';
-
-// Arrow loader
-
-export type {ArrowLoaderOptions};
-export {ArrowWorkerLoader};
-
-/** ArrowJS table loader */
-export const ArrowLoader: LoaderWithParser<
-  ArrowTable | ColumnarTable | ObjectRowTable,
-  ArrowTableBatch,
-  ArrowLoaderOptions
-> = {
-  ...ArrowWorkerLoader,
-  parse: async (arraybuffer: ArrayBuffer, options?: ArrowLoaderOptions) =>
-    parseSync(arraybuffer, options),
-  parseSync,
-  parseInBatches: parseArrowInBatches
-};
+// EXPERIMENTAL
 
 // Arrow Utils
 export type {GeoArrowEncoding} from '@loaders.gl/gis';
 // getGeometryColumnsFromArrowTable,
 // getGeoArrowEncoding
 
-export type {BinaryDataFromGeoArrow} from './geoarrow/convert-geoarrow-to-binary-geometry';
+export type {
+  BinaryDataFromGeoArrow,
+  BinaryGeometriesFromArrowOptions
+} from './geoarrow/convert-geoarrow-to-binary-geometry';
 export {
   BINARY_GEOMETRY_TEMPLATE,
-  getBinaryGeometriesFromArrow
+  getBinaryGeometriesFromArrow,
+  getTriangleIndices,
+  getMeanCentersFromBinaryGeometries
 } from './geoarrow/convert-geoarrow-to-binary-geometry';
 
+export {updateBoundsFromGeoArrowSamples} from './geoarrow/get-arrow-bounds';
+
 export {parseGeometryFromArrow} from './geoarrow/convert-geoarrow-to-geojson';
 
-export {updateBoundsFromGeoArrowSamples} from './geoarrow/get-arrow-bounds';
+export {convertArrowToGeoJSONTable} from './tables/convert-arrow-to-geojson-table';
+
+// EXPERIMENTAL WORKER
+
+export {TriangulationWorker, triangulateOnWorker} from './triangulate-on-worker';

package/src/lib/arrow-table-batch.ts

@@ -1,3 +1,6 @@
+// loaders.gl, MIT license
+// Copyright (c) vis.gl contributors
+
 import {ColumnarTableBatchAggregator} from '@loaders.gl/schema';
 import type {ArrowTableBatch} from './arrow-table';
 import * as arrow from 'apache-arrow';

package/src/lib/encode-arrow.ts

@@ -1,3 +1,6 @@
+// loaders.gl, MIT license
+// Copyright (c) vis.gl contributors
+
 import * as arrow from 'apache-arrow';
 import {AnyArrayType, VECTOR_TYPES} from '../types';
 

package/src/lib/encode-geoarrow.ts

@@ -0,0 +1,45 @@
+// loaders.gl, MIT license
+// Copyright (c) vis.gl contributors
+
+import * as arrow from 'apache-arrow';
+import {AnyArrayType, VECTOR_TYPES} from '../types';
+
+export type ColumnarTable = {
+  name: string;
+  array: AnyArrayType;
+  type: number;
+}[];
+
+/**
+ * Encodes set of arrays into the Apache Arrow columnar format
+ * https://arrow.apache.org/docs/format/Columnar.html#ipc-file-format
+ * @param data - columns data
+ * @param options - the writer options
+ * @returns - encoded ArrayBuffer
+ */
+export function encodeGeoArrowSync(data: ColumnarTable): ArrayBuffer {
+  const vectors: Record<string, arrow.Vector> = {};
+  for (const arrayData of data) {
+    const arrayVector = createVector(arrayData.array, arrayData.type);
+    vectors[arrayData.name] = arrayVector;
+  }
+  const table = new arrow.Table(vectors);
+  const arrowBuffer = arrow.tableToIPC(table);
+  return arrowBuffer;
+}
+
+/**
+ * Create Arrow arrow.Vector from given data and vector type
+ * @param array {import('../types').AnyArrayType} - columns data
+ * @param type {number} - the writer options
+ * @return a vector of one of vector's types defined in the Apache Arrow library
+ */
+function createVector(array, type): arrow.Vector {
+  switch (type) {
+    case VECTOR_TYPES.DATE:
+      return arrow.vectorFromArray(array);
+    case VECTOR_TYPES.FLOAT:
+    default:
+      return arrow.vectorFromArray(array);
+  }
+}

package/src/{lib → parsers}/parse-arrow-in-batches.ts

@@ -1,5 +1,7 @@
-// TODO - this import defeats the sophisticated typescript checking in ArrowJS
-import type {ArrowTableBatch} from './arrow-table';
+// loaders.gl, MIT license
+// Copyright (c) vis.gl contributors
+
+import type {ArrowTableBatch} from '../lib/arrow-table';
 import * as arrow from 'apache-arrow';
 // import {isIterable} from '@loaders.gl/core';
 

package/src/parsers/parse-arrow-sync.ts

@@ -0,0 +1,38 @@
+// loaders.gl, MIT license
+// Copyright (c) vis.gl contributors
+
+import type {ArrayRowTable, ColumnarTable, ObjectRowTable} from '@loaders.gl/schema';
+import type {ArrowTable} from '../lib/arrow-table';
+import {convertTable} from '@loaders.gl/schema';
+import * as arrow from 'apache-arrow';
+import {convertArrowToColumnarTable} from '../tables/convert-arrow-to-columnar-table';
+
+// Parses arrow to a columnar table
+export function parseArrowSync(
+  arrayBuffer,
+  options?: {shape?: 'arrow-table' | 'columnar-table' | 'object-row-table' | 'array-row-table'}
+): ArrowTable | ColumnarTable | ObjectRowTable | ArrayRowTable {
+  const apacheArrowTable = arrow.tableFromIPC([new Uint8Array(arrayBuffer)]);
+  const arrowTable: ArrowTable = {shape: 'arrow-table', data: apacheArrowTable};
+
+  const shape = options?.shape || 'arrow-table';
+  switch (shape) {
+    case 'arrow-table':
+      return arrowTable;
+
+    case 'columnar-table':
+      return convertArrowToColumnarTable(arrowTable);
+
+    case 'object-row-table':
+      let columnarTable = convertArrowToColumnarTable(arrowTable);
+      return convertTable(columnarTable, 'object-row-table');
+
+    case 'array-row-table':
+      columnarTable = convertArrowToColumnarTable(arrowTable);
+      return convertTable(columnarTable, 'array-row-table');
+
+    default:
+      // TODO
+      throw new Error(shape);
+  }
+}

package/src/parsers/parse-geoarrow-in-batches.ts

@@ -0,0 +1,15 @@
+// loaders.gl, MIT license
+// Copyright (c) vis.gl contributors
+
+import type {GeoJSONTableBatch} from '@loaders.gl/schema';
+import type {ArrowTableBatch} from '../lib/arrow-table';
+import {parseArrowInBatches} from './parse-arrow-in-batches';
+
+/**
+ */
+export function parseGeoArrowInBatches(
+  asyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>
+): AsyncIterable<ArrowTableBatch | GeoJSONTableBatch> {
+  // | BinaryGeometry
+  return parseArrowInBatches(asyncIterator);
+}

package/src/parsers/parse-geoarrow-sync.ts

@@ -0,0 +1,22 @@
+// loaders.gl, MIT license
+// Copyright (c) vis.gl contributors
+
+import type {GeoJSONTable} from '@loaders.gl/schema';
+import type {ArrowTable} from '../lib/arrow-table';
+import {parseArrowSync} from './parse-arrow-sync';
+import {convertArrowToGeoJSONTable} from '../tables/convert-arrow-to-geojson-table';
+
+// Parses arrow to a columnar table
+export function parseGeoArrowSync(
+  arrayBuffer,
+  options?: {shape?: 'arrow-table' | 'geojson-table' | 'binary-geometry'}
+): ArrowTable | GeoJSONTable {
+  // | BinaryGeometry
+  const table = parseArrowSync(arrayBuffer, {shape: 'arrow-table'}) as ArrowTable;
+  switch (options?.shape) {
+    case 'geojson-table':
+      return convertArrowToGeoJSONTable(table);
+    default:
+      return table;
+  }
+}

package/src/schema/convert-arrow-schema.ts

@@ -100,7 +100,15 @@ export function serializeArrowType(arrowType: arrow.DataType): DataType {
       return 'float64';
     case arrow.Utf8:
       return 'utf8';
-    case Date:
+    case arrow.Decimal:
+      const decimal = arrowType as arrow.Decimal;
+      return {
+        type: 'decimal',
+        bitWidth: decimal.bitWidth,
+        precision: decimal.precision,
+        scale: decimal.scale
+      };
+    case arrow.Date_:
       const dateUnit = (arrowType as arrow.Date_).unit;
       return dateUnit === arrow.DateUnit.DAY ? 'date-day' : 'date-millisecond';
     case arrow.DateDay:
@@ -165,6 +173,13 @@ export function serializeArrowType(arrowType: arrow.DataType): DataType {
       return 'interval-daytime';
     case arrow.IntervalYearMonth:
       return 'interval-yearmonth';
+    case arrow.Map_:
+      const mapType = arrowType as arrow.Map_;
+      return {
+        type: 'map',
+        keysSorted: mapType.keysSorted,
+        children: mapType.children.map((arrowField) => serializeArrowField(arrowField))
+      };
     case arrow.List:
       const listType = arrowType as arrow.List;
       const listField = listType.valueField;
@@ -173,15 +188,20 @@ export function serializeArrowType(arrowType: arrow.DataType): DataType {
         children: [serializeArrowField(listField)]
       };
     case arrow.FixedSizeList:
+      const fixedSizeList = arrowType as arrow.FixedSizeList;
       return {
         type: 'fixed-size-list',
-        listSize: (arrowType as arrow.FixedSizeList).listSize,
-        children: [serializeArrowField((arrowType as arrow.FixedSizeList).children[0])]
+        listSize: fixedSizeList.listSize,
+        children: [serializeArrowField(fixedSizeList.children[0])]
+      };
+    case arrow.Struct:
+      const structType = arrowType as arrow.Struct;
+      return {
+        type: 'struct',
+        children: structType.children.map((arrowField) => serializeArrowField(arrowField))
       };
-    // case arrow.Struct:
-    //   return {type: 'struct', children: (arrowType as arrow.Struct).children};
     default:
-      throw new Error('array type not supported');
+      throw new Error(`arrow type not supported: ${arrowType.constructor.name}`);
   }
 }
 
@@ -190,6 +210,11 @@ export function serializeArrowType(arrowType: arrow.DataType): DataType {
 export function deserializeArrowType(dataType: DataType): arrow.DataType {
   if (typeof dataType === 'object') {
     switch (dataType.type) {
+      case 'decimal':
+        return new arrow.Decimal(dataType.precision, dataType.scale, dataType.bitWidth);
+      case 'map':
+        let children = dataType.children.map((arrowField) => deserializeArrowField(arrowField));
+        return new arrow.Map_(children as any, dataType.keysSorted);
       case 'list':
         const field = deserializeArrowField(dataType.children[0]);
         return new arrow.List(field);
@@ -197,7 +222,7 @@ export function deserializeArrowType(dataType: DataType): arrow.DataType {
         const child = deserializeArrowField(dataType.children[0]);
         return new arrow.FixedSizeList(dataType.listSize, child);
       case 'struct':
-        const children = dataType.children.map((arrowField) => deserializeArrowField(arrowField));
+        children = dataType.children.map((arrowField) => deserializeArrowField(arrowField));
         return new arrow.Struct(children);
       default:
         throw new Error('array type not supported');

package/src/tables/convert-arrow-to-columnar-table.ts

@@ -0,0 +1,29 @@
+// loaders.gl, MIT license
+// Copyright (c) vis.gl contributors
+
+import type {ColumnarTable} from '@loaders.gl/schema';
+import type {ArrowTable} from '../lib/arrow-table';
+
+/**
+ * Convert an Apache Arrow table to a ColumnarTable
+ * @note Currently does not convert schema
+ */
+export function convertArrowToColumnarTable(table: ArrowTable): ColumnarTable {
+  // TODO - avoid calling `getColumn` on columns we are not interested in?
+  // Add options object?
+
+  const arrowTable = table.data;
+  const columnarTable = {};
+
+  for (const field of arrowTable.schema.fields) {
+    // This (is intended to) coalesce all record batches into a single typed array
+    const arrowColumn = arrowTable.getChild(field.name);
+    const values = arrowColumn?.toArray();
+    columnarTable[field.name] = values;
+  }
+
+  return {
+    shape: 'columnar-table',
+    data: columnarTable
+  };
+}

package/src/tables/convert-arrow-to-geojson-table.ts

@@ -0,0 +1,54 @@
+// loaders.gl, MIT license
+// Copyright (c) vis.gl contributors
+
+import type {Feature, GeoJSONTable} from '@loaders.gl/schema';
+import type * as arrow from 'apache-arrow';
+import type {ArrowTable} from '../lib/arrow-table';
+import {serializeArrowSchema, parseGeometryFromArrow} from '@loaders.gl/arrow';
+import {getGeometryColumnsFromSchema} from '@loaders.gl/gis';
+
+/**
+ * Wrap an apache arrow table in a loaders.gl table wrapper.
+ * From this additional conversions are available.
+ * @param arrowTable
+ * @returns
+ */
+export function convertApacheArrowToArrowTable(arrowTable: arrow.Table): ArrowTable {
+  return {
+    shape: 'arrow-table',
+    data: arrowTable
+  };
+}
+
+/**
+ * Convert an Apache Arrow table to a GeoJSONTable
+ * @note Currently does not convert schema
+ */
+export function convertArrowToGeoJSONTable(table: ArrowTable): GeoJSONTable {
+  const arrowTable = table.data;
+  const schema = serializeArrowSchema(arrowTable.schema);
+  const geometryColumns = getGeometryColumnsFromSchema(schema);
+
+  // get encoding from geometryColumns['geometry']
+  const encoding = geometryColumns.geometry.encoding;
+
+  const features: Feature[] = [];
+
+  for (let row = 0; row < arrowTable.numRows; row++) {
+    // get first geometry from arrow geometry column
+    const arrowGeometry = arrowTable.getChild('geometry')?.get(row);
+    const arrowGeometryObject = {encoding, data: arrowGeometry};
+
+    // parse arrow geometry to geojson feature
+    const feature = parseGeometryFromArrow(arrowGeometryObject);
+    if (feature) {
+      features.push(feature);
+    }
+  }
+
+  return {
+    shape: 'geojson-table',
+    type: 'FeatureCollection',
+    features
+  };
+}

package/src/tables/convert-columnar-to-row-table.ts

@@ -0,0 +1,29 @@
+// loaders.gl, MIT license
+// Copyright (c) vis.gl contributors
+
+import type {ColumnarTable, ObjectRowTable} from '@loaders.gl/schema';
+
+/**
+ *
+ * @note - should be part of schema module
+ */
+export function convertColumnarToRowFormatTable(columnarTable: ColumnarTable): ObjectRowTable {
+  const tableKeys = Object.keys(columnarTable);
+  const tableRowsCount = columnarTable[tableKeys[0]].length;
+
+  const rowFormatTable: {}[] = [];
+
+  for (let index = 0; index < tableRowsCount; index++) {
+    const tableItem = {};
+    for (let keyIndex = 0; keyIndex < tableKeys.length; keyIndex++) {
+      const fieldName = tableKeys[keyIndex];
+      tableItem[fieldName] = columnarTable[fieldName][index];
+    }
+    rowFormatTable.push(tableItem);
+  }
+
+  return {
+    shape: 'object-row-table',
+    data: rowFormatTable
+  };
+}

package/src/triangulate-on-worker.ts

@@ -0,0 +1,47 @@
+// loaders.gl, MIT license
+// Copyright (c) vis.gl contributors
+
+import type {WorkerOptions} from '@loaders.gl/worker-utils';
+import {processOnWorker} from '@loaders.gl/worker-utils';
+
+// __VERSION__ is injected by babel-plugin-version-inline
+// @ts-ignore TS2304: Cannot find name '__VERSION__'.
+const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
+
+export type TriangulationWorkerInput = TriangulateInput | {operation: 'test'; data: any};
+export type TriangulationWorkerOutput = TriangulateResult | {operation: 'test'; data: any};
+
+/** Input data for operation: 'triangulate' */
+export type TriangulateInput = {
+  operation: 'triangulate';
+  polygonIndices: Uint16Array;
+  primitivePolygonIndices: Int32Array;
+  flatCoordinateArray: Float64Array;
+  nDim: number;
+};
+
+/** Result type for operation: 'triangulate' */
+export type TriangulateResult = TriangulateInput & {
+  triangleIndices: Uint32Array;
+};
+
+/**
+ * Worker for tessellating geometries. Normally called through triangulateOnWorker
+ */
+export const TriangulationWorker = {
+  id: 'triangulation',
+  name: 'Triangulate',
+  module: 'arrow',
+  version: VERSION,
+  options: {}
+};
+
+/**
+ * Provide type safety
+ */
+export function triangulateOnWorker(
+  data: TriangulationWorkerInput,
+  options: WorkerOptions = {}
+): Promise<TriangulationWorkerOutput> {
+  return processOnWorker(TriangulationWorker, data, options);
+}

package/src/workers/arrow-worker.ts

@@ -2,6 +2,6 @@
 // Copyright (c) vis.gl contributors
 
 import {createLoaderWorker} from '@loaders.gl/loader-utils';
-import {ArrowLoader} from '../index';
+import {ArrowLoader} from '../arrow-loader';
 
 createLoaderWorker(ArrowLoader);

package/src/workers/triangulation-worker-node.ts

@@ -0,0 +1,4 @@
+// loaders.gl, MIT license
+// Copyright (c) vis.gl contributors
+
+import './triangulation-worker';

package/src/workers/triangulation-worker.ts

@@ -0,0 +1,39 @@
+// loaders.gl, MIT license
+// Copyright (c) vis.gl contributors
+
+import {createWorker} from '@loaders.gl/worker-utils';
+import {getTriangleIndices} from '../geoarrow/convert-geoarrow-to-binary-geometry';
+import type {
+  TriangulationWorkerInput,
+  TriangulateInput,
+  TriangulateResult
+} from '../triangulate-on-worker';
+
+createWorker(async (data, options = {}) => {
+  const input = data as TriangulationWorkerInput;
+  const operation = input?.operation;
+  switch (operation) {
+    case 'test':
+      return input;
+    case 'triangulate':
+      return triangulateBatch(data);
+    default:
+      throw new Error(
+        `TriangulationWorker: Unsupported operation ${operation}. Expected 'triangulate'`
+      );
+  }
+});
+
+function triangulateBatch(data: TriangulateInput): TriangulateResult {
+  // Parse any WKT/WKB geometries
+  // Build binary geometries
+  // Call earcut and triangulate
+  console.error('TriangulationWorker: tessellating batch', data);
+  const triangleIndices = getTriangleIndices(
+    data.polygonIndices,
+    data.primitivePolygonIndices,
+    data.flatCoordinateArray,
+    data.nDim
+  );
+  return {...data, triangleIndices};
+}

package/dist/lib/parse-arrow-in-batches.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"parse-arrow-in-batches.d.ts","sourceRoot":"","sources":["../../src/lib/parse-arrow-in-batches.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,eAAe,CAAC;AAInD;GACG;AACH,wBAAgB,mBAAmB,CACjC,aAAa,EAAE,aAAa,CAAC,WAAW,CAAC,GAAG,QAAQ,CAAC,WAAW,CAAC,GAChE,aAAa,CAAC,eAAe,CAAC,CAsChC"}

package/dist/lib/parse-arrow-in-batches.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"parse-arrow-in-batches.js","names":["arrow","parseArrowInBatches","asyncIterator","makeArrowAsyncIterator","readers","RecordBatchReader","readAll","reader","recordBatch","arrowTabledBatch","shape","batchType","data","Table","length"],"sources":["../../src/lib/parse-arrow-in-batches.ts"],"sourcesContent":["// TODO - this import defeats the sophisticated typescript checking in ArrowJS\nimport type {ArrowTableBatch} from './arrow-table';\nimport * as arrow from 'apache-arrow';\n// import {isIterable} from '@loaders.gl/core';\n\n/**\n */\nexport function parseArrowInBatches(\n  asyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>\n): AsyncIterable<ArrowTableBatch> {\n  // Creates the appropriate arrow.RecordBatchReader subclasses from the input\n  // This will also close the underlying source in case of early termination or errors\n\n  // As an optimization, return a non-async iterator\n  /*\n  if (isIterable(readers)) {\n    function* makeArrowIterator() {\n      for (const reader of readers) {\n        for (const batch of reader) {\n          yield processBatch(batch, reader);\n        }\n        break; // only processing one stream of batches\n      }\n    }\n    const arrowIterator = makeArrowIterator();\n  }\n  */\n\n  async function* makeArrowAsyncIterator(): AsyncIterator<ArrowTableBatch> {\n    // @ts-ignore\n    const readers = arrow.RecordBatchReader.readAll(asyncIterator);\n    for await (const reader of readers) {\n      for await (const recordBatch of reader) {\n        const arrowTabledBatch: ArrowTableBatch = {\n          shape: 'arrow-table',\n          batchType: 'data',\n          data: new arrow.Table([recordBatch]),\n          length: recordBatch.data.length\n        };\n        // processBatch(recordBatch);\n        yield arrowTabledBatch;\n      }\n      break; // only processing one stream of batches\n    }\n  }\n\n  return makeArrowAsyncIterator() as any; // as AsyncIterator<ArrowTableBatch>;\n}\n\n// function processBatch(batch: RecordBatch): ArrowTableBatch {\n//   const values = {};\n//   batch.schema.fields.forEach(({name}, index) => {\n//     values[name] = batch.getChildAt(index)?.toArray();\n//   });\n//   return {\n//   };\n// }\n"],"mappings":"AAEA,OAAO,KAAKA,KAAK,MAAM,cAAc;AAKrC,OAAO,SAASC,mBAAmBA,CACjCC,aAAiE,EACjC;EAmBhC,gBAAgBC,sBAAsBA,CAAA,EAAmC;IAEvE,MAAMC,OAAO,GAAGJ,KAAK,CAACK,iBAAiB,CAACC,OAAO,CAACJ,aAAa,CAAC;IAC9D,WAAW,MAAMK,MAAM,IAAIH,OAAO,EAAE;MAClC,WAAW,MAAMI,WAAW,IAAID,MAAM,EAAE;QACtC,MAAME,gBAAiC,GAAG;UACxCC,KAAK,EAAE,aAAa;UACpBC,SAAS,EAAE,MAAM;UACjBC,IAAI,EAAE,IAAIZ,KAAK,CAACa,KAAK,CAAC,CAACL,WAAW,CAAC,CAAC;UACpCM,MAAM,EAAEN,WAAW,CAACI,IAAI,CAACE;QAC3B,CAAC;QAED,MAAML,gBAAgB;MACxB;MACA;IACF;EACF;EAEA,OAAON,sBAAsB,CAAC,CAAC;AACjC"}

package/dist/lib/parse-arrow-sync.d.ts

@@ -1,5 +0,0 @@
-import type { ColumnarTable, ObjectRowTable } from '@loaders.gl/schema';
-import type { ArrowTable } from './arrow-table';
-import type { ArrowLoaderOptions } from '../arrow-loader';
-export default function parseArrowSync(arrayBuffer: any, options?: ArrowLoaderOptions): ArrowTable | ColumnarTable | ObjectRowTable;
-//# sourceMappingURL=parse-arrow-sync.d.ts.map

package/dist/lib/parse-arrow-sync.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"parse-arrow-sync.d.ts","sourceRoot":"","sources":["../../src/lib/parse-arrow-sync.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,aAAa,EAAE,cAAc,EAAC,MAAM,oBAAoB,CAAC;AACtE,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,eAAe,CAAC;AAG9C,OAAO,KAAK,EAAC,kBAAkB,EAAC,MAAM,iBAAiB,CAAC;AAOxD,MAAM,CAAC,OAAO,UAAU,cAAc,CACpC,WAAW,KAAA,EACX,OAAO,CAAC,EAAE,kBAAkB,GAC3B,UAAU,GAAG,aAAa,GAAG,cAAc,CAoB7C"}

package/dist/lib/parse-arrow-sync.js

@@ -1,21 +0,0 @@
-import { convertTable } from '@loaders.gl/schema';
-import * as arrow from 'apache-arrow';
-import { convertApacheArrowToArrowTable, convertArrowToColumnarTable } from "../tables/convert-arrow-to-table.js";
-export default function parseArrowSync(arrayBuffer, options) {
-  var _options$arrow;
-  const apacheArrowTable = arrow.tableFromIPC([new Uint8Array(arrayBuffer)]);
-  const arrowTable = convertApacheArrowToArrowTable(apacheArrowTable);
-  const shape = (options === null || options === void 0 ? void 0 : (_options$arrow = options.arrow) === null || _options$arrow === void 0 ? void 0 : _options$arrow.shape) || 'arrow-table';
-  switch (shape) {
-    case 'arrow-table':
-      return arrowTable;
-    case 'columnar-table':
-      return convertArrowToColumnarTable(arrowTable);
-    case 'object-row-table':
-      const columnarTable = convertArrowToColumnarTable(arrowTable);
-      return convertTable(columnarTable, 'object-row-table');
-    default:
-      throw new Error(shape);
-  }
-}
-//# sourceMappingURL=parse-arrow-sync.js.map

package/dist/lib/parse-arrow-sync.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"parse-arrow-sync.js","names":["convertTable","arrow","convertApacheArrowToArrowTable","convertArrowToColumnarTable","parseArrowSync","arrayBuffer","options","_options$arrow","apacheArrowTable","tableFromIPC","Uint8Array","arrowTable","shape","columnarTable","Error"],"sources":["../../src/lib/parse-arrow-sync.ts"],"sourcesContent":["import type {ColumnarTable, ObjectRowTable} from '@loaders.gl/schema';\nimport type {ArrowTable} from './arrow-table';\nimport {convertTable} from '@loaders.gl/schema';\nimport * as arrow from 'apache-arrow';\nimport type {ArrowLoaderOptions} from '../arrow-loader';\nimport {\n  convertApacheArrowToArrowTable,\n  convertArrowToColumnarTable\n} from '../tables/convert-arrow-to-table';\n\n// Parses arrow to a columnar table\nexport default function parseArrowSync(\n  arrayBuffer,\n  options?: ArrowLoaderOptions\n): ArrowTable | ColumnarTable | ObjectRowTable {\n  const apacheArrowTable = arrow.tableFromIPC([new Uint8Array(arrayBuffer)]);\n  const arrowTable = convertApacheArrowToArrowTable(apacheArrowTable);\n\n  const shape = options?.arrow?.shape || 'arrow-table';\n  switch (shape) {\n    case 'arrow-table':\n      return arrowTable;\n\n    case 'columnar-table':\n      return convertArrowToColumnarTable(arrowTable);\n\n    case 'object-row-table':\n      const columnarTable = convertArrowToColumnarTable(arrowTable);\n      return convertTable(columnarTable, 'object-row-table');\n\n    default:\n      // TODO\n      throw new Error(shape);\n  }\n}\n"],"mappings":"AAEA,SAAQA,YAAY,QAAO,oBAAoB;AAC/C,OAAO,KAAKC,KAAK,MAAM,cAAc;AAAC,SAGpCC,8BAA8B,EAC9BC,2BAA2B;AAI7B,eAAe,SAASC,cAAcA,CACpCC,WAAW,EACXC,OAA4B,EACiB;EAAA,IAAAC,cAAA;EAC7C,MAAMC,gBAAgB,GAAGP,KAAK,CAACQ,YAAY,CAAC,CAAC,IAAIC,UAAU,CAACL,WAAW,CAAC,CAAC,CAAC;EAC1E,MAAMM,UAAU,GAAGT,8BAA8B,CAACM,gBAAgB,CAAC;EAEnE,MAAMI,KAAK,GAAG,CAAAN,OAAO,aAAPA,OAAO,wBAAAC,cAAA,GAAPD,OAAO,CAAEL,KAAK,cAAAM,cAAA,uBAAdA,cAAA,CAAgBK,KAAK,KAAI,aAAa;EACpD,QAAQA,KAAK;IACX,KAAK,aAAa;MAChB,OAAOD,UAAU;IAEnB,KAAK,gBAAgB;MACnB,OAAOR,2BAA2B,CAACQ,UAAU,CAAC;IAEhD,KAAK,kBAAkB;MACrB,MAAME,aAAa,GAAGV,2BAA2B,CAACQ,UAAU,CAAC;MAC7D,OAAOX,YAAY,CAACa,aAAa,EAAE,kBAAkB,CAAC;IAExD;MAEE,MAAM,IAAIC,KAAK,CAACF,KAAK,CAAC;EAC1B;AACF"}

package/dist/tables/convert-arrow-to-table.d.ts

@@ -1,21 +0,0 @@
-import type { ColumnarTable, ObjectRowTable } from '@loaders.gl/schema';
-import type * as arrow from 'apache-arrow';
-import type { ArrowTable } from '../lib/arrow-table';
-/**
- * Wrap an apache arrow table in a loaders.gl table wrapper.
- * From this additional conversions are available.
- * @param arrowTable
- * @returns
- */
-export declare function convertApacheArrowToArrowTable(arrowTable: arrow.Table): ArrowTable;
-/**
- * Convert an Apache Arrow table to a ColumnarTable
- * @note Currently does not convert schema
- */
-export declare function convertArrowToColumnarTable(table: ArrowTable): ColumnarTable;
-/**
- *
- * @note - should be part of schema module
- */
-export declare function convertColumnarToRowFormatTable(columnarTable: ColumnarTable): ObjectRowTable;
-//# sourceMappingURL=convert-arrow-to-table.d.ts.map