@loaders.gl/arrow 4.0.1 → 4.0.3

package/src/geoarrow/convert-geoarrow-to-binary-geometry.ts

@@ -1,7 +1,7 @@
 // loaders.gl, MIT license
 // Copyright (c) vis.gl contributors
 
-import {Data, Vector} from 'apache-arrow';
+import * as arrow from 'apache-arrow';
 import {BinaryFeatureCollection as BinaryFeatures} from '@loaders.gl/schema';
 import {GeoArrowEncoding} from '@loaders.gl/gis';
 import {updateBoundsFromGeoArrowSamples} from './get-arrow-bounds';
@@ -24,7 +24,7 @@ type BinaryGeometryContent = {
 };
 
 // binary geometry template, see deck.gl BinaryGeometry
-const BINARY_GEOMETRY_TEMPLATE = {
+export const BINARY_GEOMETRY_TEMPLATE = {
   globalFeatureIds: {value: new Uint32Array(0), size: 1},
   positions: {value: new Float32Array(0), size: 2},
   properties: [],
@@ -40,7 +40,7 @@ const BINARY_GEOMETRY_TEMPLATE = {
  * @returns BinaryDataFromGeoArrow
  */
 export function getBinaryGeometriesFromArrow(
-  geoColumn: Vector,
+  geoColumn: arrow.Vector,
   geoEncoding: GeoArrowEncoding
 ): BinaryDataFromGeoArrow {
   const featureTypes = {
@@ -50,20 +50,18 @@ export function getBinaryGeometriesFromArrow(
   };
 
   const chunks = geoColumn.data;
-  const bounds: [number, number, number, number] = [Infinity, Infinity, -Infinity, -Infinity];
+  let bounds: [number, number, number, number] = [Infinity, Infinity, -Infinity, -Infinity];
   let globalFeatureIdOffset = 0;
   const binaryGeometries: BinaryFeatures[] = [];
 
-  for (let c = 0; c < chunks.length; c++) {
-    const geometries = chunks[c];
+  chunks.forEach((chunk) => {
     const {featureIds, flatCoordinateArray, nDim, geomOffset} = getBinaryGeometriesFromChunk(
-      geometries,
+      chunk,
       geoEncoding
     );
 
-    const numOfVertices = flatCoordinateArray.length / nDim;
-    const globalFeatureIds = new Uint32Array(numOfVertices);
-    for (let i = 0; i < numOfVertices; i++) {
+    const globalFeatureIds = new Uint32Array(featureIds.length);
+    for (let i = 0; i < featureIds.length; i++) {
       globalFeatureIds[i] = featureIds[i] + globalFeatureIdOffset;
     }
 
@@ -74,14 +72,13 @@ export function getBinaryGeometriesFromArrow(
         size: nDim
       },
       featureIds: {value: featureIds, size: 1},
-      // eslint-disable-next-line no-loop-func
-      properties: [...Array(geometries.length).keys()].map((i) => ({
+      properties: [...Array(chunk.length).keys()].map((i) => ({
         index: i + globalFeatureIdOffset
       }))
     };
 
     // TODO: check if chunks are sequentially accessed
-    globalFeatureIdOffset += geometries.length;
+    globalFeatureIdOffset += chunk.length;
 
     // NOTE: deck.gl defines the BinaryFeatures structure must have points, lines, polygons even if they are empty
     binaryGeometries.push({
@@ -114,8 +111,8 @@ export function getBinaryGeometriesFromArrow(
       }
     });
 
-    updateBoundsFromGeoArrowSamples(flatCoordinateArray, nDim, bounds);
-  }
+    bounds = updateBoundsFromGeoArrowSamples(flatCoordinateArray, nDim, bounds);
+  });
 
   return {binaryGeometries, bounds, featureTypes};
 }
@@ -127,7 +124,7 @@ export function getBinaryGeometriesFromArrow(
  * @returns BinaryGeometryContent
  */
 function getBinaryGeometriesFromChunk(
-  chunk: Data,
+  chunk: arrow.Data,
   geoEncoding: GeoArrowEncoding
 ): BinaryGeometryContent {
   switch (geoEncoding) {
@@ -151,10 +148,14 @@ function getBinaryGeometriesFromChunk(
  * @param geoEncoding the geo encoding of the geoarrow polygon column
  * @returns BinaryGeometryContent
  */
-function getBinaryPolygonsFromChunk(chunk: Data, geoEncoding: string): BinaryGeometryContent {
+function getBinaryPolygonsFromChunk(chunk: arrow.Data, geoEncoding: string): BinaryGeometryContent {
   const isMultiPolygon = geoEncoding === 'geoarrow.multipolygon';
 
   const polygonData = isMultiPolygon ? chunk.children[0] : chunk;
+  const polygonOffset = polygonData.valueOffsets;
+  const partData = isMultiPolygon
+    ? chunk.valueOffsets.map((i) => polygonOffset.at(i) || i)
+    : chunk.valueOffsets;
   const ringData = polygonData.children[0];
   const pointData = ringData.children[0];
   const coordData = pointData.children[0];
@@ -162,17 +163,16 @@ function getBinaryPolygonsFromChunk(chunk: Data, geoEncoding: string): BinaryGeo
   const geomOffset = ringData.valueOffsets;
   const flatCoordinateArray = coordData.values;
 
-  const geometryIndicies = new Uint16Array(chunk.length + 1);
-  for (let i = 0; i < chunk.length; i++) {
-    geometryIndicies[i] = geomOffset[chunk.valueOffsets[i]];
+  const geometryIndicies = new Uint16Array(polygonOffset.length);
+  for (let i = 0; i < polygonOffset.length; i++) {
+    geometryIndicies[i] = geomOffset[polygonOffset[i]];
   }
-  geometryIndicies[chunk.length] = flatCoordinateArray.length / nDim;
 
   const numOfVertices = flatCoordinateArray.length / nDim;
   const featureIds = new Uint32Array(numOfVertices);
-  for (let i = 0; i < chunk.length - 1; i++) {
-    const startIdx = geomOffset[chunk.valueOffsets[i]];
-    const endIdx = geomOffset[chunk.valueOffsets[i + 1]];
+  for (let i = 0; i < partData.length - 1; i++) {
+    const startIdx = geomOffset[partData[i]];
+    const endIdx = geomOffset[partData[i + 1]];
     for (let j = startIdx; j < endIdx; j++) {
       featureIds[j] = i;
     }
@@ -193,7 +193,7 @@ function getBinaryPolygonsFromChunk(chunk: Data, geoEncoding: string): BinaryGeo
  * @param geoEncoding the geo encoding of the geoarrow column
  * @returns BinaryGeometryContent
  */
-function getBinaryLinesFromChunk(chunk: Data, geoEncoding: string): BinaryGeometryContent {
+function getBinaryLinesFromChunk(chunk: arrow.Data, geoEncoding: string): BinaryGeometryContent {
   const isMultiLineString = geoEncoding === 'geoarrow.multilinestring';
 
   const lineData = isMultiLineString ? chunk.children[0] : chunk;
@@ -232,7 +232,7 @@ function getBinaryLinesFromChunk(chunk: Data, geoEncoding: string): BinaryGeomet
  * @param geoEncoding  geo encoding of the geoarrow column
  * @returns BinaryGeometryContent
  */
-function getBinaryPointsFromChunk(chunk: Data, geoEncoding: string): BinaryGeometryContent {
+function getBinaryPointsFromChunk(chunk: arrow.Data, geoEncoding: string): BinaryGeometryContent {
   const isMultiPoint = geoEncoding === 'geoarrow.multipoint';
 
   const pointData = isMultiPoint ? chunk.children[0] : chunk;

package/src/geoarrow/convert-geoarrow-to-geojson.ts

@@ -1,7 +1,7 @@
 // loaders.gl, MIT license
 // Copyright (c) vis.gl contributors
 
-import {Vector} from 'apache-arrow';
+import * as arrow from 'apache-arrow';
 import {
   Feature,
   MultiPolygon,
@@ -71,7 +71,7 @@ export function parseGeometryFromArrow(rawData: RawArrowFeature): Feature | null
 /**
  * convert Arrow MultiPolygon to geojson Feature
  */
-function arrowMultiPolygonToFeature(arrowMultiPolygon: Vector): MultiPolygon {
+function arrowMultiPolygonToFeature(arrowMultiPolygon: arrow.Vector): MultiPolygon {
   const multiPolygon: Position[][][] = [];
   for (let m = 0; m < arrowMultiPolygon.length; m++) {
     const arrowPolygon = arrowMultiPolygon.get(m);
@@ -98,7 +98,7 @@ function arrowMultiPolygonToFeature(arrowMultiPolygon: Vector): MultiPolygon {
 /**
  * convert Arrow Polygon to geojson Feature
  */
-function arrowPolygonToFeature(arrowPolygon: Vector): Polygon {
+function arrowPolygonToFeature(arrowPolygon: arrow.Vector): Polygon {
   const polygon: Position[][] = [];
   for (let i = 0; arrowPolygon && i < arrowPolygon.length; i++) {
     const arrowRing = arrowPolygon.get(i);
@@ -120,7 +120,7 @@ function arrowPolygonToFeature(arrowPolygon: Vector): Polygon {
 /**
  * convert Arrow MultiPoint to geojson MultiPoint
  */
-function arrowMultiPointToFeature(arrowMultiPoint: Vector): MultiPoint {
+function arrowMultiPointToFeature(arrowMultiPoint: arrow.Vector): MultiPoint {
   const multiPoint: Position[] = [];
   for (let i = 0; arrowMultiPoint && i < arrowMultiPoint.length; i++) {
     const arrowPoint = arrowMultiPoint.get(i);
@@ -139,7 +139,7 @@ function arrowMultiPointToFeature(arrowMultiPoint: Vector): MultiPoint {
 /**
  * convert Arrow Point to geojson Point
  */
-function arrowPointToFeature(arrowPoint: Vector): Point {
+function arrowPointToFeature(arrowPoint: arrow.Vector): Point {
   const point: Position = Array.from(arrowPoint);
   const geometry: Point = {
     type: 'Point',
@@ -151,7 +151,7 @@ function arrowPointToFeature(arrowPoint: Vector): Point {
 /**
  * convert Arrow MultiLineString to geojson MultiLineString
  */
-function arrowMultiLineStringToFeature(arrowMultiLineString: Vector): MultiLineString {
+function arrowMultiLineStringToFeature(arrowMultiLineString: arrow.Vector): MultiLineString {
   const multiLineString: Position[][] = [];
   for (let i = 0; arrowMultiLineString && i < arrowMultiLineString.length; i++) {
     const arrowLineString = arrowMultiLineString.get(i);
@@ -175,7 +175,7 @@ function arrowMultiLineStringToFeature(arrowMultiLineString: Vector): MultiLineS
 /**
  * convert Arrow LineString to geojson LineString
  */
-function arrowLineStringToFeature(arrowLineString: Vector): LineString {
+function arrowLineStringToFeature(arrowLineString: arrow.Vector): LineString {
   const lineString: Position[] = [];
   for (let i = 0; arrowLineString && i < arrowLineString.length; i++) {
     const arrowCoord = arrowLineString.get(i);

package/src/geoarrow/get-arrow-bounds.ts

@@ -23,7 +23,7 @@ export function updateBoundsFromGeoArrowSamples(
   for (let i = 0; i < numberOfFeatures; i += sampleStep) {
     const lng = flatCoords[i * nDim];
     const lat = flatCoords[i * nDim + 1];
-    if (lng < bounds[0]) {
+    if (lng < newBounds[0]) {
       newBounds[0] = lng;
     }
     if (lat < newBounds[1]) {
@@ -36,6 +36,5 @@ export function updateBoundsFromGeoArrowSamples(
       newBounds[3] = lat;
     }
   }
-
   return newBounds;
 }

package/src/index.ts

@@ -65,6 +65,11 @@ export type {GeoArrowEncoding} from '@loaders.gl/gis';
 // getGeoArrowEncoding
 
 export type {BinaryDataFromGeoArrow} from './geoarrow/convert-geoarrow-to-binary-geometry';
-export {getBinaryGeometriesFromArrow} from './geoarrow/convert-geoarrow-to-binary-geometry';
+export {
+  BINARY_GEOMETRY_TEMPLATE,
+  getBinaryGeometriesFromArrow
+} from './geoarrow/convert-geoarrow-to-binary-geometry';
 
 export {parseGeometryFromArrow} from './geoarrow/convert-geoarrow-to-geojson';
+
+export {updateBoundsFromGeoArrowSamples} from './geoarrow/get-arrow-bounds';

package/src/lib/arrow-table-batch.ts

@@ -1,19 +1,9 @@
 import {ColumnarTableBatchAggregator} from '@loaders.gl/schema';
 import type {ArrowTableBatch} from './arrow-table';
-import {
-  Table as ApacheArrowTable,
-  Schema,
-  Field,
-  RecordBatch,
-  Struct,
-  makeVector,
-  makeData,
-  Vector,
-  Float32
-} from 'apache-arrow';
+import * as arrow from 'apache-arrow';
 
 export class ArrowTableBatchAggregator extends ColumnarTableBatchAggregator {
-  arrowSchema: Schema | null;
+  arrowSchema: arrow.Schema | null;
 
   constructor(schema, options) {
     super(schema, options);
@@ -30,10 +20,10 @@ export class ArrowTableBatchAggregator extends ColumnarTableBatchAggregator {
       const arrowVectors = getArrowVectors(this.arrowSchema, batch.data);
 
       // Create the record batch
-      const recordBatch = new RecordBatch(
+      const recordBatch = new arrow.RecordBatch(
         this.arrowSchema,
-        makeData({
-          type: new Struct(this.arrowSchema.fields),
+        arrow.makeData({
+          type: new arrow.Struct(this.arrowSchema.fields),
           children: arrowVectors.map(({data}) => data[0])
         })
       );
@@ -41,7 +31,7 @@ export class ArrowTableBatchAggregator extends ColumnarTableBatchAggregator {
       return {
         shape: 'arrow-table',
         batchType: 'data',
-        data: new ApacheArrowTable([recordBatch]),
+        data: new arrow.Table([recordBatch]),
         length: batch.length
       };
     }
@@ -51,15 +41,15 @@ export class ArrowTableBatchAggregator extends ColumnarTableBatchAggregator {
 }
 
 // Convert from a simple loaders.gl schema to an Arrow schema
-function getArrowSchema(schema): Schema {
-  const arrowFields: Field[] = [];
+function getArrowSchema(schema): arrow.Schema {
+  const arrowFields: arrow.Field[] = [];
   for (const key in schema) {
     const field = schema[key];
     if (field.type === Float32Array) {
       // TODO - just store the original field as metadata?
       const metadata = new Map(); // field;
-      // arrow: new Field(name, nullable, metadata)
-      const arrowField = new Field(field.name, new Float32(), field.nullable, metadata);
+      // arrow: new arrow.Field(name, nullable, metadata)
+      const arrowField = new arrow.Field(field.name, new arrow.Float32(), field.nullable, metadata);
       arrowFields.push(arrowField);
     }
   }
@@ -67,16 +57,16 @@ function getArrowSchema(schema): Schema {
     throw new Error('No arrow convertible fields');
   }
 
-  return new Schema(arrowFields);
+  return new arrow.Schema(arrowFields);
 }
 
 // Convert from simple loaders.gl arrays to arrow vectors
-function getArrowVectors(arrowSchema, data): Vector[] {
+function getArrowVectors(arrowSchema, data): arrow.Vector[] {
   const arrowVectors: any[] = [];
   for (const field of arrowSchema.fields) {
     const vector = data[field.name];
     if (vector instanceof Float32Array) {
-      const arrowVector = makeVector(vector);
+      const arrowVector = arrow.makeVector(vector);
       arrowVectors.push(arrowVector);
     }
   }

package/src/lib/arrow-table.ts

@@ -2,7 +2,7 @@
 // Copyright (c) vis.gl contributors
 
 import type {Batch, Schema} from '@loaders.gl/schema';
-import type {Table as ApacheArrowTable} from 'apache-arrow';
+import type * as arrow from 'apache-arrow';
 
 /**
  * A table organized as an Apache Arrow table
@@ -11,7 +11,7 @@ import type {Table as ApacheArrowTable} from 'apache-arrow';
 export type ArrowTable = {
   shape: 'arrow-table';
   schema?: Schema;
-  data: ApacheArrowTable;
+  data: arrow.Table;
 };
 
 /**
@@ -22,6 +22,6 @@ export type ArrowTableBatch = Batch & {
   shape: 'arrow-table';
   schemaType?: 'explicit' | 'deduced';
   schema?: Schema;
-  data: ApacheArrowTable; // ApacheRecordBatch;
+  data: arrow.Table; // ApacheRecordBatch;
   length: number;
 };

package/src/lib/encode-arrow.ts

@@ -1,4 +1,4 @@
-import {Table, Vector, tableToIPC, vectorFromArray} from 'apache-arrow';
+import * as arrow from 'apache-arrow';
 import {AnyArrayType, VECTOR_TYPES} from '../types';
 
 export type ColumnarTable = {
@@ -15,28 +15,28 @@ export type ColumnarTable = {
  * @returns - encoded ArrayBuffer
  */
 export function encodeArrowSync(data: ColumnarTable): ArrayBuffer {
-  const vectors: Record<string, Vector> = {};
+  const vectors: Record<string, arrow.Vector> = {};
   for (const arrayData of data) {
     const arrayVector = createVector(arrayData.array, arrayData.type);
     vectors[arrayData.name] = arrayVector;
   }
-  const table = new Table(vectors);
-  const arrowBuffer = tableToIPC(table);
+  const table = new arrow.Table(vectors);
+  const arrowBuffer = arrow.tableToIPC(table);
   return arrowBuffer;
 }
 
 /**
- * Create Arrow Vector from given data and vector type
+ * Create Arrow arrow.Vector from given data and vector type
  * @param array {import('../types').AnyArrayType} - columns data
  * @param type {number} - the writer options
  * @return a vector of one of vector's types defined in the Apache Arrow library
  */
-function createVector(array, type): Vector {
+function createVector(array, type): arrow.Vector {
   switch (type) {
     case VECTOR_TYPES.DATE:
-      return vectorFromArray(array);
+      return arrow.vectorFromArray(array);
     case VECTOR_TYPES.FLOAT:
     default:
-      return vectorFromArray(array);
+      return arrow.vectorFromArray(array);
   }
 }

package/src/lib/parse-arrow-in-batches.ts

@@ -1,6 +1,6 @@
 // TODO - this import defeats the sophisticated typescript checking in ArrowJS
 import type {ArrowTableBatch} from './arrow-table';
-import {RecordBatchReader, Table as ApacheArrowTable} from 'apache-arrow';
+import * as arrow from 'apache-arrow';
 // import {isIterable} from '@loaders.gl/core';
 
 /**
@@ -8,7 +8,7 @@ import {RecordBatchReader, Table as ApacheArrowTable} from 'apache-arrow';
 export function parseArrowInBatches(
   asyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>
 ): AsyncIterable<ArrowTableBatch> {
-  // Creates the appropriate RecordBatchReader subclasses from the input
+  // Creates the appropriate arrow.RecordBatchReader subclasses from the input
   // This will also close the underlying source in case of early termination or errors
 
   // As an optimization, return a non-async iterator
@@ -28,13 +28,13 @@ export function parseArrowInBatches(
 
   async function* makeArrowAsyncIterator(): AsyncIterator<ArrowTableBatch> {
     // @ts-ignore
-    const readers = RecordBatchReader.readAll(asyncIterator);
+    const readers = arrow.RecordBatchReader.readAll(asyncIterator);
     for await (const reader of readers) {
       for await (const recordBatch of reader) {
         const arrowTabledBatch: ArrowTableBatch = {
           shape: 'arrow-table',
           batchType: 'data',
-          data: new ApacheArrowTable([recordBatch]),
+          data: new arrow.Table([recordBatch]),
           length: recordBatch.data.length
         };
         // processBatch(recordBatch);

package/src/lib/parse-arrow-sync.ts

@@ -1,7 +1,7 @@
 import type {ColumnarTable, ObjectRowTable} from '@loaders.gl/schema';
 import type {ArrowTable} from './arrow-table';
 import {convertTable} from '@loaders.gl/schema';
-import {tableFromIPC} from 'apache-arrow';
+import * as arrow from 'apache-arrow';
 import type {ArrowLoaderOptions} from '../arrow-loader';
 import {
   convertApacheArrowToArrowTable,
@@ -13,7 +13,7 @@ export default function parseArrowSync(
   arrayBuffer,
   options?: ArrowLoaderOptions
 ): ArrowTable | ColumnarTable | ObjectRowTable {
-  const apacheArrowTable = tableFromIPC([new Uint8Array(arrayBuffer)]);
+  const apacheArrowTable = arrow.tableFromIPC([new Uint8Array(arrayBuffer)]);
   const arrowTable = convertApacheArrowToArrowTable(apacheArrowTable);
 
   const shape = options?.arrow?.shape || 'arrow-table';

package/src/schema/arrow-type-utils.ts

@@ -2,46 +2,27 @@
 // Copyright (c) vis.gl contributors
 
 import type {TypedArray} from '@loaders.gl/schema';
-import {
-  DataType,
-  Float32,
-  Float64,
-  Int16,
-  Int32,
-  Int8,
-  Uint16,
-  Uint32,
-  Uint8
-  // Int8Vector,
-  // Uint8Vector,
-  // Int16Vector,
-  // Uint16Vector,
-  // Int32Vector,
-  // Uint32Vector,
-  // Float32Vector,
-  // Float64Vector
-} from 'apache-arrow';
-// import {AbstractVector} from 'apache-arrow/vector';
+import * as arrow from 'apache-arrow';
 
 /** Return an Apache Arrow Type instance that corresponds to the type of the elements in the supplied Typed Array */
-export function getArrowType(array: TypedArray): DataType {
+export function getArrowType(array: TypedArray): arrow.DataType {
   switch (array.constructor) {
     case Int8Array:
-      return new Int8();
+      return new arrow.Int8();
     case Uint8Array:
-      return new Uint8();
+      return new arrow.Uint8();
     case Int16Array:
-      return new Int16();
+      return new arrow.Int16();
     case Uint16Array:
-      return new Uint16();
+      return new arrow.Uint16();
     case Int32Array:
-      return new Int32();
+      return new arrow.Int32();
     case Uint32Array:
-      return new Uint32();
+      return new arrow.Uint32();
     case Float32Array:
-      return new Float32();
+      return new arrow.Float32();
     case Float64Array:
-      return new Float64();
+      return new arrow.Float64();
     default:
       throw new Error('array type not supported');
   }