@loaders.gl/arrow 4.0.4 → 4.1.0-alpha.10

package/src/geoarrow/{convert-geoarrow-to-geojson.ts → convert-geoarrow-to-geojson-geometry.ts}

@@ -1,81 +1,96 @@
-// loaders.gl, MIT license
+// loaders.gl
+// SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
-import * as arrow from 'apache-arrow';
+// import * as arrow from 'apache-arrow';
 import {
-  Feature,
   MultiPolygon,
   Position,
   Polygon,
   MultiPoint,
   Point,
   MultiLineString,
-  LineString
+  LineString,
+  Geometry,
+  BinaryGeometry
 } from '@loaders.gl/schema';
-import type {GeoArrowEncoding} from '@loaders.gl/gis';
-
-type RawArrowFeature = {
-  data: arrow.Vector;
-  encoding?: GeoArrowEncoding;
-};
+import {binaryToGeometry, type GeoArrowEncoding} from '@loaders.gl/gis';
+import {WKBLoader, WKTLoader} from '@loaders.gl/wkt';
 
 /**
  * parse geometry from arrow data that is returned from processArrowData()
- * NOTE: this function could be duplicated with the binaryToFeature() in deck.gl,
- * it is currently only used for picking because currently deck.gl returns only the index of the feature
- * So the following functions could be deprecated once deck.gl returns the feature directly for binary geojson layer
+ * NOTE: this function could be deduplicated with the binaryToFeature() in deck.gl,
+ * it is currently used for deck.gl picking because currently deck.gl returns only the index of the feature
  *
- * @param rawData the raw geometry data returned from processArrowData, which is an object with two properties: encoding and data
- * @see processArrowData
+ * @param data data extraced from arrow vector representing a geometry
+ * @param encoding the geoarrow encoding of the geometry column
  * @returns Feature or null
  */
-export function parseGeometryFromArrow(rawData: RawArrowFeature): Feature | null {
-  const encoding = rawData.encoding?.toLowerCase() as typeof rawData.encoding;
-  const data = rawData.data;
-  if (!encoding || !data) {
+export function parseGeometryFromArrow(
+  arrowCellValue: any,
+  encoding?: GeoArrowEncoding
+): Geometry | null {
+  // sanity
+  encoding = encoding?.toLowerCase() as GeoArrowEncoding;
+  if (!encoding || !arrowCellValue) {
     return null;
   }
 
-  let geometry;
+  let geometry: Geometry;
 
   switch (encoding) {
     case 'geoarrow.multipolygon':
-      geometry = arrowMultiPolygonToFeature(data);
+      geometry = arrowMultiPolygonToFeature(arrowCellValue);
       break;
     case 'geoarrow.polygon':
-      geometry = arrowPolygonToFeature(data);
+      geometry = arrowPolygonToFeature(arrowCellValue);
       break;
     case 'geoarrow.multipoint':
-      geometry = arrowMultiPointToFeature(data);
+      geometry = arrowMultiPointToFeature(arrowCellValue);
       break;
     case 'geoarrow.point':
-      geometry = arrowPointToFeature(data);
+      geometry = arrowPointToFeature(arrowCellValue);
       break;
     case 'geoarrow.multilinestring':
-      geometry = arrowMultiLineStringToFeature(data);
+      geometry = arrowMultiLineStringToFeature(arrowCellValue);
       break;
     case 'geoarrow.linestring':
-      geometry = arrowLineStringToFeature(data);
+      geometry = arrowLineStringToFeature(arrowCellValue);
       break;
     case 'geoarrow.wkb':
-      throw Error(`GeoArrow encoding not supported ${encoding}`);
+      geometry = arrowWKBToFeature(arrowCellValue);
+      break;
     case 'geoarrow.wkt':
-      throw Error(`GeoArrow encoding not supported ${encoding}`);
+      geometry = arrowWKTToFeature(arrowCellValue);
+      break;
     default: {
       throw Error(`GeoArrow encoding not supported ${encoding}`);
     }
   }
-  return {
-    type: 'Feature',
-    geometry,
-    properties: {}
-  };
+
+  return geometry;
+}
+
+function arrowWKBToFeature(arrowCellValue: any) {
+  // The actual WKB array buffer starts from byteOffset and ends at byteOffset + byteLength
+  const arrayBuffer: ArrayBuffer = arrowCellValue.buffer.slice(
+    arrowCellValue.byteOffset,
+    arrowCellValue.byteOffset + arrowCellValue.byteLength
+  );
+  const binaryGeometry = WKBLoader.parseSync?.(arrayBuffer)! as BinaryGeometry;
+  const geometry = binaryToGeometry(binaryGeometry);
+  return geometry;
+}
+
+function arrowWKTToFeature(arrowCellValue: any) {
+  const string: string = arrowCellValue;
+  return WKTLoader.parseTextSync?.(string)!;
 }
 
 /**
  * convert Arrow MultiPolygon to geojson Feature
  */
-function arrowMultiPolygonToFeature(arrowMultiPolygon: arrow.Vector): MultiPolygon {
+function arrowMultiPolygonToFeature(arrowMultiPolygon: any): MultiPolygon {
   const multiPolygon: Position[][][] = [];
   for (let m = 0; m < arrowMultiPolygon.length; m++) {
     const arrowPolygon = arrowMultiPolygon.get(m);
@@ -102,7 +117,7 @@ function arrowMultiPolygonToFeature(arrowMultiPolygon: arrow.Vector): MultiPolyg
 /**
  * convert Arrow Polygon to geojson Feature
  */
-function arrowPolygonToFeature(arrowPolygon: arrow.Vector): Polygon {
+function arrowPolygonToFeature(arrowPolygon: any): Polygon {
   const polygon: Position[][] = [];
   for (let i = 0; arrowPolygon && i < arrowPolygon.length; i++) {
     const arrowRing = arrowPolygon.get(i);
@@ -124,7 +139,7 @@ function arrowPolygonToFeature(arrowPolygon: arrow.Vector): Polygon {
 /**
  * convert Arrow MultiPoint to geojson MultiPoint
  */
-function arrowMultiPointToFeature(arrowMultiPoint: arrow.Vector): MultiPoint {
+function arrowMultiPointToFeature(arrowMultiPoint: any): MultiPoint {
   const multiPoint: Position[] = [];
   for (let i = 0; arrowMultiPoint && i < arrowMultiPoint.length; i++) {
     const arrowPoint = arrowMultiPoint.get(i);
@@ -133,29 +148,27 @@ function arrowMultiPointToFeature(arrowMultiPoint: arrow.Vector): MultiPoint {
       multiPoint.push(coord);
     }
   }
-  const geometry: MultiPoint = {
+  return {
     type: 'MultiPoint',
     coordinates: multiPoint
   };
-  return geometry;
 }
 
 /**
  * convert Arrow Point to geojson Point
  */
-function arrowPointToFeature(arrowPoint: arrow.Vector): Point {
+function arrowPointToFeature(arrowPoint: any): Point {
   const point: Position = Array.from(arrowPoint);
-  const geometry: Point = {
+  return {
     type: 'Point',
     coordinates: point
   };
-  return geometry;
 }
 
 /**
  * convert Arrow MultiLineString to geojson MultiLineString
  */
-function arrowMultiLineStringToFeature(arrowMultiLineString: arrow.Vector): MultiLineString {
+function arrowMultiLineStringToFeature(arrowMultiLineString: any): MultiLineString {
   const multiLineString: Position[][] = [];
   for (let i = 0; arrowMultiLineString && i < arrowMultiLineString.length; i++) {
     const arrowLineString = arrowMultiLineString.get(i);
@@ -169,17 +182,16 @@ function arrowMultiLineStringToFeature(arrowMultiLineString: arrow.Vector): Mult
     }
     multiLineString.push(lineString);
   }
-  const geometry: MultiLineString = {
+  return {
     type: 'MultiLineString',
     coordinates: multiLineString
   };
-  return geometry;
 }
 
 /**
  * convert Arrow LineString to geojson LineString
  */
-function arrowLineStringToFeature(arrowLineString: arrow.Vector): LineString {
+function arrowLineStringToFeature(arrowLineString: any): LineString {
   const lineString: Position[] = [];
   for (let i = 0; arrowLineString && i < arrowLineString.length; i++) {
     const arrowCoord = arrowLineString.get(i);
@@ -188,9 +200,8 @@ function arrowLineStringToFeature(arrowLineString: arrow.Vector): LineString {
       lineString.push(coords);
     }
   }
-  const geometry: LineString = {
+  return {
     type: 'LineString',
     coordinates: lineString
   };
-  return geometry;
 }

package/src/geoarrow/get-arrow-bounds.ts

@@ -1,4 +1,5 @@
-// loaders.gl, MIT license
+// loaders.gl
+// SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
 /**

package/src/geoarrow-loader.ts

@@ -1,4 +1,5 @@
-// loaders.gl, MIT license
+// loaders.gl
+// SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
 import type {Loader, LoaderWithParser, LoaderOptions} from '@loaders.gl/loader-utils';
@@ -8,10 +9,6 @@ import type {ArrowTable, ArrowTableBatch} from './lib/arrow-table';
 import {parseGeoArrowSync} from './parsers/parse-geoarrow-sync';
 import {parseGeoArrowInBatches} from './parsers/parse-geoarrow-in-batches';
 
-// __VERSION__ is injected by babel-plugin-version-inline
-// @ts-ignore TS2304: Cannot find name '__VERSION__'.
-const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
-
 export type GeoArrowLoaderOptions = LoaderOptions & {
   arrow?: {
     shape: 'arrow-table' | 'binary-geometry';

package/src/index.ts

@@ -1,4 +1,5 @@
-// loaders.gl, MIT license
+// loaders.gl
+// SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
 import type {ArrowLoaderOptions} from './arrow-loader';
@@ -52,7 +53,7 @@ export type {
   BinaryGeometriesFromArrowOptions
 } from './geoarrow/convert-geoarrow-to-binary-geometry';
 export {
-  BINARY_GEOMETRY_TEMPLATE,
+  getBinaryGeometryTemplate,
   getBinaryGeometriesFromArrow,
   getTriangleIndices,
   getMeanCentersFromBinaryGeometries
@@ -60,10 +61,16 @@ export {
 
 export {updateBoundsFromGeoArrowSamples} from './geoarrow/get-arrow-bounds';
 
-export {parseGeometryFromArrow} from './geoarrow/convert-geoarrow-to-geojson';
+export {parseGeometryFromArrow} from './geoarrow/convert-geoarrow-to-geojson-geometry';
 
 export {convertArrowToGeoJSONTable} from './tables/convert-arrow-to-geojson-table';
 
 // EXPERIMENTAL WORKER
+export {hardClone} from './workers/hard-clone';
 
-export {TriangulationWorker, triangulateOnWorker} from './triangulate-on-worker';
+export type {ParseGeoArrowInput, ParseGeoArrowResult} from './triangulate-on-worker';
+export {
+  TriangulationWorker,
+  triangulateOnWorker,
+  parseGeoArrowOnWorker
+} from './triangulate-on-worker';

package/src/lib/arrow-table-batch.ts

@@ -1,4 +1,5 @@
-// loaders.gl, MIT license
+// loaders.gl
+// SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
 import {ColumnarTableBatchAggregator} from '@loaders.gl/schema';

package/src/lib/arrow-table.ts

@@ -1,4 +1,5 @@
-// loaders.gl, MIT license
+// loaders.gl
+// SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
 import type {Batch, Schema} from '@loaders.gl/schema';

package/src/lib/encode-arrow.ts

@@ -1,4 +1,5 @@
-// loaders.gl, MIT license
+// loaders.gl
+// SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
 import * as arrow from 'apache-arrow';

package/src/lib/encode-geoarrow.ts

@@ -1,4 +1,5 @@
-// loaders.gl, MIT license
+// loaders.gl
+// SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
 import * as arrow from 'apache-arrow';

package/src/parsers/parse-arrow-in-batches.ts

@@ -1,14 +1,17 @@
-// loaders.gl, MIT license
+// loaders.gl
+// SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
 import type {ArrowTableBatch} from '../lib/arrow-table';
 import * as arrow from 'apache-arrow';
+import {ArrowLoaderOptions} from '../arrow-loader';
 // import {isIterable} from '@loaders.gl/core';
 
 /**
  */
 export function parseArrowInBatches(
-  asyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>
+  asyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>,
+  options?: ArrowLoaderOptions
 ): AsyncIterable<ArrowTableBatch> {
   // Creates the appropriate arrow.RecordBatchReader subclasses from the input
   // This will also close the underlying source in case of early termination or errors
@@ -33,6 +36,10 @@ export function parseArrowInBatches(
     const readers = arrow.RecordBatchReader.readAll(asyncIterator);
     for await (const reader of readers) {
       for await (const recordBatch of reader) {
+        // use options.batchDebounceMs to add a delay between batches if needed (use case: incremental loading)
+        if (options?.arrow?.batchDebounceMs !== undefined && options?.arrow?.batchDebounceMs > 0) {
+          await new Promise((resolve) => setTimeout(resolve, options.arrow?.batchDebounceMs || 0));
+        }
         const arrowTabledBatch: ArrowTableBatch = {
           shape: 'arrow-table',
           batchType: 'data',

package/src/parsers/parse-arrow-sync.ts

@@ -1,4 +1,5 @@
-// loaders.gl, MIT license
+// loaders.gl
+// SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
 import type {ArrayRowTable, ColumnarTable, ObjectRowTable} from '@loaders.gl/schema';
@@ -6,6 +7,7 @@ import type {ArrowTable} from '../lib/arrow-table';
 import {convertTable} from '@loaders.gl/schema';
 import * as arrow from 'apache-arrow';
 import {convertArrowToColumnarTable} from '../tables/convert-arrow-to-columnar-table';
+import {serializeArrowSchema} from '../schema/convert-arrow-schema';
 
 // Parses arrow to a columnar table
 export function parseArrowSync(
@@ -13,7 +15,11 @@ export function parseArrowSync(
   options?: {shape?: 'arrow-table' | 'columnar-table' | 'object-row-table' | 'array-row-table'}
 ): ArrowTable | ColumnarTable | ObjectRowTable | ArrayRowTable {
   const apacheArrowTable = arrow.tableFromIPC([new Uint8Array(arrayBuffer)]);
-  const arrowTable: ArrowTable = {shape: 'arrow-table', data: apacheArrowTable};
+  const arrowTable: ArrowTable = {
+    shape: 'arrow-table',
+    schema: serializeArrowSchema(apacheArrowTable.schema),
+    data: apacheArrowTable
+  };
 
   const shape = options?.shape || 'arrow-table';
   switch (shape) {

package/src/parsers/parse-geoarrow-in-batches.ts

@@ -1,4 +1,5 @@
-// loaders.gl, MIT license
+// loaders.gl
+// SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
 import type {GeoJSONTableBatch} from '@loaders.gl/schema';

package/src/parsers/parse-geoarrow-sync.ts

@@ -1,4 +1,5 @@
-// loaders.gl, MIT license
+// loaders.gl
+// SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
 import type {GeoJSONTable} from '@loaders.gl/schema';

package/src/schema/arrow-type-utils.ts

@@ -1,4 +1,5 @@
-// loaders.gl, MIT license
+// loaders.gl
+// SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
 import type {TypedArray} from '@loaders.gl/schema';

package/src/schema/convert-arrow-schema.ts

@@ -1,4 +1,5 @@
-// loaders.gl, MIT license
+// loaders.gl
+// SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
 import type {DataType, Field, Schema, SchemaMetadata} from '@loaders.gl/schema';

package/src/tables/convert-arrow-to-columnar-table.ts

@@ -1,4 +1,5 @@
-// loaders.gl, MIT license
+// loaders.gl
+// SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
 import type {ColumnarTable} from '@loaders.gl/schema';
@@ -24,6 +25,7 @@ export function convertArrowToColumnarTable(table: ArrowTable): ColumnarTable {
 
   return {
     shape: 'columnar-table',
+    schema: table.schema,
     data: columnarTable
   };
 }

package/src/tables/convert-arrow-to-geojson-table.ts

@@ -1,8 +1,9 @@
-// loaders.gl, MIT license
+// loaders.gl
+// SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
 import type {Feature, GeoJSONTable} from '@loaders.gl/schema';
-import type * as arrow from 'apache-arrow';
+import * as arrow from 'apache-arrow';
 import type {ArrowTable} from '../lib/arrow-table';
 import {serializeArrowSchema, parseGeometryFromArrow} from '@loaders.gl/arrow';
 import {getGeometryColumnsFromSchema} from '@loaders.gl/gis';
@@ -16,6 +17,7 @@ import {getGeometryColumnsFromSchema} from '@loaders.gl/gis';
 export function convertApacheArrowToArrowTable(arrowTable: arrow.Table): ArrowTable {
   return {
     shape: 'arrow-table',
+    schema: serializeArrowSchema(arrowTable.schema),
     data: arrowTable
   };
 }
@@ -34,21 +36,31 @@ export function convertArrowToGeoJSONTable(table: ArrowTable): GeoJSONTable {
 
   const features: Feature[] = [];
 
-  for (let row = 0; row < arrowTable.numRows; row++) {
-    // get first geometry from arrow geometry column
-    const arrowGeometry = arrowTable.getChild('geometry')?.get(row);
-    const arrowGeometryObject = {encoding, data: arrowGeometry};
+  // Remove geometry columns
+  const propertyColumnNames = arrowTable.schema.fields
+    .map((field) => field.name)
+    // TODO - this deletes all geometry columns
+    .filter((name) => !(name in geometryColumns));
+  const propertiesTable = arrowTable.select(propertyColumnNames);
+
+  const arrowGeometryColumn = arrowTable.getChild('geometry');
 
+  for (let row = 0; row < arrowTable.numRows; row++) {
+    // get the geometry value from arrow geometry column
+    // Note that type can vary
+    const arrowGeometry = arrowGeometryColumn?.get(row);
     // parse arrow geometry to geojson feature
-    const feature = parseGeometryFromArrow(arrowGeometryObject);
+    const feature = parseGeometryFromArrow(arrowGeometry, encoding);
     if (feature) {
-      features.push(feature);
+      const properties = propertiesTable.get(row)?.toJSON() || {};
+      features.push({type: 'Feature', geometry: feature, properties});
     }
   }
 
   return {
     shape: 'geojson-table',
     type: 'FeatureCollection',
+    schema: table.schema,
     features
   };
 }

package/src/tables/convert-columnar-to-row-table.ts

@@ -1,4 +1,5 @@
-// loaders.gl, MIT license
+// loaders.gl
+// SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
 import type {ColumnarTable, ObjectRowTable} from '@loaders.gl/schema';
@@ -24,6 +25,7 @@ export function convertColumnarToRowFormatTable(columnarTable: ColumnarTable): O
 
   return {
     shape: 'object-row-table',
+    schema: columnarTable.schema,
     data: rowFormatTable
   };
 }

package/src/tables/convert-table-to-arrow.ts

@@ -1,4 +1,5 @@
-// loaders.gl, MIT license
+// loaders.gl
+// SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
 // import {

package/src/triangulate-on-worker.ts

@@ -1,19 +1,53 @@
-// loaders.gl, MIT license
+// loaders.gl
+// SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
+import * as arrow from 'apache-arrow';
 import type {WorkerOptions} from '@loaders.gl/worker-utils';
 import {processOnWorker} from '@loaders.gl/worker-utils';
+import {BinaryDataFromGeoArrow, GeoArrowEncoding} from '@loaders.gl/arrow';
 
 // __VERSION__ is injected by babel-plugin-version-inline
 // @ts-ignore TS2304: Cannot find name '__VERSION__'.
 const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
 
-export type TriangulationWorkerInput = TriangulateInput | {operation: 'test'; data: any};
-export type TriangulationWorkerOutput = TriangulateResult | {operation: 'test'; data: any};
+export type TriangulationWorkerInput =
+  | ({operation: 'triangulate'} & TriangulateInput)
+  | ParseGeoArrowInput
+  | {operation: 'test'; data: any};
+
+export type TriangulationWorkerOutput =
+  | ({operation: 'triangulate'} & TriangulateResult)
+  | ({operation: 'parse-geoarrow'} & ParseGeoArrowResult)
+  | {operation: 'test'; data: any};
+
+type GeoArrowChunkData = {
+  type: arrow.DataType;
+  offset: number;
+  length: number;
+  nullCount: number;
+  buffers: any;
+  children: arrow.Data[];
+  dictionary?: arrow.Vector;
+};
+
+export type ParseGeoArrowInput = {
+  operation: 'parse-geoarrow';
+  chunkData: GeoArrowChunkData;
+  chunkIndex: number;
+  chunkOffset: number;
+  geometryEncoding: GeoArrowEncoding;
+  calculateMeanCenters: boolean;
+  triangle: boolean;
+};
+
+export type ParseGeoArrowResult = {
+  chunkIndex: number;
+  binaryDataFromGeoArrow: BinaryDataFromGeoArrow | null;
+};
 
 /** Input data for operation: 'triangulate' */
 export type TriangulateInput = {
-  operation: 'triangulate';
   polygonIndices: Uint16Array;
   primitivePolygonIndices: Int32Array;
   flatCoordinateArray: Float64Array;
@@ -22,7 +56,7 @@ export type TriangulateInput = {
 
 /** Result type for operation: 'triangulate' */
 export type TriangulateResult = TriangulateInput & {
-  triangleIndices: Uint32Array;
+  triangleIndices?: Uint32Array;
 };
 
 /**
@@ -37,11 +71,21 @@ export const TriangulationWorker = {
 };
 
 /**
- * Provide type safety
+ * Triangulate a set of polygons on worker, type safe API
  */
 export function triangulateOnWorker(
-  data: TriangulationWorkerInput,
+  data: TriangulateInput,
+  options: WorkerOptions = {}
+): Promise<TriangulateResult> {
+  return processOnWorker(TriangulationWorker, {...data, operation: 'triangulate'}, options);
+}
+
+/**
+ * Parse GeoArrow geometry colum on worker, type safe API
+ */
+export function parseGeoArrowOnWorker(
+  data: ParseGeoArrowInput,
   options: WorkerOptions = {}
-): Promise<TriangulationWorkerOutput> {
-  return processOnWorker(TriangulationWorker, data, options);
+): Promise<ParseGeoArrowResult> {
+  return processOnWorker(TriangulationWorker, {...data, operation: 'parse-geoarrow'}, options);
 }

package/src/types.ts

@@ -1,4 +1,5 @@
-// loaders.gl, MIT license
+// loaders.gl
+// SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
 type TypedIntArray = Int8Array | Uint8Array | Int16Array | Uint16Array | Int32Array | Uint32Array;

package/src/workers/arrow-worker.ts

@@ -1,4 +1,5 @@
-// loaders.gl, MIT license
+// loaders.gl
+// SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
 import {createLoaderWorker} from '@loaders.gl/loader-utils';