@loaders.gl/arrow 4.0.0 → 4.0.1

package/src/geoarrow/convert-geoarrow-to-binary-geometry.ts

@@ -0,0 +1,262 @@
+// loaders.gl, MIT license
+// Copyright (c) vis.gl contributors
+
+import {Data, Vector} from 'apache-arrow';
+import {BinaryFeatureCollection as BinaryFeatures} from '@loaders.gl/schema';
+import {GeoArrowEncoding} from '@loaders.gl/gis';
+import {updateBoundsFromGeoArrowSamples} from './get-arrow-bounds';
+
+/**
+ * Binary data from geoarrow column and can be used by e.g. deck.gl GeojsonLayer
+ */
+export type BinaryDataFromGeoArrow = {
+  binaryGeometries: BinaryFeatures[];
+  bounds: [number, number, number, number];
+  featureTypes: {polygon: boolean; point: boolean; line: boolean};
+};
+
+type BinaryGeometryContent = {
+  featureIds: Uint32Array;
+  flatCoordinateArray: Float64Array;
+  nDim: number;
+  geomOffset: Int32Array;
+  geometryIndicies: Uint16Array;
+};
+
+// binary geometry template, see deck.gl BinaryGeometry
+const BINARY_GEOMETRY_TEMPLATE = {
+  globalFeatureIds: {value: new Uint32Array(0), size: 1},
+  positions: {value: new Float32Array(0), size: 2},
+  properties: [],
+  numericProps: {},
+  featureIds: {value: new Uint32Array(0), size: 1}
+};
+
+/**
+ * get binary geometries from geoarrow column
+ *
+ * @param geoColumn the geoarrow column, e.g. arrowTable.getChildAt(geoColumnIndex)
+ * @param geoEncoding the geo encoding of the geoarrow column, e.g. getGeoArrowEncoding(arrowTable.schema, geoColumnName)
+ * @returns BinaryDataFromGeoArrow
+ */
+export function getBinaryGeometriesFromArrow(
+  geoColumn: Vector,
+  geoEncoding: GeoArrowEncoding
+): BinaryDataFromGeoArrow {
+  const featureTypes = {
+    polygon: geoEncoding === 'geoarrow.multipolygon' || geoEncoding === 'geoarrow.polygon',
+    point: geoEncoding === 'geoarrow.multipoint' || geoEncoding === 'geoarrow.point',
+    line: geoEncoding === 'geoarrow.multilinestring' || geoEncoding === 'geoarrow.linestring'
+  };
+
+  const chunks = geoColumn.data;
+  const bounds: [number, number, number, number] = [Infinity, Infinity, -Infinity, -Infinity];
+  let globalFeatureIdOffset = 0;
+  const binaryGeometries: BinaryFeatures[] = [];
+
+  for (let c = 0; c < chunks.length; c++) {
+    const geometries = chunks[c];
+    const {featureIds, flatCoordinateArray, nDim, geomOffset} = getBinaryGeometriesFromChunk(
+      geometries,
+      geoEncoding
+    );
+
+    const numOfVertices = flatCoordinateArray.length / nDim;
+    const globalFeatureIds = new Uint32Array(numOfVertices);
+    for (let i = 0; i < numOfVertices; i++) {
+      globalFeatureIds[i] = featureIds[i] + globalFeatureIdOffset;
+    }
+
+    const binaryContent = {
+      globalFeatureIds: {value: globalFeatureIds, size: 1},
+      positions: {
+        value: flatCoordinateArray,
+        size: nDim
+      },
+      featureIds: {value: featureIds, size: 1},
+      // eslint-disable-next-line no-loop-func
+      properties: [...Array(geometries.length).keys()].map((i) => ({
+        index: i + globalFeatureIdOffset
+      }))
+    };
+
+    // TODO: check if chunks are sequentially accessed
+    globalFeatureIdOffset += geometries.length;
+
+    // NOTE: deck.gl defines the BinaryFeatures structure must have points, lines, polygons even if they are empty
+    binaryGeometries.push({
+      shape: 'binary-feature-collection',
+      points: {
+        type: 'Point',
+        ...BINARY_GEOMETRY_TEMPLATE,
+        ...(featureTypes.point ? binaryContent : {})
+      },
+      lines: {
+        type: 'LineString',
+        ...BINARY_GEOMETRY_TEMPLATE,
+        ...(featureTypes.line ? binaryContent : {}),
+        pathIndices: {value: featureTypes.line ? geomOffset : new Uint16Array(0), size: 1}
+      },
+      polygons: {
+        type: 'Polygon',
+        ...BINARY_GEOMETRY_TEMPLATE,
+        ...(featureTypes.polygon ? binaryContent : {}),
+        polygonIndices: {
+          // TODO why deck.gl's tessellatePolygon performance is not good when using geometryIndicies
+          // even when there is no hole in any polygon
+          value: featureTypes.polygon ? geomOffset : new Uint16Array(0),
+          size: 1
+        },
+        primitivePolygonIndices: {
+          value: featureTypes.polygon ? geomOffset : new Uint16Array(0),
+          size: 1
+        }
+      }
+    });
+
+    updateBoundsFromGeoArrowSamples(flatCoordinateArray, nDim, bounds);
+  }
+
+  return {binaryGeometries, bounds, featureTypes};
+}
+
+/**
+ * get binary geometries from geoarrow column
+ * @param chunk one chunk/batch of geoarrow column
+ * @param geoEncoding geo encoding of the geoarrow column
+ * @returns BinaryGeometryContent
+ */
+function getBinaryGeometriesFromChunk(
+  chunk: Data,
+  geoEncoding: GeoArrowEncoding
+): BinaryGeometryContent {
+  switch (geoEncoding) {
+    case 'geoarrow.point':
+    case 'geoarrow.multipoint':
+      return getBinaryPointsFromChunk(chunk, geoEncoding);
+    case 'geoarrow.linestring':
+    case 'geoarrow.multilinestring':
+      return getBinaryLinesFromChunk(chunk, geoEncoding);
+    case 'geoarrow.polygon':
+    case 'geoarrow.multipolygon':
+      return getBinaryPolygonsFromChunk(chunk, geoEncoding);
+    default:
+      throw Error('invalid geoarrow encoding');
+  }
+}
+
+/**
+ * get binary polygons from geoarrow polygon column
+ * @param chunk one chunk of geoarrow polygon column
+ * @param geoEncoding the geo encoding of the geoarrow polygon column
+ * @returns BinaryGeometryContent
+ */
+function getBinaryPolygonsFromChunk(chunk: Data, geoEncoding: string): BinaryGeometryContent {
+  const isMultiPolygon = geoEncoding === 'geoarrow.multipolygon';
+
+  const polygonData = isMultiPolygon ? chunk.children[0] : chunk;
+  const ringData = polygonData.children[0];
+  const pointData = ringData.children[0];
+  const coordData = pointData.children[0];
+  const nDim = pointData.stride;
+  const geomOffset = ringData.valueOffsets;
+  const flatCoordinateArray = coordData.values;
+
+  const geometryIndicies = new Uint16Array(chunk.length + 1);
+  for (let i = 0; i < chunk.length; i++) {
+    geometryIndicies[i] = geomOffset[chunk.valueOffsets[i]];
+  }
+  geometryIndicies[chunk.length] = flatCoordinateArray.length / nDim;
+
+  const numOfVertices = flatCoordinateArray.length / nDim;
+  const featureIds = new Uint32Array(numOfVertices);
+  for (let i = 0; i < chunk.length - 1; i++) {
+    const startIdx = geomOffset[chunk.valueOffsets[i]];
+    const endIdx = geomOffset[chunk.valueOffsets[i + 1]];
+    for (let j = startIdx; j < endIdx; j++) {
+      featureIds[j] = i;
+    }
+  }
+
+  return {
+    featureIds,
+    flatCoordinateArray,
+    nDim,
+    geomOffset,
+    geometryIndicies
+  };
+}
+
+/**
+ * get binary lines from geoarrow line column
+ * @param chunk one chunk/batch of geoarrow column
+ * @param geoEncoding the geo encoding of the geoarrow column
+ * @returns BinaryGeometryContent
+ */
+function getBinaryLinesFromChunk(chunk: Data, geoEncoding: string): BinaryGeometryContent {
+  const isMultiLineString = geoEncoding === 'geoarrow.multilinestring';
+
+  const lineData = isMultiLineString ? chunk.children[0] : chunk;
+  const pointData = lineData.children[0];
+  const coordData = pointData.children[0];
+
+  const nDim = pointData.stride;
+  const geomOffset = lineData.valueOffsets;
+  const flatCoordinateArray = coordData.values;
+
+  // geometryIndicies is not needed for line string
+  const geometryIndicies = new Uint16Array(0);
+
+  const numOfVertices = flatCoordinateArray.length / nDim;
+  const featureIds = new Uint32Array(numOfVertices);
+  for (let i = 0; i < chunk.length; i++) {
+    const startIdx = geomOffset[i];
+    const endIdx = geomOffset[i + 1];
+    for (let j = startIdx; j < endIdx; j++) {
+      featureIds[j] = i;
+    }
+  }
+
+  return {
+    featureIds,
+    flatCoordinateArray,
+    nDim,
+    geomOffset,
+    geometryIndicies
+  };
+}
+
+/**
+ * get binary points from geoarrow point column
+ * @param chunk one chunk/batch of geoarrow column
+ * @param geoEncoding  geo encoding of the geoarrow column
+ * @returns BinaryGeometryContent
+ */
+function getBinaryPointsFromChunk(chunk: Data, geoEncoding: string): BinaryGeometryContent {
+  const isMultiPoint = geoEncoding === 'geoarrow.multipoint';
+
+  const pointData = isMultiPoint ? chunk.children[0] : chunk;
+  const coordData = pointData.children[0];
+
+  const nDim = pointData.stride;
+  const flatCoordinateArray = coordData.values;
+
+  // geometryIndices is not needed for point
+  const geometryIndicies = new Uint16Array(0);
+  // geomOffset is not needed for point
+  const geomOffset = new Int32Array(0);
+
+  const numOfVertices = flatCoordinateArray.length / nDim;
+  const featureIds = new Uint32Array(numOfVertices);
+  for (let i = 0; i < chunk.length; i++) {
+    featureIds[i] = i;
+  }
+
+  return {
+    featureIds,
+    flatCoordinateArray,
+    nDim,
+    geomOffset,
+    geometryIndicies
+  };
+}

package/src/geoarrow/convert-geoarrow-to-geojson.ts

@@ -0,0 +1,192 @@
+// loaders.gl, MIT license
+// Copyright (c) vis.gl contributors
+
+import {Vector} from 'apache-arrow';
+import {
+  Feature,
+  MultiPolygon,
+  Position,
+  Polygon,
+  MultiPoint,
+  Point,
+  MultiLineString,
+  LineString
+} from '@loaders.gl/schema';
+import type {GeoArrowEncoding} from '@loaders.gl/gis';
+
+type RawArrowFeature = {
+  encoding?: GeoArrowEncoding;
+  data: any;
+};
+
+/**
+ * parse geometry from arrow data that is returned from processArrowData()
+ * NOTE: this function could be duplicated with the binaryToFeature() in deck.gl,
+ * it is currently only used for picking because currently deck.gl returns only the index of the feature
+ * So the following functions could be deprecated once deck.gl returns the feature directly for binary geojson layer
+ *
+ * @param rawData the raw geometry data returned from processArrowData, which is an object with two properties: encoding and data
+ * @see processArrowData
+ * @returns Feature or null
+ */
+export function parseGeometryFromArrow(rawData: RawArrowFeature): Feature | null {
+  const encoding = rawData.encoding?.toLowerCase();
+  const data = rawData.data;
+  if (!encoding || !data) {
+    return null;
+  }
+
+  let geometry;
+
+  switch (encoding) {
+    case 'geoarrow.multipolygon':
+      geometry = arrowMultiPolygonToFeature(data);
+      break;
+    case 'geoarrow.polygon':
+      geometry = arrowPolygonToFeature(data);
+      break;
+    case 'geoarrow.multipoint':
+      geometry = arrowMultiPointToFeature(data);
+      break;
+    case 'geoarrow.point':
+      geometry = arrowPointToFeature(data);
+      break;
+    case 'geoarrow.multilinestring':
+      geometry = arrowMultiLineStringToFeature(data);
+      break;
+    case 'geoarrow.linestring':
+      geometry = arrowLineStringToFeature(data);
+      break;
+    default: {
+      throw Error(`GeoArrow encoding not supported ${encoding}`);
+    }
+  }
+  return {
+    type: 'Feature',
+    geometry,
+    properties: {}
+  };
+}
+
+/**
+ * convert Arrow MultiPolygon to geojson Feature
+ */
+function arrowMultiPolygonToFeature(arrowMultiPolygon: Vector): MultiPolygon {
+  const multiPolygon: Position[][][] = [];
+  for (let m = 0; m < arrowMultiPolygon.length; m++) {
+    const arrowPolygon = arrowMultiPolygon.get(m);
+    const polygon: Position[][] = [];
+    for (let i = 0; arrowPolygon && i < arrowPolygon?.length; i++) {
+      const arrowRing = arrowPolygon?.get(i);
+      const ring: Position[] = [];
+      for (let j = 0; arrowRing && j < arrowRing.length; j++) {
+        const arrowCoord = arrowRing.get(j);
+        const coord: Position = Array.from(arrowCoord);
+        ring.push(coord);
+      }
+      polygon.push(ring);
+    }
+    multiPolygon.push(polygon);
+  }
+  const geometry: MultiPolygon = {
+    type: 'MultiPolygon',
+    coordinates: multiPolygon
+  };
+  return geometry;
+}
+
+/**
+ * convert Arrow Polygon to geojson Feature
+ */
+function arrowPolygonToFeature(arrowPolygon: Vector): Polygon {
+  const polygon: Position[][] = [];
+  for (let i = 0; arrowPolygon && i < arrowPolygon.length; i++) {
+    const arrowRing = arrowPolygon.get(i);
+    const ring: Position[] = [];
+    for (let j = 0; arrowRing && j < arrowRing.length; j++) {
+      const arrowCoord = arrowRing.get(j);
+      const coords: Position = Array.from(arrowCoord);
+      ring.push(coords);
+    }
+    polygon.push(ring);
+  }
+  const geometry: Polygon = {
+    type: 'Polygon',
+    coordinates: polygon
+  };
+  return geometry;
+}
+
+/**
+ * convert Arrow MultiPoint to geojson MultiPoint
+ */
+function arrowMultiPointToFeature(arrowMultiPoint: Vector): MultiPoint {
+  const multiPoint: Position[] = [];
+  for (let i = 0; arrowMultiPoint && i < arrowMultiPoint.length; i++) {
+    const arrowPoint = arrowMultiPoint.get(i);
+    if (arrowPoint) {
+      const coord: Position = Array.from(arrowPoint);
+      multiPoint.push(coord);
+    }
+  }
+  const geometry: MultiPoint = {
+    type: 'MultiPoint',
+    coordinates: multiPoint
+  };
+  return geometry;
+}
+
+/**
+ * convert Arrow Point to geojson Point
+ */
+function arrowPointToFeature(arrowPoint: Vector): Point {
+  const point: Position = Array.from(arrowPoint);
+  const geometry: Point = {
+    type: 'Point',
+    coordinates: point
+  };
+  return geometry;
+}
+
+/**
+ * convert Arrow MultiLineString to geojson MultiLineString
+ */
+function arrowMultiLineStringToFeature(arrowMultiLineString: Vector): MultiLineString {
+  const multiLineString: Position[][] = [];
+  for (let i = 0; arrowMultiLineString && i < arrowMultiLineString.length; i++) {
+    const arrowLineString = arrowMultiLineString.get(i);
+    const lineString: Position[] = [];
+    for (let j = 0; arrowLineString && j < arrowLineString.length; j++) {
+      const arrowCoord = arrowLineString.get(j);
+      if (arrowCoord) {
+        const coords: Position = Array.from(arrowCoord);
+        lineString.push(coords);
+      }
+    }
+    multiLineString.push(lineString);
+  }
+  const geometry: MultiLineString = {
+    type: 'MultiLineString',
+    coordinates: multiLineString
+  };
+  return geometry;
+}
+
+/**
+ * convert Arrow LineString to geojson LineString
+ */
+function arrowLineStringToFeature(arrowLineString: Vector): LineString {
+  const lineString: Position[] = [];
+  for (let i = 0; arrowLineString && i < arrowLineString.length; i++) {
+    const arrowCoord = arrowLineString.get(i);
+    if (arrowCoord) {
+      const coords: Position = Array.from(arrowCoord);
+      lineString.push(coords);
+    }
+  }
+  const geometry: LineString = {
+    type: 'LineString',
+    coordinates: lineString
+  };
+  return geometry;
+}

package/src/geoarrow/get-arrow-bounds.ts

@@ -0,0 +1,41 @@
+// loaders.gl, MIT license
+// Copyright (c) vis.gl contributors
+
+/**
+ *  Update bounds from geoarrow sample data
+ *
+ * @param flatCoords the flattend coordinates array from one chunk of geoarrow column
+ * @param nDim the number of dimensions of the coordinates
+ * @param bounds the bounds to be updated
+ * @param sampleSize how many samples to be used to update the bounds, default is 1000 per chunk
+ * @returns the updated bounds
+ */
+export function updateBoundsFromGeoArrowSamples(
+  flatCoords: Float64Array,
+  nDim: number,
+  bounds: [number, number, number, number],
+  sampleSize: number = 100
+): [number, number, number, number] {
+  const numberOfFeatures = flatCoords.length / nDim;
+  const sampleStep = Math.max(Math.floor(numberOfFeatures / sampleSize), 1);
+
+  const newBounds: [number, number, number, number] = [...bounds];
+  for (let i = 0; i < numberOfFeatures; i += sampleStep) {
+    const lng = flatCoords[i * nDim];
+    const lat = flatCoords[i * nDim + 1];
+    if (lng < bounds[0]) {
+      newBounds[0] = lng;
+    }
+    if (lat < newBounds[1]) {
+      newBounds[1] = lat;
+    }
+    if (lng > newBounds[2]) {
+      newBounds[2] = lng;
+    }
+    if (lat > newBounds[3]) {
+      newBounds[3] = lat;
+    }
+  }
+
+  return newBounds;
+}

package/src/index.ts

@@ -1,4 +1,5 @@
-// loaders.gl, MIT
+// loaders.gl, MIT license
+// Copyright (c) vis.gl contributors
 
 import type {LoaderWithParser} from '@loaders.gl/loader-utils';
 import type {ArrowLoaderOptions} from './arrow-loader';
@@ -12,12 +13,25 @@ import {parseArrowInBatches} from './lib/parse-arrow-in-batches';
 
 import {ArrowTableBatchAggregator} from './lib/arrow-table-batch';
 
-// SCHEMA
+// Make the ArrowBatch type available
+TableBatchBuilder.ArrowBatch = ArrowTableBatchAggregator;
+
+// TYPES
 
 export {getArrowType} from './schema/arrow-type-utils';
 
-// Make the ArrowBatch type available
-TableBatchBuilder.ArrowBatch = ArrowTableBatchAggregator;
+// SCHEMA
+
+export {
+  serializeArrowSchema,
+  deserializeArrowSchema,
+  serializeArrowMetadata,
+  deserializeArrowMetadata,
+  serializeArrowField,
+  deserializeArrowField,
+  serializeArrowType,
+  deserializeArrowType
+} from './schema/convert-arrow-schema';
 
 // Types
 export type {ArrowTable, ArrowTableBatch} from './lib/arrow-table';
@@ -44,3 +58,13 @@ export const ArrowLoader: LoaderWithParser<
   parseSync,
   parseInBatches: parseArrowInBatches
 };
+
+// Arrow Utils
+export type {GeoArrowEncoding} from '@loaders.gl/gis';
+// getGeometryColumnsFromArrowTable,
+// getGeoArrowEncoding
+
+export type {BinaryDataFromGeoArrow} from './geoarrow/convert-geoarrow-to-binary-geometry';
+export {getBinaryGeometriesFromArrow} from './geoarrow/convert-geoarrow-to-binary-geometry';
+
+export {parseGeometryFromArrow} from './geoarrow/convert-geoarrow-to-geojson';

package/src/lib/parse-arrow-sync.ts

@@ -3,7 +3,10 @@ import type {ArrowTable} from './arrow-table';
 import {convertTable} from '@loaders.gl/schema';
 import {tableFromIPC} from 'apache-arrow';
 import type {ArrowLoaderOptions} from '../arrow-loader';
-import {convertApacheArrowToArrowTable, convertArrowToColumnarTable} from './convert-table';
+import {
+  convertApacheArrowToArrowTable,
+  convertArrowToColumnarTable
+} from '../tables/convert-arrow-to-table';
 
 // Parses arrow to a columnar table
 export default function parseArrowSync(

package/src/schema/arrow-type-utils.ts

@@ -23,6 +23,7 @@ import {
 } from 'apache-arrow';
 // import {AbstractVector} from 'apache-arrow/vector';
 
+/** Return an Apache Arrow Type instance that corresponds to the type of the elements in the supplied Typed Array */
 export function getArrowType(array: TypedArray): DataType {
   switch (array.constructor) {
     case Int8Array: