@loaders.gl/geoarrow 4.4.0-alpha.1

package/src/geoarrow-functions.ts

@@ -0,0 +1,258 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+/**
+ * @note Conforms to the GeoArrow memory layout
+ * @see https://geoarrow.org/format.html#memory-layouts
+ * @note this is aligned with the geoarrow-js library (MIT license)
+ * @see https://github.com/geoarrow/geoarrow-js/
+ */
+
+import {DataType} from 'apache-arrow/type';
+
+import type {
+  GeoArrowPoint,
+  GeoArrowLineString,
+  GeoArrowPolygon,
+  GeoArrowMultiPoint,
+  GeoArrowMultiLineString,
+  GeoArrowMultiPolygon,
+  GeoArrowGeometry
+} from './geoarrow-types';
+
+/** Checks whether the given Apache Arrow JS type is a Point data type */
+export function isGeoArrowPoint(type: DataType): type is GeoArrowPoint {
+  if (DataType.isFixedSizeList(type)) {
+    // Check list size
+    if (![2, 3, 4].includes(type.listSize)) {
+      return false;
+    }
+
+    // Check child of FixedSizeList is floating type
+    if (!DataType.isFloat(type.children[0])) {
+      return false;
+    }
+
+    return true;
+  }
+
+  // TODO - support separated coordinates
+  // if (DataType.isStruct(type)) {
+  //   // Check number of children
+  //   if (![2, 3, 4].includes(type.children.length)) {
+  //     return false;
+  //   }
+
+  //   // Check that children have correct field names
+  //   if (!type.children.every((field) => ['x', 'y', 'z', 'm'].includes(field.name))) {
+  //     return false;
+  //   }
+
+  //   if (!type.children.every((field) => DataType.isFloat(field))) {
+  //     return false;
+  //   }
+
+  //   return true;
+  // }
+
+  return false;
+}
+
+/** Checks whether the given Apache Arrow JS type is a Point data type */
+export function isGeoArrowLineString(type: DataType): type is GeoArrowLineString {
+  // Check the outer type is a List
+  if (!DataType.isList(type)) {
+    return false;
+  }
+
+  // Check the child is a point type
+  if (!isGeoArrowPoint(type.children[0].type)) {
+    return false;
+  }
+
+  return true;
+}
+
+/** Checks whether the given Apache Arrow JS type is a Polygon data type */
+export function isGeoArrowPolygon(type: DataType): type is GeoArrowPolygon {
+  // Check the outer vector is a List
+  if (!DataType.isList(type)) {
+    return false;
+  }
+
+  // Check the child is a linestring vector
+  if (!isGeoArrowLineString(type.children[0].type)) {
+    return false;
+  }
+
+  return true;
+}
+
+/** Checks whether the given Apache Arrow JS type is a Polygon data type */
+export function isGeoArrowMultiPoint(type: DataType): type is GeoArrowMultiPoint {
+  // Check the outer vector is a List
+  if (!DataType.isList(type)) {
+    return false;
+  }
+
+  // Check the child is a point vector
+  if (!isGeoArrowPoint(type.children[0].type)) {
+    return false;
+  }
+
+  return true;
+}
+
+/** Checks whether the given Apache Arrow JS type is a Polygon data type */
+export function isGeoArrowMultiLineString(type: DataType): type is GeoArrowMultiLineString {
+  // Check the outer vector is a List
+  if (!DataType.isList(type)) {
+    return false;
+  }
+
+  // Check the child is a linestring vector
+  if (!isGeoArrowLineString(type.children[0].type)) {
+    return false;
+  }
+
+  return true;
+}
+
+/** Checks whether the given Apache Arrow JS type is a Polygon data type */
+export function isGeoArrowMultiPolygon(type: DataType): type is GeoArrowMultiPolygon {
+  // Check the outer vector is a List
+  if (!DataType.isList(type)) {
+    return false;
+  }
+
+  // Check the child is a polygon vector
+  if (!isGeoArrowPolygon(type.children[0].type)) {
+    return false;
+  }
+
+  return true;
+}
+
+/**
+ * Checks if a given Arrow data type is a valid GeoArrowGeometry
+ * @note this is somewhat inefficient, it checks the same things multiple times
+ */
+export function isGeoArrowGeometry(type: DataType): type is GeoArrowGeometry {
+  return (
+    isGeoArrowPoint(type) ||
+    isGeoArrowLineString(type) ||
+    isGeoArrowPolygon(type) ||
+    isGeoArrowMultiPoint(type) ||
+    isGeoArrowMultiLineString(type) ||
+    isGeoArrowMultiPolygon(type)
+  );
+}
+
+// CHILD EXTRACTION
+
+/**
+ * Strongly typed accessors for children, since arrow.Data.children[] is untyped
+ 
+import { Data } from "apache-arrow/data";
+import { Vector } from "apache-arrow/vector";
+import { Float } from "apache-arrow/type";
+import {
+  LineStringData,
+  MultiLineStringData,
+  MultiPointData,
+  MultiPolygonData,
+  PointData,
+  PolygonData,
+} from "./data";
+import {
+  LineStringVector,
+  MultiLineStringVector,
+  MultiPointVector,
+  MultiPolygonVector,
+  PointVector,
+  PolygonVector,
+} from "./vector";
+
+export function getPointChild(input: PointData): Data<Float>;
+export function getPointChild(input: PointVector): Vector<Float>;
+
+export function getPointChild(
+  input: PointData | PointVector,
+): Data<Float> | Vector<Float> {
+  if ("data" in input) {
+    return input.getChildAt(0)!;
+  }
+
+  return input.children[0] as Data<Float>;
+}
+
+export function getLineStringChild(input: LineStringData): PointData;
+export function getLineStringChild(input: LineStringVector): PointVector;
+
+export function getLineStringChild(
+  input: LineStringData | LineStringVector,
+): PointData | PointVector {
+  if ("data" in input) {
+    return input.getChildAt(0)!;
+  }
+
+  return input.children[0] as PointData;
+}
+
+export function getPolygonChild(input: PolygonData): LineStringData;
+export function getPolygonChild(input: PolygonVector): LineStringVector;
+
+export function getPolygonChild(
+  input: PolygonData | PolygonVector,
+): LineStringData | LineStringVector {
+  if ("data" in input) {
+    return input.getChildAt(0)!;
+  }
+
+  return input.children[0] as LineStringData;
+}
+
+export function getMultiPointChild(input: MultiPointData): PointData;
+export function getMultiPointChild(input: MultiPointVector): PointVector;
+
+export function getMultiPointChild(
+  input: MultiPointData | MultiPointVector,
+): PointData | PointVector {
+  if ("data" in input) {
+    return input.getChildAt(0)!;
+  }
+
+  return input.children[0] as PointData;
+}
+
+export function getMultiLineStringChild(
+  input: MultiLineStringData,
+): LineStringData;
+export function getMultiLineStringChild(
+  input: MultiLineStringVector,
+): LineStringVector;
+
+export function getMultiLineStringChild(
+  input: MultiLineStringData | MultiLineStringVector,
+): LineStringData | LineStringVector {
+  if ("data" in input) {
+    return input.getChildAt(0)!;
+  }
+
+  return input.children[0] as LineStringData;
+}
+
+export function getMultiPolygonChild(input: MultiPolygonData): PolygonData;
+export function getMultiPolygonChild(input: MultiPolygonVector): PolygonVector;
+
+export function getMultiPolygonChild(
+  input: MultiPolygonData | MultiPolygonVector,
+): PolygonData | PolygonVector {
+  if ("data" in input) {
+    return input.getChildAt(0)!;
+  }
+
+  return input.children[0] as PolygonData;
+}
+*/

package/src/geoarrow-types.ts

@@ -0,0 +1,72 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+/**
+ * @note Conforms to the GeoArrow memory layout
+ * @see https://geoarrow.org/format.html#memory-layouts
+ * @note this is aligned with the geoarrow-js library (MIT license)
+ * @see https://github.com/geoarrow/geoarrow-js/
+ */
+
+import type {Struct, Float, List, FixedSizeList, Utf8, Binary} from 'apache-arrow/type';
+
+export type GeoArrowWKB = Binary;
+export type GeoArrowWKT = Utf8;
+/**
+ * @note arrow.Float (not arrow.Float64) ensures that recreating a data instance with arrow.makeData type checks using the input's data type.
+ */
+
+/** Interleaved GeoArrow coordinates */
+export type GeoArrowCoordInterleaved = FixedSizeList<Float>;
+/** Separated GeoArrow coordinates */
+export type GeoArrowCoordSeparated = Struct<{x: Float; y: Float}>;
+
+// Interleaved coords are the preferred case
+
+export type GeoArrowCoord = GeoArrowCoordInterleaved;
+
+/** Arrow type for GeoArrow geometry */
+export type GeoArrowPoint = GeoArrowCoordInterleaved;
+/** Arrow type for GeoArrow geometry */
+export type GeoArrowLineString = List<GeoArrowCoordInterleaved>;
+/** Arrow type for GeoArrow geometry */
+export type GeoArrowPolygon = List<List<GeoArrowCoordInterleaved>>;
+/** Arrow type for GeoArrow geometry */
+export type GeoArrowMultiPoint = List<GeoArrowCoordInterleaved>;
+/** Arrow type for GeoArrow geometry */
+export type GeoArrowMultiLineString = List<List<GeoArrowCoordInterleaved>>;
+/** Arrow type for GeoArrow geometry */
+export type GeoArrowMultiPolygon = List<List<List<GeoArrowCoordInterleaved>>>;
+
+/** Arrow type for GeoArrow geometry */
+export type GeoArrowGeometry =
+  | GeoArrowPoint
+  | GeoArrowLineString
+  | GeoArrowPolygon
+  | GeoArrowMultiPoint
+  | GeoArrowMultiLineString
+  | GeoArrowMultiPolygon;
+
+// Separated coords - not yet well supported
+/** Arrow type for GeoArrow geometry */
+export type GeoArrowPointSeparated = GeoArrowCoordSeparated;
+/** Arrow type for GeoArrow geometry */
+export type GeoArrowLineStringSeparated = List<GeoArrowCoordSeparated>;
+/** Arrow type for GeoArrow geometry */
+export type GeoArrowPolygonSeparated = List<List<GeoArrowCoordSeparated>>;
+/** Arrow type for GeoArrow geometry */
+export type GeoArrowMultiPointSeparated = List<GeoArrowCoordSeparated>;
+/** Arrow type for GeoArrow geometry */
+export type GeoArrowMultiLineStringSeparated = List<List<GeoArrowCoordSeparated>>;
+/** Arrow type for GeoArrow geometry */
+export type GeoArrowMultiPolygonSeparated = List<List<List<GeoArrowCoordSeparated>>>;
+
+/** Arrow type for GeoArrow geometry */
+export type GeoArrowGeometrySeparated =
+  | GeoArrowPointSeparated
+  | GeoArrowLineStringSeparated
+  | GeoArrowPolygonSeparated
+  | GeoArrowMultiPointSeparated
+  | GeoArrowMultiLineStringSeparated
+  | GeoArrowMultiPolygonSeparated;

package/src/get-arrow-bounds.ts

@@ -0,0 +1,41 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+/**
+ *  Update bounds from geoarrow sample data
+ *
+ * @param flatCoords the flattend coordinates array from one chunk of geoarrow column
+ * @param nDim the number of dimensions of the coordinates
+ * @param bounds the bounds to be updated
+ * @param sampleSize how many samples to be used to update the bounds, default is 1000 per chunk
+ * @returns the updated bounds
+ */
+export function updateBoundsFromGeoArrowSamples(
+  flatCoords: Float64Array,
+  nDim: number,
+  bounds: [number, number, number, number],
+  sampleSize: number = 100
+): [number, number, number, number] {
+  const numberOfFeatures = flatCoords.length / nDim;
+  const sampleStep = Math.max(Math.floor(numberOfFeatures / sampleSize), 1);
+
+  const newBounds: [number, number, number, number] = [...bounds];
+  for (let i = 0; i < numberOfFeatures; i += sampleStep) {
+    const lng = flatCoords[i * nDim];
+    const lat = flatCoords[i * nDim + 1];
+    if (lng < newBounds[0]) {
+      newBounds[0] = lng;
+    }
+    if (lat < newBounds[1]) {
+      newBounds[1] = lat;
+    }
+    if (lng > newBounds[2]) {
+      newBounds[2] = lng;
+    }
+    if (lat > newBounds[3]) {
+      newBounds[3] = lat;
+    }
+  }
+  return newBounds;
+}

package/src/get-geo-column-from-geoarrow.ts.disabled

@@ -0,0 +1,251 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import * as arrow from 'apache-arrow';
+import type {
+  GeoColumn,
+  GeoColumnData,
+  GeoColumnPointData,
+  GeoColumnLineData,
+  GeoColumnPolygonData
+} from './geo-column/geo-column';
+import {GeoArrowEncoding} from './metadata/geoarrow-metadata';
+
+export type GetGeoColumnOptions = {
+  /** option to specify which chunk to get binary geometries from, for progressive rendering */
+  chunkIndex?: number;
+  /** The offset (beginning index of rows) of input chunk. Used for reconstructing globalFeatureIds in web workers */
+  chunkOffset?: number;
+};
+
+/**
+ * get binary geometries from geoarrow column
+ *
+ * @param geoColumn the geoarrow column, e.g. arrowTable.getChildAt(geoColumnIndex)
+ * @param geoEncoding the geo encoding of the geoarrow column, e.g. getGeoArrowEncoding(arrowTable.schema, geoColumnName)
+ * @param options options for getting binary geometries {meanCenter: boolean}
+ * @returns BinaryDataFromGeoArrow
+ */
+export function getGeoColumnFromGeoArrowVector(
+  geoColumn: arrow.Vector,
+  geoEncoding: GeoArrowEncoding,
+  options?: GetGeoColumnOptions
+): GeoColumn {
+  const featureTypes = {
+    polygon: geoEncoding === 'geoarrow.multipolygon' || geoEncoding === 'geoarrow.polygon',
+    point: geoEncoding === 'geoarrow.multipoint' || geoEncoding === 'geoarrow.point',
+    line: geoEncoding === 'geoarrow.multilinestring' || geoEncoding === 'geoarrow.linestring'
+  };
+
+  const arrowDatas =
+    options?.chunkIndex !== undefined && options?.chunkIndex >= 0
+      ? [geoColumn.data[options?.chunkIndex]]
+      : geoColumn.data;
+
+  // const globalFeatureIdOffset = options?.chunkOffset || 0;
+
+  const data: GeoColumnData[] = [];
+
+  for (const arrowData of arrowDatas) {
+    // // const {featureIds, positions, dimension, geometryOffsets, triangles} =
+    // //   getBinaryGeometriesFromChunk(arrowData, geoEncoding, options);
+
+    // const globalFeatureIds = new Uint32Array(featureIds.length);
+    // for (let i = 0; i < featureIds.length; i++) {
+    //   globalFeatureIds[i] = featureIds[i] + globalFeatureIdOffset;
+    // }
+
+    const geoColumnData: GeoColumnData = {
+      shape: 'geo-column-data',
+      numRows: arrowData.length,
+      rowOffsets: new Uint32Array(0)
+    };
+
+    if (featureTypes.point) {
+      geoColumnData.points = getPointDataFromArrowData(arrowData, geoEncoding, options);
+    }
+
+    if (featureTypes.line) {
+      geoColumnData.lines = getLineDataFromArrowData(arrowData, geoEncoding, options);
+    }
+
+    if (featureTypes.polygon) {
+      geoColumnData.polygons = getPolygonDataFromArrowData(arrowData, geoEncoding, options);
+    }
+
+    data.push(geoColumnData);
+  }
+
+  return {
+    shape: 'geo-column',
+    data
+  };
+}
+
+/**
+ * get binary geometries from geoarrow column
+ * @param chunk one chunk/batch of geoarrow column
+ * @param geoEncoding geo encoding of the geoarrow column
+ * @param options options for getting binary geometries
+ * @returns BinaryGeometryContent
+ */
+export function getGeoColumnDataFromArrowData(
+  data: arrow.Data,
+  geoEncoding: GeoArrowEncoding,
+  options?: GetGeoColumnOptions
+): GeoColumnPolygonData | GeoColumnLineData | GeoColumnPointData {
+  switch (geoEncoding) {
+    case 'geoarrow.point':
+    case 'geoarrow.multipoint':
+      return getPointDataFromArrowData(data, geoEncoding);
+    case 'geoarrow.linestring':
+    case 'geoarrow.multilinestring':
+      return getLineDataFromArrowData(data, geoEncoding);
+    case 'geoarrow.polygon':
+    case 'geoarrow.multipolygon':
+      return getPolygonDataFromArrowData(data, geoEncoding, options);
+    default:
+      throw Error('invalid geoarrow encoding');
+  }
+}
+
+/**
+ * get binary points from geoarrow point column
+ * @param chunk one chunk/batch of geoarrow column
+ * @param geoEncoding  geo encoding of the geoarrow column
+ * @returns Flat geometry content
+ */
+function getPointDataFromArrowData(chunk: arrow.Data, geoEncoding: string): GeoColumnPointData {
+  const isMultiPoint = geoEncoding === 'geoarrow.multipoint';
+
+  const pointData = isMultiPoint ? chunk.children[0] : chunk;
+  const coordData = pointData.children[0];
+
+  const positionStride = pointData.stride as 2 | 3 | 4;
+  const positions = coordData.values;
+
+  const positionCount = positions.length / positionStride;
+
+  return {
+    shape: 'geo-column-data',
+    type: 'Point',
+
+    rowCount: chunk.length,
+    rowOffsets: chunk.valueOffsets,
+
+    positionCount,
+    positionStride,
+    positions
+  };
+}
+
+/**
+ * get binary lines from geoarrow line column
+ * @param chunk one chunk/batch of geoarrow column
+ * @param geoEncoding the geo encoding of the geoarrow column
+ * @returns Flat geometry content
+ */
+function getLineDataFromArrowData(
+  arrowData: arrow.Data,
+  geoEncoding: string,
+  options?: GetGeoColumnOptions
+): GeoColumnLineData {
+  const isMultiLineString = geoEncoding === 'geoarrow.multilinestring';
+
+  const lineData = isMultiLineString ? arrowData.children[0] : arrowData;
+  const pointData = lineData.children[0];
+  const coordData = pointData.children[0];
+
+  const dimension = pointData.stride as 2 | 3 | 4;
+  const geometryOffsets = lineData.valueOffsets;
+  const positions = coordData.values;
+
+  // geometryIndexes is not needed for line string
+  const geometryIndexes = new Uint16Array(0);
+
+  const rowCount = arrowData.length;
+  const vertexCount = positions.length / dimension;
+  const featureIds = new Uint32Array(positionCount);
+
+  return {
+    type: 'LineString',
+
+    rowCount: arrowData.length,
+    rowOffsets: geometryOffsets,
+
+    positionCount: positions.length / dimension,
+    positionStride: dimension,
+    positions
+  };
+}
+
+/**
+ * get binary polygons from geoarrow polygon column
+ * @param chunk one chunk of geoarrow polygon column
+ * @param geoEncoding the geo encoding of the geoarrow polygon column
+ * @param options options for getting binary geometries
+ * @returns BinaryGeometryContent
+ */
+function getPolygonDataFromArrowData(
+  chunk: arrow.Data,
+  geoEncoding: string,
+  options?: GetGeoColumnOptions
+): GeoColumnPolygonData {
+  const isMultiPolygon = geoEncoding === 'geoarrow.multipolygon';
+
+  const polygonData = isMultiPolygon ? chunk.children[0] : chunk;
+  const polygonOffset = polygonData.valueOffsets;
+  const partData = isMultiPolygon
+    ? chunk.valueOffsets.map((i) => polygonOffset.at(i) || i)
+    : chunk.valueOffsets;
+  const ringData = polygonData.children[0];
+  const pointData = ringData.children[0];
+  const coordData = pointData.children[0];
+  const dimension = pointData.stride;
+  const geometryOffsets = ringData.valueOffsets;
+  const positions = coordData.values;
+
+  const geometryIndexes = new Uint16Array(polygonOffset.length);
+  for (let i = 0; i < polygonOffset.length; i++) {
+    geometryIndexes[i] = geometryOffsets[polygonOffset[i]];
+  }
+
+  const positionCount = positions.length / dimension;
+  const featureIds = new Uint32Array(positionCount);
+  for (let i = 0; i < partData.length - 1; i++) {
+    const startIndex = geometryOffsets[partData[i]];
+    const endIndex = geometryOffsets[partData[i + 1]];
+    for (let j = startIndex; j < endIndex; j++) {
+      featureIds[j] = i;
+    }
+  }
+
+  return {
+    type: 'Polygon',
+    featureIds,
+    dimension,
+    positions,
+    geometryOffsets,
+    geometryIndexes
+  };
+}
+
+//   return {
+//     globalFeatureIds: {value: new Uint32Array(0), size: 1},
+//     positions: {value: new Float32Array(0), size: 2},
+//     properties: [],
+//     numericProps: {},
+//     featureIds: {value: new Uint32Array(0), size: 1}
+//     ...(featureTypes.polygon ? binaryContent : {}),
+//     polygonIndices: {
+//       // use geometryOffsets as polygonIndices same as primitivePolygonIndices since we are using earcut to get triangule indices
+//       value: featureTypes.polygon ? geometryOffsets : new Uint16Array(0),
+//       size: 1
+//     },
+//     primitivePolygonIndices: {
+//       value: featureTypes.polygon ? geometryOffsets : new Uint16Array(0),
+//       size: 1
+//     }
+//   }
+// }