@loaders.gl/geoarrow 4.4.0-alpha.1

package/src/get-geoarrow-geometry-info.ts

@@ -0,0 +1,172 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import * as arrow from 'apache-arrow';
+import type {GeoArrowEncoding} from './metadata/geoarrow-metadata';
+
+/**
+ * @see https://geoarrow.org/format.html#memory-layouts
+ */
+export type GeoArrowGeometryInfo = {
+  /** Geometry encodings that are compatible with this column (Field) */
+  compatibleEncodings: GeoArrowEncoding[];
+  /** How many levels of List<> nesting  */
+  nesting: 0 | 1 | 2 | 3;
+  /** How many values per coordinate */
+  dimension: number;
+  /**
+   * - 0: A point is just a Coordinate
+   * - 1: A line string or a multipoint is a List<Coordinate>
+   * - 2: A polygon or a multilinestring are List<List<Coordinate>>
+   * - 3: multipolygons are List<List<List<Coordinate>>>
+   */
+  /** Coordinate memory layout {x,y,...} vs [x,y,...] */
+  coordinates: 'separated' | 'interleaved';
+  /** Coordinate  */
+  valueType: 'double'; // 'float'
+};
+
+/** Helper type used to test coordinates */
+type CoordinateFieldInfo = {
+  coordinates: 'interleaved' | 'separated';
+  dimension: 2 | 3 | 4;
+  valueType: 'double';
+};
+
+/**
+ * Examines a column containing GeoArrow formatted data and returns information about the geometry type
+ * that can be useful during traversal
+ * @see https://geoarrow.org/format.html#memory-layouts
+ */
+// eslint-disable-next-line max-statements
+export function getGeoArrowGeometryInfo(arrowField: arrow.Field): GeoArrowGeometryInfo | null {
+  if (arrowField.type instanceof arrow.Utf8) {
+    return {
+      compatibleEncodings: ['geoarrow.wkt'],
+      nesting: 0,
+      /** @note: Dimension encoded in WKT */
+      dimension: 2,
+      coordinates: 'interleaved',
+      valueType: 'double'
+    };
+  }
+
+  if (arrowField.type instanceof arrow.Binary || arrowField.type instanceof arrow.LargeBinary) {
+    return {
+      compatibleEncodings: ['geoarrow.wkb'],
+      nesting: 0,
+      /** @note: Dimension encoded in WKB */
+      dimension: 2,
+      coordinates: 'interleaved',
+      valueType: 'double'
+    };
+  }
+  let coordinateInfo = getCoordinateFieldInfo(arrowField);
+  // A point is just a Coordinate
+  if (coordinateInfo) {
+    return {
+      compatibleEncodings: ['geoarrow.point'],
+      nesting: 0,
+      ...coordinateInfo
+    };
+  }
+
+  // A line string or a multipoint is a List<Coordinate>
+  if (!(arrowField.type instanceof arrow.List)) {
+    return null;
+  }
+  arrowField = arrowField.type.children[0];
+
+  coordinateInfo = getCoordinateFieldInfo(arrowField);
+  if (coordinateInfo) {
+    return {
+      compatibleEncodings: ['geoarrow.linestring', 'geoarrow.multipoint'],
+      nesting: 1,
+      ...coordinateInfo
+    };
+  }
+
+  // A polygon or a multiline string are List<List<Coordinate>>
+  if (!(arrowField.type instanceof arrow.List)) {
+    return null;
+  }
+  arrowField = arrowField.type.children[0];
+
+  coordinateInfo = getCoordinateFieldInfo(arrowField);
+  if (coordinateInfo) {
+    return {
+      compatibleEncodings: ['geoarrow.polygon', 'geoarrow.multilinestring'],
+      nesting: 2,
+      ...coordinateInfo
+    };
+  }
+
+  // A multipolygons are List<List<List<Coordinate>>>
+  if (!(arrowField.type instanceof arrow.List)) {
+    return null;
+  }
+  arrowField = arrowField.type.children[0];
+
+  coordinateInfo = getCoordinateFieldInfo(arrowField);
+  if (coordinateInfo) {
+    return {
+      compatibleEncodings: ['geoarrow.multipolygon'],
+      nesting: 3,
+      ...coordinateInfo
+    };
+  }
+
+  return null;
+}
+
+/**
+ * @see https://geoarrow.org/format.html#memory-layouts
+ */
+function getCoordinateFieldInfo(arrowField: arrow.Field): CoordinateFieldInfo | null {
+  // interleaved case
+  if (arrowField.type instanceof arrow.FixedSizeList) {
+    const dimension = arrowField.type.listSize;
+    if (dimension < 2 || dimension > 4) {
+      return null;
+    }
+
+    const child = arrowField.type.children[0];
+    // Spec currently only supports 64 bit coordinates
+    if (!(child.type instanceof arrow.Float)) {
+      return null;
+    }
+
+    return {
+      coordinates: 'interleaved',
+      dimension: dimension as 2 | 3 | 4,
+      valueType: 'double'
+    };
+  }
+
+  // separated case
+  if (arrowField.type instanceof arrow.Struct) {
+    const children = arrowField.type.children;
+
+    const dimension = children.length;
+    if (dimension < 2 || dimension > 4) {
+      return null;
+    }
+
+    // Spec currently only supports 64 bit coordinates
+    for (const child of children) {
+      if (!(child.type instanceof arrow.Float)) {
+        return null;
+      }
+    }
+
+    return {
+      coordinates: 'separated',
+      dimension: dimension as 2 | 3 | 4,
+      valueType: 'double'
+    };
+  }
+
+  // No other types are valid coordinates
+  return null;
+}

package/src/index.ts

@@ -0,0 +1,57 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+// MESH CATEGORY
+
+export {getBoundingBoxFromArrowPositions} from './mesharrow/get-bounding-box';
+export {getDeckBinaryDataFromArrowMesh} from './mesharrow/get-deck-binary-data';
+
+// GIS CATEGORY - GEOARROW
+export type {GeoArrowMetadata, GeoArrowEncoding} from './metadata/geoarrow-metadata';
+
+export type {
+  GeoArrowWKB,
+  GeoArrowWKT,
+  GeoArrowCoordInterleaved,
+  GeoArrowCoordSeparated,
+  GeoArrowCoord,
+  GeoArrowPoint,
+  GeoArrowLineString,
+  GeoArrowPolygon,
+  GeoArrowMultiPoint,
+  GeoArrowMultiLineString,
+  GeoArrowMultiPolygon,
+  GeoArrowGeometry,
+  GeoArrowPointSeparated,
+  GeoArrowLineStringSeparated,
+  GeoArrowPolygonSeparated,
+  GeoArrowMultiPointSeparated,
+  GeoArrowMultiLineStringSeparated,
+  GeoArrowMultiPolygonSeparated,
+  GeoArrowGeometrySeparated
+} from './geoarrow-types';
+
+export {
+  isGeoArrowPoint,
+  isGeoArrowLineString,
+  isGeoArrowPolygon,
+  isGeoArrowMultiPoint,
+  isGeoArrowMultiLineString,
+  isGeoArrowMultiPolygon,
+  isGeoArrowGeometry
+} from './geoarrow-functions';
+
+// GEOARROW / GEOPARQUET METADATA
+export {getGeometryColumnsFromSchema} from './metadata/geoarrow-metadata';
+
+export type {GeoColumnMetadata} from './metadata/geoparquet-metadata';
+export {
+  getGeoMetadata,
+  unpackGeoMetadata,
+  unpackJSONStringMetadata
+} from './metadata/geoparquet-metadata';
+
+export {getGeoArrowGeometryInfo} from './get-geoarrow-geometry-info';
+
+export {updateBoundsFromGeoArrowSamples} from './get-arrow-bounds';

package/src/mesharrow/arrow-fixed-size-list-utils.ts

@@ -0,0 +1,62 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import {TypedArray} from '@math.gl/types';
+import * as arrow from 'apache-arrow';
+import {getDataTypeFromArray, deserializeArrowType} from '@loaders.gl/schema-utils';
+
+export function isFixedSizeList(vector: arrow.Vector): vector is arrow.Vector<arrow.FixedSizeList> {
+  return vector.type instanceof arrow.FixedSizeList;
+}
+
+export function getFixedSizeListSize(vector: arrow.Vector): number {
+  return vector.type instanceof arrow.FixedSizeList ? vector.type.listSize : 1;
+}
+
+/** Get Arrow FixedSizeList vector from a typed array */
+export function getFixedSizeListVector(
+  typedArray: TypedArray,
+  stride: number
+): arrow.Vector<arrow.FixedSizeList> {
+  const data = getFixedSizeListData(typedArray, stride);
+  return new arrow.Vector<arrow.FixedSizeList>([data]);
+}
+
+/** Get Arrow FixedSizeList vector from a typed array */
+export function getFixedSizeListData(
+  typedArray: TypedArray,
+  stride: number
+): arrow.Data<arrow.FixedSizeList> {
+  const listType = getFixedSizeListType(typedArray, stride);
+  const nestedType = listType.children[0].type;
+  const buffers: Partial<Record<arrow.BufferType, any>> = {
+    // valueOffsets: undefined,
+    [arrow.BufferType.DATA]: typedArray // values
+    // nullBitmap: undefined,
+    // typeIds: undefined
+  };
+
+  // Note: The contiguous array of data is held by the nested "primitive type" column
+  const nestedData = new arrow.Data(nestedType, 0, typedArray.length, 0, buffers);
+
+  // Wrapped in a FixedSizeList column that provides a "strided" view of the data
+  const data = new arrow.Data<arrow.FixedSizeList>(
+    listType,
+    0,
+    typedArray.length / stride,
+    0,
+    undefined,
+    [nestedData]
+  );
+
+  return data;
+}
+
+/** Get Arrow FixedSizeList vector from a typed array */
+export function getFixedSizeListType(typedArray: TypedArray, stride: number): arrow.FixedSizeList {
+  const {type} = getDataTypeFromArray(typedArray);
+  const arrowType = deserializeArrowType(type);
+  const listType = new arrow.FixedSizeList(stride, new arrow.Field('value', arrowType));
+  return listType;
+}

package/src/mesharrow/arrow-list-of-fixed-size-list-utils.ts

@@ -0,0 +1,47 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import {TypedArray} from '@math.gl/types';
+import * as arrow from 'apache-arrow';
+import {getDataTypeFromArray, deserializeArrowType} from '@loaders.gl/schema-utils';
+
+export function isListFixedSizeList(
+  vector: arrow.Vector
+): vector is arrow.Vector<arrow.FixedSizeList> {
+  return vector.type instanceof arrow.FixedSizeList;
+}
+
+export function getListFixedSizeListSize(vector: arrow.Vector): number {
+  return vector.type instanceof arrow.FixedSizeList ? vector.type.listSize : 1;
+}
+
+/** Get Arrow FixedSizeList vector from a typed array */
+export function getListFixedSizeListVector(
+  indexes: Uint32Array,
+  typedArray: TypedArray,
+  stride: number
+): arrow.Vector<arrow.FixedSizeList> {
+  const data = getFixedSizeListData(typedArray, stride);
+  return new arrow.Vector<arrow.FixedSizeList>([data]);
+}
+
+/** Get Arrow FixedSizeList vector from a typed array */
+export function getFixedSizeListData(
+  typedArray: TypedArray,
+  stride: number
+): arrow.Data<arrow.FixedSizeList> {
+  const listType = getFixedSizeListType(typedArray, stride);
+  const data = new arrow.Data<arrow.FixedSizeList>(listType, 0, typedArray.length / stride, 0, [
+    typedArray
+  ]);
+  return data;
+}
+
+/** Get Arrow FixedSizeList vector from a typed array */
+export function getFixedSizeListType(typedArray: TypedArray, stride: number): arrow.FixedSizeList {
+  const {type} = getDataTypeFromArray(typedArray);
+  const arrowType = deserializeArrowType(type);
+  const listType = new arrow.FixedSizeList(stride, new arrow.Field('value', arrowType));
+  return listType;
+}

package/src/mesharrow/get-bounding-box.ts

@@ -0,0 +1,39 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import * as arrow from 'apache-arrow';
+
+export type BoundingBox = [[number, number, number], [number, number, number]];
+
+/** basic helper method to calculate a models upper and lower bounds */
+export function getBoundingBoxFromArrowPositions(
+  column: arrow.Vector<arrow.FixedSizeList>
+): BoundingBox {
+  const mins: [number, number, number] = [Number.MAX_VALUE, Number.MAX_VALUE, Number.MAX_VALUE];
+  const maxs: [number, number, number] = [Number.MIN_VALUE, Number.MIN_VALUE, Number.MIN_VALUE];
+
+  const valueColumn = column.getChildAt(0)!;
+  for (const data of valueColumn.data) {
+    const pointSize = 3; // attributes.POSITION.size;
+    const pointData = data.buffers[arrow.BufferType.DATA];
+    const pointCount = pointData.length / pointSize;
+
+    for (let i = 0; i < pointCount; i += pointSize) {
+      const x = pointData[i];
+      const y = pointData[i + 1];
+      const z = pointData[i + 2];
+
+      if (x < mins[0]) mins[0] = x;
+      else if (x > maxs[0]) maxs[0] = x;
+
+      if (y < mins[1]) mins[1] = y;
+      else if (y > maxs[1]) maxs[1] = y;
+
+      if (z < mins[2]) mins[2] = z;
+      else if (z > maxs[2]) maxs[2] = z;
+    }
+  }
+
+  return [mins, maxs];
+}

package/src/mesharrow/get-deck-binary-data.ts

@@ -0,0 +1,43 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import {TypedArray} from '@math.gl/types';
+import {getSizeAndValueFromMeshArrowVector} from './mesh-accessors';
+import * as arrow from 'apache-arrow';
+
+/**  */
+export type DeckBinaryData = {
+  length: number;
+  attributes: Record<
+    string,
+    {
+      value: TypedArray;
+      size: number;
+    }
+  >;
+};
+
+/**  */
+export function getDeckBinaryDataFromArrowMesh(table: arrow.Table): DeckBinaryData {
+  const positionVector = table.getChild('POSITION');
+  if (!positionVector) {
+    throw new Error('POSITION attribute not found');
+  }
+
+  const getPosition = getSizeAndValueFromMeshArrowVector(positionVector);
+
+  const deckAttributes: DeckBinaryData['attributes'] = {
+    getPosition
+  };
+
+  const colorVector = table.getChild('COLOR_0');
+  if (colorVector) {
+    deckAttributes.getColor = getSizeAndValueFromMeshArrowVector(colorVector);
+  }
+  // Check PointCloudLayer docs for other supported props?
+  return {
+    length: table.numRows,
+    attributes: deckAttributes
+  };
+}

package/src/mesharrow/mesh-accessors.ts

@@ -0,0 +1,27 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import {TypedArray} from '@math.gl/types';
+import {getFixedSizeListSize} from './arrow-fixed-size-list-utils';
+import * as arrow from 'apache-arrow';
+
+export function getSizeAndValueFromMeshArrowVector(attributeVector: arrow.Vector): {
+  size: number;
+  value: TypedArray;
+} {
+  const size = getFixedSizeListSize(attributeVector);
+  const typedArrays = getTypedArraysFromMeshArrowVector(attributeVector);
+  return {size, value: typedArrays[0]};
+}
+
+export function getTypedArraysFromMeshArrowVector(attributeVector: arrow.Vector): TypedArray[] {
+  const typedArrays: TypedArray[] = [];
+  for (const attributeData of attributeVector.data) {
+    const valueData = attributeData?.children[0];
+    const typedArray = valueData.values;
+    typedArrays.push(typedArray);
+  }
+
+  return typedArrays;
+}

package/src/metadata/geoarrow-metadata.ts

@@ -0,0 +1,102 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import {Metadata, SchemaWithMetadata, getMetadataValue} from './metadata-utils';
+
+export type GeoArrowEncoding =
+  | 'geoarrow.multipolygon'
+  | 'geoarrow.polygon'
+  | 'geoarrow.multilinestring'
+  | 'geoarrow.linestring'
+  | 'geoarrow.multipoint'
+  | 'geoarrow.point'
+  | 'geoarrow.wkb'
+  | 'geoarrow.wkt';
+
+/** Array containing all encodings */
+const GEOARROW_ENCODINGS = [
+  'geoarrow.multipolygon',
+  'geoarrow.polygon',
+  'geoarrow.multilinestring',
+  'geoarrow.linestring',
+  'geoarrow.multipoint',
+  'geoarrow.point',
+  'geoarrow.wkb',
+  'geoarrow.wkt'
+] as const satisfies GeoArrowEncoding[];
+
+const GEOARROW_ENCODING = 'ARROW:extension:name';
+const GEOARROW_METADATA = 'ARROW:extension:metadata';
+
+/**
+ * Geospatial metadata for one column, extracted from Apache Arrow metadata
+ * @see https://github.com/geoarrow/geoarrow/blob/main/extension-types.md
+ */
+export type GeoArrowMetadata = {
+  /** Encoding of geometry in this column */
+  encoding?: GeoArrowEncoding;
+  /** CRS in [PROJJSON](https://proj.org/specifications/projjson.html). Omitted if producer has no information about CRS */
+  crs?: Record<string, unknown>;
+  /** Edges are either spherical or omitted */
+  edges?: 'spherical';
+  [key: string]: unknown;
+};
+
+/**
+ * get geometry columns from arrow table
+ */
+export function getGeometryColumnsFromSchema(
+  schema: SchemaWithMetadata
+): Record<string, GeoArrowMetadata> {
+  const geometryColumns: Record<string, GeoArrowMetadata> = {};
+  for (const field of schema.fields || []) {
+    const metadata = getGeometryMetadataForField(field?.metadata || {});
+    if (metadata) {
+      geometryColumns[field.name] = metadata;
+    }
+  }
+  return geometryColumns;
+}
+/**
+ * Extracts GeoArrow metadata from a field
+ * @param field
+ * @returns
+ * @see https://github.com/geoarrow/geoarrow/blob/d2f56704414d9ae71e8a5170a8671343ed15eefe/extension-types.md
+ */
+export function getGeometryMetadataForField(fieldMetadata: Metadata): GeoArrowMetadata | null {
+  let metadata: GeoArrowMetadata | null = null;
+
+  // Check for GeoArrow column encoding
+  let geoEncoding = getMetadataValue(fieldMetadata, GEOARROW_ENCODING);
+  if (geoEncoding) {
+    geoEncoding = geoEncoding.toLowerCase();
+    // at time of testing, ogr2ogr uses WKB/WKT for encoding.
+    if (geoEncoding === 'wkb') {
+      geoEncoding = 'geoarrow.wkb';
+    }
+    if (geoEncoding === 'wkt') {
+      geoEncoding = 'geoarrow.wkt';
+    }
+    if (!GEOARROW_ENCODINGS.includes(geoEncoding as GeoArrowEncoding)) {
+      // eslint-disable-next-line no-console
+      console.warn(`Invalid GeoArrow encoding: ${geoEncoding}`);
+    } else {
+      metadata ||= {} as GeoArrowMetadata;
+      metadata.encoding = geoEncoding as GeoArrowEncoding;
+    }
+  }
+
+  // Check for GeoArrow metadata
+  const columnMetadata = getMetadataValue(fieldMetadata, GEOARROW_METADATA);
+  if (columnMetadata) {
+    try {
+      metadata = JSON.parse(columnMetadata);
+    } catch (error) {
+      // eslint-disable-next-line no-console
+      console.warn('Failed to parse GeoArrow metadata', error);
+    }
+  }
+
+  return metadata || null;
+}

package/src/metadata/geoparquet-metadata-schema.ts

@@ -0,0 +1,71 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+/* eslint-disable camelcase */
+
+/**
+ * Geoparquet JSON schema for geo metadata
+ * @see https://github.com/geoarrow/geoarrow/blob/main/metadata.md
+ * @see https://github.com/opengeospatial/geoparquet/blob/main/format-specs/geoparquet.md
+ */
+export const GEOPARQUET_METADATA_JSON_SCHEMA = {
+  $schema: 'http://json-schema.org/draft-07/schema#',
+  title: 'GeoParquet',
+  description: 'Parquet metadata included in the geo field.',
+  type: 'object',
+  required: ['version', 'primary_column', 'columns'],
+  properties: {
+    version: {type: 'string', const: '1.0.0-beta.1'},
+    primary_column: {type: 'string', minLength: 1},
+    columns: {
+      type: 'object',
+      minProperties: 1,
+      patternProperties: {
+        '.+': {
+          type: 'object',
+          required: ['encoding', 'geometry_types'],
+          properties: {
+            encoding: {type: 'string', const: 'WKB'},
+            geometry_types: {
+              type: 'array',
+              uniqueItems: true,
+              items: {
+                type: 'string',
+                pattern: '^(GeometryCollection|(Multi)?(Point|LineString|Polygon))( Z)?$'
+              }
+            },
+            crs: {
+              oneOf: [
+                {
+                  $ref: 'https://proj.org/schemas/v0.5/projjson.schema.json'
+                },
+                {type: 'null'}
+              ]
+            },
+            edges: {type: 'string', enum: ['planar', 'spherical']},
+            orientation: {type: 'string', const: 'counterclockwise'},
+            bbox: {
+              type: 'array',
+              items: {type: 'number'},
+              oneOf: [
+                {
+                  description: '2D bbox consisting of (xmin, ymin, xmax, ymax)',
+                  minItems: 4,
+                  maxItems: 4
+                },
+                {
+                  description: '3D bbox consisting of (xmin, ymin, zmin, xmax, ymax, zmax)',
+                  minItems: 6,
+                  maxItems: 6
+                }
+              ]
+            },
+            epoch: {type: 'number'}
+          }
+        }
+      },
+      additionalProperties: false
+    }
+  }
+};