@loaders.gl/geoarrow 4.4.0-alpha.1

package/dist/metadata/geoparquet-metadata.d.ts

@@ -0,0 +1,45 @@
+import { Metadata } from "./metadata-utils.js";
+/**
+ * A geoarrow / geoparquet geo metadata object
+ * (stored in stringified form in the top level metadata 'geo' key)
+ * @see https://github.com/opengeospatial/geoparquet/blob/main/format-specs/geoparquet.md
+ * @see https://github.com/geoarrow/geoarrow
+ * */
+export type GeoMetadata = {
+    version?: string;
+    primary_column?: string;
+    columns: Record<string, GeoColumnMetadata>;
+    [key: string]: unknown;
+};
+/** A geoarrow / geoparquet geo metadata for one geometry column  */
+export type GeoColumnMetadata = {
+    encoding: 'wkb' | 'wkt';
+    geometry_types: GeoParquetGeometryType[];
+    crs?: object | null;
+    orientation?: 'counterclockwise';
+    bbox?: [number, number, number, number] | [number, number, number, number, number, number];
+    edges?: 'planar' | 'spherical';
+    epoch?: number;
+    [key: string]: unknown;
+};
+/** A GeoParquet metadata geometry type */
+export type GeoParquetGeometryType = 'Point' | 'LineString' | 'Polygon' | 'MultiPoint' | 'MultiLineString' | 'MultiPolygon' | 'GeometryCollection' | 'Point Z' | 'LineString Z' | 'Polygon Z' | 'MultiPoint Z' | 'MultiLineString Z' | 'MultiPolygon Z' | 'GeometryCollection Z';
+/**
+ * Reads the GeoMetadata object from the metadata
+ * @note geoarrow / parquet schema is stringified into a single key-value pair in the parquet metadata
+ */
+export declare function getGeoMetadata(metadata: Metadata): GeoMetadata | null;
+/**
+ * Stores a geoarrow / geoparquet geo metadata object in the schema
+ * @note geoarrow / geoparquet geo metadata is a single stringified JSON field
+ */
+export declare function setGeoMetadata(metadata: Metadata, geoMetadata: GeoMetadata): void;
+/**
+ * Unpacks geo metadata into separate metadata fields (parses the long JSON string)
+ * @note geoarrow / parquet schema is stringified into a single key-value pair in the parquet metadata
+ */
+export declare function unpackGeoMetadata(metadata: Metadata): void;
+export declare function unpackJSONStringMetadata(metadata: Metadata, metadataKey: string): void;
+/** Parse a key with stringified arrow metadata */
+export declare function parseJSONStringMetadata(stringifiedMetadata: string): Record<string, unknown> | null;
+//# sourceMappingURL=geoparquet-metadata.d.ts.map

package/dist/metadata/geoparquet-metadata.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"geoparquet-metadata.d.ts","sourceRoot":"","sources":["../../src/metadata/geoparquet-metadata.ts"],"names":[],"mappings":"AAMA,OAAO,EAAC,QAAQ,EAAqC,4BAAyB;AAE9E;;;;;KAKK;AACL,MAAM,MAAM,WAAW,GAAG;IACxB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,iBAAiB,CAAC,CAAC;IAC3C,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB,CAAC;AAEF,oEAAoE;AACpE,MAAM,MAAM,iBAAiB,GAAG;IAC9B,QAAQ,EAAE,KAAK,GAAG,KAAK,CAAC;IACxB,cAAc,EAAE,sBAAsB,EAAE,CAAC;IACzC,GAAG,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACpB,WAAW,CAAC,EAAE,kBAAkB,CAAC;IACjC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;IAC3F,KAAK,CAAC,EAAE,QAAQ,GAAG,WAAW,CAAC;IAC/B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB,CAAC;AAEF,0CAA0C;AAC1C,MAAM,MAAM,sBAAsB,GAC9B,OAAO,GACP,YAAY,GACZ,SAAS,GACT,YAAY,GACZ,iBAAiB,GACjB,cAAc,GACd,oBAAoB,GACpB,SAAS,GACT,cAAc,GACd,WAAW,GACX,cAAc,GACd,mBAAmB,GACnB,gBAAgB,GAChB,sBAAsB,CAAC;AAI3B;;;GAGG;AACH,wBAAgB,cAAc,CAAC,QAAQ,EAAE,QAAQ,GAAG,WAAW,GAAG,IAAI,CAYrE;AAED;;;GAGG;AACH,wBAAgB,cAAc,CAAC,QAAQ,EAAE,QAAQ,EAAE,WAAW,EAAE,WAAW,GAAG,IAAI,CAGjF;AAED;;;GAGG;AACH,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,QAAQ,GAAG,IAAI,CA6B1D;AAED,wBAAgB,wBAAwB,CAAC,QAAQ,EAAE,QAAQ,EAAE,WAAW,EAAE,MAAM,GAAG,IAAI,CAUtF;AAqDD,kDAAkD;AAClD,wBAAgB,uBAAuB,CACrC,mBAAmB,EAAE,MAAM,GAC1B,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAchC"}

package/dist/metadata/geoparquet-metadata.js

@@ -0,0 +1,131 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+/* eslint-disable camelcase */
+import { getMetadataValue, setMetadataValue } from "./metadata-utils.js";
+// GEO METADATA
+/**
+ * Reads the GeoMetadata object from the metadata
+ * @note geoarrow / parquet schema is stringified into a single key-value pair in the parquet metadata
+ */
+export function getGeoMetadata(metadata) {
+    const stringifiedGeoMetadata = getMetadataValue(metadata, 'geo');
+    const geoMetadata = stringifiedGeoMetadata && parseJSONStringMetadata(stringifiedGeoMetadata);
+    if (!geoMetadata) {
+        return null;
+    }
+    for (const column of Object.values(geoMetadata.columns || {})) {
+        if (column.encoding) {
+            column.encoding = column.encoding.toLowerCase();
+        }
+    }
+    return geoMetadata;
+}
+/**
+ * Stores a geoarrow / geoparquet geo metadata object in the schema
+ * @note geoarrow / geoparquet geo metadata is a single stringified JSON field
+ */
+export function setGeoMetadata(metadata, geoMetadata) {
+    const stringifiedGeoMetadata = JSON.stringify(geoMetadata);
+    setMetadataValue(metadata, 'geo', stringifiedGeoMetadata);
+}
+/**
+ * Unpacks geo metadata into separate metadata fields (parses the long JSON string)
+ * @note geoarrow / parquet schema is stringified into a single key-value pair in the parquet metadata
+ */
+export function unpackGeoMetadata(metadata) {
+    const geoMetadata = getGeoMetadata(metadata);
+    if (!geoMetadata) {
+        return;
+    }
+    // Store Parquet Schema Level Metadata
+    const { version, primary_column, columns } = geoMetadata;
+    if (version) {
+        setMetadataValue(metadata, 'geo.version', version);
+    }
+    if (primary_column) {
+        setMetadataValue(metadata, 'geo.primary_column', primary_column);
+    }
+    // store column names as comma separated list
+    setMetadataValue(metadata, 'geo.columns', Object.keys(columns || {}).join(''));
+    // for (const [columnName, columnMetadata] of Object.entries(columns || {})) {
+    //   const field = schema.fields.find((field) => field.name === columnName);
+    //   if (field) {
+    //     if (field.name === primary_column) {
+    //       setFieldMetadata(field, 'geo.primary_field', 'true');
+    //     }
+    //     unpackGeoFieldMetadata(field, columnMetadata);
+    //   }
+    // }
+}
+export function unpackJSONStringMetadata(metadata, metadataKey) {
+    const stringifiedGeoMetadata = getMetadataValue(metadata, 'geo');
+    const json = stringifiedGeoMetadata && parseJSONStringMetadata(stringifiedGeoMetadata);
+    for (const [key, value] of Object.entries(json || {})) {
+        setMetadataValue(metadata, `${metadataKey}.${key}`, typeof value === 'string' ? value : JSON.stringify(value));
+    }
+}
+// eslint-disable-next-line complexity
+// function unpackGeoFieldMetadata(fieldMetadata: Metadata, columnMetadata): void {
+//   for (const [key, value] of Object.entries(columnMetadata || {})) {
+//     switch (key) {
+//       case 'geometry_types':
+//         setFieldMetadata(field, `geo.${key}`, (value as string[]).join(','));
+//         break;
+//       case 'bbox':
+//         setFieldMetadata(field, `geo.crs.${key}`, JSON.stringify(value));
+//         break;
+//       case 'crs':
+//         // @ts-ignore
+//         for (const [crsKey, crsValue] of Object.entries(value || {})) {
+//           switch (crsKey) {
+//             case 'id':
+//               // prettier-ignore
+//               const crsId =
+//                 typeof crsValue === 'object'
+//                   ? // @ts-ignore
+//                   `${crsValue?.authority}:${crsValue?.code}`
+//                   : JSON.stringify(crsValue);
+//               setFieldMetadata(field, `geo.crs.${crsKey}`, crsId);
+//               break;
+//             default:
+//               setFieldMetadata(
+//                 field,
+//                 `geo.crs.${crsKey}`,
+//                 typeof crsValue === 'string' ? crsValue : JSON.stringify(crsValue)
+//               );
+//               break;
+//           }
+//         }
+//         break;
+//       case 'edges':
+//       default:
+//         setFieldMetadata(
+//           field,
+//           `geo.${key}`,
+//           typeof value === 'string' ? value : JSON.stringify(value)
+//         );
+//     }
+//   }
+// }
+// function setFieldMetadata(field: Field, key: string, value: string): void {
+//   field.metadata = field.metadata || {};
+//   field.metadata[key] = value;
+// }
+// HELPERS
+/** Parse a key with stringified arrow metadata */
+export function parseJSONStringMetadata(stringifiedMetadata) {
+    if (!stringifiedMetadata) {
+        return null;
+    }
+    try {
+        const metadata = JSON.parse(stringifiedMetadata);
+        if (!metadata || typeof metadata !== 'object') {
+            return null;
+        }
+        return metadata;
+    }
+    catch {
+        return null;
+    }
+}

package/dist/metadata/metadata-utils.d.ts

@@ -0,0 +1,21 @@
+/**
+ * A type that matches a string-to-string metadata map
+ * in the form of a map or an object.
+ */
+export type Metadata = Map<string, string> | Record<string, string>;
+/**
+ * A type that matches a variety of Apache arrow like schemas.
+ *
+ */
+export type SchemaWithMetadata = {
+    /** Schema level metadata */
+    metadata?: Metadata;
+    /** Field top-level metadata */
+    fields?: {
+        name: string;
+        metadata?: Metadata;
+    }[];
+};
+export declare function getMetadataValue(metadata: Metadata, key: string): string | null;
+export declare function setMetadataValue(metadata: Metadata, key: string, value: string): void;
+//# sourceMappingURL=metadata-utils.d.ts.map

package/dist/metadata/metadata-utils.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"metadata-utils.d.ts","sourceRoot":"","sources":["../../src/metadata/metadata-utils.ts"],"names":[],"mappings":"AAIA;;;GAGG;AACH,MAAM,MAAM,QAAQ,GAAG,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AAEpE;;;GAGG;AACH,MAAM,MAAM,kBAAkB,GAAG;IAC/B,4BAA4B;IAC5B,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,+BAA+B;IAC/B,MAAM,CAAC,EAAE;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,QAAQ,CAAA;KAAC,EAAE,CAAC;CAChD,CAAC;AAEF,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,QAAQ,EAAE,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAE/E;AAED,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,QAAQ,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI,CAMrF"}

package/dist/metadata/metadata-utils.js

@@ -0,0 +1,14 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+export function getMetadataValue(metadata, key) {
+    return metadata instanceof Map ? metadata.get(key) || null : metadata[key] || null;
+}
+export function setMetadataValue(metadata, key, value) {
+    if (metadata instanceof Map) {
+        metadata.set('key', key);
+    }
+    else {
+        metadata.key = key;
+    }
+}

package/package.json

@@ -0,0 +1,63 @@
+{
+  "name": "@loaders.gl/geoarrow",
+  "version": "4.4.0-alpha.1",
+  "description": "GeoArrow columnar geometry encoding and decoding",
+  "license": "MIT",
+  "type": "module",
+  "publishConfig": {
+    "access": "public"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/visgl/loaders.gl"
+  },
+  "keywords": [
+    "loader",
+    "parser",
+    "writer",
+    "encoder",
+    "geoarrow",
+    "apache-arrow",
+    "arrow",
+    "binary columnar",
+    "cloud native",
+    "webgl",
+    "webgpu"
+  ],
+  "types": "dist/index.d.ts",
+  "main": "dist/index.cjs",
+  "module": "dist/index.js",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js",
+      "require": "./dist/index.cjs"
+    },
+    "./exports/*": {
+      "types": "./dist/exports/*.d.ts",
+      "import": "./dist/exports/*.js"
+    }
+  },
+  "sideEffects": false,
+  "files": [
+    "src",
+    "dist",
+    "README.md"
+  ],
+  "browser": {
+    "fs": false
+  },
+  "scripts": {
+    "pre-build": "",
+    "build-bundle": "ocular-bundle ./bundle.ts --output=dist/dist.min.js",
+    "build-bundle-dev": "ocular-bundle ./bundle.ts --env=dev --output=dist/dist.dev.js"
+  },
+  "dependencies": {
+    "@math.gl/polygon": "^4.1.0",
+    "apache-arrow": ">= 16.1.0"
+  },
+  "peerDependencies": {
+    "@loaders.gl/core": "4.4.0-alpha.0"
+  },
+  "gitHead": "f1732de45907bd500bf4eedb4803beca8bf4bfb0"
+}

package/src/geo-column/geo-column-info.ts

@@ -0,0 +1,114 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+// import type {GeoColumn} from './geo-column';
+// import {getGeoParquetMetadata, GeoParquetMetadata} from '../metadata/geoparquet-metadata';
+// import {getGeoArrowMetadata, GeoArrowMetadata} from '../metadata_3/geoarrow-metadata';
+
+/**
+ * Information about a binary geometry
+ */
+export type geoColumnInfo = {
+  type: 'Point' | 'LineString' | 'Polygon';
+  /** The GeoJSON style geometry type corresponding to this particular binary geometry */
+  multiGeometryType:
+    | 'Point'
+    | 'LineString'
+    | 'Polygon'
+    | 'MultiPoint'
+    | 'MultiLineString'
+    | 'MultiPolygon';
+  /** Is this a "Multi" version of the binary geometry? */
+  isMultiGeometry: boolean;
+  /** How many dimensions are the coordinates? */
+  dimension: number;
+  /** How many points does this geometry have? */
+  pointCount: number;
+  /** How many coordinates does this geometry have? */
+  coordinateCount: number;
+};
+
+/**
+ * @returns information about a binary geometry
+ */
+// export function getGeoColumnInfo(geometry: GeoColumn): GeoColumnInfo {
+
+// }
+
+// // loaders.gl
+// // SPDX-License-Identifier: MIT
+// // Copyright (c) vis.gl contributors
+
+// import type {Schema, Field, GeoArrowMetadata, GeoArrowEncoding, Geometry} from '@loaders.gl/schema';
+// import {convertArrowToSchema} from '@loaders.gl/schema-utils';
+// import * as arrow from 'apache-arrow';
+// import {getGeometryColumnsFromSchema, getGeometryMetadataForField} from '../geoarrow/geoarrow-metadata';
+// import { getGeoMetadata as getParquetMe} from './geoparquet-metadata';
+
+// /**
+// * A geoarrow / geoparquet geo metadata object
+// * (stored in stringified form in the top level metadata 'geo' key)
+// * @see https://github.com/opengeospatial/geoparquet/blob/main/format-specs/geoparquet.md
+// * @see https://github.com/geoarrow/geoarrow
+// * */
+// export type GeoTableMetadata = {
+//  version?: string;
+//  primaryGeometryColumn?: string;
+//  columns: Record<string, GeoColumnInfo>;
+//  [key: string]: unknown;
+// };
+
+// /** A geoarrow / geoparquet geo metadata for one geometry column  */
+// export type GeoColumnInfo = {
+//  encoding: 'wkb' | 'wkt' | 'none';
+//  geometryTypes: Geometry['type'][];
+//  dimension: number;
+//  crs?: object | null;
+//  orientation?: 'counterclockwise';
+//  bbox?: [number, number, number, number] | [number, number, number, number, number, number];
+//  edges?: 'planar' | 'spherical';
+//  epoch?: number;
+
+//  [key: string]: unknown;
+//  // geoParquetGeometryType:
+// };
+
+// export type GeoArrowInfo = {
+//   geometryColumns: Record<string, GeometryColumnInfo>
+// };
+
+// /**
+//  * get geometry columns from arrow table
+//  */
+// export function getGeoArrowInfo(arrowTable: arrow.Table): GeoArrowInfo {
+//  const schema = convertArrowToSchema(arrowTable.schema);
+//   const geometryColumns = getGeometryColumnsFromSchema(schema);
+
+//   // get encoding from geometryColumns['geometry']
+//   const encoding = geometryColumns.geometry.encoding;
+
+//   // Remove geometry columns
+//   const propertyColumnNames = arrowTable.schema.fields
+//     .map((field) => field.name)
+//     // TODO - this deletes all geometry columns
+//     .filter((name) => name in geometryColumns);
+//   const propertiesTable = arrowTable.select(propertyColumnNames);
+
+//   const arrowGeometryColumn = arrowTable.getChild('geometry');
+
+//   for (let row = 0; row < arrowTable.numRows; row++) {
+//     // get the geometry value from arrow geometry column
+//     // Note that type can vary
+//     const arrowGeometry = arrowGeometryColumn?.get(row);
+//     // parse arrow geometry to geojson feature
+//     const feature = convertGeoArrowGeometryToGeoJSON(arrowGeometry, encoding);
+//     if (feature) {
+//       const properties = propertiesTable.get(row)?.toJSON() || {};
+//       features.push({type: 'Feature', geometry: feature, properties});
+//     }
+//   }
+
+//   return {
+//   };
+// }

package/src/geo-column/geo-column.ts

@@ -0,0 +1,85 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+// GIS
+import type {TypedArray} from '@math.gl/types';
+
+// BINARY FORMAT GEOMETRY
+
+export type GeoColumnType = 'Point' | 'LineString' | 'Polygon';
+
+/** A geometry column can be built from list of geometry column data chunks */
+export type GeoColumn<MetadataT = Record<string, unknown>> = {
+  shape: 'geo-column';
+  metadata?: MetadataT;
+  data: GeoColumnData[];
+};
+
+export type GeoColumnData = GeoColumnPointData | GeoColumnLineData | GeoColumnPolygonData;
+
+/**
+ * Describes a single contiguous chunk of geometries in binary columnar format
+ * Extracts the nested position and offset arrays from a GeoArrow column
+ * @note Designed to be a cheap operation: does not create any new arrays, just holds extracted references the existing arrays
+ */
+export type GeoColumnCommonData = {
+  shape: 'geo-column-data';
+
+  /** Number of rows geometries (can be less than pointCount if representing multipoints) */
+  numRows: number;
+  /** Offset to the end of the next geometry (length = rowCount) */
+  rowOffsets: Uint32Array;
+
+  /** Number of positions */
+  numPositions: number;
+  /** Data for coordinates, either interleaved or separate arrays */
+  positions: GeoColumnPositions;
+
+  /** Stores the data for the respective geometry types */
+  // points?: GeoColumnPointData;
+  // lines?: GeoColumnLineData;
+  // polygons?: GeoColumnPolygonData;
+};
+
+/** Columnar point geometry: an array of positions */
+export type GeoColumnPointData = GeoColumnCommonData & {
+  type: 'Point';
+};
+
+/** Columnar line geometry, array of positions and indices to the start of each line */
+export type GeoColumnLineData = GeoColumnCommonData & {
+  type: 'LineString';
+
+  /** Offset to the next path within the multi feature */
+  pathOffsets: Uint32Array;
+};
+
+/** Columnar polygon geometry, an array of positions to each primitite polygon and polygon */
+export type GeoColumnPolygonData = GeoColumnCommonData & {
+  type: 'Polygon';
+  /** Offset to next polygon */
+  polygonOffsets: Uint32Array;
+  /** Offset to next primitive polygon */
+  ringOffsets: Uint32Array;
+};
+
+export type GeoColumnPositions = GeoColumnInterleavedPositions | GeoColumnSeparatePositions;
+
+/** Positions are in a single coordinate array */
+export type GeoColumnInterleavedPositions = {
+  layout: 'interleaved';
+  /** Dimension, i.e. number of coordinates per position: 2, 3 or 4 */
+  stride: 2 | 3 | 4;
+  /** Flat array of position coordinates (length = positionCount * positionStride */
+  coordinates: TypedArray;
+};
+
+/** Positions are in separate coordinate arrays */
+export type GeoColumnSeparatePositions = {
+  layout: 'separate';
+  /** Dimension, i.e. number of coordinates per position: 2, 3 or 4 */
+  stride: 2 | 3 | 4;
+  /** Flat array of position coordinates (length = positionCount * positionStride */
+  coordinates: TypedArray[];
+};