@loaders.gl/parquet 4.0.0 → 4.0.2

package/dist/lib/geo/geoparquet-metadata-schema.json

@@ -1,60 +0,0 @@
-{
-  "$schema": "http://json-schema.org/draft-07/schema#",
-  "title": "GeoParquet",
-  "description": "Parquet metadata included in the geo field.",
-  "type": "object",
-  "required": ["version", "primary_column", "columns"],
-  "properties": {
-    "version": {"type": "string", "const": "1.0.0-beta.1"},
-    "primary_column": {"type": "string", "minLength": 1},
-    "columns": {
-      "type": "object",
-      "minProperties": 1,
-      "patternProperties": {
-        ".+": {
-          "type": "object",
-          "required": ["encoding", "geometry_types"],
-          "properties": {
-            "encoding": {"type": "string", "const": "WKB"},
-            "geometry_types": {
-              "type": "array",
-              "uniqueItems": true,
-              "items": {
-                "type": "string",
-                "pattern": "^(GeometryCollection|(Multi)?(Point|LineString|Polygon))( Z)?$"
-              }
-            },
-            "crs": {
-              "oneOf": [
-                {
-                  "$ref": "https://proj.org/schemas/v0.5/projjson.schema.json"
-                },
-                {"type": "null"}
-              ]
-            },
-            "edges": {"type": "string", "enum": ["planar", "spherical"]},
-            "orientation": {"type": "string", "const": "counterclockwise"},
-            "bbox": {
-              "type": "array",
-              "items": {"type": "number"},
-              "oneOf": [
-                {
-                  "description": "2D bbox consisting of (xmin, ymin, xmax, ymax)",
-                  "minItems": 4,
-                  "maxItems": 4
-                },
-                {
-                  "description": "3D bbox consisting of (xmin, ymin, zmin, xmax, ymax, zmax)",
-                  "minItems": 6,
-                  "maxItems": 6
-                }
-              ]
-            },
-            "epoch": {"type": "number"}
-          }
-        }
-      },
-      "additionalProperties": false
-    }
-  }
-}

package/src/lib/geo/decode-geo-column.ts

@@ -1,55 +0,0 @@
-// loaders.gl, MIT license
-// Copyright (c) vis.gl contributors
-
-import type {ArrayRowTable, GeoJSONTable, ObjectRowTable, Schema} from '@loaders.gl/schema';
-import type {Feature, Geometry} from '@loaders.gl/schema';
-import {getTableLength, getTableRowAsObject} from '@loaders.gl/schema';
-// import {binaryToGeometry} from '@loaders.gl/gis';
-import {WKBLoader, WKTLoader} from '@loaders.gl/wkt';
-
-import {GeoColumnMetadata, getGeoMetadata} from './decode-geo-metadata';
-
-/** TODO - move to loaders.gl/gis? */
-export function convertWKBTableToGeoJSON(
-  table: ArrayRowTable | ObjectRowTable,
-  schema: Schema
-): GeoJSONTable {
-  const geoMetadata = getGeoMetadata(schema);
-  const primaryColumn = geoMetadata?.primary_column;
-  if (!primaryColumn) {
-    throw new Error('no geometry column');
-  }
-  const columnMetadata = geoMetadata.columns[primaryColumn];
-
-  const features: Feature[] = [];
-
-  const length = getTableLength(table);
-  for (let rowIndex = 0; rowIndex < length; rowIndex++) {
-    const row = getTableRowAsObject(table, rowIndex);
-    const geometry = parseGeometry(row[primaryColumn], columnMetadata);
-    delete row[primaryColumn];
-    const feature: Feature = {type: 'Feature', geometry: geometry!, properties: row};
-    features.push(feature);
-  }
-
-  return {shape: 'geojson-table', schema, type: 'FeatureCollection', features};
-}
-
-function parseGeometry(geometry: unknown, columnMetadata: GeoColumnMetadata): Geometry | null {
-  switch (columnMetadata.encoding) {
-    case 'wkt':
-      return WKTLoader.parseTextSync?.(geometry as string) || null;
-    case 'wkb':
-    default:
-      const arrayBuffer = ArrayBuffer.isView(geometry)
-        ? geometry.buffer.slice(geometry.byteOffset, geometry.byteOffset + geometry.byteLength)
-        : (geometry as ArrayBuffer);
-      const geojson = WKBLoader.parseSync?.(arrayBuffer, {
-        wkb: {shape: 'geometry'}
-      }) as unknown as Geometry;
-      return geojson; // binaryGeometry ? binaryToGeometry(binaryGeometry) : null;
-    // const binaryGeometry = WKBLoader.parseSync?.(geometry);
-    // ts-ignore
-    // return binaryGeometry ? binaryToGeometry(binaryGeometry) : null;
-  }
-}

package/src/lib/geo/decode-geo-metadata.ts

@@ -1,177 +0,0 @@
-// loaders.gl, MIT license
-// Copyright (c) vis.gl contributors
-import {Schema, Field} from '@loaders.gl/schema';
-
-/* eslint-disable camelcase */
-
-type GeometryType =
-  | 'Point'
-  | 'LineString'
-  | 'Polygon'
-  | 'MultiPoint'
-  | 'MultiLineString'
-  | 'MultiPolygon'
-  | 'GeometryCollection'
-  | 'Point Z'
-  | 'LineString Z'
-  | 'Polygon Z'
-  | 'MultiPoint Z'
-  | 'MultiLineString Z'
-  | 'MultiPolygon Z'
-  | 'GeometryCollection Z';
-
-/**
- * A geoarrow / geoparquet geo metadata object
- * (stored in stringified form in the top level metadata 'geo' key)
- * @see https://github.com/opengeospatial/geoparquet/blob/main/format-specs/geoparquet.md
- * @see https://github.com/geoarrow/geoarrow/blob/main/metadata.md
- * */
-export type GeoMetadata = {
-  version?: string;
-  primary_column?: string;
-  columns: Record<string, GeoColumnMetadata>;
-  [key: string]: unknown;
-};
-
-/** A geoarrow / geoparquet geo metadata for one geometry column  */
-export type GeoColumnMetadata = {
-  encoding: 'wkb' | 'wkt';
-  geometry_types: GeometryType[];
-  crs?: object | null;
-  orientation?: 'counterclockwise';
-  bbox?: [number, number, number, number] | [number, number, number, number, number, number];
-  edges?: 'planar' | 'spherical';
-  epoch?: number;
-  [key: string]: unknown;
-};
-
-/** Parse a key with stringified arrow metadata */
-export function parseJSONStringMetadata(
-  schema: Schema,
-  metadataKey: string
-): Record<string, unknown> | null {
-  const stringifiedMetadata = schema.metadata[metadataKey];
-  if (!stringifiedMetadata) {
-    return null;
-  }
-
-  try {
-    const metadata = JSON.parse(stringifiedMetadata);
-    if (!metadata || typeof metadata !== 'object') {
-      return null;
-    }
-    return metadata;
-  } catch {
-    return null;
-  }
-}
-
-export function unpackJSONStringMetadata(schema: Schema, metadataKey: string): void {
-  const json = parseJSONStringMetadata(schema, metadataKey);
-  for (const [key, value] of Object.entries(json || {})) {
-    schema.metadata[`${metadataKey}.${key}`] =
-      typeof value === 'string' ? value : JSON.stringify(value);
-  }
-}
-
-// GEO METADATA
-
-/**
- * Reads the GeoMetadata object from the metadata
- * @note geoarrow / parquet schema is stringified into a single key-value pair in the parquet metadata */
-export function getGeoMetadata(schema: Schema): GeoMetadata | null {
-  const geoMetadata = parseJSONStringMetadata(schema, 'geo') as GeoMetadata;
-  return geoMetadata;
-}
-
-/**
- * Stores a geoarrow / geoparquet geo metadata object in the schema
- * @note geoarrow / geoparquet geo metadata is a single stringified JSON field
- */
-export function setGeoMetadata(schema: Schema, geoMetadata: GeoMetadata): void {
-  const stringifiedGeoMetadata = JSON.stringify(geoMetadata);
-  schema.metadata.geo = stringifiedGeoMetadata;
-}
-
-/**
- * Unpacks geo metadata into separate metadata fields (parses the long JSON string)
- * @note geoarrow / parquet schema is stringified into a single key-value pair in the parquet metadata
- */
-export function unpackGeoMetadata(schema: Schema): void {
-  const geoMetadata = getGeoMetadata(schema);
-  if (!geoMetadata) {
-    return;
-  }
-
-  // Store Parquet Schema Level Metadata
-
-  const {version, primary_column, columns} = geoMetadata;
-  if (version) {
-    schema.metadata['geo.version'] = version;
-  }
-
-  if (primary_column) {
-    schema.metadata['geo.primary_column'] = primary_column;
-  }
-
-  // store column names as comma separated list
-  schema.metadata['geo.columns'] = Object.keys(columns || {}).join('');
-
-  for (const [columnName, columnMetadata] of Object.entries(columns || {})) {
-    const field = schema.fields.find((field) => field.name === columnName);
-    if (field) {
-      if (field.name === primary_column) {
-        setFieldMetadata(field, 'geo.primary_field', 'true');
-      }
-      unpackGeoFieldMetadata(field, columnMetadata);
-    }
-  }
-}
-
-// eslint-disable-next-line complexity
-function unpackGeoFieldMetadata(field: Field, columnMetadata): void {
-  for (const [key, value] of Object.entries(columnMetadata || {})) {
-    switch (key) {
-      case 'geometry_type':
-        setFieldMetadata(field, `geo.${key}`, (value as string[]).join(','));
-        break;
-      case 'bbox':
-        setFieldMetadata(field, `geo.crs.${key}`, JSON.stringify(value));
-        break;
-      case 'crs':
-        // @ts-ignore
-        for (const [crsKey, crsValue] of Object.entries(value || {})) {
-          switch (crsKey) {
-            case 'id':
-              const crsId =
-                typeof crsValue === 'object'
-                  ? // @ts-ignore
-                    `${crsValue?.authority}:${crsValue?.code}`
-                  : JSON.stringify(crsValue);
-              setFieldMetadata(field, `geo.crs.${crsKey}`, crsId);
-              break;
-            default:
-              setFieldMetadata(
-                field,
-                `geo.crs.${crsKey}`,
-                typeof crsValue === 'string' ? crsValue : JSON.stringify(crsValue)
-              );
-              break;
-          }
-        }
-        break;
-      case 'edges':
-      default:
-        setFieldMetadata(
-          field,
-          `geo.${key}`,
-          typeof value === 'string' ? value : JSON.stringify(value)
-        );
-    }
-  }
-}
-
-function setFieldMetadata(field: Field, key: string, value: string): void {
-  field.metadata = field.metadata || {};
-  field.metadata[key] = value;
-}

package/src/lib/geo/geoparquet-metadata-schema.json

@@ -1,60 +0,0 @@
-{
-  "$schema": "http://json-schema.org/draft-07/schema#",
-  "title": "GeoParquet",
-  "description": "Parquet metadata included in the geo field.",
-  "type": "object",
-  "required": ["version", "primary_column", "columns"],
-  "properties": {
-    "version": {"type": "string", "const": "1.0.0-beta.1"},
-    "primary_column": {"type": "string", "minLength": 1},
-    "columns": {
-      "type": "object",
-      "minProperties": 1,
-      "patternProperties": {
-        ".+": {
-          "type": "object",
-          "required": ["encoding", "geometry_types"],
-          "properties": {
-            "encoding": {"type": "string", "const": "WKB"},
-            "geometry_types": {
-              "type": "array",
-              "uniqueItems": true,
-              "items": {
-                "type": "string",
-                "pattern": "^(GeometryCollection|(Multi)?(Point|LineString|Polygon))( Z)?$"
-              }
-            },
-            "crs": {
-              "oneOf": [
-                {
-                  "$ref": "https://proj.org/schemas/v0.5/projjson.schema.json"
-                },
-                {"type": "null"}
-              ]
-            },
-            "edges": {"type": "string", "enum": ["planar", "spherical"]},
-            "orientation": {"type": "string", "const": "counterclockwise"},
-            "bbox": {
-              "type": "array",
-              "items": {"type": "number"},
-              "oneOf": [
-                {
-                  "description": "2D bbox consisting of (xmin, ymin, xmax, ymax)",
-                  "minItems": 4,
-                  "maxItems": 4
-                },
-                {
-                  "description": "3D bbox consisting of (xmin, ymin, zmin, xmax, ymax, zmax)",
-                  "minItems": 6,
-                  "maxItems": 6
-                }
-              ]
-            },
-            "epoch": {"type": "number"}
-          }
-        }
-      },
-      "additionalProperties": false
-    }
-  }
-}

package/src/lib/geo/geoparquet-metadata-schema.ts

@@ -1,70 +0,0 @@
-// loaders.gl, MIT license
-// Copyright (c) vis.gl contributors
-
-/* eslint-disable camelcase */
-
-/**
- * Geoparquet JSON schema for geo metadata
- * @see https://github.com/geoarrow/geoarrow/blob/main/metadata.md
- * @see https://github.com/opengeospatial/geoparquet/blob/main/format-specs/geoparquet.md
- */
-export const GEOPARQUET_METADATA_JSON_SCHEMA = {
-  $schema: 'http://json-schema.org/draft-07/schema#',
-  title: 'GeoParquet',
-  description: 'Parquet metadata included in the geo field.',
-  type: 'object',
-  required: ['version', 'primary_column', 'columns'],
-  properties: {
-    version: {type: 'string', const: '1.0.0-beta.1'},
-    primary_column: {type: 'string', minLength: 1},
-    columns: {
-      type: 'object',
-      minProperties: 1,
-      patternProperties: {
-        '.+': {
-          type: 'object',
-          required: ['encoding', 'geometry_types'],
-          properties: {
-            encoding: {type: 'string', const: 'WKB'},
-            geometry_types: {
-              type: 'array',
-              uniqueItems: true,
-              items: {
-                type: 'string',
-                pattern: '^(GeometryCollection|(Multi)?(Point|LineString|Polygon))( Z)?$'
-              }
-            },
-            crs: {
-              oneOf: [
-                {
-                  $ref: 'https://proj.org/schemas/v0.5/projjson.schema.json'
-                },
-                {type: 'null'}
-              ]
-            },
-            edges: {type: 'string', enum: ['planar', 'spherical']},
-            orientation: {type: 'string', const: 'counterclockwise'},
-            bbox: {
-              type: 'array',
-              items: {type: 'number'},
-              oneOf: [
-                {
-                  description: '2D bbox consisting of (xmin, ymin, xmax, ymax)',
-                  minItems: 4,
-                  maxItems: 4
-                },
-                {
-                  description: '3D bbox consisting of (xmin, ymin, zmin, xmax, ymax, zmax)',
-                  minItems: 6,
-                  maxItems: 6
-                }
-              ]
-            },
-            epoch: {type: 'number'}
-          }
-        }
-      },
-      additionalProperties: false
-    }
-  }
-};