@loaders.gl/core 5.0.0-alpha.11 → 5.0.0-alpha.13

package/src/schema/geometry/geoarrow.ts

@@ -0,0 +1,116 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+/** Canonical GeoArrow extension names. */
+export type GeoArrowEncoding =
+  | 'geoarrow.geometry'
+  | 'geoarrow.geometrycollection'
+  | 'geoarrow.multipolygon'
+  | 'geoarrow.polygon'
+  | 'geoarrow.multilinestring'
+  | 'geoarrow.linestring'
+  | 'geoarrow.multipoint'
+  | 'geoarrow.point'
+  | 'geoarrow.box'
+  | 'geoarrow.wkb'
+  | 'geoarrow.wkt';
+
+/** Preferred output policy for loader-facing GeoArrow conversions. */
+export type GeoArrowEncodingPreference = 'geoarrow.wkb' | 'geoarrow.geometry' | 'optimized';
+
+/** Coordinate dimensions understood by GeoArrow and GeoParquet. */
+export type GeoArrowDimension = 'xy' | 'xyz' | 'xym' | 'xyzm';
+
+/** Coordinate memory layouts understood by GeoArrow. */
+export type GeoArrowCoordinateLayout = 'interleaved' | 'separated';
+
+/** Offset widths supported by variable-length GeoArrow arrays. */
+export type GeoArrowOffsetType = 'int32' | 'int64';
+
+/** One axis-aligned GeoArrow Box value in canonical child order. */
+export type GeoArrowBox = {
+  xmin: number;
+  ymin: number;
+  xmax: number;
+  ymax: number;
+  zmin?: number;
+  zmax?: number;
+  mmin?: number;
+  mmax?: number;
+};
+
+/** Non-planar edge interpretations used by geospatial metadata. */
+export type GeoArrowEdgeType = 'spherical' | 'vincenty' | 'thomas' | 'andoyer' | 'karney';
+
+/** Geometry type names used by GeoParquet metadata. */
+export type GeoParquetGeometryType =
+  | 'Point'
+  | 'LineString'
+  | 'Polygon'
+  | 'MultiPoint'
+  | 'MultiLineString'
+  | 'MultiPolygon'
+  | 'GeometryCollection'
+  | 'Point Z'
+  | 'LineString Z'
+  | 'Polygon Z'
+  | 'MultiPoint Z'
+  | 'MultiLineString Z'
+  | 'MultiPolygon Z'
+  | 'GeometryCollection Z'
+  | 'Point M'
+  | 'LineString M'
+  | 'Polygon M'
+  | 'MultiPoint M'
+  | 'MultiLineString M'
+  | 'MultiPolygon M'
+  | 'GeometryCollection M'
+  | 'Point ZM'
+  | 'LineString ZM'
+  | 'Polygon ZM'
+  | 'MultiPoint ZM'
+  | 'MultiLineString ZM'
+  | 'MultiPolygon ZM'
+  | 'GeometryCollection ZM';
+
+/** JSON-compatible CRS value accepted by GeoArrow extension metadata. */
+export type GeoArrowCRSValue = Record<string, unknown> | string;
+
+/** CRS representation tag used by GeoArrow extension metadata. */
+export type GeoArrowCRSType = 'projjson' | 'wkt2:2019' | 'authority_code' | 'srid';
+
+/** CRS metadata attached to a GeoArrow field. */
+export type GeoArrowCRSMetadata =
+  | {crs?: undefined; crs_type?: undefined}
+  | {crs: Record<string, unknown>; crs_type?: 'projjson'}
+  | {crs: string; crs_type: 'wkt2:2019' | 'authority_code' | 'srid'}
+  | {crs: string; crs_type?: undefined};
+
+/** Metadata attached to a GeoArrow extension field. */
+export type GeoArrowMetadata = GeoArrowCRSMetadata & {
+  encoding?: GeoArrowEncoding;
+  edges?: GeoArrowEdgeType;
+  geometry_types?: GeoParquetGeometryType[];
+  [key: string]: unknown;
+};
+
+/** Metadata for one geometry column in a GeoParquet file. */
+export type GeoColumnMetadata = {
+  encoding: string;
+  geometry_types: GeoParquetGeometryType[];
+  crs?: Record<string, unknown> | null;
+  orientation?: 'counterclockwise';
+  bbox?: number[];
+  edges?: 'planar' | GeoArrowEdgeType;
+  epoch?: number;
+  [key: string]: unknown;
+};
+
+/** File-level GeoParquet metadata stored under the `geo` key. */
+export type GeoMetadata = {
+  version?: string;
+  primary_column?: string;
+  columns: Record<string, GeoColumnMetadata>;
+  [key: string]: unknown;
+};

package/src/schema/geospatial-metadata-zod-schema.ts

@@ -0,0 +1,163 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright vis.gl contributors
+
+import {z} from 'zod';
+import type {
+  GeoArrowMetadata,
+  GeoColumnMetadata,
+  GeoMetadata,
+  GeoParquetGeometryType
+} from './geometry/geoarrow';
+
+/** GeoParquet geometry type names, including optional Z dimensions. */
+export const GeoParquetGeometryTypeSchema = z.enum([
+  'Point',
+  'LineString',
+  'Polygon',
+  'MultiPoint',
+  'MultiLineString',
+  'MultiPolygon',
+  'GeometryCollection',
+  'Point Z',
+  'LineString Z',
+  'Polygon Z',
+  'MultiPoint Z',
+  'MultiLineString Z',
+  'MultiPolygon Z',
+  'GeometryCollection Z',
+  'Point M',
+  'LineString M',
+  'Polygon M',
+  'MultiPoint M',
+  'MultiLineString M',
+  'MultiPolygon M',
+  'GeometryCollection M',
+  'Point ZM',
+  'LineString ZM',
+  'Polygon ZM',
+  'MultiPoint ZM',
+  'MultiLineString ZM',
+  'MultiPolygon ZM',
+  'GeometryCollection ZM'
+]) satisfies z.ZodType<GeoParquetGeometryType>;
+
+const GeoParquetGeometryTypesSchema = z
+  .array(GeoParquetGeometryTypeSchema)
+  .refine(geometryTypes => new Set(geometryTypes).size === geometryTypes.length, {
+    message: 'GeoParquet geometry_types entries must be unique'
+  });
+
+const PROJJSONCRSSchema = z.record(z.string(), z.unknown());
+
+/** Zod schema for metadata attached to one GeoParquet geometry column. */
+export const GeoParquetColumnMetadataSchema = z
+  .object({
+    encoding: z.enum([
+      'WKB',
+      'wkb',
+      'GEOMETRY',
+      'geometry',
+      'GEOGRAPHY',
+      'geography',
+      'point',
+      'linestring',
+      'polygon',
+      'multipoint',
+      'multilinestring',
+      'multipolygon'
+    ]),
+    geometry_types: GeoParquetGeometryTypesSchema,
+    crs: z.union([PROJJSONCRSSchema, z.null()]).optional(),
+    orientation: z.literal('counterclockwise').optional(),
+    bbox: z
+      .union([
+        z.tuple([z.number(), z.number(), z.number(), z.number()]),
+        z.tuple([z.number(), z.number(), z.number(), z.number(), z.number(), z.number()]),
+        z.tuple([
+          z.number(),
+          z.number(),
+          z.number(),
+          z.number(),
+          z.number(),
+          z.number(),
+          z.number(),
+          z.number()
+        ])
+      ])
+      .optional(),
+    edges: z.enum(['planar', 'spherical', 'vincenty', 'thomas', 'andoyer', 'karney']).optional(),
+    epoch: z.number().finite().optional()
+  })
+  .passthrough() as unknown as z.ZodType<GeoColumnMetadata>;
+
+/** Zod schema for the GeoParquet metadata stored in a Parquet file's `geo` key. */
+export const GeoParquetMetadataSchema = z
+  .object({
+    version: z.string().min(1),
+    primary_column: z.string().min(1),
+    columns: z.record(z.string(), GeoParquetColumnMetadataSchema)
+  })
+  .passthrough()
+  .refine(metadata => Boolean(metadata.columns[metadata.primary_column]), {
+    message: 'GeoParquet primary_column must name an entry in columns',
+    path: ['primary_column']
+  })
+  .superRefine((metadata, context) => {
+    if (!metadata.version.startsWith('2.')) return;
+    for (const [columnName, column] of Object.entries(metadata.columns)) {
+      const encoding = column.encoding.toUpperCase();
+      if (encoding !== 'WKB' && encoding !== 'GEOMETRY' && encoding !== 'GEOGRAPHY') {
+        context.addIssue({
+          code: 'custom',
+          message: 'GeoParquet 2.x geometry columns must use WKB, GEOMETRY, or GEOGRAPHY encoding',
+          path: ['columns', columnName, 'encoding']
+        });
+      }
+    }
+  }) satisfies z.ZodType<GeoMetadata>;
+
+/** Zod schema for metadata stored on one GeoArrow extension field. */
+export const GeoArrowMetadataSchema = z
+  .object({
+    encoding: z
+      .enum([
+        'geoarrow.geometry',
+        'geoarrow.geometrycollection',
+        'geoarrow.multipolygon',
+        'geoarrow.polygon',
+        'geoarrow.multilinestring',
+        'geoarrow.linestring',
+        'geoarrow.multipoint',
+        'geoarrow.point',
+        'geoarrow.box',
+        'geoarrow.wkb',
+        'geoarrow.wkt'
+      ])
+      .optional(),
+    crs: z.union([PROJJSONCRSSchema, z.string()]).optional(),
+    crs_type: z.enum(['projjson', 'wkt2:2019', 'authority_code', 'srid']).optional(),
+    edges: z.enum(['spherical', 'vincenty', 'thomas', 'andoyer', 'karney']).optional(),
+    geometry_types: GeoParquetGeometryTypesSchema.optional()
+  })
+  .passthrough()
+  .superRefine((metadata, context) => {
+    const {crs, crs_type: crsType} = metadata;
+    const valid =
+      (crs === undefined && crsType === undefined) ||
+      (typeof crs === 'object' &&
+        crs !== null &&
+        (crsType === undefined || crsType === 'projjson')) ||
+      (typeof crs === 'string' &&
+        (crsType === undefined ||
+          crsType === 'wkt2:2019' ||
+          crsType === 'authority_code' ||
+          crsType === 'srid'));
+    if (!valid) {
+      context.addIssue({
+        code: 'custom',
+        message: 'GeoArrow crs and crs_type must describe the same CRS representation',
+        path: ['crs_type']
+      });
+    }
+  }) as z.ZodType<GeoArrowMetadata>;

package/src/schema/index.ts

@@ -0,0 +1,169 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+// COMMON CATEGORY
+export type {
+  TypedArray,
+  BigTypedArray,
+  TypedArrayConstructor,
+  BigTypedArrayConstructor,
+  NumberArray,
+  ArrayType,
+  AnyArray
+} from './types/types';
+export {
+  createFloat16Array,
+  convertColorArrayToFloat16,
+  convertColorArrayToFloat32,
+  encodeFloat16,
+  getFloat16Storage,
+  getFloat16Value,
+  isNativeFloat16Array,
+  setFloat16Value
+} from './float16';
+
+// SCHEMAS AND DATA TYPES
+
+export type {Schema, Field, DataType, KeyType, SchemaMetadata, FieldMetadata} from './types/schema';
+export type {Metadata, SchemaWithMetadata} from './metadata';
+export {
+  getGeoMetadata,
+  getMetadataValue,
+  parseJSONStringMetadata,
+  setGeoMetadata,
+  setMetadataValue,
+  unpackGeoMetadata,
+  unpackJSONStringMetadata
+} from './metadata';
+export type {Batch} from './types/batch';
+export type {
+  GeoArrowEncoding,
+  GeoArrowEncodingPreference,
+  GeoArrowDimension,
+  GeoArrowCoordinateLayout,
+  GeoArrowOffsetType,
+  GeoArrowBox,
+  GeoArrowEdgeType,
+  GeoArrowCRSValue,
+  GeoArrowCRSType,
+  GeoArrowCRSMetadata,
+  GeoArrowMetadata,
+  GeoParquetGeometryType,
+  GeoColumnMetadata,
+  GeoMetadata
+} from './geometry/geoarrow';
+
+// TABLE CATEGORY TYPES
+export type {
+  Table,
+  RowTable,
+  ArrayRowTable,
+  ObjectRowTable,
+  GeoJSONTable,
+  ColumnarTable,
+  ArrowTable,
+  Tables
+} from './categories/category-table';
+export type {
+  TableBatch,
+  ArrayRowTableBatch,
+  ObjectRowTableBatch,
+  GeoJSONTableBatch,
+  ColumnarTableBatch,
+  ArrowTableBatch
+} from './categories/category-table';
+
+// MESH CATEGORY
+export type {
+  MeshTable,
+  MeshArrowTable,
+  MeshArrowColumns,
+  IndexedMeshArrowColumns,
+  MeshArrowTableData,
+  IndexedMeshArrowTableData,
+  Mesh,
+  MeshGeometry,
+  MeshAttribute,
+  MeshAttributeOctahedronTransform,
+  MeshAttributeQuantizationTransform,
+  MeshAttributeTransform,
+  MeshAttributes,
+  MeshTopology
+} from './categories/category-mesh';
+export {meshArrowSchema, indexedMeshArrowSchema} from './categories/category-mesh';
+/** @deprecated Import `getMeshBoundingBox` from `@loaders.gl/core/schema-utils`. */
+export {getMeshBoundingBox} from './deprecated/mesh-utils';
+// TEXTURES
+export type {
+  TextureMetadata,
+  Texture,
+  Texture1D,
+  Texture2D,
+  Texture3D,
+  TextureCube,
+  Texture2DArray,
+  TextureCubeArray,
+  TextureLevel,
+  GPUTextureFormat,
+  TextureFormat
+} from './categories/category-texture';
+
+// IMAGES
+export type {ImageDataType, ImageType, ImageTypeEnum} from './categories/category-image';
+
+// GIS CATEGORY - GEOARROW
+// export type {GeoArrowMetadata, GeoArrowEncoding} from './geometry/geoarrow-metadata';
+
+// GIS CATEGORY - GEOJSON
+export type {
+  GeoJSON,
+  Feature,
+  FeatureCollection,
+  Geometry,
+  Position,
+  GeoJsonProperties,
+  Point,
+  MultiPoint,
+  LineString,
+  MultiLineString,
+  Polygon,
+  MultiPolygon,
+  GeometryCollection
+} from './categories/category-gis';
+
+// GIS CATEGORY - FLAT GEOJSON
+export type {
+  FlatFeature,
+  FlatIndexedGeometry,
+  FlatGeometry,
+  FlatGeometryType,
+  FlatPoint,
+  FlatLineString,
+  FlatPolygon
+} from './categories/category-gis';
+
+// GIS CATEGORY - BINARY
+export type {
+  BinaryGeometryType,
+  BinaryGeometry,
+  BinaryPointGeometry,
+  BinaryLineGeometry,
+  BinaryPolygonGeometry,
+  BinaryAttribute
+} from './categories/category-gis';
+export type {
+  BinaryFeatureCollection,
+  BinaryFeature,
+  BinaryPointFeature,
+  BinaryLineFeature,
+  BinaryPolygonFeature
+} from './categories/category-gis';
+export type {
+  ArrowBinaryFeatureTable,
+  ArrowBinaryPointFeature,
+  ArrowBinaryLineFeature,
+  ArrowBinaryPolygonFeature,
+  ArrowBinaryFeatureCollection,
+  BinaryFeatureCollectionLike
+} from './categories/category-gis';

package/src/schema/metadata.ts

@@ -0,0 +1,115 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {GeoMetadata} from './geometry/geoarrow';
+
+/** Metadata container compatible with Apache Arrow metadata maps and plain objects. */
+export type Metadata = Map<string, string> | Record<string, string>;
+
+/** Arrow-like schema shape that exposes schema and field metadata. */
+export type SchemaWithMetadata = {
+  /** Schema-level metadata. */
+  metadata?: Metadata;
+  /** Top-level fields with optional metadata. */
+  fields?: {name: string; metadata?: Metadata}[];
+};
+
+/**
+ * Reads one metadata value from an Arrow-like metadata container.
+ *
+ * @param metadata Metadata map or object.
+ * @param key Metadata key.
+ * @returns Stored value or `null` when absent.
+ */
+export function getMetadataValue(metadata: Metadata, key: string): string | null {
+  return metadata instanceof Map ? metadata.get(key) || null : metadata[key] || null;
+}
+
+/**
+ * Writes one metadata value to an Arrow-like metadata container.
+ *
+ * @param metadata Metadata map or object.
+ * @param key Metadata key.
+ * @param value Metadata value.
+ */
+export function setMetadataValue(metadata: Metadata, key: string, value: string): void {
+  if (metadata instanceof Map) {
+    metadata.set(key, value);
+  } else {
+    metadata[key] = value;
+  }
+}
+
+/** Reads GeoParquet metadata from a schema metadata container. */
+export function getGeoMetadata(metadata: Metadata | undefined): GeoMetadata | null {
+  if (!metadata) {
+    return null;
+  }
+
+  const stringifiedGeoMetadata = getMetadataValue(metadata, 'geo');
+  const geoMetadata = stringifiedGeoMetadata && parseJSONStringMetadata(stringifiedGeoMetadata);
+  if (!geoMetadata) {
+    return null;
+  }
+
+  for (const column of Object.values(geoMetadata.columns || {})) {
+    if (column && typeof column === 'object' && typeof column.encoding === 'string') {
+      column.encoding = column.encoding.toLowerCase();
+    }
+  }
+
+  return geoMetadata as GeoMetadata;
+}
+
+/** Stores GeoParquet metadata under the top-level `geo` key. */
+export function setGeoMetadata(metadata: Metadata, geoMetadata: GeoMetadata): void {
+  setMetadataValue(metadata, 'geo', JSON.stringify(geoMetadata));
+}
+
+/** Unpacks top-level GeoParquet metadata into flattened metadata entries. */
+export function unpackGeoMetadata(metadata: Metadata): void {
+  const geoMetadata = getGeoMetadata(metadata);
+  if (!geoMetadata) {
+    return;
+  }
+
+  const {version, primary_column: primaryColumn, columns} = geoMetadata;
+  if (version) {
+    setMetadataValue(metadata, 'geo.version', version);
+  }
+  if (primaryColumn) {
+    setMetadataValue(metadata, 'geo.primary_column', primaryColumn);
+  }
+  setMetadataValue(metadata, 'geo.columns', Object.keys(columns || {}).join(','));
+}
+
+/** Unpacks one JSON metadata value into flattened metadata entries. */
+export function unpackJSONStringMetadata(metadata: Metadata, metadataKey: string): void {
+  const stringifiedMetadata = getMetadataValue(metadata, metadataKey);
+  const json = stringifiedMetadata && parseJSONStringMetadata(stringifiedMetadata);
+
+  for (const [key, value] of Object.entries(json || {})) {
+    setMetadataValue(
+      metadata,
+      `${metadataKey}.${key}`,
+      typeof value === 'string' ? value : JSON.stringify(value)
+    );
+  }
+}
+
+/** Parses one JSON metadata value into an object. */
+export function parseJSONStringMetadata(
+  stringifiedMetadata: string
+): Record<string, unknown> | null {
+  if (!stringifiedMetadata) {
+    return null;
+  }
+
+  try {
+    const metadata = JSON.parse(stringifiedMetadata);
+    return metadata && typeof metadata === 'object' ? metadata : null;
+  } catch {
+    return null;
+  }
+}

package/src/schema/types/batch.ts

@@ -0,0 +1,47 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import {Schema} from './schema';
+// import type {RecordBatch} from 'apache-arrow';
+
+type ApacheRecordBatch = unknown;
+
+/**
+ * A batch of data (or metadata/schema), from a streaming loader
+ * @see parseInBatches()
+ * @see loadInBatches()
+ */
+export type Batch<MetadataT = unknown> = {
+  /** A batch can contain metadata, data, or in case of unstructured data (JSON) */
+  batchType: 'data' | 'metadata' | 'partial-result' | 'final-result';
+  /** Metadata for this batch if batchType === 'metadata' */
+  metadata?: MetadataT;
+  /** A string identifying the shape of data in this batch (table, etc) */
+  shape: string;
+  /** Schema of the data in this batch */
+  schema?: Schema;
+  /** Data in this batch */
+  data?: unknown;
+  /**  If this is an arrow table. @deprecated Use `data` */
+  recordBatch?: ApacheRecordBatch;
+  /** Length of data in this batch */
+  length: number;
+
+  /** A count of batches received */
+  batch?: number;
+
+  /** A count of batches received */
+  count?: number;
+
+  /** Bytes used so far */
+  bytesUsed?: number;
+  /** cursor is the  */
+  cursor?: number;
+
+  /** MIME type of the data generating this batch */
+  mimeType?: string;
+
+  /** Any other data */
+  [key: string]: unknown;
+};

package/src/schema/types/schema.ts

@@ -0,0 +1,86 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+/** For dictionary type */
+export type KeyType = 'int8' | 'int16' | 'int32' | 'uint8' | 'uint16' | 'uint32';
+
+/** ArrowLike DataType class */
+export type DataType =
+  | 'null'
+  // Primitive types
+  | 'bool'
+  | 'int'
+  | 'int8'
+  | 'int16'
+  | 'int32'
+  | 'int64'
+  | 'uint8'
+  | 'uint16'
+  | 'uint32'
+  | 'uint64'
+  | 'float'
+  | 'float16'
+  | 'float32'
+  | 'float64'
+  | 'binary'
+  | 'binary-view'
+  | 'utf8'
+  | 'utf8-view'
+  | 'date-day'
+  | 'date-millisecond'
+  | 'time-second'
+  | 'time-millisecond'
+  | 'time-microsecond'
+  | 'time-nanosecond'
+  | 'timestamp-second'
+  | 'timestamp-millisecond'
+  | 'timestamp-microsecond'
+  | 'timestamp-nanosecond'
+  | 'interval-daytime'
+  | 'interval-yearmonth'
+  // Composite types
+  | {type: 'decimal'; bitWidth: number; precision: number; scale: number}
+  | {type: 'list'; children: Field[]} // one child only
+  | {type: 'large-list'; children: Field[]} // one child only, with 64-bit offsets
+  | {type: 'struct'; children: Field[]}
+  | {
+      type: 'sparse-union';
+      typeIds: Int32Array;
+      children: Field[];
+      typeIdToChildIndex: {[key: number]: number};
+    }
+  | {
+      type: 'dense-union';
+      typeIds: Int32Array;
+      children: Field[];
+      typeIdToChildIndex: {[key: number]: number};
+    }
+  | {type: 'fixed-size-binary'; byteWidth: number}
+  | {type: 'fixed-size-list'; listSize: number; children: Field[]}
+  | {type: 'map'; keysSorted: boolean; children: Field[]} // Field is "struct"
+  | {type: 'dictionary'; id: number; indices: KeyType; dictionary: DataType; isOrdered: boolean};
+// TODO - unions etc
+
+/**
+ * SchemaMetadata
+ */
+export type SchemaMetadata = Record<string, string>;
+export type FieldMetadata = Record<string, string>;
+
+export type Field = {
+  name: string;
+  type: DataType;
+  nullable?: boolean;
+  metadata?: FieldMetadata;
+};
+
+/**
+ * `Schema` type that that can hold all data required by an Arrow Schema
+ * but is fully serializable. Helper functions make it easy to convert to and from arrow schemas
+ * https://loaders.gl/arrowjs/docs/api-reference/schema
+ */
+export type Schema = {
+  fields: Field[];
+  metadata: SchemaMetadata;
+};