@loaders.gl/parquet 4.5.3 → 5.0.0-alpha.10

package/src/lib/geo/geospatial-metadata.ts

@@ -0,0 +1,547 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import * as arrow from 'apache-arrow';
+
+import {
+  getGeoMetadata,
+  getMetadataValue,
+  setGeoMetadata,
+  setMetadataValue,
+  type GeoArrowEncoding,
+  type GeoArrowMetadata,
+  type GeoColumnMetadata,
+  type GeoMetadata,
+  type GeoParquetGeometryType,
+  type Field,
+  type Schema
+} from '@loaders.gl/schema';
+import {
+  GeoArrowMetadataSchema,
+  GeoParquetMetadataSchema
+} from '@loaders.gl/gis/geospatial-metadata-zod-schema';
+import {convertArrowToSchema} from '@loaders.gl/schema-utils';
+
+const GEOARROW_EXTENSION_NAME_KEY = 'ARROW:extension:name';
+const GEOARROW_EXTENSION_METADATA_KEY = 'ARROW:extension:metadata';
+const GEOPARQUET_VERSION = '1.1.0';
+
+const GEOARROW_ENCODINGS = [
+  'geoarrow.geometry',
+  'geoarrow.geometrycollection',
+  'geoarrow.multipolygon',
+  'geoarrow.polygon',
+  'geoarrow.multilinestring',
+  'geoarrow.linestring',
+  'geoarrow.multipoint',
+  'geoarrow.point',
+  'geoarrow.box',
+  'geoarrow.wkb',
+  'geoarrow.wkt'
+] as const satisfies GeoArrowEncoding[];
+
+const GEOPARQUET_TO_GEOARROW_ENCODINGS = {
+  wkb: 'geoarrow.wkb',
+  geometry: 'geoarrow.wkb',
+  geography: 'geoarrow.wkb',
+  point: 'geoarrow.point',
+  linestring: 'geoarrow.linestring',
+  polygon: 'geoarrow.polygon',
+  multipoint: 'geoarrow.multipoint',
+  multilinestring: 'geoarrow.multilinestring',
+  multipolygon: 'geoarrow.multipolygon'
+} as const satisfies Record<string, GeoArrowEncoding>;
+
+const GEOARROW_TO_GEOPARQUET_ENCODINGS = {
+  'geoarrow.wkb': 'wkb',
+  'geoarrow.point': 'point',
+  'geoarrow.linestring': 'linestring',
+  'geoarrow.polygon': 'polygon',
+  'geoarrow.multipoint': 'multipoint',
+  'geoarrow.multilinestring': 'multilinestring',
+  'geoarrow.multipolygon': 'multipolygon'
+} as const;
+
+const GEOARROW_TO_GEOMETRY_TYPE = {
+  'geoarrow.point': 'Point',
+  'geoarrow.linestring': 'LineString',
+  'geoarrow.polygon': 'Polygon',
+  'geoarrow.multipoint': 'MultiPoint',
+  'geoarrow.multilinestring': 'MultiLineString',
+  'geoarrow.multipolygon': 'MultiPolygon'
+} as const satisfies Record<string, GeoParquetGeometryType>;
+
+/** Returns GeoArrow geometry field metadata keyed by column name. */
+function getGeometryColumnsFromSchema(schema: Schema): Record<string, GeoArrowMetadata> {
+  const geometryColumns: Record<string, GeoArrowMetadata> = {};
+  for (const field of schema.fields || []) {
+    const metadata = getGeometryMetadataForField(field.metadata || {});
+    if (metadata) {
+      geometryColumns[field.name] = metadata;
+    }
+  }
+  return geometryColumns;
+}
+
+/** Extracts GeoArrow metadata from one schema field metadata object. */
+function getGeometryMetadataForField(
+  fieldMetadata: Record<string, string>
+): GeoArrowMetadata | null {
+  let metadata: GeoArrowMetadata | null = null;
+  let geoEncoding = getMetadataValue(fieldMetadata, GEOARROW_EXTENSION_NAME_KEY);
+  if (geoEncoding) {
+    geoEncoding = geoEncoding.toLowerCase();
+    if (geoEncoding === 'wkb') {
+      geoEncoding = 'geoarrow.wkb';
+    }
+    if (geoEncoding === 'wkt') {
+      geoEncoding = 'geoarrow.wkt';
+    }
+    if (GEOARROW_ENCODINGS.includes(geoEncoding as GeoArrowEncoding)) {
+      metadata ||= {};
+      metadata.encoding = geoEncoding as GeoArrowEncoding;
+    }
+  }
+
+  const columnMetadata = getMetadataValue(fieldMetadata, GEOARROW_EXTENSION_METADATA_KEY);
+  if (columnMetadata) {
+    try {
+      const parsedMetadata = parseGeoArrowMetadata(JSON.parse(columnMetadata));
+      metadata = {
+        ...(metadata || {}),
+        ...parsedMetadata
+      };
+    } catch {
+      return metadata;
+    }
+  }
+
+  return metadata;
+}
+
+/** Validates GeoArrow metadata, treating a mislabeled string CRS as opaque metadata. */
+function parseGeoArrowMetadata(value: unknown): GeoArrowMetadata {
+  const result = GeoArrowMetadataSchema.safeParse(value);
+  if (result.success) {
+    return result.data;
+  }
+  if (value && typeof value === 'object' && typeof (value as {crs?: unknown}).crs === 'string') {
+    const {crs_type: _untrustedCRSType, ...opaqueMetadata} = value as Record<string, unknown>;
+    const opaqueResult = GeoArrowMetadataSchema.safeParse(opaqueMetadata);
+    if (opaqueResult.success) {
+      return opaqueResult.data;
+    }
+  }
+  throw result.error;
+}
+
+/**
+ * Applies GeoParquet schema metadata to matching geometry fields as GeoArrow field metadata.
+ *
+ * @param schema - Schema to annotate.
+ * @returns The same schema reference.
+ */
+export function applyGeoParquetToFieldMetadata(schema: Schema): Schema {
+  const geoMetadata = getGeoMetadata(schema.metadata);
+  if (!geoMetadata) {
+    return schema;
+  }
+
+  for (const field of schema.fields) {
+    const columnMetadata = geoMetadata.columns?.[field.name];
+    if (!columnMetadata) {
+      continue;
+    }
+
+    const fieldMetadata = getGeoArrowMetadataFromGeoParquetField(field, columnMetadata);
+    if (!fieldMetadata) {
+      continue;
+    }
+
+    field.metadata ||= {};
+    setMetadataValue(field.metadata, GEOARROW_EXTENSION_NAME_KEY, fieldMetadata.extensionName);
+
+    if (fieldMetadata.extensionMetadata) {
+      setMetadataValue(
+        field.metadata,
+        GEOARROW_EXTENSION_METADATA_KEY,
+        JSON.stringify(fieldMetadata.extensionMetadata)
+      );
+    }
+  }
+
+  return schema;
+}
+
+/**
+ * Creates an Arrow JS table whose schema metadata matches a loaders.gl schema.
+ *
+ * @param table - Arrow JS table to rewrap.
+ * @param schema - loaders.gl schema containing desired field/schema metadata.
+ * @returns Table using the updated Arrow JS schema.
+ */
+export function applyLoadersSchemaToArrowTable(table: arrow.Table, schema: Schema): arrow.Table {
+  const arrowSchema = createArrowJsSchema(table.schema, schema);
+  return new arrow.Table(arrowSchema, table.batches);
+}
+
+/**
+ * Normalizes an Arrow wrapper so GeoParquet metadata is preserved and GeoArrow field metadata is attached.
+ *
+ * @param arrowTable - loaders.gl Arrow table wrapper.
+ * @param schemaMetadata - Optional schema metadata override from the Parquet footer.
+ * @returns Arrow table wrapper with synchronized schema metadata.
+ */
+export function normalizeArrowTableGeoMetadata(
+  arrowTable: {shape: 'arrow-table'; data: arrow.Table; schema?: Schema},
+  schemaMetadata?: Map<string, string> | Record<string, string>
+): {shape: 'arrow-table'; data: arrow.Table; schema: Schema} {
+  const schema = convertArrowToSchema(arrowTable.data.schema);
+  mergeSchemaMetadata(schema, schemaMetadata);
+  applyGeoParquetToFieldMetadata(schema);
+
+  return {
+    shape: 'arrow-table',
+    data: applyLoadersSchemaToArrowTable(arrowTable.data, schema),
+    schema
+  };
+}
+
+/**
+ * Synthesizes GeoParquet schema metadata from GeoArrow field metadata when needed.
+ *
+ * @param schema - Schema to inspect and update.
+ * @returns The same schema reference.
+ */
+export function ensureGeoParquetMetadata(schema: Schema): Schema {
+  schema.metadata ||= {};
+  const rawExistingGeoMetadata = getGeoMetadata(schema.metadata);
+  if (isValidGeoParquetMetadata(rawExistingGeoMetadata)) {
+    return schema;
+  }
+  const existingGeoMetadata: GeoMetadata | undefined = rawExistingGeoMetadata || undefined;
+
+  const geometryColumns = getGeometryColumnsFromSchema(schema);
+  const geometryColumnNames = Object.keys(geometryColumns);
+  if (geometryColumnNames.length === 0) {
+    return schema;
+  }
+
+  const geoMetadata: GeoMetadata = {
+    version: GEOPARQUET_VERSION,
+    primary_column: geometryColumnNames[0],
+    columns: {}
+  };
+
+  for (const geometryColumnName of geometryColumnNames) {
+    const geometryMetadata = geometryColumns[geometryColumnName];
+    const synthesizedColumnMetadata = synthesizeGeoParquetColumnMetadata(geometryMetadata);
+    if (!synthesizedColumnMetadata) {
+      continue;
+    }
+
+    const existingColumnMetadata = existingGeoMetadata?.columns?.[geometryColumnName];
+    geoMetadata.columns[geometryColumnName] = {
+      ...pickValidOptionalGeoParquetColumnMetadata(existingColumnMetadata),
+      ...synthesizedColumnMetadata
+    };
+  }
+
+  if (Object.keys(geoMetadata.columns).length === 0) {
+    return schema;
+  }
+
+  setGeoMetadata(schema.metadata, geoMetadata);
+  return schema;
+}
+
+/**
+ * Ensures an Arrow table wrapper carries valid GeoParquet schema metadata when GeoArrow fields are present.
+ *
+ * @param arrowTable - Arrow table wrapper to inspect and update.
+ * @returns Updated Arrow table wrapper.
+ */
+export function ensureGeoParquetMetadataOnArrowTable(arrowTable: {
+  shape: 'arrow-table';
+  data: arrow.Table;
+  schema?: Schema;
+}): {shape: 'arrow-table'; data: arrow.Table; schema: Schema} {
+  const schema = arrowTable.schema || convertArrowToSchema(arrowTable.data.schema);
+  ensureGeoParquetMetadata(schema);
+
+  return {
+    shape: 'arrow-table',
+    data: applyLoadersSchemaToArrowTable(arrowTable.data, schema),
+    schema
+  };
+}
+
+/**
+ * Returns Parquet file key/value metadata as a plain map.
+ *
+ * @param parquetMetadata - parquet-wasm metadata object.
+ * @returns Metadata map.
+ */
+export function getParquetFileMetadataMap(parquetMetadata: {
+  fileMetadata(): {keyValueMetadata(): Map<string, string>};
+}): Map<string, string> {
+  return parquetMetadata.fileMetadata().keyValueMetadata();
+}
+
+function mergeSchemaMetadata(
+  schema: Schema,
+  schemaMetadata?: Map<string, string> | Record<string, string>
+): void {
+  if (!schemaMetadata) {
+    return;
+  }
+
+  schema.metadata ||= {};
+  for (const [metadataKey, metadataValue] of getMetadataEntries(schemaMetadata)) {
+    setMetadataValue(schema.metadata, metadataKey, metadataValue);
+  }
+}
+
+function createArrowJsSchema(baseSchema: arrow.Schema, schema: Schema): arrow.Schema {
+  const fields = baseSchema.fields.map(baseField => {
+    const nextField = schema.fields.find(field => field.name === baseField.name);
+    if (!nextField?.metadata) {
+      return baseField;
+    }
+
+    return baseField.clone({
+      metadata: createMetadataMap(nextField.metadata)
+    });
+  });
+
+  return new arrow.Schema(
+    fields,
+    createMetadataMap(schema.metadata),
+    baseSchema.dictionaries,
+    baseSchema.metadataVersion
+  );
+}
+
+function getGeoArrowMetadataFromGeoParquetField(
+  field: Field,
+  columnMetadata: GeoColumnMetadata
+): {extensionName: GeoArrowEncoding; extensionMetadata?: Record<string, unknown>} | null {
+  const normalizedEncoding = String(columnMetadata.encoding || '').toLowerCase();
+  const extensionName =
+    GEOPARQUET_TO_GEOARROW_ENCODINGS[
+      normalizedEncoding as keyof typeof GEOPARQUET_TO_GEOARROW_ENCODINGS
+    ];
+
+  if (!extensionName) {
+    return null;
+  }
+
+  const isBinaryGeospatialEncoding =
+    normalizedEncoding === 'wkb' ||
+    normalizedEncoding === 'geometry' ||
+    normalizedEncoding === 'geography';
+  if (isBinaryGeospatialEncoding && !isBinaryFieldType(field.type)) {
+    return null;
+  }
+
+  if (
+    !isBinaryGeospatialEncoding &&
+    (isBinaryFieldType(field.type) || isStringFieldType(field.type))
+  ) {
+    return null;
+  }
+
+  const extensionMetadata: Record<string, unknown> = {};
+  if (columnMetadata.crs === undefined) {
+    extensionMetadata.crs = 'OGC:CRS84';
+    extensionMetadata.crs_type = 'authority_code';
+  } else if (columnMetadata.crs !== null) {
+    extensionMetadata.crs = columnMetadata.crs;
+    extensionMetadata.crs_type = 'projjson';
+  }
+  if (columnMetadata.edges && columnMetadata.edges !== 'planar') {
+    extensionMetadata.edges = columnMetadata.edges;
+  }
+  if (normalizedEncoding === 'geography' && !extensionMetadata.edges) {
+    extensionMetadata.edges = 'spherical';
+  }
+  for (const [metadataKey, metadataValue] of Object.entries(columnMetadata)) {
+    if (
+      metadataValue !== undefined &&
+      metadataValue !== null &&
+      !['encoding', 'geometry_types', 'crs', 'edges'].includes(metadataKey)
+    ) {
+      extensionMetadata[metadataKey] = metadataValue;
+    }
+  }
+
+  return Object.keys(extensionMetadata).length > 0
+    ? {extensionName, extensionMetadata}
+    : {extensionName};
+}
+
+function isValidGeoParquetMetadata(geoMetadata: GeoMetadata | null): boolean {
+  return GeoParquetMetadataSchema.safeParse(geoMetadata).success;
+}
+
+function synthesizeGeoParquetColumnMetadata(
+  geometryMetadata: GeoArrowMetadata
+): GeoColumnMetadata | null {
+  const encoding = geometryMetadata.encoding
+    ? GEOARROW_TO_GEOPARQUET_ENCODINGS[
+        geometryMetadata.encoding as keyof typeof GEOARROW_TO_GEOPARQUET_ENCODINGS
+      ]
+    : null;
+
+  if (!encoding) {
+    return null;
+  }
+
+  const geometryTypes = Array.isArray(geometryMetadata.geometry_types)
+    ? geometryMetadata.geometry_types
+    : inferGeometryTypesFromGeoArrowEncoding(geometryMetadata.encoding);
+
+  const columnMetadata: GeoColumnMetadata = {
+    encoding,
+    geometry_types: geometryTypes
+  };
+
+  const preservedMetadata = preserveUnknownGeoParquetColumnMetadata(geometryMetadata);
+
+  if (geometryMetadata.crs === undefined) {
+    columnMetadata.crs = null;
+  } else if (typeof geometryMetadata.crs === 'object') {
+    columnMetadata.crs = geometryMetadata.crs;
+  } else if (
+    typeof geometryMetadata.crs !== 'string' ||
+    !isDefaultCRS84Identifier(geometryMetadata.crs)
+  ) {
+    return null;
+  }
+  if (geometryMetadata.edges) {
+    columnMetadata.edges = geometryMetadata.edges;
+  }
+
+  return {...preservedMetadata, ...columnMetadata};
+}
+
+/** Returns whether a compact GeoArrow CRS identifier has GeoParquet's default CRS semantics. */
+function isDefaultCRS84Identifier(crs: string): boolean {
+  const normalizedCRS = crs.toUpperCase();
+  return normalizedCRS === 'OGC:CRS84' || normalizedCRS === 'EPSG:4326';
+}
+
+function inferGeometryTypesFromGeoArrowEncoding(
+  encoding?: GeoArrowEncoding
+): GeoParquetGeometryType[] {
+  const geometryType = encoding
+    ? GEOARROW_TO_GEOMETRY_TYPE[encoding as keyof typeof GEOARROW_TO_GEOMETRY_TYPE]
+    : null;
+  return geometryType ? [geometryType] : [];
+}
+
+function pickValidOptionalGeoParquetColumnMetadata(
+  columnMetadata: GeoColumnMetadata | undefined
+): Partial<GeoColumnMetadata> {
+  if (!columnMetadata) {
+    return {};
+  }
+
+  const nextColumnMetadata = preserveUnknownGeoParquetColumnMetadata(columnMetadata);
+
+  if (columnMetadata.orientation === 'counterclockwise') {
+    nextColumnMetadata.orientation = columnMetadata.orientation;
+  }
+
+  if (isValidBBox(columnMetadata.bbox)) {
+    nextColumnMetadata.bbox = columnMetadata.bbox;
+  }
+
+  if (typeof columnMetadata.epoch === 'number' && Number.isFinite(columnMetadata.epoch)) {
+    nextColumnMetadata.epoch = columnMetadata.epoch;
+  }
+
+  if (columnMetadata.covering && typeof columnMetadata.covering === 'object') {
+    nextColumnMetadata.covering = columnMetadata.covering;
+  }
+
+  return nextColumnMetadata;
+}
+
+/** Preserves vendor and future GeoParquet column keys without trusting core fields. */
+function preserveUnknownGeoParquetColumnMetadata(
+  columnMetadata: Record<string, unknown> | undefined
+): Record<string, unknown> {
+  if (!columnMetadata) {
+    return {};
+  }
+
+  const preservedMetadata: Record<string, unknown> = {};
+  for (const [metadataKey, metadataValue] of Object.entries(columnMetadata)) {
+    if (
+      metadataValue !== undefined &&
+      ![
+        'encoding',
+        'geometry_types',
+        'crs',
+        'edges',
+        'orientation',
+        'bbox',
+        'epoch',
+        'covering'
+      ].includes(metadataKey)
+    ) {
+      preservedMetadata[metadataKey] = metadataValue;
+    }
+  }
+  return preservedMetadata;
+}
+
+/** Returns whether an Arrow field is a binary value capable of carrying WKB bytes. */
+function isBinaryFieldType(fieldType: Field['type']): boolean {
+  return typeof fieldType === 'string' && (fieldType === 'binary' || fieldType === 'binary-view');
+}
+
+/** Returns whether an Arrow field is a UTF-8 value capable of carrying WKT text. */
+function isStringFieldType(fieldType: Field['type']): boolean {
+  return typeof fieldType === 'string' && (fieldType === 'utf8' || fieldType === 'utf8-view');
+}
+
+function isValidBBox(
+  bbox: GeoColumnMetadata['bbox']
+): bbox is NonNullable<GeoColumnMetadata['bbox']> {
+  if (!Array.isArray(bbox) || ![4, 6, 8].includes(bbox.length)) {
+    return false;
+  }
+
+  return bbox.every(value => typeof value === 'number' && Number.isFinite(value));
+}
+
+function getMetadataEntries(
+  metadata: Map<string, string> | Record<string, string>
+): [string, string][] {
+  if (metadata instanceof Map) {
+    return [...metadata.entries()];
+  }
+  return Object.entries(metadata);
+}
+
+function createMetadataMap(
+  metadata: Map<string, string> | Record<string, string> | undefined
+): Map<string, string> {
+  if (!metadata) {
+    return new Map();
+  }
+
+  if (metadata instanceof Map) {
+    return new Map(metadata);
+  }
+
+  return new Map(
+    Object.entries(metadata).map(([metadataKey, metadataValue]) => [
+      metadataKey,
+      String(metadataValue)
+    ])
+  );
+}

package/src/lib/parquet-bloom-filter-planner.ts

@@ -0,0 +1,86 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {
+  ParquetColumnChunkMetadata,
+  ParquetPredicate,
+  ParquetPredicateValue,
+  ParquetRowGroupMetadata
+} from '../parquet-source-types';
+
+/** One conservative Bloom-filter probe that can be applied before decoding a row group. */
+export type ParquetBloomFilterProbe = {
+  /** Column chunk containing the split-block Bloom filter. */
+  readonly column: ParquetColumnChunkMetadata;
+  /** Plain-encoded values to hash and probe. */
+  readonly values: readonly ParquetPredicateValue[];
+};
+
+/**
+ * Finds equality and membership predicates backed by row-group Bloom-filter metadata.
+ *
+ * Only leaves and conjunctions are returned. Disjunctions and negations are intentionally
+ * excluded because probing an individual branch could otherwise create a false negative.
+ */
+export function getParquetBloomFilterProbes(
+  predicate: ParquetPredicate,
+  rowGroup: ParquetRowGroupMetadata
+): ParquetBloomFilterProbe[] {
+  const probes: ParquetBloomFilterProbe[] = [];
+  collectParquetBloomFilterProbes(predicate, rowGroup, probes);
+  const uniqueProbes = new Map<string, ParquetBloomFilterProbe>();
+  for (const probe of probes) {
+    const key = probe.column.path.join('\0');
+    const previousProbe = uniqueProbes.get(key);
+    if (previousProbe) {
+      uniqueProbes.set(key, {
+        column: probe.column,
+        values: [...previousProbe.values, ...probe.values]
+      });
+    } else {
+      uniqueProbes.set(key, probe);
+    }
+  }
+  return [...uniqueProbes.values()];
+}
+
+function collectParquetBloomFilterProbes(
+  predicate: ParquetPredicate,
+  rowGroup: ParquetRowGroupMetadata,
+  probes: ParquetBloomFilterProbe[]
+): void {
+  if (predicate.op === 'and') {
+    for (const child of predicate.args) {
+      collectParquetBloomFilterProbes(child, rowGroup, probes);
+    }
+    return;
+  }
+  if (predicate.op !== '=' && predicate.op !== 'in') {
+    return;
+  }
+  const path =
+    typeof predicate.args[0].property === 'string'
+      ? [predicate.args[0].property]
+      : [...predicate.args[0].property];
+  const column = rowGroup.columns.find(
+    candidate =>
+      candidate.path.length === path.length &&
+      candidate.path.every((part, index) => part === path[index])
+  );
+  if (
+    !column ||
+    column.bloomFilterOffset === undefined ||
+    column.bloomFilterByteLength === undefined ||
+    column.bloomFilterByteLength <= 0
+  ) {
+    return;
+  }
+  probes.push({
+    column,
+    values:
+      predicate.op === '='
+        ? [predicate.args[1]]
+        : [...(predicate.args[1] as readonly ParquetPredicateValue[])]
+  });
+}