@loaders.gl/tiles 5.0.0-alpha.12 → 5.0.0-alpha.13

package/src/mvt/lib/parse-mvt.ts

@@ -0,0 +1,145 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright vis.gl contributors
+
+import type {ArrowTable, FlatFeature, BinaryFeatureCollection} from '@loaders.gl/core/schema';
+import {convertFeaturesToWKBArrowTable} from '@loaders.gl/arrow-geometry/feature-collection-to-arrow';
+import {
+  convertFlatGeojsonToBinaryFeatureCollection as flatGeojsonToBinary,
+  type GeojsonGeometryInfo
+} from '@loaders.gl/arrow-geometry/legacy-binary';
+import {log} from '@loaders.gl/core/loader-utils';
+import Protobuf from 'pbf';
+
+import {VectorTile} from './vector-tile/vector-tile';
+import {VectorTileFeature} from './vector-tile/vector-tile-feature';
+import {parseMVTGeoJSON, parseToGeoJSONFeatures} from './parse-mvt-geojson';
+
+import type {MVTLoaderOptions} from '../mvt-loader';
+type MVTOptions = Required<MVTLoaderOptions>['mvt'];
+
+/**
+ * Parse MVT arrayBuffer and return GeoJSON.
+ *
+ * @param arrayBuffer A MVT arrayBuffer
+ * @param options
+ * @returns A GeoJSON geometry object or a binary representation
+ */
+export function parseMVT(arrayBuffer: ArrayBuffer, options?: MVTLoaderOptions) {
+  const mvtOptions = checkOptions(options);
+
+  const shape: string | undefined =
+    options?.mvt?.shape || (options as {shape?: string} | undefined)?.shape || 'arrow-table';
+  switch (shape) {
+    case 'columnar-table': // binary + some JS arrays
+      return {shape: 'columnar-table', data: parseToBinary(arrayBuffer, mvtOptions)};
+    case 'geojson-table':
+      return parseMVTGeoJSON(arrayBuffer, {mvt: mvtOptions});
+    case 'arrow-table': {
+      const table: ArrowTable = convertFeaturesToWKBArrowTable(
+        parseToGeoJSONFeatures(arrayBuffer, mvtOptions),
+        {
+          encodingPreference:
+            options?.geoarrow?.encodingPreference || mvtOptions.geoarrow?.encodingPreference
+        }
+      );
+      return table;
+    }
+    case 'binary-geometry':
+      return parseToBinary(arrayBuffer, mvtOptions);
+    default:
+      throw new Error(shape || 'undefined shape');
+  }
+}
+
+function parseToBinary(arrayBuffer: ArrayBuffer, options: MVTOptions): BinaryFeatureCollection {
+  const [flatGeoJsonFeatures, geometryInfo] = parseToFlatGeoJson(arrayBuffer, options);
+
+  const binaryData = flatGeojsonToBinary(flatGeoJsonFeatures, geometryInfo);
+  // Add the original byteLength (as a reasonable approximation of the size of the binary data)
+  // TODO decide where to store extra fields like byteLength (header etc) and document
+  // @ts-ignore
+  binaryData.byteLength = arrayBuffer.byteLength;
+  return binaryData;
+}
+
+function parseToFlatGeoJson(
+  arrayBuffer: ArrayBuffer,
+  options: MVTOptions
+): [FlatFeature[], GeojsonGeometryInfo] {
+  const features: FlatFeature[] = [];
+  const geometryInfo: GeojsonGeometryInfo = {
+    coordLength: 2,
+    pointPositionsCount: 0,
+    pointFeaturesCount: 0,
+    linePositionsCount: 0,
+    linePathsCount: 0,
+    lineFeaturesCount: 0,
+    polygonPositionsCount: 0,
+    polygonObjectsCount: 0,
+    polygonRingsCount: 0,
+    polygonFeaturesCount: 0
+  };
+
+  if (arrayBuffer.byteLength <= 0) {
+    return [features, geometryInfo];
+  }
+
+  const tile = new VectorTile(new Protobuf(arrayBuffer));
+
+  const selectedLayers =
+    options && Array.isArray(options.layers) ? options.layers : Object.keys(tile.layers);
+
+  selectedLayers.forEach((layerName: string) => {
+    const vectorTileLayer = tile.layers[layerName];
+    if (!vectorTileLayer) {
+      return;
+    }
+
+    for (let i = 0; i < vectorTileLayer.length; i++) {
+      const vectorTileFeature = vectorTileLayer.getBinaryFeature(i, geometryInfo);
+      const decodedFeature = getDecodedFeatureBinary(vectorTileFeature, options, layerName);
+      features.push(decodedFeature);
+    }
+  });
+
+  return [features, geometryInfo];
+}
+
+/** Check that options are good */
+function checkOptions(options?: MVTLoaderOptions): MVTOptions {
+  const mvtOptions = options?.mvt as MVTOptions | undefined;
+  if (!mvtOptions) {
+    throw new Error('mvt options required');
+  }
+
+  if (mvtOptions.coordinates === 'wgs84' && !mvtOptions.tileIndex) {
+    throw new Error('MVT Loader: WGS84 coordinates need tileIndex property');
+  }
+
+  if (options?.gis) {
+    log.warn('MVTLoader: "options.gis" is deprecated, use "options.mvt.shape" instead')();
+  }
+
+  return mvtOptions;
+}
+
+/**
+ * @param feature
+ * @param options
+ * @returns decoded binary feature
+ */
+function getDecodedFeatureBinary(
+  feature: VectorTileFeature,
+  options: MVTOptions,
+  layerName: string
+): FlatFeature {
+  const decodedFeature = feature.toBinaryFeature(options.coordinates || 'local', options.tileIndex);
+
+  // Add layer name to GeoJSON properties
+  if (options.layerProperty && decodedFeature.properties) {
+    decodedFeature.properties[options.layerProperty] = layerName;
+  }
+
+  return decodedFeature;
+}

package/src/mvt/lib/parse-tilejson.ts

@@ -0,0 +1,397 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import {Schema} from '@loaders.gl/core/schema';
+import {getSchemaFromTileJSONLayer} from './get-schemas-from-tilejson';
+import {
+  TileJSONMetadataSchema,
+  TilestatsSchema,
+  type TileJSONMetadata,
+  type TilestatsLayer,
+  type TilestatsLayerAttribute
+} from '../tilejson-zod-schema';
+
+export type TileJSONOptions = {
+  /** max number of values. If not provided, include all values in the source tilestats */
+  maxValues?: number;
+};
+
+/** Parsed and typed TileJSON, merges Tilestats information if present */
+export type TileJSON = {
+  /** Name of the tileset (for presentation in UI) */
+  name?: string;
+  /** A description of the contents or purpose of the tileset */
+  description?: string;
+  /** The version of the tileset */
+  version?: string;
+
+  tileFormat?: string;
+  tilesetType?: string;
+
+  /** Generating application. Tippecanoe adds this. */
+  generator?: string;
+  /** Generating application options. Tippecanoe adds this. */
+  generatorOptions?: string;
+
+  /** Tile indexing scheme */
+  scheme?: 'xyz' | 'tms';
+  /** Sharded URLs */
+  tiles?: string[];
+  /** `[[w, s], [e, n]]`, indicates the limits of the bounding box using the axis units and order of the specified CRS. */
+  boundingBox?: [min: [w: number, s: number], max: [e: number, n: number]];
+  /** May be set to the maxZoom of the first layer */
+  maxZoom?: number | null;
+  /** May be set to the minZoom of the first layer */
+  minZoom?: number | null;
+  center?: number[] | null;
+  htmlAttribution?: string;
+  htmlLegend?: string;
+
+  // Combination of tilestats (if present) and tilejson layer information
+  layers?: TileJSONLayer[];
+
+  /** Any nested JSON metadata */
+  metaJson?: any | null;
+};
+
+export type TileJSONLayer = {
+  /** The name (id) of this layer (tilejson.vector_layers[].id / tilestats.layers[].layer) */
+  name: string;
+
+  /** The description of this layer (tilejson.layer.description) */
+  description?: string;
+
+  // tilestats
+
+  /** The number of features in this layer (tilestats.layer.count) */
+  featureCount?: number;
+  /** The dominant geometry type in this layer (tilestats.layer.geometry) */
+  dominantGeometry?: string;
+  /** An array of details about the first 100 attributes in this layer */
+
+  /**  */
+  minZoom?: number;
+  maxZoom?: number;
+  fields: TileJSONField[];
+
+  schema?: Schema;
+};
+
+export type TileJSONField = {
+  /** The name of this attribute */
+  name: string;
+  description?: string;
+
+  // tilestats
+
+  type: string;
+  /** min value (if there are *any* numbers in the values) */
+  min?: number;
+  /** max value (if there are *any* numbers in the values) */
+  max?: number;
+  /** Number of unique values across the tileset */
+  uniqueValueCount?: number;
+  /** An array of this attribute's first 100 unique values */
+  values?: unknown[];
+};
+
+/**
+ * Parse TileJSON from metadata
+ * @param jsonMetadata - metadata object
+ * @param options - options
+ * @returns - parsed TileJSON
+ */
+// eslint-disable-next-line complexity
+export function parseTileJSON(jsonMetadata: unknown, options: TileJSONOptions): TileJSON | null {
+  if (!jsonMetadata || typeof jsonMetadata !== 'object' || Array.isArray(jsonMetadata)) {
+    return null;
+  }
+
+  const metadata: TileJSONMetadata = TileJSONMetadataSchema.parse(jsonMetadata);
+
+  let tileJSON: TileJSON = {
+    name: metadata.name || '',
+    description: metadata.description || '',
+    ...(metadata.scheme ? {scheme: metadata.scheme} : {})
+  };
+
+  // tippecanoe
+
+  if (typeof metadata.generator === 'string') {
+    tileJSON.generator = metadata.generator;
+  }
+  if (typeof metadata.generator_options === 'string') {
+    tileJSON.generatorOptions = metadata.generator_options;
+  }
+
+  // Tippecanoe emits `antimeridian_adjusted_bounds` instead of `bounds`
+  tileJSON.boundingBox =
+    parseBounds(metadata.bounds) || parseBounds(metadata.antimeridian_adjusted_bounds);
+
+  // TODO - can be undefined - we could set to center of bounds...
+  tileJSON.center = parseCenter(metadata.center);
+  // TODO - can be undefined, we could extract from layers...
+  tileJSON.maxZoom = safeParseFloat(metadata.maxzoom);
+  // TODO - can be undefined, we could extract from layers...
+  tileJSON.minZoom = safeParseFloat(metadata.minzoom);
+
+  // Look for nested metadata embedded in .json field
+  // TODO - document what source this applies to, when is this needed?
+  if (typeof metadata.json === 'string') {
+    // try to parse json
+    try {
+      tileJSON.metaJson = JSON.parse(metadata.json);
+    } catch (_error) {
+      // do nothing
+    }
+  }
+
+  // Look for fields in tilestats
+
+  const tilestats = metadata.tilestats || tileJSON.metaJson?.tilestats;
+  const tileStatsLayers = parseTilestatsLayers(tilestats, options);
+  const tileJSONlayers = parseTileJSONLayers(metadata.vector_layers); // eslint-disable-line camelcase
+  // TODO - merge in description from tilejson
+  const layers = mergeLayers(tileJSONlayers, tileStatsLayers);
+
+  tileJSON = {
+    ...tileJSON,
+    layers
+  };
+
+  if (tileJSON.maxZoom === null && layers.length > 0) {
+    tileJSON.maxZoom = layers[0].maxZoom || null;
+  }
+
+  if (tileJSON.minZoom === null && layers.length > 0) {
+    tileJSON.minZoom = layers[0].minZoom || null;
+  }
+
+  return tileJSON;
+}
+
+function parseTileJSONLayers(layers: TileJSONMetadata['vector_layers']): TileJSONLayer[] {
+  // Look for fields in vector_layers
+  if (!Array.isArray(layers)) {
+    return [];
+  }
+  return layers.map(layer => parseTileJSONLayer(layer));
+}
+
+function parseTileJSONLayer(
+  layer: NonNullable<TileJSONMetadata['vector_layers']>[number]
+): TileJSONLayer {
+  const fields = Object.entries(layer.fields || []).map(([key, datatype]) => ({
+    name: key,
+    ...attributeTypeToFieldType(String(datatype))
+  }));
+  const layer2 = {...layer};
+  delete layer2.fields;
+  return {
+    name: layer.id || '',
+    ...layer2,
+    fields
+  };
+}
+
+/** parse Layers array from tilestats */
+function parseTilestatsLayers(tilestats: unknown, options: TileJSONOptions): TileJSONLayer[] {
+  const result = TilestatsSchema.safeParse(tilestats);
+  if (result.success && Array.isArray(result.data?.layers)) {
+    // we are in luck!
+    return result.data.layers.map(layer => parseTilestatsForLayer(layer, options));
+  }
+  return [];
+}
+
+function parseTilestatsForLayer(layer: TilestatsLayer, options: TileJSONOptions): TileJSONLayer {
+  const fields: TileJSONField[] = [];
+  const indexedAttributes: {[key: string]: TilestatsLayerAttribute[]} = {};
+
+  const attributes = layer.attributes || [];
+  for (const attribute of attributes) {
+    const name = attribute.attribute;
+    if (typeof name === 'string') {
+      // TODO - code copied from kepler.gl, need sample tilestats files to test
+      if (name.split('|').length > 1) {
+        // indexed field
+        const fname = name.split('|')[0];
+        indexedAttributes[fname] = indexedAttributes[fname] || [];
+        indexedAttributes[fname].push(attribute);
+      } else if (!fields[name]) {
+        fields.push(attributeToField(attribute, options));
+      } else {
+        // return (fields[name], attribute);
+      }
+    }
+  }
+  return {
+    name: layer.layer || '',
+    dominantGeometry: layer.geometry,
+    fields
+  };
+}
+
+function mergeLayers(layers: TileJSONLayer[], tilestatsLayers: TileJSONLayer[]): TileJSONLayer[] {
+  return layers.map((layer: TileJSONLayer): TileJSONLayer => {
+    const tilestatsLayer = tilestatsLayers.find(tsLayer => tsLayer.name === layer.name);
+    const fields = tilestatsLayer?.fields || layer.fields || [];
+    const mergedLayer = {
+      ...layer,
+      ...tilestatsLayer,
+      fields
+    } as TileJSONLayer;
+    mergedLayer.schema = getSchemaFromTileJSONLayer(mergedLayer);
+    return mergedLayer;
+  });
+}
+
+/**
+ * bounds should be [minLng, minLat, maxLng, maxLat]
+ *`[[w, s], [e, n]]`, indicates the limits of the bounding box using the axis units and order of the specified CRS.
+ */
+function parseBounds(
+  bounds: string | number[] | undefined
+): [[east: number, south: number], [west: number, north: number]] | undefined {
+  // supported formats
+  // string: "-96.657715,40.126127,-90.140061,43.516689",
+  // array: [ -180, -85.05112877980659, 180, 85.0511287798066 ]
+  const result = fromArrayOrString(bounds);
+  // validate bounds
+  if (
+    Array.isArray(result) &&
+    result.length === 4 &&
+    [result[0], result[2]].every(isLng) &&
+    [result[1], result[3]].every(isLat)
+  ) {
+    return [
+      [result[0], result[1]],
+      [result[2], result[3]]
+    ];
+  }
+  return undefined;
+}
+
+function parseCenter(center: string | number[] | undefined): number[] | null {
+  // supported formats
+  // string: "-96.657715,40.126127,-90.140061,43.516689",
+  // array: [-91.505127,41.615442,14]
+  const result = fromArrayOrString(center);
+  if (
+    Array.isArray(result) &&
+    result.length === 3 &&
+    isLng(result[0]) &&
+    isLat(result[1]) &&
+    isZoom(result[2])
+  ) {
+    return result;
+  }
+  return null;
+}
+
+function safeParseFloat(input: unknown): number | null {
+  const result =
+    typeof input === 'string' ? parseFloat(input) : typeof input === 'number' ? input : null;
+  return result === null || Number.isNaN(result) ? null : result;
+}
+
+// https://github.com/mapbox/tilejson-spec/tree/master/2.2.0
+function isLat(num: any): boolean {
+  return Number.isFinite(num) && num <= 90 && num >= -90;
+}
+function isLng(num: any): boolean {
+  return Number.isFinite(num) && num <= 180 && num >= -180;
+}
+function isZoom(num: any): boolean {
+  return Number.isFinite(num) && num >= 0 && num <= 22;
+}
+function fromArrayOrString(data: string | number[] | undefined): number[] | null {
+  if (typeof data === 'string') {
+    return data.split(',').map(parseFloat);
+  } else if (Array.isArray(data)) {
+    return data;
+  }
+  return null;
+}
+
+// possible types https://github.com/mapbox/tippecanoe#modifying-feature-attributes
+const attrTypeMap = {
+  number: {
+    type: 'float32'
+  },
+  numeric: {
+    type: 'float32'
+  },
+  string: {
+    type: 'utf8'
+  },
+  vachar: {
+    type: 'utf8'
+  },
+  float: {
+    type: 'float32'
+  },
+  int: {
+    type: 'int32'
+  },
+  int4: {
+    type: 'int32'
+  },
+  boolean: {
+    type: 'boolean'
+  },
+  bool: {
+    type: 'boolean'
+  }
+};
+
+function attributeToField(
+  attribute: TilestatsLayerAttribute = {},
+  options: TileJSONOptions
+): TileJSONField {
+  const fieldTypes = attributeTypeToFieldType(attribute.type!);
+  const field: TileJSONField = {
+    name: attribute.attribute as string,
+    // what happens if attribute type is string...
+    // filterProps: getFilterProps(fieldTypes.type, attribute),
+    ...fieldTypes
+  };
+
+  // attribute: "_season_peaks_color"
+  // count: 1000
+  // max: 0.95
+  // min: 0.24375
+  // type: "number"
+
+  if (typeof attribute.min === 'number') {
+    field.min = attribute.min;
+  }
+  if (typeof attribute.max === 'number') {
+    field.max = attribute.max;
+  }
+  if (typeof attribute.count === 'number') {
+    field.uniqueValueCount = attribute.count;
+  }
+  if (attribute.values) {
+    // Too much data? Add option?
+    field.values = attribute.values;
+  }
+
+  if (field.values && typeof options.maxValues === 'number') {
+    // Too much data? Add option?
+    field.values = field.values?.slice(0, options.maxValues);
+  }
+
+  return field;
+}
+
+function attributeTypeToFieldType(aType: string): {type: string} {
+  const type = aType.toLowerCase();
+  if (!type || !attrTypeMap[type]) {
+    // console.warn(
+    //   `cannot convert attribute type ${type} to loaders.gl data type, use string by default`
+    // );
+  }
+  return attrTypeMap[type] || {type: 'string'};
+}

package/src/mvt/lib/resolve-table-tile-parameters.ts

@@ -0,0 +1,23 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+import type {GetTileParameters} from '@loaders.gl/core/loader-utils';
+
+/** Resolves a generated table tiler's native numeric level ID before tiling or coordinate conversion. */
+export function resolveTableTileParameters(
+  parameters: GetTileParameters,
+  maxZoom: number
+): GetTileParameters {
+  if (parameters.signal?.aborted) throw new DOMException('The operation was aborted', 'AbortError');
+  if (parameters.tileMatrix === undefined) return parameters;
+  const zoom = Number(parameters.tileMatrix);
+  if (
+    !Number.isInteger(zoom) ||
+    zoom < 0 ||
+    zoom > maxZoom ||
+    String(zoom) !== parameters.tileMatrix
+  ) {
+    throw new RangeError(`Unknown table tile matrix "${parameters.tileMatrix}"`);
+  }
+  return {...parameters, z: zoom};
+}

package/src/mvt/lib/types.ts

@@ -0,0 +1,90 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {Feature, GeoJSONTable} from '@loaders.gl/core/schema';
+
+/** GeoJSON feature with optional vector-tile provenance outside authored properties. */
+export type MVTGeoJSONFeature = Feature & {
+  /** Name of the vector-tile layer that contained this feature. */
+  sourceLayer?: string;
+};
+
+/** GeoJSON output with optional per-feature vector-tile provenance. */
+export type MVTGeoJSONTable = Omit<GeoJSONTable, 'features'> & {
+  /** Decoded features, optionally carrying their original vector-tile layer names. */
+  features: MVTGeoJSONFeature[];
+};
+
+/** Options shared by the full MVT parser and the GeoJSON-only entry. */
+export type MVTGeoJSONOptions = {
+  /** Vector-tile decoding options. */
+  mvt?: {
+    /** Return tile-local or geographic coordinates. */
+    coordinates?: 'wgs84' | 'local';
+    /** Tile index required when requesting geographic coordinates. */
+    tileIndex?: {x: number; y: number; z: number};
+    /** Legacy property receiving the source-layer name, or null to disable injection. */
+    layerProperty?: string | number | null;
+    /** Include source-layer provenance outside authored feature properties. */
+    sourceLayer?: boolean;
+    /** Restrict decoding to these vector-tile layers. */
+    layers?: string[];
+  };
+};
+
+/** For local coordinates, the tileIndex is not required */
+type MVTLocalCoordinatesOptions = {
+  /**
+   * When set to `local`, the parser will return a flat array of GeoJSON objects with local coordinates decoded from tile origin.
+   */
+  coordinates: 'local';
+  tileIndex: null;
+};
+
+/** In WGS84 coordinates, the tileIndex is required */
+type MVTWgs84CoordinatesOptions = {
+  /**
+   * When set to `wgs84`, the parser will return a flat array of GeoJSON objects with coordinates in longitude, latitude decoded from the provided tile index.
+   */
+  coordinates?: 'wgs84';
+
+  /**
+   * Mandatory with `wgs84` coordinates option. An object containing tile index values (`x`, `y`,
+   * `z`) to reproject features' coordinates into WGS84.
+   */
+  tileIndex?: {x: number; y: number; z: number};
+};
+
+export type MVTOptions = (MVTLocalCoordinatesOptions | MVTWgs84CoordinatesOptions) & {
+  shape?: 'geojson-table' | 'columnar-table' | 'binary-geometry';
+  /**
+   * When non-`null`, the layer name of each feature is added to
+   * `feature.properties[layerProperty]`. (A `feature.properties` object is created if the feature
+   * has no existing properties). If set to `null`, a layer name property will not be added.
+   */
+  layerProperty?: string | number | null;
+
+  /**
+   * Optional list of layer names. If not `null`, only features belonging to the named layers will
+   * be included in the output. If `null`, features from all layers are returned.
+   */
+  layers?: string[];
+};
+
+export type MVTMapboxGeometry = {
+  type?: string;
+  id?: number;
+  length: number;
+  coordinates?: any[];
+};
+
+export type MVTMapboxCoordinates = {
+  type: string;
+  geometry: {
+    type: string;
+    coordinates: MVTMapboxGeometry;
+  };
+  properties: {[x: string]: string | number | boolean | null};
+  id?: number;
+};