@loaders.gl/mvt 4.0.0-alpha.9 → 4.0.0-beta.2

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@loaders.gl/mvt",
   "description": "Loader for Mapbox Vector Tiles",
-  "version": "4.0.0-alpha.9",
+  "version": "4.0.0-beta.2",
   "license": "MIT",
   "publishConfig": {
     "access": "public"
@@ -32,14 +32,14 @@
     "build-worker": "esbuild src/workers/mvt-worker.ts --bundle --outfile=dist/mvt-worker.js --define:__VERSION__=\\\"$npm_package_version\\\""
   },
   "dependencies": {
-    "@loaders.gl/gis": "4.0.0-alpha.9",
-    "@loaders.gl/loader-utils": "4.0.0-alpha.9",
-    "@loaders.gl/schema": "4.0.0-alpha.9",
+    "@loaders.gl/gis": "4.0.0-beta.2",
+    "@loaders.gl/loader-utils": "4.0.0-beta.2",
+    "@loaders.gl/schema": "4.0.0-beta.2",
     "@math.gl/polygon": "^3.5.1",
     "pbf": "^3.2.1"
   },
   "devDependencies": {
     "@types/pbf": "^3.0.2"
   },
-  "gitHead": "03ff81ab468f20f3bddeec787aa88d477a7e1c72"
+  "gitHead": "79c2033f755e88e11bc30a04428e3666b177b8fc"
 }

package/src/index.ts

@@ -1,7 +1,12 @@
 // loaders.gl, MIT license
 
+export type {MVTLoaderOptions} from './lib/types';
 export {MVTLoader, MVTWorkerLoader} from './mvt-loader';
 
+export type {TileJSON} from './lib/parse-tilejson';
+export type {TileJSONLoaderOptions} from './tilejson-loader';
+export {TileJSONLoader} from './tilejson-loader';
+
 // GeoJSONTiler
 
 export type {GeoJSONTilerOptions} from './lib/geojson-tiler/geojson-tiler';

package/src/lib/parse-mvt.ts

@@ -3,8 +3,8 @@ import type {
   FlatFeature,
   Feature,
   GeojsonGeometryInfo,
-  BinaryFeatures,
-  GeoJSONRowTable
+  BinaryFeatureCollection,
+  GeoJSONTable
 } from '@loaders.gl/schema';
 import Protobuf from 'pbf';
 
@@ -25,29 +25,31 @@ import VectorTileFeatureMapBox from './mapbox-vector-tile/vector-tile-feature';
 export default function parseMVT(arrayBuffer: ArrayBuffer, options?: MVTLoaderOptions) {
   const mvtOptions = normalizeOptions(options);
 
-  const shape = options?.gis?.format || options?.mvt?.shape;
+  const shape: string | undefined =
+    options?.gis?.format || options?.mvt?.shape || (options?.shape as string);
   switch (shape) {
     case 'columnar-table': // binary + some JS arrays
       return {shape: 'columnar-table', data: parseToBinary(arrayBuffer, mvtOptions)};
-    case 'geojson-row-table': {
-      const table: GeoJSONRowTable = {
-        shape: 'geojson-row-table',
-        data: parseToGeojson(arrayBuffer, mvtOptions)
+    case 'geojson-table': {
+      const table: GeoJSONTable = {
+        shape: 'geojson-table',
+        type: 'FeatureCollection',
+        features: parseToGeojsonFeatures(arrayBuffer, mvtOptions)
       };
       return table;
     }
     case 'geojson':
-      return parseToGeojson(arrayBuffer, mvtOptions);
+      return parseToGeojsonFeatures(arrayBuffer, mvtOptions);
     case 'binary-geometry':
       return parseToBinary(arrayBuffer, mvtOptions);
     case 'binary':
       return parseToBinary(arrayBuffer, mvtOptions);
     default:
-      throw new Error(shape);
+      throw new Error(shape || 'undefined shape');
   }
 }
 
-function parseToBinary(arrayBuffer: ArrayBuffer, options: MVTOptions): BinaryFeatures {
+function parseToBinary(arrayBuffer: ArrayBuffer, options: MVTOptions): BinaryFeatureCollection {
   const [flatGeoJsonFeatures, geometryInfo] = parseToFlatGeoJson(arrayBuffer, options);
 
   const binaryData = flatGeojsonToBinary(flatGeoJsonFeatures, geometryInfo);
@@ -101,7 +103,7 @@ function parseToFlatGeoJson(
   return [features, geometryInfo];
 }
 
-function parseToGeojson(arrayBuffer: ArrayBuffer, options: MVTOptions): Feature[] {
+function parseToGeojsonFeatures(arrayBuffer: ArrayBuffer, options: MVTOptions): Feature[] {
   if (arrayBuffer.byteLength <= 0) {
     return [];
   }
@@ -159,6 +161,7 @@ function getDecodedFeature(
   layerName: string
 ): MVTMapboxCoordinates {
   const decodedFeature = feature.toGeoJSON(
+    // @ts-expect-error What is going on here?
     options.coordinates === 'wgs84' ? options.tileIndex : transformToLocalCoordinates
   );
 
@@ -181,6 +184,7 @@ function getDecodedFeatureBinary(
   layerName: string
 ): FlatFeature {
   const decodedFeature = feature.toBinaryCoordinates(
+    // @ts-expect-error What is going on here?
     options.coordinates === 'wgs84' ? options.tileIndex : transformToLocalCoordinatesBinary
   );
 

package/src/lib/parse-tilejson.ts

@@ -0,0 +1,314 @@
+// loaders.gl, MIT license
+
+/** Parsed and typed TileJSON, merges Tilestats information if present */
+export type TileJSON = {
+  name?: string;
+  description?: string;
+  version?: string;
+  scheme?: 'xyz' | 'tms';
+  tiles?: string[];
+  /** `[[w, s], [e, n]]`, indicates the limits of the bounding box using the axis units and order of the specified CRS. */
+  boundingBox?: [min: [w: number, s: number], max: [e: number, n: number]];
+  center: number[] | null;
+  maxZoom: number | null;
+  minZoom: number | null;
+  htmlAttribution?: string;
+  htmlLegend?: string;
+  layers?: TileJSONLayer[];
+  metaJson?: any | null;
+};
+
+export type TileJSONLayer = {
+  /** The name (id) of this layer (tilejson.vector_layers[].id / tilestats.layers[].layer) */
+  name: string;
+
+  /** The description of this layer (tilejson.layer.description) */
+  description?: string;
+
+  // tilestats
+
+  /** The number of features in this layer (tilestats.layer.count) */
+  featureCount?: number;
+  /** The dominant geometry type in this layer (tilestats.layer.geometry) */
+  dominantGeometry?: string;
+  /** An array of details about the first 100 attributes in this layer */
+
+  /**  */
+  minZoom?: number;
+  maxZoom?: number;
+  fields: TileJSONField[];
+};
+
+export type TileJSONField = {
+  /** The name of this attribute */
+  name: string;
+  description?: string;
+
+  // tilestats
+
+  type: string;
+  /** min value (if there are *any* numbers in the values) */
+  min?: number;
+  /** max value (if there are *any* numbers in the values) */
+  max?: number;
+  /** An array of this attribute's first 100 unique values */
+  values?: unknown[];
+};
+
+/**
+ * The raw/unparsed tilestats layer type
+ * @see https://github.com/mapbox/mapbox-geostats#output-the-stats
+ */
+type TilestatsLayer = {
+  /** The name of this layer */
+  layer: string;
+  /** The number of features in this layer */
+  count: number;
+  /** The dominant geometry type in this layer */
+  geometry: string;
+  /** The number of unique attributes in this layer (max. 1000) */
+  attributeCount: number;
+  /** Fields for this layer */
+  attributes?: TilestatsLayerAttribute[];
+};
+
+/**
+ * The raw/unparsed tilestats attribute type
+ * @see https://github.com/mapbox/mapbox-geostats#output-the-stats
+ */
+type TilestatsLayerAttribute = {
+  /** The name of this layer */
+  attribute?: string;
+  /** Each attribute has one of the following types:
+   * - 'string' if all its values are strings (or null).
+   * - 'number' if all its values are numbers (or null).
+   * - 'boolean' if all its values are booleans (or null).
+   * - 'null' if its only value is null.
+   * - 'mixed' if it has values of multiple types.
+   * - Array and object values are coerced to strings.
+   */
+  type?: string;
+  /** min value (if there are *any* numbers in the values) */
+  min?: number;
+  /** max value (if there are *any* numbers in the values) */
+  max?: number;
+};
+
+const isObject: (x: unknown) => boolean = (x) => x !== null && typeof x === 'object';
+
+export function parseTileJSON(jsonMetadata: any): TileJSON | null {
+  if (!jsonMetadata || !isObject(jsonMetadata)) {
+    return null;
+  }
+
+  const boundingBox = parseBounds(jsonMetadata.bounds);
+  const center = parseCenter(jsonMetadata.center);
+  const maxZoom = safeParseFloat(jsonMetadata.maxzoom);
+  const minZoom = safeParseFloat(jsonMetadata.minzoom);
+
+  let tileJSON: TileJSON = {
+    name: jsonMetadata.name || '',
+    description: jsonMetadata.description || '',
+    boundingBox,
+    center,
+    maxZoom,
+    minZoom,
+    layers: []
+  };
+
+  // try to parse json
+  if (typeof jsonMetadata?.json === 'string') {
+    try {
+      tileJSON.metaJson = JSON.parse(jsonMetadata.json);
+    } catch (err) {
+      // do nothing
+    }
+  }
+
+  let layers = parseTilestatsLayers(tileJSON.metaJson?.tilestats);
+  // TODO - merge in description from tilejson
+  if (layers.length === 0) {
+    layers = parseTileJSONLayers(jsonMetadata.vector_layers); // eslint-disable-line camelcase
+  }
+
+  tileJSON = {
+    ...tileJSON,
+    layers
+  };
+
+  return tileJSON;
+}
+
+function parseTileJSONLayers(layers: any[]): TileJSONLayer[] {
+  // Look for fields in vector_layers
+  if (!Array.isArray(layers)) {
+    return [];
+  }
+  return layers.map((layer) => ({
+    name: layer.id || '',
+    fields: Object.entries(layer.fields || []).map(([key, datatype]) => ({
+      name: key,
+      ...attributeTypeToFieldType(String(datatype))
+    }))
+  }));
+}
+
+/** parse Layers array from tilestats */
+function parseTilestatsLayers(tilestats: any): TileJSONLayer[] {
+  if (isObject(tilestats) && Array.isArray(tilestats.layers)) {
+    // we are in luck!
+    return tilestats.layers.map((layer) => parseTilestatsForLayer(layer));
+  }
+  return [];
+}
+
+function parseTilestatsForLayer(layer: TilestatsLayer): TileJSONLayer {
+  const fields: TileJSONField[] = [];
+  const indexedAttributes: {[key: string]: TilestatsLayerAttribute[]} = {};
+
+  const attributes = layer.attributes || [];
+  for (const attr of attributes) {
+    const name = attr.attribute;
+    if (typeof name === 'string') {
+      if (name.split('|').length > 1) {
+        // indexed field
+        const fname = name.split('|')[0];
+        indexedAttributes[fname] = indexedAttributes[fname] || [];
+        indexedAttributes[fname].push(attr);
+      } else if (!fields[name]) {
+        fields[name] = attributeToField(attr);
+      } else {
+        // return (fields[name], attr);
+      }
+    }
+  }
+  return {
+    name: layer.layer || '',
+    dominantGeometry: layer.geometry,
+    fields
+  };
+}
+
+/**
+ * bounds should be [minLng, minLat, maxLng, maxLat]
+ *`[[w, s], [e, n]]`, indicates the limits of the bounding box using the axis units and order of the specified CRS.
+ */
+function parseBounds(
+  bounds: string | number[]
+): [[east: number, south: number], [west: number, north: number]] | undefined {
+  // supported formats
+  // string: "-96.657715,40.126127,-90.140061,43.516689",
+  // array: [ -180, -85.05112877980659, 180, 85.0511287798066 ]
+  const result = fromArrayOrString(bounds);
+  // validate bounds
+  if (
+    Array.isArray(result) &&
+    result.length === 4 &&
+    [result[0], result[2]].every(isLng) &&
+    [result[1], result[3]].every(isLat)
+  ) {
+    return [
+      [result[0], result[1]],
+      [result[2], result[3]]
+    ];
+  }
+  return undefined;
+}
+
+function parseCenter(center: string | number[]): number[] | null {
+  // supported formats
+  // string: "-96.657715,40.126127,-90.140061,43.516689",
+  // array: [-91.505127,41.615442,14]
+  const result = fromArrayOrString(center);
+  if (
+    Array.isArray(result) &&
+    result.length === 3 &&
+    isLng(result[0]) &&
+    isLat(result[1]) &&
+    isZoom(result[2])
+  ) {
+    return result;
+  }
+  return null;
+}
+
+function safeParseFloat(input: unknown): number | null {
+  const result =
+    typeof input === 'string' ? parseFloat(input) : typeof input === 'number' ? input : null;
+  return result === null || isNaN(result) ? null : result;
+}
+
+// https://github.com/mapbox/tilejson-spec/tree/master/2.2.0
+function isLat(num: any): boolean {
+  return Number.isFinite(num) && num <= 90 && num >= -90;
+}
+function isLng(num: any): boolean {
+  return Number.isFinite(num) && num <= 180 && num >= -180;
+}
+function isZoom(num: any): boolean {
+  return Number.isFinite(num) && num >= 0 && num <= 22;
+}
+function fromArrayOrString(data: string | number[]): number[] | null {
+  if (typeof data === 'string') {
+    return data.split(',').map(parseFloat);
+  } else if (Array.isArray(data)) {
+    return data;
+  }
+  return null;
+}
+
+// possible types https://github.com/mapbox/tippecanoe#modifying-feature-attributes
+const attrTypeMap = {
+  number: {
+    type: 'float32'
+  },
+  numeric: {
+    type: 'float32'
+  },
+  string: {
+    type: 'utf8'
+  },
+  vachar: {
+    type: 'utf8'
+  },
+  float: {
+    type: 'float32'
+  },
+  int: {
+    type: 'int32'
+  },
+  int4: {
+    type: 'int32'
+  },
+  boolean: {
+    type: 'boolean'
+  },
+  bool: {
+    type: 'boolean'
+  }
+};
+
+function attributeToField(attribute: TilestatsLayerAttribute = {}): TileJSONField {
+  // attribute: "_season_peaks_color"
+  // count: 1000
+  // max: 0.95
+  // min: 0.24375
+  // type: "number"
+  const fieldTypes = attributeTypeToFieldType(attribute.type!);
+  return {
+    name: attribute.attribute as string,
+    // what happens if attribute type is string...
+    // filterProps: getFilterProps(fieldTypes.type, attribute),
+    ...fieldTypes
+  };
+}
+
+function attributeTypeToFieldType(aType: string): {type: string} {
+  const type = aType.toLowerCase();
+  if (!type || !attrTypeMap[type]) {
+    // console.warn(
+    //   `cannot convert attribute type ${type} to loaders.gl data type, use string by default`
+    // );
+  }
+  return attrTypeMap[type] || {type: 'string'};
+}

package/src/lib/types.ts

@@ -14,13 +14,13 @@ type MVTWgs84CoordinatesOptions = {
   /**
    * When set to `wgs84`, the parser will return a flat array of GeoJSON objects with coordinates in longitude, latitude decoded from the provided tile index.
    */
-  coordinates: 'wgs84';
+  coordinates?: 'wgs84';
 
   /**
    * Mandatory with `wgs84` coordinates option. An object containing tile index values (`x`, `y`,
    * `z`) to reproject features' coordinates into WGS84.
    */
-  tileIndex: {x: number; y: number; z: number};
+  tileIndex?: {x: number; y: number; z: number};
 };
 
 export type MVTOptions = (MVTLocalCoordinatesOptions | MVTWgs84CoordinatesOptions) & {
@@ -36,7 +36,7 @@ export type MVTOptions = (MVTLocalCoordinatesOptions | MVTWgs84CoordinatesOption
    * be included in the output. If `null`, features from all layers are returned.
    */
   layers?: string[];
-  shape?: 'geojson-row-table' | 'columnar-table' | 'geojson' | 'binary' | 'binary-geometry';
+  shape?: 'geojson-table' | 'columnar-table' | 'geojson' | 'binary' | 'binary-geometry';
 };
 
 export type MVTMapboxGeometry = {
@@ -64,6 +64,6 @@ export type MVTLoaderOptions = LoaderOptions & {
      */
     binary?: boolean;
     /** @deprecated. Use options.mvt.shape */
-    format?: 'geojson-row-table' | 'columnar-table' | 'geojson' | 'binary' | 'binary-geometry';
+    format?: 'geojson-table' | 'columnar-table' | 'geojson' | 'binary' | 'binary-geometry';
   };
 };

package/src/mvt-loader.ts

@@ -1,25 +1,26 @@
 import type {Loader, LoaderWithParser} from '@loaders.gl/loader-utils';
 import type {MVTLoaderOptions} from './lib/types';
+// import type {
+//   Feature,
+//   BinaryFeatureCollection,
+//   GeoJSONTable,
+//   Geometry,
+//   GeoJsonProperties
+// } from '@loaders.gl/schema';
 import parseMVT from './lib/parse-mvt';
 
 // __VERSION__ is injected by babel-plugin-version-inline
 // @ts-ignore TS2304: Cannot find name '__VERSION__'.
 const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
 
-const DEFAULT_MVT_LOADER_OPTIONS: MVTLoaderOptions = {
-  mvt: {
-    shape: 'geojson',
-    coordinates: 'local',
-    layerProperty: 'layerName',
-    layers: undefined,
-    tileIndex: null
-  }
-};
-
 /**
  * Worker loader for the Mapbox Vector Tile format
  */
-export const MVTWorkerLoader: Loader = {
+export const MVTWorkerLoader: Loader<
+  any, // BinaryFeatureCollection | GeoJSONTable | Feature<Geometry, GeoJsonProperties>,
+  never,
+  MVTLoaderOptions
+> = {
   name: 'Mapbox Vector Tile',
   id: 'mvt',
   module: 'mvt',
@@ -27,19 +28,32 @@ export const MVTWorkerLoader: Loader = {
   // Note: ArcGIS uses '.pbf' extension and 'application/octet-stream'
   extensions: ['mvt', 'pbf'],
   mimeTypes: [
+    // https://www.iana.org/assignments/media-types/application/vnd.mapbox-vector-tile
     'application/vnd.mapbox-vector-tile',
     'application/x-protobuf'
     // 'application/octet-stream'
   ],
   worker: true,
   category: 'geometry',
-  options: DEFAULT_MVT_LOADER_OPTIONS
+  options: {
+    mvt: {
+      shape: 'geojson',
+      coordinates: 'local',
+      layerProperty: 'layerName',
+      layers: undefined,
+      tileIndex: null
+    }
+  }
 };
 
 /**
  * Loader for the Mapbox Vector Tile format
  */
-export const MVTLoader: LoaderWithParser = {
+export const MVTLoader: LoaderWithParser<
+  any, // BinaryFeatureCollection | GeoJSONTable | Feature<Geometry, GeoJsonProperties>,
+  never,
+  MVTLoaderOptions
+> = {
   ...MVTWorkerLoader,
   parse: async (arrayBuffer, options?: MVTLoaderOptions) => parseMVT(arrayBuffer, options),
   parseSync: parseMVT,

package/src/tilejson-loader.ts

@@ -0,0 +1,39 @@
+// loaders.gl, MIT license
+
+import type {LoaderWithParser, LoaderOptions} from '@loaders.gl/loader-utils';
+import type {TileJSON} from './lib/parse-tilejson';
+import {parseTileJSON} from './lib/parse-tilejson';
+
+// __VERSION__ is injected by babel-plugin-version-inline
+// @ts-ignore TS2304: Cannot find name '__VERSION__'.
+const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
+
+export type TileJSONLoaderOptions = LoaderOptions & {
+  tilejson?: {};
+};
+
+/**
+ * Loader for TileJSON metadata
+ */
+export const TileJSONLoader: LoaderWithParser<TileJSON, never, TileJSONLoaderOptions> = {
+  name: 'TileJSON',
+  id: 'tilejson',
+  module: 'pmtiles',
+  version: VERSION,
+  worker: true,
+  extensions: ['json'],
+  mimeTypes: ['application/json'],
+  text: true,
+  options: {
+    tilejson: {}
+  },
+  parse: async (arrayBuffer, options) => {
+    const jsonString = new TextDecoder().decode(arrayBuffer);
+    const json = JSON.parse(jsonString);
+    return parseTileJSON(json) as TileJSON;
+  },
+  parseTextSync: (text, options) => {
+    const json = JSON.parse(text);
+    return parseTileJSON(json) as TileJSON;
+  }
+};

package/dist/bundle.js

@@ -1,5 +0,0 @@
-"use strict";
-// @ts-nocheck
-const moduleExports = require('./index');
-globalThis.loaders = globalThis.loaders || {};
-module.exports = Object.assign(globalThis.loaders, moduleExports);