@loaders.gl/mvt 4.0.0-beta.8 → 4.0.1

package/src/lib/parse-tilejson.ts

@@ -1,21 +1,42 @@
 // loaders.gl, MIT license
 // Copyright (c) vis.gl contributors
 
+export type TileJSONOptions = {
+  maxValues?: number | false;
+};
+
 /** Parsed and typed TileJSON, merges Tilestats information if present */
 export type TileJSON = {
   name?: string;
   description?: string;
   version?: string;
+
+  tileFormat?: string;
+  tilesetType?: string;
+
+  /** Generating application. Tippecanoe adds this. */
+  generator?: string;
+  /** Generating application options. Tippecanoe adds this. */
+  generatorOptions?: string;
+
+  /** Tile indexing scheme */
   scheme?: 'xyz' | 'tms';
+  /** Sharded URLs */
   tiles?: string[];
   /** `[[w, s], [e, n]]`, indicates the limits of the bounding box using the axis units and order of the specified CRS. */
   boundingBox?: [min: [w: number, s: number], max: [e: number, n: number]];
-  center: number[] | null;
-  maxZoom: number | null;
-  minZoom: number | null;
+  /** May be set to the maxZoom of the first layer */
+  maxZoom?: number | null;
+  /** May be set to the minZoom of the first layer */
+  minZoom?: number | null;
+  center?: number[] | null;
   htmlAttribution?: string;
   htmlLegend?: string;
+
+  // Combination of tilestats (if present) and tilejson layer information
   layers?: TileJSONLayer[];
+
+  /** Any nested JSON metadata */
   metaJson?: any | null;
 };
 
@@ -52,6 +73,8 @@ export type TileJSONField = {
   min?: number;
   /** max value (if there are *any* numbers in the values) */
   max?: number;
+  /** Number of unique values across the tileset */
+  uniqueValueCount?: number;
   /** An array of this attribute's first 100 unique values */
   values?: unknown[];
 };
@@ -93,50 +116,77 @@ type TilestatsLayerAttribute = {
   min?: number;
   /** max value (if there are *any* numbers in the values) */
   max?: number;
+  /** Number of unique values */
+  count?: number;
+  /** First 100 values */
+  values?: unknown[];
 };
 
 const isObject: (x: unknown) => boolean = (x) => x !== null && typeof x === 'object';
 
-export function parseTileJSON(jsonMetadata: any): TileJSON | null {
+export function parseTileJSON(jsonMetadata: any, options: TileJSONOptions): TileJSON | null {
   if (!jsonMetadata || !isObject(jsonMetadata)) {
     return null;
   }
 
-  const boundingBox = parseBounds(jsonMetadata.bounds);
-  const center = parseCenter(jsonMetadata.center);
-  const maxZoom = safeParseFloat(jsonMetadata.maxzoom);
-  const minZoom = safeParseFloat(jsonMetadata.minzoom);
-
   let tileJSON: TileJSON = {
     name: jsonMetadata.name || '',
-    description: jsonMetadata.description || '',
-    boundingBox,
-    center,
-    maxZoom,
-    minZoom,
-    layers: []
+    description: jsonMetadata.description || ''
   };
 
-  // try to parse json
+  // tippecanoe
+
+  if (typeof jsonMetadata.generator === 'string') {
+    tileJSON.generator = jsonMetadata.generator;
+  }
+  if (typeof jsonMetadata.generator_options === 'string') {
+    tileJSON.generatorOptions = jsonMetadata.generator_options;
+  }
+
+  // Tippecanoe emits `antimeridian_adjusted_bounds` instead of `bounds`
+  tileJSON.boundingBox =
+    parseBounds(jsonMetadata.bounds) || parseBounds(jsonMetadata.antimeridian_adjusted_bounds);
+
+  // TODO - can be undefined - we could set to center of bounds...
+  tileJSON.center = parseCenter(jsonMetadata.center);
+  // TODO - can be undefined, we could extract from layers...
+  tileJSON.maxZoom = safeParseFloat(jsonMetadata.maxzoom);
+  // TODO - can be undefined, we could extract from layers...
+  tileJSON.minZoom = safeParseFloat(jsonMetadata.minzoom);
+
+  // Look for nested metadata embedded in .json field
+  // TODO - document what source this applies to, when is this needed?
   if (typeof jsonMetadata?.json === 'string') {
+    // try to parse json
     try {
       tileJSON.metaJson = JSON.parse(jsonMetadata.json);
-    } catch (err) {
+    } catch (error) {
+      console.warn('Failed to parse tilejson.json field', error);
       // do nothing
     }
   }
 
-  let layers = parseTilestatsLayers(tileJSON.metaJson?.tilestats);
+  // Look for fields in tilestats
+
+  const tilestats = jsonMetadata.tilestats || tileJSON.metaJson?.tilestats;
+  const tileStatsLayers = parseTilestatsLayers(tilestats, options);
+  const tileJSONlayers = parseTileJSONLayers(jsonMetadata.vector_layers); // eslint-disable-line camelcase
   // TODO - merge in description from tilejson
-  if (layers.length === 0) {
-    layers = parseTileJSONLayers(jsonMetadata.vector_layers); // eslint-disable-line camelcase
-  }
+  const layers = mergeLayers(tileJSONlayers, tileStatsLayers);
 
   tileJSON = {
     ...tileJSON,
     layers
   };
 
+  if (tileJSON.maxZoom === null && layers.length > 0) {
+    tileJSON.maxZoom = layers[0].maxZoom || null;
+  }
+
+  if (tileJSON.minZoom === null && layers.length > 0) {
+    tileJSON.minZoom = layers[0].minZoom || null;
+  }
+
   return tileJSON;
 }
 
@@ -145,41 +195,52 @@ function parseTileJSONLayers(layers: any[]): TileJSONLayer[] {
   if (!Array.isArray(layers)) {
     return [];
   }
-  return layers.map((layer) => ({
-    name: layer.id || '',
-    fields: Object.entries(layer.fields || []).map(([key, datatype]) => ({
-      name: key,
-      ...attributeTypeToFieldType(String(datatype))
-    }))
+  return layers.map((layer) => parseTileJSONLayer(layer));
+}
+
+function parseTileJSONLayer(layer: any): TileJSONLayer {
+  const fields = Object.entries(layer.fields || []).map(([key, datatype]) => ({
+    name: key,
+    ...attributeTypeToFieldType(String(datatype))
   }));
+  const layer2 = {...layer};
+  delete layer2.fields;
+  return {
+    name: layer.id || '',
+    ...layer2,
+    fields
+  };
 }
 
 /** parse Layers array from tilestats */
-function parseTilestatsLayers(tilestats: any): TileJSONLayer[] {
+function parseTilestatsLayers(tilestats: any, options: TileJSONOptions): TileJSONLayer[] {
   if (isObject(tilestats) && Array.isArray(tilestats.layers)) {
     // we are in luck!
-    return tilestats.layers.map((layer) => parseTilestatsForLayer(layer));
+    return tilestats.layers.map((layer) => parseTilestatsForLayer(layer, options));
   }
   return [];
 }
 
-function parseTilestatsForLayer(layer: TilestatsLayer): TileJSONLayer {
+function parseTilestatsForLayer(layer: TilestatsLayer, options: TileJSONOptions): TileJSONLayer {
   const fields: TileJSONField[] = [];
   const indexedAttributes: {[key: string]: TilestatsLayerAttribute[]} = {};
 
   const attributes = layer.attributes || [];
-  for (const attr of attributes) {
-    const name = attr.attribute;
+  for (const attribute of attributes) {
+    const name = attribute.attribute;
     if (typeof name === 'string') {
+      // TODO - code copied from kepler.gl, need sample tilestats files to test
       if (name.split('|').length > 1) {
         // indexed field
         const fname = name.split('|')[0];
         indexedAttributes[fname] = indexedAttributes[fname] || [];
-        indexedAttributes[fname].push(attr);
+        indexedAttributes[fname].push(attribute);
+        // eslint-disable-next-line no-console
+        console.warn('ignoring tilestats indexed field', fname);
       } else if (!fields[name]) {
-        fields[name] = attributeToField(attr);
+        fields.push(attributeToField(attribute, options));
       } else {
-        // return (fields[name], attr);
+        // return (fields[name], attribute);
       }
     }
   }
@@ -190,6 +251,21 @@ function parseTilestatsForLayer(layer: TilestatsLayer): TileJSONLayer {
   };
 }
 
+function mergeLayers(layers: TileJSONLayer[], tilestatsLayers: TileJSONLayer[]): TileJSONLayer[] {
+  return layers.map((layer) => {
+    const tilestatsLayer = tilestatsLayers.find((tsLayer) => tsLayer.name === layer.name);
+    // For aesthetics in JSON dumps, we preserve field order (make sure layers is last)
+    const fields = tilestatsLayer?.fields || [];
+    const layer2: Partial<TileJSONLayer> = {...layer};
+    delete layer2.fields;
+    return {
+      ...layer2,
+      ...tilestatsLayer,
+      fields
+    } as TileJSONLayer;
+  });
+}
+
 /**
  * bounds should be [minLng, minLat, maxLng, maxLat]
  *`[[w, s], [e, n]]`, indicates the limits of the bounding box using the axis units and order of the specified CRS.
@@ -289,19 +365,38 @@ const attrTypeMap = {
   }
 };
 
-function attributeToField(attribute: TilestatsLayerAttribute = {}): TileJSONField {
-  // attribute: "_season_peaks_color"
-  // count: 1000
-  // max: 0.95
-  // min: 0.24375
-  // type: "number"
+function attributeToField(
+  attribute: TilestatsLayerAttribute = {},
+  options: TileJSONOptions
+): TileJSONField {
   const fieldTypes = attributeTypeToFieldType(attribute.type!);
-  return {
+  const field: TileJSONField = {
     name: attribute.attribute as string,
     // what happens if attribute type is string...
     // filterProps: getFilterProps(fieldTypes.type, attribute),
     ...fieldTypes
   };
+
+  // attribute: "_season_peaks_color"
+  // count: 1000
+  // max: 0.95
+  // min: 0.24375
+  // type: "number"
+
+  if (typeof attribute.min === 'number') {
+    field.min = attribute.min;
+  }
+  if (typeof attribute.max === 'number') {
+    field.max = attribute.max;
+  }
+  if (typeof attribute.count === 'number') {
+    field.uniqueValueCount = attribute.count;
+  }
+  if (options.maxValues !== false && attribute.values) {
+    // Too much data? Add option?
+    field.values = attribute.values?.slice(0, options.maxValues);
+  }
+  return field;
 }
 
 function attributeTypeToFieldType(aType: string): {type: string} {

package/src/mvt-source.ts

@@ -4,7 +4,7 @@
 import type {GetTileParameters, ImageType, DataSourceProps} from '@loaders.gl/loader-utils';
 import type {ImageTileSource, VectorTileSource} from '@loaders.gl/loader-utils';
 import {DataSource, resolvePath} from '@loaders.gl/loader-utils';
-import {ImageLoader} from '@loaders.gl/images';
+import {ImageLoader, getBinaryImageMetadata} from '@loaders.gl/images';
 import {MVTLoader, MVTLoaderOptions, TileJSONLoader, TileJSON} from '@loaders.gl/mvt';
 
 import {TileLoadParameters} from '@loaders.gl/loader-utils';
@@ -21,13 +21,17 @@ export type MVTSourceProps = DataSourceProps & {
 export class MVTSource extends DataSource implements ImageTileSource, VectorTileSource {
   props: MVTSourceProps;
   url: string;
+  data: string;
   schema: 'tms' | 'xyz' = 'tms';
   metadata: Promise<TileJSON | null>;
+  extension = '.png';
+  mimeType: string | null = null;
 
   constructor(props: MVTSourceProps) {
     super(props);
     this.props = props;
     this.url = resolvePath(props.url);
+    this.data = this.url;
     this.getTileData = this.getTileData.bind(this);
     this.metadata = this.getMetadata();
   }
@@ -35,16 +39,31 @@ export class MVTSource extends DataSource implements ImageTileSource, VectorTile
   // @ts-ignore - Metadata type misalignment
   async getMetadata(): Promise<TileJSON | null> {
     const metadataUrl = this.getMetadataUrl();
-    const response = await this.fetch(metadataUrl);
+    let response: Response;
+    try {
+      // Annoyingly, fetch throws on CORS errors which is common when requesting an unavailable resource
+      response = await this.fetch(metadataUrl);
+    } catch (error: unknown) {
+      console.error((error as TypeError).message);
+      return null;
+    }
     if (!response.ok) {
+      console.error(response.statusText);
       return null;
     }
     const tileJSON = await response.text();
     const metadata = TileJSONLoader.parseTextSync?.(JSON.stringify(tileJSON)) || null;
     // metadata.attributions = [...this.props.attributions, ...(metadata.attributions || [])];
+    // if (metadata?.mimeType) {
+    //   this.mimeType = metadata?.tileMIMEType;
+    // }
     return metadata;
   }
 
+  getTileMIMEType(): string | null {
+    return this.mimeType;
+  }
+
   async getTile(tileParams: GetTileParameters): Promise<ArrayBuffer | null> {
     const {x, y, zoom: z} = tileParams;
     const tileUrl = this.getTileURL(x, y, z);
@@ -61,27 +80,48 @@ export class MVTSource extends DataSource implements ImageTileSource, VectorTile
 
   async getTileData(tileParams: TileLoadParameters): Promise<unknown | null> {
     const {x, y, z} = tileParams.index;
-    const metadata = await this.metadata;
-    // @ts-expect-error
-    switch (metadata.mimeType || 'application/vnd.mapbox-vector-tile') {
+    // const metadata = await this.metadata;
+    // mimeType = metadata?.tileMIMEType || 'application/vnd.mapbox-vector-tile';
+
+    const arrayBuffer = await this.getTile({x, y, zoom: z, layers: []});
+    if (arrayBuffer === null) {
+      return null;
+    }
+
+    const imageMetadata = getBinaryImageMetadata(arrayBuffer);
+    this.mimeType =
+      this.mimeType || imageMetadata?.mimeType || 'application/vnd.mapbox-vector-tile';
+    switch (this.mimeType) {
       case 'application/vnd.mapbox-vector-tile':
-        return await this.getVectorTile({x, y, zoom: z, layers: []});
+        return await this.parseVectorTile(arrayBuffer, {x, y, zoom: z, layers: []});
       default:
-        return await this.getImageTile({x, y, zoom: z, layers: []});
+        return await this.parseImageTile(arrayBuffer);
     }
   }
+  x;
 
   // ImageTileSource interface implementation
 
   async getImageTile(tileParams: GetTileParameters): Promise<ImageType | null> {
     const arrayBuffer = await this.getTile(tileParams);
-    return arrayBuffer ? await ImageLoader.parse(arrayBuffer, this.loadOptions) : null;
+    return arrayBuffer ? this.parseImageTile(arrayBuffer) : null;
+  }
+
+  protected async parseImageTile(arrayBuffer: ArrayBuffer): Promise<ImageType> {
+    return await ImageLoader.parse(arrayBuffer, this.loadOptions);
   }
 
   // VectorTileSource interface implementation
 
   async getVectorTile(tileParams: GetTileParameters): Promise<unknown | null> {
     const arrayBuffer = await this.getTile(tileParams);
+    return arrayBuffer ? this.parseVectorTile(arrayBuffer, tileParams) : null;
+  }
+
+  protected async parseVectorTile(
+    arrayBuffer: ArrayBuffer,
+    tileParams: GetTileParameters
+  ): Promise<unknown | null> {
     const loadOptions: MVTLoaderOptions = {
       shape: 'geojson-table',
       mvt: {
@@ -92,7 +132,7 @@ export class MVTSource extends DataSource implements ImageTileSource, VectorTile
       ...this.loadOptions
     };
 
-    return arrayBuffer ? await MVTLoader.parse(arrayBuffer, loadOptions) : null;
+    return await MVTLoader.parse(arrayBuffer, loadOptions);
   }
 
   getMetadataUrl(): string {
@@ -102,10 +142,10 @@ export class MVTSource extends DataSource implements ImageTileSource, VectorTile
   getTileURL(x: number, y: number, z: number) {
     switch (this.schema) {
       case 'xyz':
-        return `${this.url}/${x}/${y}/${z}`;
+        return `${this.url}/${x}/${y}/${z}${this.extension}`;
       case 'tms':
       default:
-        return `${this.url}/${z}/${x}/${y}`;
+        return `${this.url}/${z}/${x}/${y}${this.extension}`;
     }
   }
 }

package/src/tilejson-loader.ts

@@ -10,7 +10,9 @@ import {parseTileJSON} from './lib/parse-tilejson';
 const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
 
 export type TileJSONLoaderOptions = LoaderOptions & {
-  tilejson?: {};
+  tilejson?: {
+    maxValues?: number | false;
+  };
 };
 
 /**
@@ -26,15 +28,19 @@ export const TileJSONLoader: LoaderWithParser<TileJSON, never, TileJSONLoaderOpt
   mimeTypes: ['application/json'],
   text: true,
   options: {
-    tilejson: {}
+    tilejson: {
+      maxValues: 10
+    }
   },
-  parse: async (arrayBuffer, options) => {
+  parse: async (arrayBuffer, options?: TileJSONLoaderOptions) => {
     const jsonString = new TextDecoder().decode(arrayBuffer);
     const json = JSON.parse(jsonString);
-    return parseTileJSON(json) as TileJSON;
+    const tilejsonOptions = {...TileJSONLoader.options.tilejson, ...options?.tilejson};
+    return parseTileJSON(json, tilejsonOptions) as TileJSON;
   },
   parseTextSync: (text, options) => {
     const json = JSON.parse(text);
-    return parseTileJSON(json) as TileJSON;
+    const tilejsonOptions = {...TileJSONLoader.options.tilejson, ...options?.tilejson};
+    return parseTileJSON(json, tilejsonOptions) as TileJSON;
   }
 };