@deck.gl/carto 8.9.0-alpha.2 → 8.9.0-alpha.3

package/src/layers/carto-tile-layer.ts

@@ -1,6 +1,8 @@
-/* global TextDecoder */
-import Protobuf from 'pbf';
-import {log, DefaultProps} from '@deck.gl/core';
+import {registerLoaders} from '@loaders.gl/core';
+import CartoVectorTileLoader from './schema/carto-vector-tile-loader';
+registerLoaders([CartoVectorTileLoader]);
+
+import {DefaultProps} from '@deck.gl/core';
 import {ClipExtension} from '@deck.gl/extensions';
 import {
   MVTLayer,
@@ -11,81 +13,16 @@ import {
   _TileLoadProps as TileLoadProps
 } from '@deck.gl/geo-layers';
 import {GeoJsonLayer} from '@deck.gl/layers';
-import {geojsonToBinary} from '@loaders.gl/gis';
-import {KeyValueProperties, Tile, TileReader} from './schema/carto-tile';
-import {TileFormat, TILE_FORMATS} from '../api/maps-api-common';
-import {LoaderOptions, LoaderWithParser} from '@loaders.gl/loader-utils';
+import {binaryToGeojson, geojsonToBinary} from '@loaders.gl/gis';
 import type {BinaryFeatures} from '@loaders.gl/schema';
+import {TileFormat, TILE_FORMATS} from '../api/maps-api-common';
 import type {Feature} from 'geojson';
 
-function parseJSON(arrayBuffer: ArrayBuffer): any {
-  return JSON.parse(new TextDecoder().decode(arrayBuffer));
-}
-
-function parsePbf(buffer: ArrayBuffer): Tile {
-  const pbf = new Protobuf(buffer);
-  const tile = TileReader.read(pbf);
-  return tile;
-}
-
-function unpackProperties(properties: KeyValueProperties[]) {
-  if (!properties || !properties.length) {
-    return [];
-  }
-  return properties.map(item => {
-    const currentRecord: Record<string, unknown> = {};
-    item.data.forEach(({key, value}) => {
-      currentRecord[key] = value;
-    });
-    return currentRecord;
-  });
-}
-
-function parseCartoTile(arrayBuffer: ArrayBuffer, options?: LoaderOptions): BinaryFeatures | null {
-  if (!arrayBuffer) return null;
-  const formatTiles = options && options.cartoTile && options.cartoTile.formatTiles;
-  if (formatTiles === TILE_FORMATS.GEOJSON) return geojsonToBinary(parseJSON(arrayBuffer).features);
-
-  const tile = parsePbf(arrayBuffer);
-
-  const {points, lines, polygons} = tile;
-  const data = {
-    points: {...points, properties: unpackProperties(points.properties)},
-    lines: {...lines, properties: unpackProperties(lines.properties)},
-    polygons: {...polygons, properties: unpackProperties(polygons.properties)}
-  };
-
-  // Note: there is slight, difference in `numericProps` type, however geojson/mvtlayer can cope with this
-  return data as unknown as BinaryFeatures;
-}
-
 const defaultTileFormat = TILE_FORMATS.BINARY;
 
-const CartoTileLoader: LoaderWithParser = {
-  name: 'CARTO Tile',
-  version: '1',
-  id: 'cartoTile',
-  module: 'carto',
-  extensions: ['pbf'],
-  mimeTypes: [
-    'application/vnd.carto-vector-tile',
-    'application/x-protobuf' // Back-compatibility
-  ],
-  category: 'geometry',
-  worker: false,
-  parse: async (arrayBuffer, options) => parseCartoTile(arrayBuffer, options),
-  parseSync: parseCartoTile,
-  options: {
-    cartoTile: {
-      formatTiles: defaultTileFormat
-    }
-  }
-};
-
 const defaultProps: DefaultProps<CartoTileLayerProps> = {
   ...MVTLayer.defaultProps,
-  formatTiles: defaultTileFormat,
-  loaders: [CartoTileLoader]
+  formatTiles: defaultTileFormat
 };
 
 /** All properties supported by CartoTileLayer. */
@@ -110,6 +47,12 @@ export default class CartoTileLayer<
   static layerName = 'CartoTileLayer';
   static defaultProps = defaultProps;
 
+  initializeState(): void {
+    super.initializeState();
+    const binary = this.props.formatTiles === TILE_FORMATS.BINARY;
+    this.setState({binary});
+  }
+
   getTileData(tile: TileLoadProps) {
     const url = _getURLFromTemplate(this.state.data, tile);
     if (!url) {
@@ -120,17 +63,13 @@ export default class CartoTileLayer<
     const {fetch, formatTiles} = this.props;
     const {signal} = tile;
 
-    loadOptions = {
-      ...loadOptions,
-      mimeType: 'application/x-protobuf'
-    };
-
-    if (formatTiles) {
-      log.assert(
-        Object.values(TILE_FORMATS).includes(formatTiles),
-        `Invalid value for formatTiles: ${formatTiles}. Use value from TILE_FORMATS`
-      );
-      loadOptions.cartoTile = {formatTiles};
+    // The backend doesn't yet support our custom mime-type, so force it here
+    // TODO remove once backend sends the correct mime-type
+    if (formatTiles === TILE_FORMATS.BINARY) {
+      loadOptions = {
+        ...loadOptions,
+        mimeType: 'application/vnd.carto-vector-tile'
+      };
     }
 
     return fetch(url, {propName: 'data', layer: this, loadOptions, signal});
@@ -161,4 +100,15 @@ export default class CartoTileLayer<
     const subLayer = new GeoJsonLayer(subLayerProps);
     return subLayer;
   }
+
+  getPickingInfo(params) {
+    const info = super.getPickingInfo(params);
+
+    if (this.state.binary && info.index !== -1) {
+      const {data} = params.sourceLayer!.props;
+      info.object = binaryToGeojson(data as BinaryFeatures, {globalFeatureId: info.index}) as DataT;
+    }
+
+    return info;
+  }
 }

package/src/layers/h3-tile-layer.ts

@@ -94,7 +94,13 @@ export default class H3TileLayer<DataT = any, ExtraPropsT = {}> extends Composit
         renderSubLayers,
         // minZoom and maxZoom are H3 resolutions, however we must use this naming as that is what the Tileset2D class expects
         minZoom: minresolution,
-        maxZoom: maxresolution
+        maxZoom: maxresolution,
+        loadOptions: {
+          ...this.getLoadOptions(),
+          cartoSpatialTile: {
+            scheme: 'h3'
+          }
+        }
       })
     ];
   }

package/src/layers/quadbin-layer.ts

@@ -19,7 +19,7 @@ type _QuadbinLayerProps<DataT> = {
    *
    * By default, it reads `quadbin` property of data object.
    */
-  getQuadbin?: AccessorFunction<DataT, string>;
+  getQuadbin?: AccessorFunction<DataT, bigint>;
 };
 
 export default class QuadbinLayer<DataT = any, ExtraProps = {}> extends GeoCellLayer<

package/src/layers/quadbin-tile-layer.ts

@@ -64,7 +64,13 @@ export default class QuadbinTileLayer<DataT = any, ExtraProps = {}> extends Comp
         // TODO: Tileset2D should be generic over TileIndex type
         TilesetClass: QuadbinTileset2D as any,
         renderSubLayers,
-        maxZoom
+        maxZoom,
+        loadOptions: {
+          ...this.getLoadOptions(),
+          cartoSpatialTile: {
+            scheme: 'quadbin'
+          }
+        }
       })
     ];
   }

package/src/layers/quadbin-tileset-2d.ts

@@ -1,34 +1,36 @@
 import {_Tileset2D as Tileset2D} from '@deck.gl/geo-layers';
-import {tileToQuadbin, quadbinToTile, quadbinParent, quadbinZoom} from './quadbin-utils';
+import {bigIntToHex, cellToParent, cellToTile, getResolution, tileToCell} from 'quadbin';
 
-type QuadbinTileIndex = {i: string};
+// For calculations bigint representation is used, but
+// for constructing URL also provide the hexidecimal value
+type QuadbinTileIndex = {q: bigint; i?: string};
 
 export default class QuadbinTileset2D extends Tileset2D {
   // @ts-expect-error for spatial indices, TileSet2d should be parametrized by TileIndexT
   getTileIndices(opts): QuadbinTileIndex[] {
     return super
       .getTileIndices(opts)
-      .map(tileToQuadbin)
-      .map(i => ({i}));
+      .map(tileToCell)
+      .map(q => ({q, i: bigIntToHex(q)}));
   }
 
   // @ts-expect-error TileIndex must be generic
-  getTileId({i}: QuadbinTileIndex): string {
-    return i;
+  getTileId({q, i}: QuadbinTileIndex): string {
+    return i || bigIntToHex(q);
   }
 
   // @ts-expect-error TileIndex must be generic
-  getTileMetadata({i}: QuadbinTileIndex) {
-    return super.getTileMetadata(quadbinToTile(i));
+  getTileMetadata({q}: QuadbinTileIndex) {
+    return super.getTileMetadata(cellToTile(q));
   }
 
   // @ts-expect-error TileIndex must be generic
-  getTileZoom({i}: QuadbinTileIndex): number {
-    return Number(quadbinZoom(i));
+  getTileZoom({q}: QuadbinTileIndex): number {
+    return Number(getResolution(q));
   }
 
   // @ts-expect-error TileIndex must be generic
-  getParentIndex({i}: QuadbinTileIndex): QuadbinTileIndex {
-    return {i: quadbinParent(i)};
+  getParentIndex({q}: QuadbinTileIndex): QuadbinTileIndex {
+    return {q: cellToParent(q)};
   }
 }

package/src/layers/quadbin-utils.ts

@@ -1,91 +1,10 @@
 import {worldToLngLat} from '@math.gl/web-mercator';
+import {cellToTile} from 'quadbin';
 
 const TILE_SIZE = 512;
 
-const B = [
-  0x5555555555555555n,
-  0x3333333333333333n,
-  0x0f0f0f0f0f0f0f0fn,
-  0x00ff00ff00ff00ffn,
-  0x0000ffff0000ffffn,
-  0x00000000ffffffffn
-];
-const S = [0n, 1n, 2n, 4n, 8n, 16n];
-
-export function indexToBigInt(index: string): bigint {
-  return BigInt(`0x${index}`);
-}
-
-export function bigIntToIndex(quadbin: bigint): string {
-  return quadbin.toString(16);
-}
-
-export function tileToQuadbin(tile): string {
-  if (tile.z < 0 || tile.z > 26) {
-    throw new Error('Wrong zoom');
-  }
-  const z = BigInt(tile.z);
-  let x = BigInt(tile.x) << (32n - z);
-  let y = BigInt(tile.y) << (32n - z);
-
-  for (let i = 0; i < 5; i++) {
-    const s = S[5 - i];
-    const b = B[4 - i];
-    x = (x | (x << s)) & b;
-    y = (y | (y << s)) & b;
-  }
-
-  const quadbin =
-    0x4000000000000000n |
-    (1n << 59n) | // | (mode << 59) | (mode_dep << 57)
-    (z << 52n) |
-    ((x | (y << 1n)) >> 12n) |
-    (0xfffffffffffffn >> (z * 2n));
-  return bigIntToIndex(quadbin);
-}
-
-export function quadbinToTile(index: string) {
-  const quadbin = indexToBigInt(index);
-  const mode = (quadbin >> 59n) & 7n;
-  const modeDep = (quadbin >> 57n) & 3n;
-  const z = (quadbin >> 52n) & 0x1fn;
-  const q = (quadbin & 0xfffffffffffffn) << 12n;
-
-  if (mode !== 1n && modeDep !== 0n) {
-    throw new Error('Wrong mode');
-  }
-
-  let x = q;
-  let y = q >> 1n;
-
-  for (let i = 0; i < 6; i++) {
-    const s = S[i];
-    const b = B[i];
-    x = (x | (x >> s)) & b;
-    y = (y | (y >> s)) & b;
-  }
-
-  x = x >> (32n - z);
-  y = y >> (32n - z);
-
-  return {z: Number(z), x: Number(x), y: Number(y)};
-}
-
-export function quadbinZoom(index: string) {
-  const quadbin = indexToBigInt(index);
-  return (quadbin >> 52n) & 0x1fn;
-}
-
-export function quadbinParent(index: string) {
-  const quadbin = indexToBigInt(index);
-  const zparent = quadbinZoom(index) - 1n;
-  const parent =
-    (quadbin & ~(0x1fn << 52n)) | (zparent << 52n) | (0xfffffffffffffn >> (zparent * 2n));
-  return bigIntToIndex(parent);
-}
-
-export function quadbinToWorldBounds(quadbin: string): [number[], number[]] {
-  const {x, y, z} = quadbinToTile(quadbin);
+export function quadbinToWorldBounds(quadbin: bigint): [number[], number[]] {
+  const {x, y, z} = cellToTile(quadbin);
   const mask = 1 << z;
   const scale = mask / TILE_SIZE;
   return [
@@ -94,7 +13,7 @@ export function quadbinToWorldBounds(quadbin: string): [number[], number[]] {
   ];
 }
 
-export function getQuadbinPolygon(quadbin: string): number[] {
+export function getQuadbinPolygon(quadbin: bigint): number[] {
   const [topLeft, bottomRight] = quadbinToWorldBounds(quadbin);
   const [w, n] = worldToLngLat(topLeft);
   const [e, s] = worldToLngLat(bottomRight);

package/src/layers/schema/carto-spatial-tile-loader.ts

@@ -1,9 +1,8 @@
-import Protobuf from 'pbf';
 import {LoaderOptions, LoaderWithParser} from '@loaders.gl/loader-utils';
 
-import {KeyValueProperties} from './carto-tile';
-import {binaryToSpatialjson, Properties, SpatialJson} from './spatialjson-utils';
 import {Tile, TileReader} from './carto-spatial-tile';
+import {parsePbf, unpackProperties} from './tile-loader-utils';
+import {IndexScheme, binaryToSpatialjson, SpatialJson} from './spatialjson-utils';
 
 const CartoSpatialTileLoader: LoaderWithParser = {
   name: 'CARTO Spatial Tile',
@@ -11,46 +10,30 @@ const CartoSpatialTileLoader: LoaderWithParser = {
   id: 'cartoSpatialTile',
   module: 'carto',
   extensions: ['pbf'],
-  mimeTypes: [
-    'application/vnd.carto-spatial-tile',
-    'application/x-protobuf' // Back-compatibility
-  ],
+  mimeTypes: ['application/vnd.carto-spatial-tile'],
   category: 'geometry',
   worker: false,
   parse: async (arrayBuffer, options) => parseCartoSpatialTile(arrayBuffer, options),
   parseSync: parseCartoSpatialTile,
-  options: {}
-};
-
-function parsePbf(buffer: ArrayBuffer): Tile {
-  const pbf = new Protobuf(buffer);
-  const tile = TileReader.read(pbf);
-  return tile;
-}
-
-function unpackProperties(properties: KeyValueProperties[]): Properties[] {
-  if (!properties || !properties.length) {
-    return [];
+  options: {
+    cartoSpatialTile: {
+      scheme: 'quadbin'
+    } as {scheme: IndexScheme}
   }
-  return properties.map(item => {
-    const currentRecord: Properties = {};
-    item.data.forEach(({key, value}) => {
-      currentRecord[key] = value;
-    });
-    return currentRecord;
-  });
-}
+};
 
 function parseCartoSpatialTile(
   arrayBuffer: ArrayBuffer,
   options?: LoaderOptions
 ): SpatialJson | null {
   if (!arrayBuffer) return null;
-  const tile = parsePbf(arrayBuffer);
+  const tile: Tile = parsePbf(arrayBuffer, TileReader);
 
   const {cells} = tile;
+  const scheme = options?.cartoSpatialTile?.scheme;
   const data = {
-    cells: {...cells, properties: unpackProperties(cells.properties)}
+    cells: {...cells, properties: unpackProperties(cells.properties)},
+    scheme
   };
 
   return binaryToSpatialjson(data);

package/src/layers/schema/carto-tile.ts

@@ -67,6 +67,21 @@ class IntsReader {
   }
 }
 
+// Fields ========================================
+
+interface Fields {
+  id: number;
+}
+
+class FieldsReader {
+  static read(pbf, end?: number): Fields {
+    return pbf.readFields(FieldsReader._readField, {id: 0}, end);
+  }
+  static _readField(this: void, tag: number, obj: Fields, pbf) {
+    if (tag === 1) obj.id = pbf.readVarint();
+  }
+}
+
 // NumericProp ========================================
 
 export interface NumericProp {
@@ -106,13 +121,21 @@ interface Points {
   featureIds: Ints;
   properties: KeyValueProperties[];
   numericProps: Record<string, NumericProp>;
+  fields: Fields[];
 }
 
 class PointsReader {
   static read(pbf, end?: number): Points {
     return pbf.readFields(
       PointsReader._readField,
-      {positions: null, globalFeatureIds: null, featureIds: null, properties: [], numericProps: {}},
+      {
+        positions: null,
+        globalFeatureIds: null,
+        featureIds: null,
+        properties: [],
+        numericProps: {},
+        fields: []
+      },
       end
     );
   }
@@ -124,7 +147,7 @@ class PointsReader {
     else if (tag === 5) {
       const entry = NumericPropKeyValueReader.read(pbf, pbf.readVarint() + pbf.pos);
       obj.numericProps[entry.key] = entry.value;
-    }
+    } else if (tag === 6) obj.fields.push(FieldsReader.read(pbf, pbf.readVarint() + pbf.pos));
   }
 }
 
@@ -143,7 +166,8 @@ class LinesReader {
         globalFeatureIds: null,
         featureIds: null,
         properties: [],
-        numericProps: {}
+        numericProps: {},
+        fields: []
       },
       end
     );
@@ -157,7 +181,7 @@ class LinesReader {
     else if (tag === 6) {
       const entry = NumericPropKeyValueReader.read(pbf, pbf.readVarint() + pbf.pos);
       obj.numericProps[entry.key] = entry.value;
-    }
+    } else if (tag === 7) obj.fields.push(FieldsReader.read(pbf, pbf.readVarint() + pbf.pos));
   }
 }
 // Polygons ========================================
@@ -180,7 +204,8 @@ class PolygonsReader {
         primitivePolygonIndices: null,
         triangles: null,
         properties: [],
-        numericProps: {}
+        numericProps: {},
+        fields: []
       },
       end
     );
@@ -197,7 +222,7 @@ class PolygonsReader {
     else if (tag === 8) {
       const entry = NumericPropKeyValueReader.read(pbf, pbf.readVarint() + pbf.pos);
       obj.numericProps[entry.key] = entry.value;
-    }
+    } else if (tag === 9) obj.fields.push(FieldsReader.read(pbf, pbf.readVarint() + pbf.pos));
   }
 }
 

package/src/layers/schema/carto-vector-tile-loader.ts

@@ -0,0 +1,39 @@
+import {LoaderOptions, LoaderWithParser} from '@loaders.gl/loader-utils';
+import type {BinaryFeatures} from '@loaders.gl/schema';
+
+import {TileReader} from './carto-tile';
+import {parsePbf, unpackProperties} from './tile-loader-utils';
+
+const CartoVectorTileLoader: LoaderWithParser = {
+  name: 'CARTO Vector Tile',
+  version: '1',
+  id: 'cartoVectorTile',
+  module: 'carto',
+  extensions: ['pbf'],
+  mimeTypes: ['application/vnd.carto-vector-tile'],
+  category: 'geometry',
+  worker: false,
+  parse: async (arrayBuffer, options) => parseCartoVectorTile(arrayBuffer, options),
+  parseSync: parseCartoVectorTile,
+  options: {}
+};
+
+function parseCartoVectorTile(
+  arrayBuffer: ArrayBuffer,
+  options?: LoaderOptions
+): BinaryFeatures | null {
+  if (!arrayBuffer) return null;
+  const tile = parsePbf(arrayBuffer, TileReader);
+
+  const {points, lines, polygons} = tile;
+  const data = {
+    points: {...points, properties: unpackProperties(points.properties)},
+    lines: {...lines, properties: unpackProperties(lines.properties)},
+    polygons: {...polygons, properties: unpackProperties(polygons.properties)}
+  };
+
+  // Note: there is slight, difference in `numericProps` type, however geojson/mvtlayer can cope with this
+  return data as unknown as BinaryFeatures;
+}
+
+export default CartoVectorTileLoader;

package/src/layers/schema/spatialjson-utils.ts

@@ -1,4 +1,4 @@
-import {bigIntToIndex} from '../quadbin-utils';
+import {bigIntToHex} from 'quadbin';
 
 export type IndexScheme = 'h3' | 'quadbin';
 type TypedArray = Float32Array | Float64Array;
@@ -13,16 +13,16 @@ export type Cells = {
 };
 export type SpatialBinary = {scheme?: IndexScheme; cells: Cells};
 export type SpatialJson = {
-  id: string;
+  id: string | bigint;
   properties: Properties;
 }[];
 
 export function binaryToSpatialjson(binary: SpatialBinary): SpatialJson {
-  const {cells} = binary;
+  const {cells, scheme} = binary;
   const count = cells.indices.value.length;
   const spatial: any[] = [];
   for (let i = 0; i < count; i++) {
-    const id = bigIntToIndex(cells.indices.value[i]);
+    const id = scheme === 'h3' ? bigIntToHex(cells.indices.value[i]) : cells.indices.value[i];
 
     const properties = {...cells.properties[i]};
     for (const key of Object.keys(cells.numericProps)) {

package/src/layers/schema/tile-loader-utils.ts

@@ -0,0 +1,22 @@
+import Protobuf from 'pbf';
+import {KeyValueProperties} from './carto-tile';
+import {Properties} from './spatialjson-utils';
+
+export function parsePbf(buffer: ArrayBuffer, TileReader) {
+  const pbf = new Protobuf(buffer);
+  const tile = TileReader.read(pbf);
+  return tile;
+}
+
+export function unpackProperties(properties: KeyValueProperties[]): Properties[] {
+  if (!properties || !properties.length) {
+    return [];
+  }
+  return properties.map(item => {
+    const currentRecord: Properties = {};
+    item.data.forEach(({key, value}) => {
+      currentRecord[key] = value;
+    });
+    return currentRecord;
+  });
+}

package/src/layers/spatial-index-tile-layer.ts

@@ -3,7 +3,13 @@ import CartoSpatialTileLoader from './schema/carto-spatial-tile-loader';
 registerLoaders([CartoSpatialTileLoader]);
 
 import {PickingInfo} from '@deck.gl/core';
-import {TileLayer, _Tile2DHeader as Tile2DHeader} from '@deck.gl/geo-layers';
+import {
+  TileLayer,
+  _getURLFromTemplate,
+  _Tile2DHeader as Tile2DHeader,
+  _TileLoadProps as TileLoadProps
+} from '@deck.gl/geo-layers';
+import {TILE_FORMATS} from '../api/maps-api-common';
 
 function isFeatureIdDefined(value: unknown): boolean {
   return value !== undefined && value !== null && value !== '';
@@ -12,6 +18,36 @@ function isFeatureIdDefined(value: unknown): boolean {
 export default class SpatialIndexTileLayer<ExtraProps = {}> extends TileLayer<any, ExtraProps> {
   static layerName = 'SpatialIndexTileLayer';
 
+  getTileData(tile: TileLoadProps) {
+    const {data, getTileData, fetch} = this.props;
+    const {signal} = tile;
+
+    tile.url =
+      typeof data === 'string' || Array.isArray(data) ? _getURLFromTemplate(data, tile) : null;
+    if (!tile.url) {
+      return Promise.reject('Invalid URL');
+    }
+
+    if (getTileData) {
+      return getTileData(tile);
+    }
+
+    let loadOptions = this.getLoadOptions();
+    // @ts-ignore
+    const {formatTiles} = this.props;
+
+    // The backend doesn't yet support our custom mime-type, so force it here
+    // TODO remove entire `getTileData` method once backend sends the correct mime-type
+    if (formatTiles === TILE_FORMATS.BINARY) {
+      loadOptions = {
+        ...loadOptions,
+        mimeType: 'application/vnd.carto-spatial-tile'
+      };
+    }
+
+    return fetch(tile.url, {propName: 'data', layer: this, loadOptions, signal});
+  }
+
   protected _updateAutoHighlight(info: PickingInfo): void {
     const {hoveredFeatureId} = this.state;
     const hoveredFeature = info.object;