@deck.gl/carto 8.7.0-beta.1 → 8.7.0-beta.5

package/package.json

@@ -2,7 +2,7 @@
   "name": "@deck.gl/carto",
   "description": "CARTO official integration with Deck.gl. Build geospatial applications using CARTO and Deck.gl.",
   "license": "MIT",
-  "version": "8.7.0-beta.1",
+  "version": "8.7.0-beta.5",
   "publishConfig": {
     "access": "public"
   },
@@ -35,13 +35,14 @@
     "@loaders.gl/loader-utils": "^3.1.5",
     "@loaders.gl/mvt": "^3.1.5",
     "@loaders.gl/tiles": "^3.1.5",
-    "@math.gl/web-mercator": "^3.5.6",
+    "@math.gl/web-mercator": "^3.5.7",
     "cartocolor": "^4.0.2",
     "d3-array": "^2.8.0",
     "d3-color": "^2.0.0",
     "d3-format": "^2.0.0",
     "d3-scale": "^3.2.3",
-    "moment-timezone": "^0.5.33"
+    "moment-timezone": "^0.5.33",
+    "pbf": "^3.2.1"
   },
   "peerDependencies": {
     "@deck.gl/core": "^8.0.0",
@@ -49,5 +50,5 @@
     "@deck.gl/layers": "^8.0.0",
     "@loaders.gl/core": "^3.1.5"
   },
-  "gitHead": "0297ca9d2c85d980b075e2af4cfbeb75e31db55a"
+  "gitHead": "02b67ab8085c6283c0cae45e37a9b70b24f4434b"
 }

package/src/api/index.js

@@ -1,3 +1,3 @@
-export {FORMATS, MAP_TYPES, API_VERSIONS} from './maps-api-common';
+export {FORMATS, TILE_FORMATS, MAP_TYPES, API_VERSIONS} from './maps-api-common';
 export {getDataV2, CONNECTIONS} from './maps-client';
 export {fetchLayerData, fetchMap, getData, mapInstantiation} from './maps-v3-client';

package/src/api/layer-map.js

@@ -11,8 +11,10 @@ import {
 } from 'd3-scale';
 import {format as d3Format} from 'd3-format';
 import moment from 'moment-timezone';
+import {log} from '@deck.gl/core';
 import {H3HexagonLayer, MVTLayer} from '@deck.gl/geo-layers';
 import {GeoJsonLayer} from '@deck.gl/layers';
+import {_CartoDynamicTileLayer, TILE_FORMATS} from '@deck.gl/carto';
 
 const SCALE_FUNCS = {
   linear: scaleLinear,
@@ -62,37 +64,60 @@ function mergePropMaps(a, b) {
   return {...a, ...b, visConfig: {...a.visConfig, ...b.visConfig}};
 }
 
-export const LAYER_MAP = {
-  point: {
-    Layer: GeoJsonLayer,
-    propMap: mergePropMaps(sharedPropMap, {
-      visConfig: {outline: 'stroked'}
-    }),
-    defaultProps
-  },
-  geojson: {
-    Layer: GeoJsonLayer,
-    propMap: sharedPropMap,
-    defaultProps: {...defaultProps, lineWidthScale: 2}
-  },
-  hexagonId: {
-    Layer: H3HexagonLayer,
-    propMap: mergePropMaps(sharedPropMap, {
-      visConfig: {coverage: 'coverage', elevationScale: 'elevationScale'}
-    }),
-    defaultProps: {...defaultProps, getHexagon: d => d.h3}
-  },
-  mvt: {
-    Layer: MVTLayer,
+export function getLayer(type, config, dataset) {
+  if (type === 'mvt' || type === 'tileset') {
+    return getTileLayer(dataset);
+  }
+
+  const hexagonId = config.columns?.hex_id;
+  const layer = {
+    point: {
+      Layer: GeoJsonLayer,
+      propMap: mergePropMaps(sharedPropMap, {
+        visConfig: {outline: 'stroked'}
+      }),
+      defaultProps
+    },
+    geojson: {
+      Layer: GeoJsonLayer,
+      propMap: sharedPropMap,
+      defaultProps: {...defaultProps, lineWidthScale: 2}
+    },
+    hexagonId: {
+      Layer: H3HexagonLayer,
+      propMap: mergePropMaps(sharedPropMap, {
+        visConfig: {coverage: 'coverage', elevationScale: 'elevationScale'}
+      }),
+      defaultProps: {...defaultProps, getHexagon: d => d[hexagonId]}
+    }
+  }[type];
+
+  log.assert(layer, `Unsupported layer type: ${type}`);
+
+  return layer;
+}
+
+function getTileLayer(dataset) {
+  const {
+    data: {
+      tiles: [tileUrl]
+    }
+  } = dataset;
+  /* global URL */
+  const formatTiles = new URL(tileUrl).searchParams.get('formatTiles') || TILE_FORMATS.MVT;
+
+  return {
+    Layer: formatTiles === TILE_FORMATS.MVT ? MVTLayer : _CartoDynamicTileLayer,
     propMap: sharedPropMap,
     defaultProps: {
       ...defaultProps,
       pointRadiusScale: 0.3,
       lineWidthScale: 2,
-      uniqueIdProperty: 'geoid'
+      uniqueIdProperty: 'geoid',
+      formatTiles
     }
-  }
-};
+  };
+}
 
 function domainFromAttribute(attribute, scaleType) {
   if (scaleType === 'ordinal' || scaleType === 'point') {

package/src/api/maps-api-common.js

@@ -26,6 +26,13 @@ export const FORMATS = {
   JSON: 'json'
 };
 
+// AVAILABLE FORMATS FOR TILES
+export const TILE_FORMATS = {
+  MVT: 'mvt',
+  GEOJSON: 'geojson',
+  BINARY: 'binary'
+};
+
 /**
  * Simple encode parameter
  */

package/src/api/maps-v3-client.js

@@ -255,7 +255,7 @@ export async function getData({
   return layerData.data;
 }
 
-/* global clearInterval, setInterval, URLSearchParams */
+/* global clearInterval, setInterval, URL */
 async function _fetchMapDataset(dataset, accessToken, credentials, clientId) {
   // First fetch metadata
   const {connectionName: connection, source, type} = dataset;
@@ -268,7 +268,7 @@ async function _fetchMapDataset(dataset, accessToken, credentials, clientId) {
   });
 
   // Extract the last time the data changed
-  const cache = parseInt(new URLSearchParams(url).get('cache'), 10);
+  const cache = parseInt(new URL(url).searchParams.get('cache'), 10);
   if (cache && dataset.cache === cache) {
     return false;
   }
@@ -291,6 +291,7 @@ export async function fetchMap({cartoMapId, clientId, credentials, autoRefresh,
     ...(defaultCredentials.apiVersion === API_VERSIONS.V3 && defaultCredentials),
     ...credentials
   };
+  const {accessToken} = localCreds;
 
   log.assert(cartoMapId, `Must define CARTO map id: fetchMap({cartoMapId: 'XXXX-XXXX-XXXX'})`);
 
@@ -307,7 +308,7 @@ export async function fetchMap({cartoMapId, clientId, credentials, autoRefresh,
   }
 
   const url = `${localCreds.mapsUrl}/public/${cartoMapId}`;
-  const map = await request({url});
+  const map = await request({url, accessToken});
 
   // Periodically check if the data has changed. Note that this
   // will not update when a map is published.

package/src/api/parseMap.js

@@ -1,6 +1,6 @@
 import GL from '@luma.gl/constants';
 import {
-  LAYER_MAP,
+  getLayer,
   getColorAccessor,
   getSizeAccessor,
   getTextAccessor,
@@ -24,14 +24,12 @@ export function parseMap(json) {
     mapStyle,
     layers: extractTextLayers(layers.reverse()).map(({id, type, config, visualChannels}) => {
       try {
-        log.assert(type in LAYER_MAP, `Unsupported layer type: ${type}`);
-        const {Layer, propMap, defaultProps} = LAYER_MAP[type];
-
         const {dataId} = config;
         const dataset = datasets.find(d => d.id === dataId);
         log.assert(dataset, `No dataset matching dataId: ${dataId}`);
         const {data} = dataset;
         log.assert(data, `No data loaded for dataId: ${dataId}`);
+        const {Layer, propMap, defaultProps} = getLayer(type, config, dataset);
         return new Layer({
           id,
           data,

package/src/index.js

@@ -1,11 +1,13 @@
 export {getDefaultCredentials, setDefaultCredentials} from './config.js';
 export {default as CartoLayer} from './layers/carto-layer';
+export {default as _CartoDynamicTileLayer} from './layers/carto-dynamic-tile-layer';
 export {default as BASEMAP} from './basemap';
 export {default as colorBins} from './style/color-bins-style';
 export {default as colorCategories} from './style/color-categories-style';
 export {default as colorContinuous} from './style/color-continuous-style';
 export {
   FORMATS,
+  TILE_FORMATS,
   MAP_TYPES,
   API_VERSIONS,
   fetchLayerData,

package/src/layers/carto-dynamic-tile-layer.js

@@ -1,20 +1,51 @@
 /* global TextDecoder */
+import Protobuf from 'pbf';
 import {log} from '@deck.gl/core';
 import {ClipExtension} from '@deck.gl/extensions';
 import {MVTLayer, _getURLFromTemplate} from '@deck.gl/geo-layers';
 import {GeoJsonLayer} from '@deck.gl/layers';
 import {geojsonToBinary} from '@loaders.gl/gis';
-import {encodeParameter} from '../api/maps-api-common';
+import {Tile} from './schema/carto-dynamic-tile';
+import {TILE_FORMATS} from '../api/maps-api-common';
 
 function parseJSON(arrayBuffer) {
   return JSON.parse(new TextDecoder().decode(arrayBuffer));
 }
 
+function parsePbf(buffer) {
+  const pbf = new Protobuf(buffer);
+  const tile = Tile.read(pbf);
+  return tile;
+}
+
+function unpackProperties(properties) {
+  if (!properties || !properties.length) {
+    return [];
+  }
+  return properties.map(item => {
+    const currentRecord = {};
+    item.data.forEach(({key, value}) => {
+      currentRecord[key] = value;
+    });
+    return currentRecord;
+  });
+}
+
 function parseCartoDynamicTile(arrayBuffer, options) {
   if (!arrayBuffer) return null;
   const formatTiles = options && options.cartoDynamicTile && options.cartoDynamicTile.formatTiles;
-  if (formatTiles === 'geojson') return geojsonToBinary(parseJSON(arrayBuffer).features);
-  return null;
+  if (formatTiles === TILE_FORMATS.GEOJSON) return geojsonToBinary(parseJSON(arrayBuffer).features);
+
+  const tile = parsePbf(arrayBuffer);
+
+  const {points, lines, polygons} = tile;
+  const data = {
+    points: {...points, properties: unpackProperties(points.properties)},
+    lines: {...lines, properties: unpackProperties(lines.properties)},
+    polygons: {...polygons, properties: unpackProperties(polygons.properties)}
+  };
+
+  return data;
 }
 
 const CartoDynamicTileLoader = {
@@ -29,7 +60,8 @@ const CartoDynamicTileLoader = {
   parseSync: parseCartoDynamicTile,
   options: {
     cartoDynamicTile: {
-      formatTiles: 'geojson'
+      // TODO change to BINARY for 8.7 prod release
+      formatTiles: TILE_FORMATS.GEOJSON
     }
   }
 };
@@ -52,11 +84,14 @@ export default class CartoDynamicTileLayer extends MVTLayer {
 
     if (formatTiles) {
       log.assert(
-        ['geojson'].includes(formatTiles),
-        `Invalid value for formatTiles: ${formatTiles}`
+        Object.values(TILE_FORMATS).includes(formatTiles),
+        `Invalid value for formatTiles: ${formatTiles}. Use value from TILE_FORMATS`
       );
-      url += `&${encodeParameter('formatTiles', formatTiles)}`;
-      loadOptions.cartoBinaryTile = {formatTiles};
+      // eslint-disable-next-line no-undef
+      const newUrl = new URL(url);
+      newUrl.searchParams.set('formatTiles', formatTiles);
+      url = newUrl.toString();
+      loadOptions.cartoDynamicTile = {formatTiles};
     }
 
     return fetch(url, {propName: 'data', layer: this, loadOptions, signal});
@@ -72,7 +107,7 @@ export default class CartoDynamicTileLayer extends MVTLayer {
     const {
       bbox: {west, south, east, north}
     } = props.tile;
-    props.extensions = [new ClipExtension()];
+    props.extensions = [new ClipExtension(), ...(props.extensions || [])];
     props.clipBounds = [west, south, east, north];
 
     const subLayer = new GeoJsonLayer({

package/src/layers/carto-layer.js

@@ -3,7 +3,7 @@ import CartoDynamicTileLayer from './carto-dynamic-tile-layer';
 import {MVTLayer} from '@deck.gl/geo-layers';
 import {GeoJsonLayer} from '@deck.gl/layers';
 import {fetchLayerData, getDataV2, API_VERSIONS} from '../api';
-import {FORMATS, MAP_TYPES} from '../api/maps-api-common';
+import {FORMATS, MAP_TYPES, TILE_FORMATS} from '../api/maps-api-common';
 import {getDefaultCredentials} from '../config';
 
 const defaultProps = {
@@ -27,6 +27,9 @@ const defaultProps = {
   // (String, optional): format of data
   format: null,
 
+  // (String, optional): force format of data for tiles
+  formatTiles: null,
+
   // (String, optional): clientId identifier used for internal tracing, place here a string to identify the client who is doing the request.
   clientId: null,
 
@@ -123,29 +126,37 @@ export default class CartoLayer extends CompositeLayer {
     }
   }
 
-  renderLayers() {
+  _getRenderLayer() {
     const {data, format, apiVersion} = this.state;
 
-    if (!data) return null;
+    if (apiVersion === API_VERSIONS.V1 || apiVersion === API_VERSIONS.V2) {
+      return MVTLayer;
+    }
 
-    const {type, updateTriggers} = this.props;
+    if (format === FORMATS.TILEJSON) {
+      /* global URL */
+      const formatTiles =
+        this.props.formatTiles ||
+        new URL(data.tiles[0]).searchParams.get('formatTiles') ||
+        TILE_FORMATS.MVT;
+      return formatTiles === TILE_FORMATS.MVT ? MVTLayer : CartoDynamicTileLayer;
+    }
 
-    let layer;
+    // It's a geojson layer
+    return GeoJsonLayer;
+  }
 
-    if (apiVersion === API_VERSIONS.V3 && type === MAP_TYPES.TABLE && format === FORMATS.TILEJSON) {
-      layer = CartoDynamicTileLayer;
-    } else if (
-      apiVersion === API_VERSIONS.V1 ||
-      apiVersion === API_VERSIONS.V2 ||
-      format === FORMATS.TILEJSON
-    ) {
-      layer = MVTLayer;
-    } else {
-      layer = GeoJsonLayer;
-    }
+  renderLayers() {
+    const {data} = this.state;
+
+    if (!data) return null;
+
+    const {updateTriggers} = this.props;
+
+    const layer = this._getRenderLayer();
 
-    const {uniqueIdProperty} = defaultProps;
-    const props = {uniqueIdProperty, ...this.props};
+    const {formatTiles, uniqueIdProperty} = defaultProps;
+    const props = {formatTiles, uniqueIdProperty, ...this.props};
     delete props.data;
 
     // eslint-disable-next-line new-cap

package/src/layers/schema/carto-dynamic-tile.js

@@ -0,0 +1,195 @@
+// KeyValueObject ========================================
+
+const KeyValueObject = {};
+
+KeyValueObject.read = function (pbf, end) {
+  return pbf.readFields(KeyValueObject._readField, {key: '', value: ''}, end);
+};
+KeyValueObject._readField = function (tag, obj, pbf) {
+  if (tag === 1) obj.key = pbf.readString();
+  else if (tag === 2) obj.value = pbf.readString();
+};
+
+// Properties ========================================
+
+const Properties = {};
+
+Properties.read = function (pbf, end) {
+  return pbf.readFields(Properties._readField, {data: []}, end);
+};
+Properties._readField = function (tag, obj, pbf) {
+  if (tag === 1) obj.data.push(KeyValueObject.read(pbf, pbf.readVarint() + pbf.pos));
+};
+
+// Doubles ========================================
+
+const Doubles = {};
+
+Doubles.read = function (pbf, end) {
+  // TODO perhaps we can do better and directly map from the source
+  // ArrayBuffer using ArrayBuffer.slice()
+  const {value, size} = pbf.readFields(Doubles._readField, {value: [], size: 0}, end);
+  return {value: new Float32Array(value), size};
+};
+Doubles._readField = function (tag, obj, pbf) {
+  if (tag === 1) pbf.readPackedDouble(obj.value);
+  else if (tag === 2) obj.size = pbf.readVarint(true);
+};
+
+// Ints ========================================
+
+const Ints = {};
+
+Ints.read = function (pbf, end) {
+  const {value, size} = pbf.readFields(Ints._readField, {value: [], size: 0}, end);
+  return {value: new Uint32Array(value), size};
+};
+Ints._readField = function (tag, obj, pbf) {
+  if (tag === 1) pbf.readPackedVarint(obj.value);
+  else if (tag === 2) obj.size = pbf.readVarint(true);
+};
+
+// NumericProp ========================================
+
+const NumericProp = {};
+
+NumericProp.read = function (pbf, end) {
+  return pbf.readFields(NumericProp._readField, {value: []}, end);
+};
+NumericProp._readField = function (tag, obj, pbf) {
+  if (tag === 1) pbf.readPackedDouble(obj.value);
+};
+
+// NumericPropKeyValue ========================================
+
+const NumericPropKeyValue = {};
+
+NumericPropKeyValue.read = function (pbf, end) {
+  return pbf.readFields(NumericPropKeyValue._readField, {key: '', value: null}, end);
+};
+NumericPropKeyValue._readField = function (tag, obj, pbf) {
+  if (tag === 1) obj.key = pbf.readString();
+  else if (tag === 2) obj.value = NumericProp.read(pbf, pbf.readVarint() + pbf.pos);
+};
+
+// NumericProp._FieldEntry1 ========================================
+
+NumericProp._FieldEntry1 = {};
+
+NumericProp._FieldEntry1.read = function (pbf, end) {
+  return pbf.readFields(NumericProp._FieldEntry1._readField, {key: '', value: 0}, end);
+};
+NumericProp._FieldEntry1._readField = function (tag, obj, pbf) {
+  if (tag === 1) obj.key = pbf.readString();
+  else if (tag === 2) obj.value = pbf.readDouble();
+};
+
+// StringProp ========================================
+
+const StringProp = {};
+
+StringProp.read = function (pbf, end) {
+  return pbf.readFields(StringProp._readField, {value: []}, end);
+};
+StringProp._readField = function (tag, obj, pbf) {
+  if (tag === 1) obj.value.push(pbf.readString());
+};
+
+// Points ========================================
+
+const Points = {};
+
+Points.read = function (pbf, end) {
+  return pbf.readFields(
+    Points._readField,
+    {positions: null, globalFeatureIds: null, featureIds: null, properties: [], numericProps: {}},
+    end
+  );
+};
+Points._readField = function (tag, obj, pbf) {
+  if (tag === 1) obj.positions = Doubles.read(pbf, pbf.readVarint() + pbf.pos);
+  else if (tag === 2) obj.globalFeatureIds = Ints.read(pbf, pbf.readVarint() + pbf.pos);
+  else if (tag === 3) obj.featureIds = Ints.read(pbf, pbf.readVarint() + pbf.pos);
+  else if (tag === 4) obj.properties.push(Properties.read(pbf, pbf.readVarint() + pbf.pos));
+  else if (tag === 5) {
+    const entry = NumericPropKeyValue.read(pbf, pbf.readVarint() + pbf.pos);
+    obj.numericProps[entry.key] = entry.value;
+  }
+};
+
+// Lines ========================================
+
+const Lines = {};
+
+Lines.read = function (pbf, end) {
+  return pbf.readFields(
+    Lines._readField,
+    {
+      positions: null,
+      pathIndices: null,
+      globalFeatureIds: null,
+      featureIds: null,
+      properties: [],
+      numericProps: {}
+    },
+    end
+  );
+};
+Lines._readField = function (tag, obj, pbf) {
+  if (tag === 1) obj.positions = Doubles.read(pbf, pbf.readVarint() + pbf.pos);
+  else if (tag === 2) obj.pathIndices = Ints.read(pbf, pbf.readVarint() + pbf.pos);
+  else if (tag === 3) obj.globalFeatureIds = Ints.read(pbf, pbf.readVarint() + pbf.pos);
+  else if (tag === 4) obj.featureIds = Ints.read(pbf, pbf.readVarint() + pbf.pos);
+  else if (tag === 5) obj.properties.push(Properties.read(pbf, pbf.readVarint() + pbf.pos));
+  else if (tag === 6) {
+    const entry = NumericPropKeyValue.read(pbf, pbf.readVarint() + pbf.pos);
+    obj.numericProps[entry.key] = entry.value;
+  }
+};
+
+// Polygons ========================================
+
+const Polygons = {};
+
+Polygons.read = function (pbf, end) {
+  return pbf.readFields(
+    Polygons._readField,
+    {
+      positions: null,
+      polygonIndices: null,
+      globalFeatureIds: null,
+      featureIds: null,
+      primitivePolygonIndices: null,
+      triangles: null,
+      properties: [],
+      numericProps: {}
+    },
+    end
+  );
+};
+Polygons._readField = function (tag, obj, pbf) {
+  if (tag === 1) obj.positions = Doubles.read(pbf, pbf.readVarint() + pbf.pos);
+  else if (tag === 2) obj.polygonIndices = Ints.read(pbf, pbf.readVarint() + pbf.pos);
+  else if (tag === 3) obj.globalFeatureIds = Ints.read(pbf, pbf.readVarint() + pbf.pos);
+  else if (tag === 4) obj.featureIds = Ints.read(pbf, pbf.readVarint() + pbf.pos);
+  else if (tag === 5) obj.primitivePolygonIndices = Ints.read(pbf, pbf.readVarint() + pbf.pos);
+  else if (tag === 6) obj.triangles = Ints.read(pbf, pbf.readVarint() + pbf.pos);
+  else if (tag === 7) obj.properties.push(Properties.read(pbf, pbf.readVarint() + pbf.pos));
+  else if (tag === 8) {
+    const entry = NumericPropKeyValue.read(pbf, pbf.readVarint() + pbf.pos);
+    obj.numericProps[entry.key] = entry.value;
+  }
+};
+
+// Tile ========================================
+
+export const Tile = {};
+
+Tile.read = function (pbf, end) {
+  return pbf.readFields(Tile._readField, {points: null, lines: null, polygons: null}, end);
+};
+Tile._readField = function (tag, obj, pbf) {
+  if (tag === 1) obj.points = Points.read(pbf, pbf.readVarint() + pbf.pos);
+  else if (tag === 2) obj.lines = Lines.read(pbf, pbf.readVarint() + pbf.pos);
+  else if (tag === 3) obj.polygons = Polygons.read(pbf, pbf.readVarint() + pbf.pos);
+};