@deck.gl/carto 8.8.9 → 8.8.11

package/package.json

@@ -2,7 +2,7 @@
   "name": "@deck.gl/carto",
   "description": "CARTO official integration with Deck.gl. Build geospatial applications using CARTO and Deck.gl.",
   "license": "MIT",
-  "version": "8.8.9",
+  "version": "8.8.11",
   "publishConfig": {
     "access": "public"
   },
@@ -61,5 +61,5 @@
     "@deck.gl/layers": "^8.0.0",
     "@loaders.gl/core": "^3.0.0"
   },
-  "gitHead": "1cceaed3dad056c3fb51864e7f56585271ed1e5a"
+  "gitHead": "f70e8c354aee34949134f9e802f59cf60db4de19"
 }

package/src/api/layer-map.ts

@@ -13,7 +13,7 @@ import {
 import {format as d3Format} from 'd3-format';
 import moment from 'moment-timezone';
 
-import {Layer, _ConstructorOf as ConstructorOf} from '@deck.gl/core';
+import {Accessor, Layer, _ConstructorOf as ConstructorOf} from '@deck.gl/core';
 import {CPUGridLayer, HeatmapLayer, HexagonLayer} from '@deck.gl/aggregation-layers';
 import {GeoJsonLayer} from '@deck.gl/layers';
 import {H3HexagonLayer, MVTLayer} from '@deck.gl/geo-layers';
@@ -23,6 +23,15 @@ import H3TileLayer from '../layers/h3-tile-layer';
 import QuadbinTileLayer from '../layers/quadbin-tile-layer';
 import {TILE_FORMATS} from './maps-api-common';
 import {assert} from '../utils';
+import {
+  CustomMarkersRange,
+  MapDataset,
+  MapTextSubLayerConfig,
+  TextLabel,
+  VisConfig,
+  VisualChannelField,
+  VisualChannels
+} from './types';
 
 const SCALE_FUNCS = {
   linear: scaleLinear,
@@ -36,6 +45,10 @@ const SCALE_FUNCS = {
 };
 export type SCALE_TYPE = keyof typeof SCALE_FUNCS;
 
+function identity<T>(v: T): T {
+  return v;
+}
+
 const UNKNOWN_COLOR = '#868d91';
 
 export const AGGREGATION = {
@@ -88,6 +101,13 @@ const sharedPropMap = {
   }
 };
 
+const customMarkersPropsMap = {
+  color: 'getIconColor',
+  visConfig: {
+    radius: 'getIconSize'
+  }
+};
+
 const aggregationVisConfig = {
   colorAggregation: x => ({colorAggregation: AGGREGATION[x] || AGGREGATION.sum}),
   colorRange: x => ({colorRange: x.colors.map(hexToRGBA)}),
@@ -110,11 +130,16 @@ function mergePropMaps(a: Record<string, any> = {}, b: Record<string, any> = {})
 
 export function getLayer(
   type: string,
-  config,
-  dataset
+  config: MapTextSubLayerConfig,
+  dataset: MapDataset
 ): {Layer: ConstructorOf<Layer>; propMap: any; defaultProps: any} {
+  let basePropMap: any = sharedPropMap;
+
+  if (config.visConfig?.customMarkers && !config.textLabel) {
+    basePropMap = mergePropMaps(sharedPropMap, customMarkersPropsMap);
+  }
   if (type === 'mvt' || type === 'tileset' || type === 'h3' || type === 'quadbin') {
-    return getTileLayer(dataset);
+    return getTileLayer(dataset, basePropMap);
   }
 
   const geoColumn = dataset?.geoColumn;
@@ -160,7 +185,7 @@ export function getLayer(
   assert(layer, `Unsupported layer type: ${type}`);
   return {
     ...layer,
-    propMap: mergePropMaps(sharedPropMap, layer.propMap),
+    propMap: mergePropMaps(basePropMap, layer.propMap),
     defaultProps: {...defaultProps, ...layer.defaultProps}
   };
 }
@@ -183,7 +208,7 @@ export function layerFromTileDataset(
   return CartoTileLayer;
 }
 
-function getTileLayer(dataset) {
+function getTileLayer(dataset: MapDataset, basePropMap) {
   const {
     aggregationExp,
     aggregationResLevel,
@@ -197,7 +222,7 @@ function getTileLayer(dataset) {
 
   return {
     Layer: layerFromTileDataset(formatTiles, scheme),
-    propMap: sharedPropMap,
+    propMap: basePropMap,
     defaultProps: {
       ...defaultProps,
       ...(aggregationExp && {aggregationExp}),
@@ -225,7 +250,7 @@ function domainFromAttribute(attribute, scaleType: SCALE_TYPE, scaleLength: numb
 }
 
 function domainFromValues(values, scaleType: SCALE_TYPE) {
-  if (scaleType === 'ordinal') {
+  if (scaleType === 'ordinal' || scaleType === 'point') {
     return groupSort(
       values,
       g => -g.length,
@@ -268,7 +293,7 @@ function normalizeAccessor(accessor, data) {
   return accessor;
 }
 
-export function opacityToAlpha(opacity) {
+export function opacityToAlpha(opacity?: number) {
   return opacity !== undefined ? Math.round(255 * Math.pow(opacity, 1 / 2.2)) : 255;
 }
 
@@ -339,16 +364,71 @@ export function getColorAccessor(
   return normalizeAccessor(accessor, data);
 }
 
+const FALLBACK_ICON =
+  'data:image/svg+xml;charset=utf-8;base64,PHN2ZyB2aWV3Qm94PSIwIDAgMTAwIDEwMCIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIj4NCiAgPGNpcmNsZSBjeD0iNTAiIGN5PSI1MCIgcj0iNTAiLz4NCjwvc3ZnPg==';
+
+export function getIconUrlAccessor(
+  field: VisualChannelField | null | undefined,
+  fallbackUrl: string | null | undefined,
+  range: CustomMarkersRange | null | undefined,
+  maxIconSize: number,
+  data: any
+) {
+  const urlToUnpackedIcon = (url: string) => ({
+    id: `${url}@@${maxIconSize}`,
+    url,
+    width: maxIconSize,
+    height: maxIconSize,
+    mask: true
+  });
+  let unknownValue = fallbackUrl || FALLBACK_ICON;
+
+  if (range?.othersMarker) {
+    unknownValue = range.othersMarker;
+  }
+
+  const unknownIcon = urlToUnpackedIcon(unknownValue);
+  if (!range || !field) {
+    return () => unknownIcon;
+  }
+
+  const mapping: Record<string, any> = {};
+  for (const {value, markerUrl} of range.markerMap) {
+    if (markerUrl) {
+      mapping[value] = urlToUnpackedIcon(markerUrl);
+    }
+  }
+
+  const accessor = properties => {
+    const propertyValue = properties[field.name];
+    return mapping[propertyValue] || unknownIcon;
+  };
+  return normalizeAccessor(accessor, data);
+}
+
+export function getMaxMarkerSize(visConfig: VisConfig, visualChannels: VisualChannels): number {
+  const {radiusRange, radius} = visConfig;
+  const {radiusField, sizeField} = visualChannels;
+  const field = radiusField || sizeField;
+  return Math.ceil(radiusRange && field ? radiusRange[1] : radius);
+}
+
+export function negateAccessor<T>(accessor: Accessor<T, number>): Accessor<T, number> {
+  return typeof accessor === 'function' ? (d, i) => -accessor(d, i) : -accessor;
+}
+
 export function getSizeAccessor(
   {name},
-  scaleType: SCALE_TYPE,
+  scaleType: SCALE_TYPE | undefined,
   aggregation,
-  range: Iterable<Range>,
+  range: Iterable<Range> | undefined,
   data: any
 ) {
-  const scale = SCALE_FUNCS[scaleType as any]();
-  scale.domain(calculateDomain(data, name, scaleType));
-  scale.range(range);
+  const scale = scaleType ? SCALE_FUNCS[scaleType as any]() : identity;
+  if (scaleType) {
+    scale.domain(calculateDomain(data, name, scaleType));
+    scale.range(range);
+  }
 
   let accessorKeys = getAccessorKeys(name, aggregation);
   const accessor = properties => {
@@ -369,7 +449,7 @@ const FORMATS: Record<string, (value: any) => string> = {
   default: String
 };
 
-export function getTextAccessor({name, type}, data) {
+export function getTextAccessor({name, type}: VisualChannelField, data) {
   const format = FORMATS[type] || FORMATS.default;
   const accessor = properties => {
     return format(properties[name]);
@@ -377,13 +457,19 @@ export function getTextAccessor({name, type}, data) {
   return normalizeAccessor(accessor, data);
 }
 
-export function getTextPixelOffsetAccessor({alignment, anchor, size}, radius) {
+export function getTextPixelOffsetAccessor(
+  {alignment, anchor, size}: TextLabel,
+  radius
+): Accessor<unknown, [number, number]> {
   const padding = 20;
   const signX = anchor === 'middle' ? 0 : anchor === 'start' ? 1 : -1;
   const signY = alignment === 'center' ? 0 : alignment === 'bottom' ? 1 : -1;
   const sizeOffset = alignment === 'center' ? 0 : size;
 
-  const calculateOffset = r => [signX * (r + padding), signY * (r + padding + sizeOffset)];
+  const calculateOffset = (r: number): [number, number] => [
+    signX * (r + padding),
+    signY * (r + padding + sizeOffset)
+  ];
 
   return typeof radius === 'function'
     ? d => {

package/src/api/maps-v3-client.ts

@@ -405,8 +405,7 @@ async function _fetchMapDataset(
   accessToken: string,
   credentials: CloudNativeCredentials,
   clientId?: string,
-  headers?: Headers,
-  queryParameters?: QueryParameters
+  headers?: Headers
 ) {
   const {
     aggregationExp,
@@ -416,7 +415,8 @@ async function _fetchMapDataset(
     format,
     geoColumn,
     source,
-    type
+    type,
+    queryParameters
   } = dataset;
   // First fetch metadata
   const {url, mapFormat} = await _fetchDataUrl({

package/src/api/parseMap.ts

@@ -8,10 +8,14 @@ import {
   getTextAccessor,
   getTextPixelOffsetAccessor,
   OPACITY_MAP,
-  opacityToAlpha
+  opacityToAlpha,
+  getIconUrlAccessor,
+  negateAccessor,
+  getMaxMarkerSize
 } from './layer-map';
 import {_flatten as flatten, log} from '@deck.gl/core';
 import {assert} from '../utils';
+import {MapDataset, MapTextSubLayerConfig, VisualChannels} from './types';
 
 export function parseMap(json) {
   const {keplerMapConfig, datasets} = json;
@@ -30,7 +34,7 @@ export function parseMap(json) {
     layers: extractTextLayers(layers.reverse()).map(({id, type, config, visualChannels}) => {
       try {
         const {dataId} = config;
-        const dataset = datasets.find(d => d.id === dataId);
+        const dataset: MapDataset | null = datasets.find(d => d.id === dataId);
         assert(dataset, `No dataset matching dataId: ${dataId}`);
         const {data} = dataset;
         assert(data, `No data loaded for dataId: ${dataId}`);
@@ -128,7 +132,7 @@ function mapProps(source, target, mapping) {
   }
 }
 
-function createStyleProps(config, mapping) {
+function createStyleProps(config: MapTextSubLayerConfig, mapping) {
   const result: Record<string, any> = {};
   mapProps(config, result, mapping);
 
@@ -153,9 +157,22 @@ function createStyleProps(config, mapping) {
 }
 
 /* eslint-disable complexity, max-statements */
-function createChannelProps(visualChannels, type, config, data) {
-  const {colorField, colorScale, sizeField, sizeScale, strokeColorField, strokeColorScale} =
-    visualChannels;
+function createChannelProps(
+  visualChannels: VisualChannels,
+  type: string,
+  config: MapTextSubLayerConfig,
+  data
+) {
+  const {
+    colorField,
+    colorScale,
+    radiusField,
+    radiusScale,
+    sizeField,
+    sizeScale,
+    strokeColorField,
+    strokeColorScale
+  } = visualChannels;
   let {heightField, heightScale} = visualChannels;
   if (type === 'hexagonId') {
     heightField = sizeField;
@@ -179,6 +196,7 @@ function createChannelProps(visualChannels, type, config, data) {
     const {colorAggregation: aggregation, colorRange: range} = visConfig;
     result.getFillColor = getColorAccessor(
       colorField,
+      // @ts-ignore
       colorScale,
       {aggregation, range},
       visConfig.opacity,
@@ -186,6 +204,18 @@ function createChannelProps(visualChannels, type, config, data) {
     );
   }
 
+  if (radiusField || sizeField) {
+    result.getPointRadius = getSizeAccessor(
+      // @ts-ignore
+      radiusField || sizeField,
+      // @ts-ignore
+      radiusScale || sizeScale,
+      visConfig.sizeAggregation,
+      visConfig.radiusRange || visConfig.sizeRange,
+      data
+    );
+  }
+
   if (strokeColorField) {
     const fallbackOpacity = type === 'point' ? visConfig.opacity : 1;
     const opacity =
@@ -193,35 +223,70 @@ function createChannelProps(visualChannels, type, config, data) {
     const {strokeColorAggregation: aggregation, strokeColorRange: range} = visConfig;
     result.getLineColor = getColorAccessor(
       strokeColorField,
+      // @ts-ignore
       strokeColorScale,
+      // @ts-ignore
       {aggregation, range},
       opacity,
       data
     );
   }
+
   if (heightField) {
     result.getElevation = getSizeAccessor(
       heightField,
+      // @ts-ignore
       heightScale,
       visConfig.heightAggregation,
       visConfig.heightRange || visConfig.sizeRange,
       data
     );
   }
-  if (sizeField) {
-    result.getPointRadius = getSizeAccessor(
-      sizeField,
-      sizeScale,
-      visConfig.sizeAggregation,
-      visConfig.radiusRange || visConfig.sizeRange,
-      data
-    );
-  }
+
   if (textLabelField) {
     result.getText = getTextAccessor(textLabelField, data);
     result.pointType = 'text';
     const radius = result.getPointRadius || visConfig.radius;
     result.getTextPixelOffset = getTextPixelOffsetAccessor(textLabel, radius);
+  } else if (visConfig.customMarkers) {
+    const maxIconSize = getMaxMarkerSize(visConfig, visualChannels);
+    const {getPointRadius, getFillColor} = result;
+    result.pointType = 'icon';
+    result.getIcon = getIconUrlAccessor(
+      visualChannels.customMarkersField,
+      visConfig.customMarkersUrl,
+      visConfig.customMarkersRange,
+      maxIconSize,
+      data
+    );
+    result._subLayerProps = {
+      'points-icon': {
+        loadOptions: {
+          image: {
+            type: 'imagebitmap'
+          },
+          imagebitmap: {
+            resizeWidth: maxIconSize,
+            resizeHeight: maxIconSize,
+            resizeQuality: 'high'
+          }
+        }
+      }
+    };
+
+    if (getFillColor) {
+      result.getIconColor = getFillColor;
+    }
+
+    if (getPointRadius) {
+      result.getIconSize = getPointRadius;
+    }
+
+    if (visualChannels.rotationField) {
+      result.getIconAngle = negateAccessor(
+        getSizeAccessor(visualChannels.rotationField, undefined, null, undefined, data)
+      );
+    }
   }
 
   return result;

package/src/api/types.ts

@@ -0,0 +1,108 @@
+import {SCALE_TYPE} from './layer-map';
+
+export type VisualChannelField = {
+  name: string;
+  type: string;
+};
+
+export type VisualChannels = {
+  colorField?: VisualChannelField;
+  colorScale?: SCALE_TYPE;
+
+  customMarkersField?: VisualChannelField;
+  customMarkersScale?: SCALE_TYPE;
+
+  radiusField?: VisualChannelField;
+  radiusScale?: SCALE_TYPE;
+
+  rotationScale?: SCALE_TYPE;
+  rotationField?: VisualChannelField;
+
+  sizeField?: VisualChannelField;
+  sizeScale?: SCALE_TYPE;
+
+  strokeColorField?: VisualChannelField;
+  strokeColorScale?: SCALE_TYPE;
+
+  heightField?: VisualChannelField;
+  heightScale?: SCALE_TYPE;
+};
+
+export type ColorRange = {
+  category: string;
+  colors: string[];
+  colorMap: string[][] | undefined;
+  name: string;
+  type: string;
+};
+
+export type CustomMarkersRange = {
+  markerMap: {
+    value: string;
+    markerUrl?: string;
+  }[];
+  othersMarker?: string;
+};
+
+export type VisConfig = {
+  filled?: boolean;
+  opacity?: number;
+  enable3d?: boolean;
+
+  colorAggregation?: any;
+  colorRange: ColorRange;
+
+  customMarkers?: boolean;
+  customMarkersRange?: CustomMarkersRange | null;
+  customMarkersUrl?: string | null;
+
+  radius: number;
+  radiusRange?: number[];
+
+  sizeAggregation?: any;
+  sizeRange?: any;
+
+  strokeColorAggregation?: any;
+  strokeOpacity?: number;
+  strokeColorRange?: ColorRange;
+
+  heightRange?: any;
+  heightAggregation?: any;
+};
+
+export type TextLabel = {
+  field: VisualChannelField | null | undefined;
+  alignment?: 'center' | 'bottom' | 'top';
+  anchor?: 'middle' | 'start' | 'end';
+  size: number;
+  color?: number[];
+  offset?: [number, number];
+};
+
+export type MapLayerConfig = {
+  columns?: Record<string, any>;
+  color?: number[];
+  label?: string;
+  dataId: string;
+  textLabel: TextLabel[];
+  visConfig: VisConfig;
+};
+
+export type MapTextSubLayerConfig = Omit<MapLayerConfig, 'textLabel'> & {
+  textLabel?: TextLabel;
+};
+
+export type MapConfigLayer = {
+  type: string;
+  id: string;
+  config: MapLayerConfig;
+  visualChannels: VisualChannels;
+};
+
+export type MapDataset = {
+  id: string;
+  data: any;
+  aggregationExp: string | null;
+  aggregationResLevel: number | null;
+  geoColumn: string;
+};

package/src/layers/carto-tile-layer.ts

@@ -1,6 +1,8 @@
-/* global TextDecoder */
-import Protobuf from 'pbf';
-import {log, DefaultProps} from '@deck.gl/core';
+import {registerLoaders} from '@loaders.gl/core';
+import CartoVectorTileLoader from './schema/carto-vector-tile-loader';
+registerLoaders([CartoVectorTileLoader]);
+
+import {DefaultProps} from '@deck.gl/core';
 import {ClipExtension} from '@deck.gl/extensions';
 import {
   MVTLayer,
@@ -11,81 +13,16 @@ import {
   _TileLoadProps as TileLoadProps
 } from '@deck.gl/geo-layers';
 import {GeoJsonLayer} from '@deck.gl/layers';
-import {geojsonToBinary} from '@loaders.gl/gis';
-import {KeyValueProperties, Tile, TileReader} from './schema/carto-tile';
-import {TileFormat, TILE_FORMATS} from '../api/maps-api-common';
-import {LoaderOptions, LoaderWithParser} from '@loaders.gl/loader-utils';
+import {binaryToGeojson, geojsonToBinary} from '@loaders.gl/gis';
 import type {BinaryFeatures} from '@loaders.gl/schema';
+import {TileFormat, TILE_FORMATS} from '../api/maps-api-common';
 import type {Feature} from 'geojson';
 
-function parseJSON(arrayBuffer: ArrayBuffer): any {
-  return JSON.parse(new TextDecoder().decode(arrayBuffer));
-}
-
-function parsePbf(buffer: ArrayBuffer): Tile {
-  const pbf = new Protobuf(buffer);
-  const tile = TileReader.read(pbf);
-  return tile;
-}
-
-function unpackProperties(properties: KeyValueProperties[]) {
-  if (!properties || !properties.length) {
-    return [];
-  }
-  return properties.map(item => {
-    const currentRecord: Record<string, unknown> = {};
-    item.data.forEach(({key, value}) => {
-      currentRecord[key] = value;
-    });
-    return currentRecord;
-  });
-}
-
-function parseCartoTile(arrayBuffer: ArrayBuffer, options?: LoaderOptions): BinaryFeatures | null {
-  if (!arrayBuffer) return null;
-  const formatTiles = options && options.cartoTile && options.cartoTile.formatTiles;
-  if (formatTiles === TILE_FORMATS.GEOJSON) return geojsonToBinary(parseJSON(arrayBuffer).features);
-
-  const tile = parsePbf(arrayBuffer);
-
-  const {points, lines, polygons} = tile;
-  const data = {
-    points: {...points, properties: unpackProperties(points.properties)},
-    lines: {...lines, properties: unpackProperties(lines.properties)},
-    polygons: {...polygons, properties: unpackProperties(polygons.properties)}
-  };
-
-  // Note: there is slight, difference in `numericProps` type, however geojson/mvtlayer can cope with this
-  return data as unknown as BinaryFeatures;
-}
-
 const defaultTileFormat = TILE_FORMATS.BINARY;
 
-const CartoTileLoader: LoaderWithParser = {
-  name: 'CARTO Tile',
-  version: '1',
-  id: 'cartoTile',
-  module: 'carto',
-  extensions: ['pbf'],
-  mimeTypes: [
-    'application/vnd.carto-vector-tile',
-    'application/x-protobuf' // Back-compatibility
-  ],
-  category: 'geometry',
-  worker: false,
-  parse: async (arrayBuffer, options) => parseCartoTile(arrayBuffer, options),
-  parseSync: parseCartoTile,
-  options: {
-    cartoTile: {
-      formatTiles: defaultTileFormat
-    }
-  }
-};
-
 const defaultProps: DefaultProps<CartoTileLayerProps> = {
   ...MVTLayer.defaultProps,
-  formatTiles: defaultTileFormat,
-  loaders: [CartoTileLoader]
+  formatTiles: defaultTileFormat
 };
 
 /** All properties supported by CartoTileLayer. */
@@ -110,6 +47,12 @@ export default class CartoTileLayer<
   static layerName = 'CartoTileLayer';
   static defaultProps = defaultProps;
 
+  initializeState(): void {
+    super.initializeState();
+    const binary = this.props.formatTiles === TILE_FORMATS.BINARY;
+    this.setState({binary});
+  }
+
   getTileData(tile: TileLoadProps) {
     const url = _getURLFromTemplate(this.state.data, tile);
     if (!url) {
@@ -120,17 +63,13 @@ export default class CartoTileLayer<
     const {fetch, formatTiles} = this.props;
     const {signal} = tile;
 
-    loadOptions = {
-      ...loadOptions,
-      mimeType: 'application/x-protobuf'
-    };
-
-    if (formatTiles) {
-      log.assert(
-        Object.values(TILE_FORMATS).includes(formatTiles),
-        `Invalid value for formatTiles: ${formatTiles}. Use value from TILE_FORMATS`
-      );
-      loadOptions.cartoTile = {formatTiles};
+    // The backend doesn't yet support our custom mime-type, so force it here
+    // TODO remove once backend sends the correct mime-type
+    if (formatTiles === TILE_FORMATS.BINARY) {
+      loadOptions = {
+        ...loadOptions,
+        mimeType: 'application/vnd.carto-vector-tile'
+      };
     }
 
     return fetch(url, {propName: 'data', layer: this, loadOptions, signal});
@@ -161,4 +100,15 @@ export default class CartoTileLayer<
     const subLayer = new GeoJsonLayer(subLayerProps);
     return subLayer;
   }
+
+  getPickingInfo(params) {
+    const info = super.getPickingInfo(params);
+
+    if (this.state.binary && info.index !== -1) {
+      const {data} = params.sourceLayer!.props;
+      info.object = binaryToGeojson(data as BinaryFeatures, {globalFeatureId: info.index}) as DataT;
+    }
+
+    return info;
+  }
 }