@deck.gl/carto 8.8.2 → 8.8.5

package/package.json

@@ -2,7 +2,7 @@
   "name": "@deck.gl/carto",
   "description": "CARTO official integration with Deck.gl. Build geospatial applications using CARTO and Deck.gl.",
   "license": "MIT",
-  "version": "8.8.2",
+  "version": "8.8.5",
   "publishConfig": {
     "access": "public"
   },
@@ -32,12 +32,12 @@
     "prepublishOnly": "npm run build-bundle && npm run build-bundle -- --env.dev"
   },
   "dependencies": {
-    "@loaders.gl/gis": "^3.2.0",
-    "@loaders.gl/loader-utils": "^3.2.0",
-    "@loaders.gl/mvt": "^3.2.0",
-    "@loaders.gl/tiles": "^3.2.0",
-    "@luma.gl/constants": "^8.5.14",
-    "@math.gl/web-mercator": "^3.6.0",
+    "@loaders.gl/gis": "^3.2.5",
+    "@loaders.gl/loader-utils": "^3.2.5",
+    "@loaders.gl/mvt": "^3.2.5",
+    "@loaders.gl/tiles": "^3.2.5",
+    "@luma.gl/constants": "^8.5.16",
+    "@math.gl/web-mercator": "^3.6.2",
     "cartocolor": "^4.0.2",
     "d3-array": "^2.8.0",
     "d3-color": "^2.0.0",
@@ -48,7 +48,7 @@
     "pbf": "^3.2.1"
   },
   "devDependencies": {
-    "@loaders.gl/schema": "^3.2.0",
+    "@loaders.gl/schema": "^3.2.5",
     "@types/d3-array": "^3.0.2",
     "@types/d3-color": "^1.4.2",
     "@types/d3-scale": "^3.0.0"
@@ -61,5 +61,5 @@
     "@deck.gl/layers": "^8.0.0",
     "@loaders.gl/core": "^3.0.0"
   },
-  "gitHead": "17d768baed9e2149d3b62863e9db2b9d23bbf5ba"
+  "gitHead": "96af77adfd71563e0da68acd4e7ae318f551c5de"
 }

package/src/api/layer-map.ts

@@ -19,6 +19,8 @@ import {GeoJsonLayer} from '@deck.gl/layers';
 import {H3HexagonLayer, MVTLayer} from '@deck.gl/geo-layers';
 
 import CartoTileLayer from '../layers/carto-tile-layer';
+import H3TileLayer from '../layers/h3-tile-layer';
+import QuadbinTileLayer from '../layers/quadbin-tile-layer';
 import {TILE_FORMATS} from './maps-api-common';
 import {assert} from '../utils';
 
@@ -111,7 +113,7 @@ export function getLayer(
   config,
   dataset
 ): {Layer: ConstructorOf<Layer>; propMap: any; defaultProps: any} {
-  if (type === 'mvt' || type === 'tileset') {
+  if (type === 'mvt' || type === 'tileset' || type === 'h3' || type === 'quadbin') {
     return getTileLayer(dataset);
   }
 
@@ -163,31 +165,58 @@ export function getLayer(
   };
 }
 
+export function layerFromTileDataset(
+  formatTiles: string | null = TILE_FORMATS.MVT,
+  scheme: string
+): typeof CartoTileLayer | typeof H3TileLayer | typeof MVTLayer | typeof QuadbinTileLayer {
+  if (scheme === 'h3') {
+    return H3TileLayer;
+  }
+  if (scheme === 'quadbin') {
+    return QuadbinTileLayer;
+  }
+  if (formatTiles === TILE_FORMATS.MVT) {
+    return MVTLayer;
+  }
+
+  // formatTiles === BINARY|JSON|GEOJSON
+  return CartoTileLayer;
+}
+
 function getTileLayer(dataset) {
   const {
+    aggregationExp,
+    aggregationResLevel,
     data: {
+      scheme,
       tiles: [tileUrl]
     }
   } = dataset;
   /* global URL */
-  const formatTiles = new URL(tileUrl).searchParams.get('formatTiles') || TILE_FORMATS.MVT;
+  const formatTiles = new URL(tileUrl).searchParams.get('formatTiles');
 
   return {
-    Layer: formatTiles === TILE_FORMATS.MVT ? MVTLayer : CartoTileLayer,
+    Layer: layerFromTileDataset(formatTiles, scheme),
     propMap: sharedPropMap,
     defaultProps: {
       ...defaultProps,
-      uniqueIdProperty: 'geoid',
-      formatTiles
+      ...(aggregationExp && {aggregationExp}),
+      ...(aggregationResLevel && {aggregationResLevel}),
+      formatTiles,
+      uniqueIdProperty: 'geoid'
     }
   };
 }
 
-function domainFromAttribute(attribute, scaleType: SCALE_TYPE) {
+function domainFromAttribute(attribute, scaleType: SCALE_TYPE, scaleLength: number) {
   if (scaleType === 'ordinal' || scaleType === 'point') {
     return attribute.categories.map(c => c.category).filter(c => c !== undefined && c !== null);
   }
 
+  if (scaleType === 'quantile' && attribute.quantiles) {
+    return attribute.quantiles[scaleLength];
+  }
+
   let {min} = attribute;
   if (scaleType === 'log' && min === 0) {
     min = 1e-5;
@@ -211,12 +240,12 @@ function domainFromValues(values, scaleType: SCALE_TYPE) {
   return extent(values);
 }
 
-function calculateDomain(data, name, scaleType) {
+function calculateDomain(data, name, scaleType, scaleLength?) {
   if (data.tilestats) {
     // Tileset data type
     const {attributes} = data.tilestats.layers[0];
     const attribute = attributes.find(a => a.attribute === name);
-    return domainFromAttribute(attribute, scaleType);
+    return domainFromAttribute(attribute, scaleType, scaleLength);
   } else if (data.features) {
     // GeoJSON data type
     const values = data.features.map(({properties}) => properties[name]);
@@ -243,6 +272,25 @@ export function opacityToAlpha(opacity) {
   return opacity !== undefined ? Math.round(255 * Math.pow(opacity, 1 / 2.2)) : 255;
 }
 
+function getAccessorKeys(name: string, aggregation: string | undefined): string[] {
+  let keys = [name];
+  if (aggregation) {
+    // Snowflake will capitalized the keys, need to check lower and upper case version
+    keys = keys.concat([aggregation, aggregation.toUpperCase()].map(a => `${name}_${a}`));
+  }
+  return keys;
+}
+
+function findAccessorKey(keys: string[], properties): string[] {
+  for (const key of keys) {
+    if (key in properties) {
+      return [key];
+    }
+  }
+
+  throw new Error(`Could not find property for any accessor key: ${keys}`);
+}
+
 export function getColorValueAccessor({name}, colorAggregation, data: any) {
   const aggregator = AGGREGATION_FUNC[colorAggregation];
   const accessor = values => aggregator(values, p => p[name]);
@@ -252,7 +300,7 @@ export function getColorValueAccessor({name}, colorAggregation, data: any) {
 export function getColorAccessor(
   {name},
   scaleType: SCALE_TYPE,
-  {colors, colorMap},
+  {aggregation, range: {colors, colorMap}},
   opacity: number | undefined,
   data: any
 ) {
@@ -266,7 +314,7 @@ export function getColorAccessor(
       scaleColor.push(color);
     });
   } else {
-    domain = calculateDomain(data, name, scaleType);
+    domain = calculateDomain(data, name, scaleType, colors.length);
     scaleColor = colors;
   }
 
@@ -279,21 +327,36 @@ export function getColorAccessor(
   scale.unknown(UNKNOWN_COLOR);
   const alpha = opacityToAlpha(opacity);
 
+  let accessorKeys = getAccessorKeys(name, aggregation);
   const accessor = properties => {
-    const propertyValue = properties[name];
+    if (!(accessorKeys[0] in properties)) {
+      accessorKeys = findAccessorKey(accessorKeys, properties);
+    }
+    const propertyValue = properties[accessorKeys[0]];
     const {r, g, b} = rgb(scale(propertyValue));
     return [r, g, b, propertyValue === null ? 0 : alpha];
   };
   return normalizeAccessor(accessor, data);
 }
 
-export function getSizeAccessor({name}, scaleType: SCALE_TYPE, range: Iterable<Range>, data: any) {
+export function getSizeAccessor(
+  {name},
+  scaleType: SCALE_TYPE,
+  aggregation,
+  range: Iterable<Range>,
+  data: any
+) {
   const scale = SCALE_FUNCS[scaleType as any]();
   scale.domain(calculateDomain(data, name, scaleType));
   scale.range(range);
 
+  let accessorKeys = getAccessorKeys(name, aggregation);
   const accessor = properties => {
-    return scale(properties[name]);
+    if (!(accessorKeys[0] in properties)) {
+      accessorKeys = findAccessorKey(accessorKeys, properties);
+    }
+    const propertyValue = properties[accessorKeys[0]];
+    return scale(propertyValue);
   };
   return normalizeAccessor(accessor, data);
 }

package/src/api/maps-v3-client.ts

@@ -1,7 +1,12 @@
 /**
  * Maps API Client for Carto 3
  */
-import {getDefaultCredentials, buildMapsUrlFromBase, CloudNativeCredentials} from '../config';
+import {
+  getDefaultCredentials,
+  buildMapsUrlFromBase,
+  buildStatsUrlFromBase,
+  CloudNativeCredentials
+} from '../config';
 import {
   API_VERSIONS,
   COLUMNS_SUPPORT,
@@ -144,6 +149,9 @@ function getParameters({
   }
   if (aggregationExp) {
     parameters.push(encodeParameter('aggregationExp', aggregationExp));
+  } else if (isSpatialIndexGeoColumn(geoColumn)) {
+    // Default aggregationExp required for spatial index layers
+    parameters.push(encodeParameter('aggregationExp', '1 AS value'));
   }
   if (aggregationResLevel) {
     parameters.push(encodeParameter('aggregationResLevel', aggregationResLevel));
@@ -152,6 +160,11 @@ function getParameters({
   return parameters.join('&');
 }
 
+function isSpatialIndexGeoColumn(geoColumn: string | undefined) {
+  const spatialIndex = geoColumn?.split(':')[0];
+  return spatialIndex === 'h3' || spatialIndex === 'quadbin';
+}
+
 export async function mapInstantiation({
   type,
   source,
@@ -170,8 +183,8 @@ export async function mapInstantiation({
     geoColumn,
     columns,
     clientId,
-    aggregationExp,
-    aggregationResLevel
+    aggregationResLevel,
+    aggregationExp
   })}`;
   const {accessToken} = credentials;
 
@@ -363,9 +376,20 @@ async function _fetchMapDataset(
   credentials: CloudNativeCredentials,
   clientId?: string
 ) {
-  const {connectionName: connection, columns, format, geoColumn, source, type} = dataset;
+  const {
+    aggregationExp,
+    aggregationResLevel,
+    connectionName: connection,
+    columns,
+    format,
+    geoColumn,
+    source,
+    type
+  } = dataset;
   // First fetch metadata
   const {url, mapFormat} = await _fetchDataUrl({
+    aggregationExp,
+    aggregationResLevel,
     clientId,
     credentials: {...credentials, accessToken},
     connection,
@@ -389,6 +413,31 @@ async function _fetchMapDataset(
   return true;
 }
 
+async function _fetchTilestats(
+  attribute,
+  dataset,
+  accessToken: string,
+  credentials: CloudNativeCredentials
+) {
+  const {connectionName: connection, source, type} = dataset;
+
+  const statsUrl = buildStatsUrlFromBase(credentials.apiBaseUrl);
+  let url = `${statsUrl}/${connection}/`;
+  if (type === MAP_TYPES.QUERY) {
+    url += `${attribute}?q=${source}`;
+  } else {
+    // MAP_TYPE.TABLE
+    url += `${source}/${attribute}`;
+  }
+  const stats = await requestData({url, format: FORMATS.JSON, accessToken});
+
+  // Replace tilestats for attribute with value from API
+  const {attributes} = dataset.data.tilestats.layers[0];
+  const index = attributes.findIndex(d => d.attribute === attribute);
+  attributes[index] = stats;
+  return true;
+}
+
 async function fillInMapDatasets(
   {datasets, token},
   clientId: string,
@@ -398,6 +447,42 @@ async function fillInMapDatasets(
   return await Promise.all(promises);
 }
 
+async function fillInTileStats(
+  {datasets, keplerMapConfig, token},
+  credentials: CloudNativeCredentials
+) {
+  const attributes: {attribute?: string; dataset?: any}[] = [];
+  const {layers} = keplerMapConfig.config.visState;
+  for (const layer of layers) {
+    for (const channel of Object.keys(layer.visualChannels)) {
+      const attribute = layer.visualChannels[channel]?.name;
+      if (attribute) {
+        const dataset = datasets.find(d => d.id === layer.config.dataId);
+        if (dataset.data.tilestats && dataset.type !== MAP_TYPES.TILESET) {
+          // Only fetch stats for QUERY & TABLE map types
+          attributes.push({attribute, dataset});
+        }
+      }
+    }
+  }
+  // Remove duplicates to avoid repeated requests
+  const filteredAttributes: {attribute?: string; dataset?: any}[] = [];
+  for (const a of attributes) {
+    if (
+      !filteredAttributes.find(
+        ({attribute, dataset}) => attribute === a.attribute && dataset === a.dataset
+      )
+    ) {
+      filteredAttributes.push(a);
+    }
+  }
+
+  const promises = filteredAttributes.map(({attribute, dataset}) =>
+    _fetchTilestats(attribute, dataset, token, credentials)
+  );
+  return await Promise.all(promises);
+}
+
 export async function fetchMap({
   cartoMapId,
   clientId,
@@ -466,6 +551,9 @@ export async function fetchMap({
 
   // Mutates map.datasets so that dataset.data contains data
   await fillInMapDatasets(map, clientId, localCreds);
+
+  // Mutates attributes in visualChannels to contain tile stats
+  await fillInTileStats(map, localCreds);
   return {
     ...parseMap(map),
     ...{stopAutoRefresh}

package/src/api/parseMap.ts

@@ -176,10 +176,11 @@ function createChannelProps(visualChannels, type, config, data) {
       }
     }
   } else if (colorField) {
+    const {colorAggregation: aggregation, colorRange: range} = visConfig;
     result.getFillColor = getColorAccessor(
       colorField,
       colorScale,
-      visConfig.colorRange,
+      {aggregation, range},
       visConfig.opacity,
       data
     );
@@ -189,10 +190,11 @@ function createChannelProps(visualChannels, type, config, data) {
     const fallbackOpacity = type === 'point' ? visConfig.opacity : 1;
     const opacity =
       visConfig.strokeOpacity !== undefined ? visConfig.strokeOpacity : fallbackOpacity;
+    const {strokeColorAggregation: aggregation, strokeColorRange: range} = visConfig;
     result.getLineColor = getColorAccessor(
       strokeColorField,
       strokeColorScale,
-      visConfig.strokeColorRange,
+      {aggregation, range},
       opacity,
       data
     );
@@ -201,6 +203,7 @@ function createChannelProps(visualChannels, type, config, data) {
     result.getElevation = getSizeAccessor(
       heightField,
       heightScale,
+      visConfig.heightAggregation,
       visConfig.heightRange || visConfig.sizeRange,
       data
     );
@@ -209,6 +212,7 @@ function createChannelProps(visualChannels, type, config, data) {
     result.getPointRadius = getSizeAccessor(
       sizeField,
       sizeScale,
+      visConfig.sizeAggregation,
       visConfig.radiusRange || visConfig.sizeRange,
       data
     );

package/src/config.ts

@@ -50,11 +50,21 @@ export function getDefaultCredentials(): Credentials {
   return credentials;
 }
 
-export function buildMapsUrlFromBase(apiBaseUrl: string): string {
-  let suffix = '/v3/maps';
+export type V3Endpoint = 'maps' | 'stats';
+
+function buildUrlFromBase(apiBaseUrl: string, endpoint: V3Endpoint) {
+  let suffix = `/v3/${endpoint}`;
   if (apiBaseUrl.endsWith('/')) {
     suffix = suffix.substring(1);
   }
 
   return `${apiBaseUrl}${suffix}`;
 }
+
+export function buildMapsUrlFromBase(apiBaseUrl: string): string {
+  return buildUrlFromBase(apiBaseUrl, 'maps');
+}
+
+export function buildStatsUrlFromBase(apiBaseUrl: string): string {
+  return buildUrlFromBase(apiBaseUrl, 'stats');
+}

package/src/layers/carto-layer.ts

@@ -1,24 +1,22 @@
 import {
   CompositeLayer,
-  Layer,
-  log,
   CompositeLayerProps,
+  Layer,
   LayerProps,
-  ChangeFlags
+  log,
+  UpdateParameters
 } from '@deck.gl/core';
-import CartoTileLayer from './carto-tile-layer';
-import H3TileLayer from './h3-tile-layer';
-import QuadbinTileLayer from './quadbin-tile-layer';
+
 import {MVTLayer} from '@deck.gl/geo-layers';
 import {fetchLayerData, getDataV2, API_VERSIONS} from '../api';
+import {layerFromTileDataset} from '../api/layer-map';
 import {
   COLUMNS_SUPPORT,
   FORMATS,
   GEO_COLUMN_SUPPORT,
   MapType,
   MAP_TYPES,
-  TileFormat,
-  TILE_FORMATS
+  TileFormat
 } from '../api/maps-api-common';
 import {
   ClassicCredentials,
@@ -209,16 +207,7 @@ export default class CartoLayer<ExtraProps = {}> extends CompositeLayer<
     }
   }
 
-  updateState({
-    props,
-    oldProps,
-    changeFlags
-  }: {
-    props: CartoLayerProps;
-    oldProps: CartoLayerProps;
-    context: any;
-    changeFlags: ChangeFlags;
-  }): void {
+  updateState({props, oldProps, changeFlags}: UpdateParameters<this>) {
     this._checkProps(props);
     const shouldUpdateData =
       changeFlags.dataChanged ||
@@ -294,19 +283,10 @@ export default class CartoLayer<ExtraProps = {}> extends CompositeLayer<
 
     /* global URL */
     const tileUrl = new URL(data.tiles[0]);
-
     props.formatTiles =
-      props.formatTiles ||
-      (tileUrl.searchParams.get('formatTiles') as TileFormat) ||
-      TILE_FORMATS.MVT;
+      props.formatTiles || (tileUrl.searchParams.get('formatTiles') as TileFormat);
 
-    if (data.scheme === 'h3') {
-      return [H3TileLayer, props];
-    }
-    if (data.scheme === 'quadbin') {
-      return [QuadbinTileLayer, props];
-    }
-    return props.formatTiles === TILE_FORMATS.MVT ? [MVTLayer, props] : [CartoTileLayer, props];
+    return [layerFromTileDataset(props.formatTiles, data.scheme), props];
   }
 
   renderLayers(): Layer | null {

package/src/style/color-bins-style.ts

@@ -1,18 +1,40 @@
 import {scaleThreshold} from 'd3-scale';
+import {AccessorFunction} from '@deck.gl/core';
+import {Feature} from 'geojson';
 import getPalette, {Color, DEFAULT_PALETTE, NULL_COLOR} from './palette';
 import {assert, AttributeSelector, getAttrValue} from './utils';
 
-export default function colorBins({
+/**
+ * Helper function for quickly creating a color bins style based on `d3` `scaleThreshold`.
+ *
+ * Data values of each attribute are rounded down to the nearest value in the domain and are then
+ * styled with the corresponding color.
+ *
+ * @return accessor that maps objects to `Color` values
+ */
+export default function colorBins<DataT = Feature>({
   attr,
   domain,
   colors = DEFAULT_PALETTE,
   nullColor = NULL_COLOR
 }: {
-  attr: AttributeSelector;
+  /** Attribute or column to symbolize by. */
+  attr: AttributeSelector<DataT, number>;
+
+  /** Category list. Must be a valid list of categories. */
   domain: number[];
+
+  /**
+   * Color assigned to each domain value.
+   *
+   * Either Array of colors in RGBA or valid named CARTOColors palette.
+   * @default `PurpOr`
+   */
   colors?: string | Color[];
+
+  /** Color for null values. @default: [204, 204, 204] */
   nullColor?: Color;
-}): AttributeSelector {
+}): AccessorFunction<DataT, Color> {
   assert(Array.isArray(domain), 'Expected "domain" to be an array of numbers');
 
   const palette = typeof colors === 'string' ? getPalette(colors, domain.length + 1) : colors;

package/src/style/color-categories-style.ts

@@ -1,19 +1,43 @@
+import {AccessorFunction} from '@deck.gl/core';
+import {Feature} from 'geojson';
 import getPalette, {Color, DEFAULT_PALETTE, NULL_COLOR, OTHERS_COLOR} from './palette';
 import {assert, AttributeSelector, getAttrValue} from './utils';
 
-export default function colorCategories({
+/**
+ * Helper function for quickly creating a color category style.
+ *
+ * Data values of each attribute listed in the domain are mapped one to one
+ * with corresponding colors in the range.
+ *
+ * @return accessor that maps objects to `Color` values
+ */
+export default function colorCategories<DataT = Feature>({
   attr,
   domain,
   colors = DEFAULT_PALETTE,
   nullColor = NULL_COLOR,
   othersColor = OTHERS_COLOR
 }: {
-  attr: AttributeSelector;
-  domain: number[];
+  /** Attribute or column to symbolize by */
+  attr: string | AttributeSelector<DataT, number | string>;
+
+  /** Category list. Must be a valid list of categories. */
+  domain: number[] | string[];
+
+  /**
+   * Color assigned to each domain value.
+   *
+   * Either Array of colors in RGBA or valid named CARTOColors palette.
+   * @default `PurpOr`
+   */
   colors: string | Color[];
+
+  /** Color for null values. @default: [204, 204, 204] */
   nullColor?: Color;
+
+  /** Fallback color for a category not correctly assigned. @default: [119, 119, 119] */
   othersColor?: Color;
-}): AttributeSelector {
+}): AccessorFunction<DataT, Color> {
   assert(Array.isArray(domain), 'Expected "domain" to be an array of numbers or strings');
 
   const colorsByCategory = {};

package/src/style/color-continuous-style.ts

@@ -1,18 +1,40 @@
+import {AccessorFunction} from '@deck.gl/core';
 import {scaleLinear} from 'd3-scale';
+import {Feature} from 'geojson';
 import getPalette, {Color, DEFAULT_PALETTE, NULL_COLOR} from './palette';
 import {assert, AttributeSelector, getAttrValue} from './utils';
 
-export default function colorContinuous({
+/**
+ * Helper function for quickly creating a color continuous style.
+ *
+ * Data values of each field are interpolated linearly across values in the domain and
+ * are then styled with a blend of the corresponding color in the range.
+ *
+ * @return accessor that maps objects to `Color` values
+ */
+export default function colorContinuous<DataT = Feature>({
   attr,
   domain,
   colors = DEFAULT_PALETTE,
   nullColor = NULL_COLOR
 }: {
-  attr: AttributeSelector;
+  /** Attribute or column to symbolize by. */
+  attr: AttributeSelector<DataT, number>;
+
+  /** Attribute domain to define the data range. */
   domain: number[];
+
+  /**
+   * Color assigned to each domain value.
+   *
+   * Either Array of colors in RGBA or valid named CARTOColors palette.
+   * @default `PurpOr`
+   */
   colors?: string | Color[];
+
+  /** Color for null values. @default: [204, 204, 204]*/
   nullColor?: Color;
-}): AttributeSelector {
+}): AccessorFunction<DataT, Color> {
   assert(Array.isArray(domain), 'Expected "domain" to be an array of numbers');
 
   const palette = typeof colors === 'string' ? getPalette(colors, domain.length) : colors;

package/src/style/utils.ts

@@ -1,9 +1,13 @@
+import {Feature} from 'geojson';
+
 const ALLOWED_ATTR_TYPES = Object.freeze(['function', 'string']);
 
-type Row = {properties: Record<string, unknown>};
-export type AttributeSelector = string | ((d: Row) => unknown);
+export type AttributeSelector<DataT = Feature, OutT = any> = string | ((d: DataT) => OutT);
 
-export function getAttrValue(attr: AttributeSelector, d: Row): unknown {
+export function getAttrValue<DataT = Feature, OutT = any>(
+  attr: string | AttributeSelector<DataT, OutT>,
+  d: DataT
+): OutT {
   assert(typeof d === 'object', 'Expected "data" to be an object');
   assert(ALLOWED_ATTR_TYPES.includes(typeof attr), 'Expected "attr" to be a function or string');
 
@@ -11,7 +15,7 @@ export function getAttrValue(attr: AttributeSelector, d: Row): unknown {
   if (typeof attr === 'function') {
     return attr(d);
   }
-  return d.properties[attr];
+  return (d as unknown as Feature)?.properties?.[attr] as OutT;
 }
 
 export function assert(condition, message = ''): asserts condition {