@deck.gl/carto 8.8.4 → 8.8.7

package/package.json

@@ -2,7 +2,7 @@
   "name": "@deck.gl/carto",
   "description": "CARTO official integration with Deck.gl. Build geospatial applications using CARTO and Deck.gl.",
   "license": "MIT",
-  "version": "8.8.4",
+  "version": "8.8.7",
   "publishConfig": {
     "access": "public"
   },
@@ -61,5 +61,5 @@
     "@deck.gl/layers": "^8.0.0",
     "@loaders.gl/core": "^3.0.0"
   },
-  "gitHead": "faab03adb63f2e2470d9c202dd740610be2ebc22"
+  "gitHead": "4485a8be47b76ba5f28447af304c683a00bbb4eb"
 }

package/src/api/index.ts

@@ -1,3 +1,4 @@
 export {FORMATS, TILE_FORMATS, MAP_TYPES, API_VERSIONS} from './maps-api-common';
+export type {QueryParameters} from './maps-api-common';
 export {getDataV2, CONNECTIONS} from './maps-client';
 export {fetchLayerData, fetchMap, mapInstantiation} from './maps-v3-client';

package/src/api/layer-map.ts

@@ -19,6 +19,8 @@ import {GeoJsonLayer} from '@deck.gl/layers';
 import {H3HexagonLayer, MVTLayer} from '@deck.gl/geo-layers';
 
 import CartoTileLayer from '../layers/carto-tile-layer';
+import H3TileLayer from '../layers/h3-tile-layer';
+import QuadbinTileLayer from '../layers/quadbin-tile-layer';
 import {TILE_FORMATS} from './maps-api-common';
 import {assert} from '../utils';
 
@@ -111,7 +113,7 @@ export function getLayer(
   config,
   dataset
 ): {Layer: ConstructorOf<Layer>; propMap: any; defaultProps: any} {
-  if (type === 'mvt' || type === 'tileset') {
+  if (type === 'mvt' || type === 'tileset' || type === 'h3' || type === 'quadbin') {
     return getTileLayer(dataset);
   }
 
@@ -163,31 +165,58 @@ export function getLayer(
   };
 }
 
+export function layerFromTileDataset(
+  formatTiles: string | null = TILE_FORMATS.MVT,
+  scheme: string
+): typeof CartoTileLayer | typeof H3TileLayer | typeof MVTLayer | typeof QuadbinTileLayer {
+  if (scheme === 'h3') {
+    return H3TileLayer;
+  }
+  if (scheme === 'quadbin') {
+    return QuadbinTileLayer;
+  }
+  if (formatTiles === TILE_FORMATS.MVT) {
+    return MVTLayer;
+  }
+
+  // formatTiles === BINARY|JSON|GEOJSON
+  return CartoTileLayer;
+}
+
 function getTileLayer(dataset) {
   const {
+    aggregationExp,
+    aggregationResLevel,
     data: {
+      scheme,
       tiles: [tileUrl]
     }
   } = dataset;
   /* global URL */
-  const formatTiles = new URL(tileUrl).searchParams.get('formatTiles') || TILE_FORMATS.MVT;
+  const formatTiles = new URL(tileUrl).searchParams.get('formatTiles');
 
   return {
-    Layer: formatTiles === TILE_FORMATS.MVT ? MVTLayer : CartoTileLayer,
+    Layer: layerFromTileDataset(formatTiles, scheme),
     propMap: sharedPropMap,
     defaultProps: {
       ...defaultProps,
-      uniqueIdProperty: 'geoid',
-      formatTiles
+      ...(aggregationExp && {aggregationExp}),
+      ...(aggregationResLevel && {aggregationResLevel}),
+      formatTiles,
+      uniqueIdProperty: 'geoid'
     }
   };
 }
 
-function domainFromAttribute(attribute, scaleType: SCALE_TYPE) {
+function domainFromAttribute(attribute, scaleType: SCALE_TYPE, scaleLength: number) {
   if (scaleType === 'ordinal' || scaleType === 'point') {
     return attribute.categories.map(c => c.category).filter(c => c !== undefined && c !== null);
   }
 
+  if (scaleType === 'quantile' && attribute.quantiles) {
+    return attribute.quantiles[scaleLength];
+  }
+
   let {min} = attribute;
   if (scaleType === 'log' && min === 0) {
     min = 1e-5;
@@ -211,12 +240,12 @@ function domainFromValues(values, scaleType: SCALE_TYPE) {
   return extent(values);
 }
 
-function calculateDomain(data, name, scaleType) {
+function calculateDomain(data, name, scaleType, scaleLength?) {
   if (data.tilestats) {
     // Tileset data type
     const {attributes} = data.tilestats.layers[0];
     const attribute = attributes.find(a => a.attribute === name);
-    return domainFromAttribute(attribute, scaleType);
+    return domainFromAttribute(attribute, scaleType, scaleLength);
   } else if (data.features) {
     // GeoJSON data type
     const values = data.features.map(({properties}) => properties[name]);
@@ -243,6 +272,25 @@ export function opacityToAlpha(opacity) {
   return opacity !== undefined ? Math.round(255 * Math.pow(opacity, 1 / 2.2)) : 255;
 }
 
+function getAccessorKeys(name: string, aggregation: string | undefined): string[] {
+  let keys = [name];
+  if (aggregation) {
+    // Snowflake will capitalized the keys, need to check lower and upper case version
+    keys = keys.concat([aggregation, aggregation.toUpperCase()].map(a => `${name}_${a}`));
+  }
+  return keys;
+}
+
+function findAccessorKey(keys: string[], properties): string[] {
+  for (const key of keys) {
+    if (key in properties) {
+      return [key];
+    }
+  }
+
+  throw new Error(`Could not find property for any accessor key: ${keys}`);
+}
+
 export function getColorValueAccessor({name}, colorAggregation, data: any) {
   const aggregator = AGGREGATION_FUNC[colorAggregation];
   const accessor = values => aggregator(values, p => p[name]);
@@ -252,7 +300,7 @@ export function getColorValueAccessor({name}, colorAggregation, data: any) {
 export function getColorAccessor(
   {name},
   scaleType: SCALE_TYPE,
-  {colors, colorMap},
+  {aggregation, range: {colors, colorMap}},
   opacity: number | undefined,
   data: any
 ) {
@@ -266,7 +314,7 @@ export function getColorAccessor(
       scaleColor.push(color);
     });
   } else {
-    domain = calculateDomain(data, name, scaleType);
+    domain = calculateDomain(data, name, scaleType, colors.length);
     scaleColor = colors;
   }
 
@@ -279,21 +327,36 @@ export function getColorAccessor(
   scale.unknown(UNKNOWN_COLOR);
   const alpha = opacityToAlpha(opacity);
 
+  let accessorKeys = getAccessorKeys(name, aggregation);
   const accessor = properties => {
-    const propertyValue = properties[name];
+    if (!(accessorKeys[0] in properties)) {
+      accessorKeys = findAccessorKey(accessorKeys, properties);
+    }
+    const propertyValue = properties[accessorKeys[0]];
     const {r, g, b} = rgb(scale(propertyValue));
     return [r, g, b, propertyValue === null ? 0 : alpha];
   };
   return normalizeAccessor(accessor, data);
 }
 
-export function getSizeAccessor({name}, scaleType: SCALE_TYPE, range: Iterable<Range>, data: any) {
+export function getSizeAccessor(
+  {name},
+  scaleType: SCALE_TYPE,
+  aggregation,
+  range: Iterable<Range>,
+  data: any
+) {
   const scale = SCALE_FUNCS[scaleType as any]();
   scale.domain(calculateDomain(data, name, scaleType));
   scale.range(range);
 
+  let accessorKeys = getAccessorKeys(name, aggregation);
   const accessor = properties => {
-    return scale(properties[name]);
+    if (!(accessorKeys[0] in properties)) {
+      accessorKeys = findAccessorKey(accessorKeys, properties);
+    }
+    const propertyValue = properties[accessorKeys[0]];
+    return scale(propertyValue);
   };
   return normalizeAccessor(accessor, data);
 }

package/src/api/maps-api-common.ts

@@ -77,3 +77,11 @@ type MapInstantiationFormats = Record<
 export function encodeParameter(name: string, value: string | boolean | number): string {
   return `${name}=${encodeURIComponent(value)}`;
 }
+
+export type QueryParameterValue = string | number | boolean | Array<QueryParameterValue> | object;
+
+export type NamedQueryParameter = Record<string, QueryParameterValue>;
+
+export type PositionalQueryParameter = QueryParameterValue[];
+
+export type QueryParameters = NamedQueryParameter | PositionalQueryParameter;

package/src/api/maps-v3-client.ts

@@ -1,7 +1,12 @@
 /**
  * Maps API Client for Carto 3
  */
-import {getDefaultCredentials, buildMapsUrlFromBase, CloudNativeCredentials} from '../config';
+import {
+  getDefaultCredentials,
+  buildMapsUrlFromBase,
+  buildStatsUrlFromBase,
+  CloudNativeCredentials
+} from '../config';
 import {
   API_VERSIONS,
   COLUMNS_SUPPORT,
@@ -12,6 +17,7 @@ import {
   MapInstantiation,
   MapType,
   MAP_TYPES,
+  QueryParameters,
   SchemaField,
   TileFormat,
   TILE_FORMATS
@@ -23,9 +29,11 @@ import {assert} from '../utils';
 const MAX_GET_LENGTH = 2048;
 const DEFAULT_CLIENT = 'deck-gl-carto';
 
+export type Headers = Record<string, string>;
 interface RequestParams {
   method?: string;
   url: string;
+  headers?: Headers;
   accessToken?: string;
   body?: any;
 }
@@ -33,8 +41,15 @@ interface RequestParams {
 /**
  * Request against Maps API
  */
-async function request({method, url, accessToken, body}: RequestParams): Promise<Response> {
-  const headers: Record<string, string> = {
+async function request({
+  method,
+  url,
+  headers: customHeaders,
+  accessToken,
+  body
+}: RequestParams): Promise<Response> {
+  const headers: Headers = {
+    ...customHeaders,
     Accept: 'application/json'
   };
 
@@ -61,10 +76,11 @@ async function request({method, url, accessToken, body}: RequestParams): Promise
 async function requestJson<T = unknown>({
   method,
   url,
+  headers,
   accessToken,
   body
 }: RequestParams): Promise<T> {
-  const response = await request({method, url, accessToken, body});
+  const response = await request({method, url, headers, accessToken, body});
   const json = await response.json();
 
   if (!response.ok) {
@@ -115,8 +131,10 @@ type FetchLayerDataParams = {
   clientId?: string;
   format?: Format;
   formatTiles?: TileFormat;
+  headers?: Headers;
   aggregationExp?: string;
   aggregationResLevel?: number;
+  queryParameters?: QueryParameters;
 };
 
 /**
@@ -129,13 +147,18 @@ function getParameters({
   columns,
   clientId,
   aggregationExp,
-  aggregationResLevel
+  aggregationResLevel,
+  queryParameters
 }: Omit<FetchLayerDataParams, 'connection' | 'credentials'>) {
   const parameters = [encodeParameter('client', clientId || DEFAULT_CLIENT)];
 
   const sourceName = type === MAP_TYPES.QUERY ? 'q' : 'name';
   parameters.push(encodeParameter(sourceName, source));
 
+  if (queryParameters) {
+    parameters.push(encodeParameter('queryParameters', JSON.stringify(queryParameters)));
+  }
+
   if (geoColumn) {
     parameters.push(encodeParameter('geo_column', geoColumn));
   }
@@ -144,6 +167,9 @@ function getParameters({
   }
   if (aggregationExp) {
     parameters.push(encodeParameter('aggregationExp', aggregationExp));
+  } else if (isSpatialIndexGeoColumn(geoColumn)) {
+    // Default aggregationExp required for spatial index layers
+    parameters.push(encodeParameter('aggregationExp', '1 AS value'));
   }
   if (aggregationResLevel) {
     parameters.push(encodeParameter('aggregationResLevel', aggregationResLevel));
@@ -152,6 +178,11 @@ function getParameters({
   return parameters.join('&');
 }
 
+function isSpatialIndexGeoColumn(geoColumn: string | undefined) {
+  const spatialIndex = geoColumn?.split(':')[0];
+  return spatialIndex === 'h3' || spatialIndex === 'quadbin';
+}
+
 export async function mapInstantiation({
   type,
   source,
@@ -160,8 +191,10 @@ export async function mapInstantiation({
   geoColumn,
   columns,
   clientId,
+  headers,
   aggregationExp,
-  aggregationResLevel
+  aggregationResLevel,
+  queryParameters
 }: FetchLayerDataParams): Promise<MapInstantiation> {
   const baseUrl = `${credentials.mapsUrl}/${connection}/${type}`;
   const url = `${baseUrl}?${getParameters({
@@ -170,8 +203,9 @@ export async function mapInstantiation({
     geoColumn,
     columns,
     clientId,
+    aggregationResLevel,
     aggregationExp,
-    aggregationResLevel
+    queryParameters
   })}`;
   const {accessToken} = credentials;
 
@@ -179,12 +213,13 @@ export async function mapInstantiation({
     // need to be a POST request
     const body = JSON.stringify({
       q: source,
-      client: clientId || DEFAULT_CLIENT
+      client: clientId || DEFAULT_CLIENT,
+      queryParameters
     });
-    return await requestJson({method: 'POST', url: baseUrl, accessToken, body});
+    return await requestJson({method: 'POST', url: baseUrl, headers, accessToken, body});
   }
 
-  return await requestJson({url, accessToken});
+  return await requestJson({url, headers, accessToken});
 }
 
 function getUrlFromMetadata(metadata: MapInstantiation, format: Format): string | null {
@@ -254,8 +289,10 @@ export async function fetchLayerData({
   format,
   formatTiles,
   clientId,
+  headers,
   aggregationExp,
-  aggregationResLevel
+  aggregationResLevel,
+  queryParameters
 }: FetchLayerDataParams): Promise<FetchLayerDataResult> {
   // Internally we split data fetching into two parts to allow us to
   // conditionally fetch the actual data, depending on the metadata state
@@ -269,8 +306,10 @@ export async function fetchLayerData({
     format,
     formatTiles,
     clientId,
+    headers,
     aggregationExp,
-    aggregationResLevel
+    aggregationResLevel,
+    queryParameters
   });
 
   const data = await requestData({url, format: mapFormat, accessToken});
@@ -288,8 +327,10 @@ async function _fetchDataUrl({
   format,
   formatTiles,
   clientId,
+  headers,
   aggregationExp,
-  aggregationResLevel
+  aggregationResLevel,
+  queryParameters
 }: FetchLayerDataParams) {
   const defaultCredentials = getDefaultCredentials();
   // Only pick up default credentials if they have been defined for
@@ -321,8 +362,10 @@ async function _fetchDataUrl({
     geoColumn,
     columns,
     clientId,
+    headers,
     aggregationExp,
-    aggregationResLevel
+    aggregationResLevel,
+    queryParameters
   });
   let url: string | null = null;
   let mapFormat: Format | undefined;
@@ -361,19 +404,34 @@ async function _fetchMapDataset(
   dataset,
   accessToken: string,
   credentials: CloudNativeCredentials,
-  clientId?: string
+  clientId?: string,
+  headers?: Headers,
+  queryParameters?: QueryParameters
 ) {
-  const {connectionName: connection, columns, format, geoColumn, source, type} = dataset;
+  const {
+    aggregationExp,
+    aggregationResLevel,
+    connectionName: connection,
+    columns,
+    format,
+    geoColumn,
+    source,
+    type
+  } = dataset;
   // First fetch metadata
   const {url, mapFormat} = await _fetchDataUrl({
+    aggregationExp,
+    aggregationResLevel,
     clientId,
     credentials: {...credentials, accessToken},
     connection,
     columns,
     format,
     geoColumn,
+    headers,
     source,
-    type
+    type,
+    queryParameters
   });
 
   // Extract the last time the data changed
@@ -389,12 +447,76 @@ async function _fetchMapDataset(
   return true;
 }
 
+async function _fetchTilestats(
+  attribute,
+  dataset,
+  accessToken: string,
+  credentials: CloudNativeCredentials
+) {
+  const {connectionName: connection, source, type} = dataset;
+
+  const statsUrl = buildStatsUrlFromBase(credentials.apiBaseUrl);
+  let url = `${statsUrl}/${connection}/`;
+  if (type === MAP_TYPES.QUERY) {
+    url += `${attribute}?q=${source}`;
+  } else {
+    // MAP_TYPE.TABLE
+    url += `${source}/${attribute}`;
+  }
+  const stats = await requestData({url, format: FORMATS.JSON, accessToken});
+
+  // Replace tilestats for attribute with value from API
+  const {attributes} = dataset.data.tilestats.layers[0];
+  const index = attributes.findIndex(d => d.attribute === attribute);
+  attributes[index] = stats;
+  return true;
+}
+
 async function fillInMapDatasets(
   {datasets, token},
   clientId: string,
+  credentials: CloudNativeCredentials,
+  headers?: Headers
+) {
+  const promises = datasets.map(dataset =>
+    _fetchMapDataset(dataset, token, credentials, clientId, headers)
+  );
+  return await Promise.all(promises);
+}
+
+async function fillInTileStats(
+  {datasets, keplerMapConfig, token},
   credentials: CloudNativeCredentials
 ) {
-  const promises = datasets.map(dataset => _fetchMapDataset(dataset, token, credentials, clientId));
+  const attributes: {attribute?: string; dataset?: any}[] = [];
+  const {layers} = keplerMapConfig.config.visState;
+  for (const layer of layers) {
+    for (const channel of Object.keys(layer.visualChannels)) {
+      const attribute = layer.visualChannels[channel]?.name;
+      if (attribute) {
+        const dataset = datasets.find(d => d.id === layer.config.dataId);
+        if (dataset.data.tilestats && dataset.type !== MAP_TYPES.TILESET) {
+          // Only fetch stats for QUERY & TABLE map types
+          attributes.push({attribute, dataset});
+        }
+      }
+    }
+  }
+  // Remove duplicates to avoid repeated requests
+  const filteredAttributes: {attribute?: string; dataset?: any}[] = [];
+  for (const a of attributes) {
+    if (
+      !filteredAttributes.find(
+        ({attribute, dataset}) => attribute === a.attribute && dataset === a.dataset
+      )
+    ) {
+      filteredAttributes.push(a);
+    }
+  }
+
+  const promises = filteredAttributes.map(({attribute, dataset}) =>
+    _fetchTilestats(attribute, dataset, token, credentials)
+  );
   return await Promise.all(promises);
 }
 
@@ -402,12 +524,14 @@ export async function fetchMap({
   cartoMapId,
   clientId,
   credentials,
+  headers,
   autoRefresh,
   onNewData
 }: {
   cartoMapId: string;
   clientId: string;
   credentials?: CloudNativeCredentials;
+  headers?: Headers;
   autoRefresh?: number;
   onNewData?: (map: any) => void;
 }) {
@@ -436,7 +560,7 @@ export async function fetchMap({
   }
 
   const url = `${localCreds.mapsUrl}/public/${cartoMapId}`;
-  const map = await requestJson<any>({url, accessToken});
+  const map = await requestJson<any>({url, headers, accessToken});
 
   // Periodically check if the data has changed. Note that this
   // will not update when a map is published.
@@ -444,7 +568,7 @@ export async function fetchMap({
   if (autoRefresh) {
     // eslint-disable-next-line @typescript-eslint/no-misused-promises
     const intervalId = setInterval(async () => {
-      const changed = await fillInMapDatasets(map, clientId, localCreds);
+      const changed = await fillInMapDatasets(map, clientId, localCreds, headers);
       if (onNewData && changed.some(v => v === true)) {
         onNewData(parseMap(map));
       }
@@ -465,7 +589,10 @@ export async function fetchMap({
   });
 
   // Mutates map.datasets so that dataset.data contains data
-  await fillInMapDatasets(map, clientId, localCreds);
+  await fillInMapDatasets(map, clientId, localCreds, headers);
+
+  // Mutates attributes in visualChannels to contain tile stats
+  await fillInTileStats(map, localCreds);
   return {
     ...parseMap(map),
     ...{stopAutoRefresh}

package/src/api/parseMap.ts

@@ -176,10 +176,11 @@ function createChannelProps(visualChannels, type, config, data) {
       }
     }
   } else if (colorField) {
+    const {colorAggregation: aggregation, colorRange: range} = visConfig;
     result.getFillColor = getColorAccessor(
       colorField,
       colorScale,
-      visConfig.colorRange,
+      {aggregation, range},
       visConfig.opacity,
       data
     );
@@ -189,10 +190,11 @@ function createChannelProps(visualChannels, type, config, data) {
     const fallbackOpacity = type === 'point' ? visConfig.opacity : 1;
     const opacity =
       visConfig.strokeOpacity !== undefined ? visConfig.strokeOpacity : fallbackOpacity;
+    const {strokeColorAggregation: aggregation, strokeColorRange: range} = visConfig;
     result.getLineColor = getColorAccessor(
       strokeColorField,
       strokeColorScale,
-      visConfig.strokeColorRange,
+      {aggregation, range},
       opacity,
       data
     );
@@ -201,6 +203,7 @@ function createChannelProps(visualChannels, type, config, data) {
     result.getElevation = getSizeAccessor(
       heightField,
       heightScale,
+      visConfig.heightAggregation,
       visConfig.heightRange || visConfig.sizeRange,
       data
     );
@@ -209,6 +212,7 @@ function createChannelProps(visualChannels, type, config, data) {
     result.getPointRadius = getSizeAccessor(
       sizeField,
       sizeScale,
+      visConfig.sizeAggregation,
       visConfig.radiusRange || visConfig.sizeRange,
       data
     );

package/src/config.ts

@@ -50,11 +50,21 @@ export function getDefaultCredentials(): Credentials {
   return credentials;
 }
 
-export function buildMapsUrlFromBase(apiBaseUrl: string): string {
-  let suffix = '/v3/maps';
+export type V3Endpoint = 'maps' | 'stats';
+
+function buildUrlFromBase(apiBaseUrl: string, endpoint: V3Endpoint) {
+  let suffix = `/v3/${endpoint}`;
   if (apiBaseUrl.endsWith('/')) {
     suffix = suffix.substring(1);
   }
 
   return `${apiBaseUrl}${suffix}`;
 }
+
+export function buildMapsUrlFromBase(apiBaseUrl: string): string {
+  return buildUrlFromBase(apiBaseUrl, 'maps');
+}
+
+export function buildStatsUrlFromBase(apiBaseUrl: string): string {
+  return buildUrlFromBase(apiBaseUrl, 'stats');
+}

package/src/index.ts

@@ -22,5 +22,5 @@ export {
   getDataV2 as _getDataV2,
   mapInstantiation as _mapInstantiation
 } from './api';
-
+export type {QueryParameters} from './api';
 export type {CartoLayerProps} from './layers/carto-layer';