@deck.gl/carto 8.6.3 → 8.7.0-alpha.10

package/package.json

@@ -2,7 +2,7 @@
   "name": "@deck.gl/carto",
   "description": "CARTO official integration with Deck.gl. Build geospatial applications using CARTO and Deck.gl.",
   "license": "MIT",
-  "version": "8.6.3",
+  "version": "8.7.0-alpha.10",
   "publishConfig": {
     "access": "public"
   },
@@ -31,18 +31,23 @@
     "prepublishOnly": "npm run build-bundle && npm run build-bundle -- --env.dev"
   },
   "dependencies": {
-    "@loaders.gl/loader-utils": "^3.0.8",
-    "@loaders.gl/mvt": "^3.0.8",
-    "@loaders.gl/tiles": "^3.0.8",
-    "@math.gl/web-mercator": "^3.5.4",
+    "@loaders.gl/gis": "^3.1.3",
+    "@loaders.gl/loader-utils": "^3.1.3",
+    "@loaders.gl/mvt": "^3.1.3",
+    "@loaders.gl/tiles": "^3.1.3",
+    "@math.gl/web-mercator": "^3.5.6",
     "cartocolor": "^4.0.2",
-    "d3-scale": "^3.2.3"
+    "d3-array": "^2.8.0",
+    "d3-color": "^2.0.0",
+    "d3-format": "^2.0.0",
+    "d3-scale": "^3.2.3",
+    "moment-timezone": "^0.5.33"
   },
   "peerDependencies": {
     "@deck.gl/core": "^8.0.0",
     "@deck.gl/geo-layers": "^8.0.0",
     "@deck.gl/layers": "^8.0.0",
-    "@loaders.gl/core": "^3.0.0"
+    "@loaders.gl/core": "^3.1.3"
   },
-  "gitHead": "938ee03697eecbaced24c9df637d53ab42375df1"
+  "gitHead": "b6c48fb68b66755743c28776a0642b56a2df086a"
 }

package/src/api/index.js

@@ -1,3 +1,3 @@
 export {FORMATS, MAP_TYPES, API_VERSIONS} from './maps-api-common';
 export {getDataV2, CONNECTIONS} from './maps-client';
-export {fetchLayerData, getData, mapInstantiation} from './maps-v3-client';
+export {fetchLayerData, fetchMap, getData, mapInstantiation} from './maps-v3-client';

package/src/api/layer-map.js

@@ -0,0 +1,195 @@
+import {extent} from 'd3-array';
+import {rgb} from 'd3-color';
+import {
+  scaleLinear,
+  scaleOrdinal,
+  scaleLog,
+  scalePoint,
+  scaleQuantile,
+  scaleQuantize,
+  scaleSqrt
+} from 'd3-scale';
+import {format as d3Format} from 'd3-format';
+import moment from 'moment-timezone';
+import {H3HexagonLayer, MVTLayer} from '@deck.gl/geo-layers';
+import {GeoJsonLayer} from '@deck.gl/layers';
+
+const SCALE_FUNCS = {
+  linear: scaleLinear,
+  ordinal: scaleOrdinal,
+  log: scaleLog,
+  point: scalePoint,
+  quantile: scaleQuantile,
+  quantize: scaleQuantize,
+  sqrt: scaleSqrt
+};
+
+// Kepler -> Deck.gl
+const sharedPropMap = {
+  color: 'getFillColor',
+  isVisible: 'visible',
+  label: 'cartoLabel',
+  textLabel: {
+    alignment: 'getTextAlignmentBaseline',
+    anchor: 'getTextAnchor',
+    color: 'getTextColor',
+    size: 'getTextSize'
+  },
+  visConfig: {
+    enable3d: 'extruded',
+    filled: 'filled',
+    opacity: 'opacity',
+    strokeColor: 'getLineColor',
+    stroked: 'stroked',
+    thickness: 'getLineWidth',
+    radius: 'getPointRadius',
+    wireframe: 'wireframe'
+  }
+};
+
+const RADIUS_DOWNSCALE = 0.2;
+
+const defaultProps = {
+  lineMiterLimit: 2,
+  lineWidthUnits: 'pixels',
+  pointRadiusScale: RADIUS_DOWNSCALE,
+  pointRadiusUnits: 'pixels',
+  rounded: true,
+  wrapLongitude: false
+};
+
+function mergePropMaps(a, b) {
+  return {...a, ...b, visConfig: {...a.visConfig, ...b.visConfig}};
+}
+
+export const LAYER_MAP = {
+  point: {
+    Layer: GeoJsonLayer,
+    propMap: mergePropMaps(sharedPropMap, {
+      visConfig: {outline: 'stroked'}
+    }),
+    defaultProps
+  },
+  geojson: {
+    Layer: GeoJsonLayer,
+    propMap: sharedPropMap,
+    defaultProps: {...defaultProps, lineWidthScale: 2}
+  },
+  hexagonId: {
+    Layer: H3HexagonLayer,
+    propMap: mergePropMaps(sharedPropMap, {
+      visConfig: {coverage: 'coverage', elevationScale: 'elevationScale'}
+    }),
+    defaultProps: {...defaultProps, getHexagon: d => d.h3}
+  },
+  mvt: {
+    Layer: MVTLayer,
+    propMap: sharedPropMap,
+    defaultProps: {...defaultProps, pointRadiusScale: 0.3, lineWidthScale: 2}
+  }
+};
+
+function domainFromAttribute(attribute, scaleType) {
+  if (scaleType === 'ordinal' || scaleType === 'point') {
+    return attribute.categories.map(c => c.category).filter(c => c !== undefined && c !== null);
+  }
+
+  let {min} = attribute;
+  if (scaleType === 'log' && min === 0) {
+    min = 1e-5;
+  }
+  return [min, attribute.max];
+}
+
+function domainFromValues(values, scaleType) {
+  if (scaleType === 'ordinal') {
+    return [...new Set(values)].sort();
+  }
+  return extent(values);
+}
+
+function calculateDomain(data, name, scaleType) {
+  if (data.tilestats) {
+    // Tileset data type
+    const {attributes} = data.tilestats.layers[0];
+    const attribute = attributes.find(a => a.attribute === name);
+    return domainFromAttribute(attribute, scaleType);
+  } else if (data.features) {
+    // GeoJSON data type
+    const values = data.features.map(({properties}) => properties[name]);
+    return domainFromValues(values, scaleType);
+  } else if (Array.isArray(data) && data[0][name] !== undefined) {
+    // JSON data type
+    const values = data.map(properties => properties[name]);
+    return domainFromValues(values, scaleType);
+  }
+
+  return [0, 1];
+}
+
+function normalizeAccessor(accessor, data) {
+  if (data.features || data.tilestats) {
+    return ({properties}) => {
+      return accessor(properties);
+    };
+  }
+  return accessor;
+}
+
+export function getColorAccessor({name}, scaleType, {colors}, opacity, data) {
+  const scale = SCALE_FUNCS[scaleType]();
+  scale.domain(calculateDomain(data, name, scaleType));
+  scale.range(colors);
+  const alpha = opacity !== undefined ? Math.round(255 * Math.pow(opacity, 1 / 2.2)) : 255;
+
+  const accessor = properties => {
+    const rgba = rgb(scale(properties[name]));
+    return [rgba.r, rgba.g, rgba.b, alpha];
+  };
+  return normalizeAccessor(accessor, data);
+}
+
+export function getSizeAccessor({name}, scaleType, range, data) {
+  const scale = SCALE_FUNCS[scaleType]();
+  scale.domain(calculateDomain(data, name, scaleType));
+  scale.range(range);
+
+  const accessor = properties => {
+    return scale(properties[name]);
+  };
+  return normalizeAccessor(accessor, data);
+}
+
+const FORMATS = {
+  date: s => moment.utc(s).format('MM/DD/YY HH:mm:ssa'),
+  integer: d3Format('i'),
+  float: d3Format('.5f'),
+  timestamp: s => moment.utc(s).format('X'),
+  default: String
+};
+
+export function getTextAccessor({name, type}, data) {
+  const format = FORMATS[type] || FORMATS.default;
+  const accessor = properties => {
+    return format(properties[name]);
+  };
+  return normalizeAccessor(accessor, data);
+}
+
+export function getTextPixelOffsetAccessor({alignment, anchor, size}, radius) {
+  const padding = 20;
+  const signX = anchor === 'middle' ? 0 : anchor === 'start' ? 1 : -1;
+  const signY = alignment === 'center' ? 0 : alignment === 'bottom' ? 1 : -1;
+  const sizeOffset = alignment === 'center' ? 0 : size;
+
+  const calculateOffset = r => {
+    r = RADIUS_DOWNSCALE * r;
+    return [signX * (r + padding), signY * (r + padding + sizeOffset)];
+  };
+
+  return typeof radius === 'function'
+    ? d => {
+        return calculateOffset(radius(d));
+      }
+    : calculateOffset(radius);
+}

package/src/api/maps-client.js

@@ -56,7 +56,7 @@ export async function getDataV2({type, source, credentials}) {
 
     default:
       throw new Error(
-        `Invalid maps API version. It shoud be ${API_VERSIONS.V1} or ${API_VERSIONS.V2}`
+        `Invalid maps API version. It should be ${API_VERSIONS.V1} or ${API_VERSIONS.V2}`
       );
   }
 }
@@ -95,15 +95,11 @@ function dealWithError({response, json, credentials}) {
   switch (response.status) {
     case 401:
       throw new Error(
-        `Unauthorized access to Maps API: invalid combination of user ('${
-          credentials.username
-        }') and apiKey ('${credentials.apiKey}')`
+        `Unauthorized access to Maps API: invalid combination of user ('${credentials.username}') and apiKey ('${credentials.apiKey}')`
       );
     case 403:
       throw new Error(
-        `Unauthorized access to dataset: the provided apiKey('${
-          credentials.apiKey
-        }') doesn't provide access to the requested data`
+        `Unauthorized access to dataset: the provided apiKey('${credentials.apiKey}') doesn't provide access to the requested data`
       );
 
     default:

package/src/api/maps-v3-client.js

@@ -3,9 +3,11 @@
  */
 import {getDefaultCredentials, buildMapsUrlFromBase} from '../config';
 import {API_VERSIONS, encodeParameter, FORMATS, MAP_TYPES} from './maps-api-common';
+import {parseMap} from './parseMap';
 import {log} from '@deck.gl/core';
 
 const MAX_GET_LENGTH = 2048;
+const DEFAULT_CLIENT = 'deck-gl-carto';
 
 /**
  * Request against Maps API
@@ -68,8 +70,8 @@ function dealWithError({response, error}) {
 /**
  * Build a URL with all required parameters
  */
-function getParameters({type, source, geoColumn, columns, schema}) {
-  const parameters = [encodeParameter('client', 'deck-gl-carto')];
+function getParameters({type, source, geoColumn, columns, schema, clientId}) {
+  const parameters = [encodeParameter('client', clientId || DEFAULT_CLIENT)];
   if (schema) {
     parameters.push(encodeParameter('schema', true));
   }
@@ -96,10 +98,11 @@ export async function mapInstantiation({
   credentials,
   geoColumn,
   columns,
-  schema
+  schema,
+  clientId
 }) {
   const baseUrl = `${credentials.mapsUrl}/${connection}/${type}`;
-  const url = `${baseUrl}?${getParameters({type, source, geoColumn, columns, schema})}`;
+  const url = `${baseUrl}?${getParameters({type, source, geoColumn, columns, schema, clientId})}`;
   const {accessToken} = credentials;
 
   const format = 'json';
@@ -108,7 +111,7 @@ export async function mapInstantiation({
     // need to be a POST request
     const body = JSON.stringify({
       q: source,
-      client: 'deck-gl-carto'
+      client: clientId || DEFAULT_CLIENT
     });
     return await request({method: 'POST', url: baseUrl, format, accessToken, body});
   }
@@ -126,7 +129,7 @@ function getUrlFromMetadata(metadata, format) {
   return null;
 }
 
-function checkGetLayerDataParameters({type, source, connection, localCreds}) {
+function checkFetchLayerDataParameters({type, source, connection, localCreds}) {
   log.assert(connection, 'Must define connection');
   log.assert(type, 'Must define a type');
   log.assert(source, 'Must define a source');
@@ -144,7 +147,42 @@ export async function fetchLayerData({
   geoColumn,
   columns,
   format,
-  schema
+  schema,
+  clientId
+}) {
+  // Internally we split data fetching into two parts to allow us to
+  // conditionally fetch the actual data, depending on the metadata state
+  const {url, accessToken, mapFormat, metadata} = await _fetchDataUrl({
+    type,
+    source,
+    connection,
+    credentials,
+    geoColumn,
+    columns,
+    format,
+    schema,
+    clientId
+  });
+
+  const data = await request({url, format: mapFormat, accessToken});
+  const result = {data, format: mapFormat};
+  if (schema) {
+    result.schema = metadata.schema;
+  }
+
+  return result;
+}
+
+async function _fetchDataUrl({
+  type,
+  source,
+  connection,
+  credentials,
+  geoColumn,
+  columns,
+  format,
+  schema,
+  clientId
 }) {
   const defaultCredentials = getDefaultCredentials();
   // Only pick up default credentials if they have been defined for
@@ -153,7 +191,7 @@ export async function fetchLayerData({
     ...(defaultCredentials.apiVersion === API_VERSIONS.V3 && defaultCredentials),
     ...credentials
   };
-  checkGetLayerDataParameters({type, source, connection, localCreds});
+  checkFetchLayerDataParameters({type, source, connection, localCreds});
 
   if (!localCreds.mapsUrl) {
     localCreds.mapsUrl = buildMapsUrlFromBase(localCreds.apiBaseUrl);
@@ -166,7 +204,8 @@ export async function fetchLayerData({
     credentials: localCreds,
     geoColumn,
     columns,
-    schema
+    schema,
+    clientId
   });
   let url;
   let mapFormat;
@@ -177,7 +216,7 @@ export async function fetchLayerData({
     log.assert(url, `Format ${format} not available`);
   } else {
     // guess map format
-    const prioritizedFormats = [FORMATS.GEOJSON, FORMATS.NDJSON, FORMATS.TILEJSON];
+    const prioritizedFormats = [FORMATS.GEOJSON, FORMATS.JSON, FORMATS.NDJSON, FORMATS.TILEJSON];
     for (const f of prioritizedFormats) {
       url = getUrlFromMetadata(metadata, f);
       if (url) {
@@ -188,17 +227,20 @@ export async function fetchLayerData({
   }
 
   const {accessToken} = localCreds;
-
-  const data = await request({url, format: mapFormat, accessToken});
-  const result = {data, format: mapFormat};
-  if (schema) {
-    result.schema = metadata.schema;
-  }
-
-  return result;
+  return {url, accessToken, mapFormat, metadata};
 }
 
-export async function getData({type, source, connection, credentials, geoColumn, columns, format}) {
+export async function getData({
+  type,
+  source,
+  connection,
+  credentials,
+  geoColumn,
+  columns,
+  format,
+  clientId
+}) {
+  log.deprecated('getData', 'fetchLayerData')();
   const layerData = await fetchLayerData({
     type,
     source,
@@ -207,7 +249,85 @@ export async function getData({type, source, connection, credentials, geoColumn,
     geoColumn,
     columns,
     format,
-    schema: false
+    schema: false,
+    clientId
   });
   return layerData.data;
 }
+
+/* global clearInterval, setInterval, URLSearchParams */
+async function _fetchMapDataset(dataset, accessToken, credentials, clientId) {
+  // First fetch metadata
+  const {connectionName: connection, source, type} = dataset;
+  const {url, mapFormat} = await _fetchDataUrl({
+    credentials: {...credentials, accessToken},
+    connection,
+    source,
+    type,
+    clientId
+  });
+
+  // Extract the last time the data changed
+  const cache = parseInt(new URLSearchParams(url).get('cache'), 10);
+  if (cache && dataset.cache === cache) {
+    return false;
+  }
+  dataset.cache = cache;
+
+  // Only fetch if the data has changed
+  const data = await request({url, format: mapFormat, accessToken});
+  dataset.data = data;
+  return true;
+}
+
+async function fillInMapDatasets({datasets, token}, clientId, credentials) {
+  const promises = datasets.map(dataset => _fetchMapDataset(dataset, token, credentials, clientId));
+  return await Promise.all(promises);
+}
+
+export async function fetchMap({cartoMapId, clientId, credentials, autoRefresh, onNewData}) {
+  const defaultCredentials = getDefaultCredentials();
+  const localCreds = {
+    ...(defaultCredentials.apiVersion === API_VERSIONS.V3 && defaultCredentials),
+    ...credentials
+  };
+
+  log.assert(cartoMapId, `Must define CARTO map id: fetchMap({cartoMapId: 'XXXX-XXXX-XXXX'})`);
+
+  log.assert(localCreds.apiVersion === API_VERSIONS.V3, 'Method only available for v3');
+  log.assert(localCreds.apiBaseUrl, 'Must define apiBaseUrl');
+  if (!localCreds.mapsUrl) {
+    localCreds.mapsUrl = buildMapsUrlFromBase(localCreds.apiBaseUrl);
+  }
+
+  if (autoRefresh || onNewData) {
+    log.assert(onNewData, 'Must define `onNewData` when using autoRefresh');
+    log.assert(typeof onNewData === 'function', '`onNewData` must be a function');
+    log.assert(autoRefresh > 0, '`autoRefresh` must be a positive number');
+  }
+
+  const url = `${localCreds.mapsUrl}/public/${cartoMapId}`;
+  const map = await request({url});
+
+  // Periodically check if the data has changed. Note that this
+  // will not update when a map is published.
+  let stopAutoRefresh;
+  if (autoRefresh) {
+    const intervalId = setInterval(async () => {
+      const changed = await fillInMapDatasets(map, clientId, credentials);
+      if (changed.some(v => v === true)) {
+        onNewData(parseMap(map));
+      }
+    }, autoRefresh * 1000);
+    stopAutoRefresh = () => {
+      clearInterval(intervalId);
+    };
+  }
+
+  // Mutates map.datasets so that dataset.data contains data
+  await fillInMapDatasets(map, clientId, credentials);
+  return {
+    ...parseMap(map),
+    ...{stopAutoRefresh}
+  };
+}

package/src/api/parseMap.js

@@ -0,0 +1,183 @@
+import GL from '@luma.gl/constants';
+import {
+  LAYER_MAP,
+  getColorAccessor,
+  getSizeAccessor,
+  getTextAccessor,
+  getTextPixelOffsetAccessor
+} from './layer-map';
+import {_flatten as flatten, log} from '@deck.gl/core';
+
+export function parseMap(json) {
+  const {keplerMapConfig, datasets} = json;
+  log.assert(keplerMapConfig.version === 'v1', 'Only support Kepler v1');
+  const {mapState, mapStyle} = keplerMapConfig.config;
+  const {layers, layerBlending, interactionConfig} = keplerMapConfig.config.visState;
+
+  return {
+    id: json.id,
+    title: json.title,
+    description: json.description,
+    createdAt: json.createdAt,
+    updatedAt: json.updatedAt,
+    initialViewState: mapState,
+    mapStyle,
+    layers: extractTextLayers(layers.reverse()).map(({id, type, config, visualChannels}) => {
+      log.assert(type in LAYER_MAP, `Unsupported layer type: ${type}`);
+      const {Layer, propMap, defaultProps} = LAYER_MAP[type];
+
+      const {dataId} = config;
+      const dataset = datasets.find(d => d.id === dataId);
+      log.assert(dataset, `No dataset matching dataId: ${dataId}`);
+      const {data} = dataset;
+      log.assert(data, `No data loaded for dataId: ${dataId}`);
+      return new Layer({
+        id,
+        data,
+        ...defaultProps,
+        ...createBlendingProps(layerBlending),
+        ...(!config.textLabel && createInteractionProps(interactionConfig)),
+        ...createStyleProps(config, propMap),
+        ...createChannelProps(visualChannels, type, config, data) // Must come after style
+      });
+    })
+  };
+}
+
+function extractTextLayers(layers) {
+  return flatten(
+    layers.map(({id, config, ...rest}) => {
+      const {textLabel, ...configRest} = config;
+      return [
+        // Original layer without textLabel
+        {id, config: configRest, ...rest},
+
+        // One layer per valid text label, with full opacity
+        ...textLabel
+          .filter(t => t.field)
+          .map(t => {
+            return {
+              id: `${id}-label-${t.field.name}`,
+              config: {
+                textLabel: t,
+                ...configRest,
+                label: `${config.label}-label-${t.field.name}`,
+                visConfig: {...configRest.visConfig, opacity: 1}
+              },
+              ...rest
+            };
+          })
+      ];
+    })
+  );
+}
+
+function createBlendingProps(layerBlending) {
+  if (layerBlending === 'additive') {
+    return {
+      parameters: {
+        blendFunc: [GL.SRC_ALPHA, GL.DST_ALPHA],
+        blendEquation: GL.FUNC_ADD
+      }
+    };
+  } else if (layerBlending === 'subtractive') {
+    return {
+      parameters: {
+        blendFunc: [GL.ONE, GL.ONE_MINUS_DST_COLOR, GL.SRC_ALPHA, GL.DST_ALPHA],
+        blendEquation: [GL.FUNC_SUBTRACT, GL.FUNC_ADD]
+      }
+    };
+  }
+
+  return {};
+}
+
+function createInteractionProps(interactionConfig) {
+  const pickable = interactionConfig && interactionConfig.tooltip.enabled;
+  return {
+    autoHighlight: pickable,
+    pickable
+  };
+}
+
+function mapProps(source, target, mapping) {
+  for (const sourceKey in mapping) {
+    const sourceValue = source[sourceKey];
+    const targetKey = mapping[sourceKey];
+    if (sourceValue === undefined) {
+      // eslint-disable-next-line no-continue
+      continue;
+    }
+    if (typeof targetKey === 'string') {
+      target[targetKey] = sourceValue;
+    } else if (typeof targetKey === 'object') {
+      // Nested definition, recurse down one level
+      mapProps(sourceValue, target, targetKey);
+    }
+  }
+}
+
+function createStyleProps(config, mapping) {
+  const result = {};
+  mapProps(config, result, mapping);
+  result.highlightColor = config.visConfig.enable3d ? [255, 255, 255, 60] : [252, 242, 26, 255];
+  return result;
+}
+
+/* eslint-disable complexity */
+function createChannelProps(visualChannels, type, config, data) {
+  const {colorField, colorScale, sizeField, sizeScale, strokeColorField, strokeColorScale} =
+    visualChannels;
+  let {heightField, heightScale} = visualChannels;
+  if (type === 'hexagonId') {
+    heightField = sizeField;
+    heightScale = sizeScale;
+  }
+  const {textLabel, visConfig} = config;
+  const result = {};
+  const textLabelField = textLabel && textLabel.field;
+  if (colorField) {
+    result.getFillColor = getColorAccessor(
+      colorField,
+      colorScale,
+      visConfig.colorRange,
+      1, // Rely on layer opacity
+      data
+    );
+  }
+  if (strokeColorField) {
+    const opacity =
+      visConfig.strokeOpacity !== undefined ? visConfig.strokeOpacity / visConfig.opacity : 1;
+    result.getLineColor = getColorAccessor(
+      strokeColorField,
+      strokeColorScale,
+      visConfig.colorRange,
+      opacity,
+      data
+    );
+  }
+  if (heightField) {
+    result.getElevation = getSizeAccessor(
+      heightField,
+      heightScale,
+      visConfig.heightRange || visConfig.sizeRange,
+      data
+    );
+  }
+  if (sizeField) {
+    result.getPointRadius = getSizeAccessor(
+      sizeField,
+      sizeScale,
+      visConfig.radiusRange || visConfig.sizeRange,
+      data
+    );
+  }
+  if (textLabelField) {
+    result.getText = getTextAccessor(textLabelField, data);
+    result.pointType = 'text';
+    const radius = result.getPointRadius || visConfig.radius;
+    result.getTextPixelOffset = getTextPixelOffsetAccessor(textLabel, radius);
+  }
+
+  return result;
+}