@carto/kepler.gl 0.0.76 → 0.0.78-0

package/src/utils/data-utils.js

@@ -1,447 +0,0 @@
-// Copyright (c) 2022 Uber Technologies, Inc.
-//
-// Permission is hereby granted, free of charge, to any person obtaining a copy
-// of this software and associated documentation files (the "Software"), to deal
-// in the Software without restriction, including without limitation the rights
-// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-// copies of the Software, and to permit persons to whom the Software is
-// furnished to do so, subject to the following conditions:
-//
-// The above copyright notice and this permission notice shall be included in
-// all copies or substantial portions of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-// THE SOFTWARE.
-
-import assert from 'assert';
-import {ALL_FIELD_TYPES} from 'constants/default-settings';
-import {TOOLTIP_FORMATS, TOOLTIP_FORMAT_TYPES, TOOLTIP_KEY} from 'constants/tooltip';
-import {format as d3Format} from 'd3-format';
-import {bisectLeft} from 'd3-array';
-import moment from 'moment-timezone';
-
-const MAX_LATITUDE = 90;
-const MIN_LATITUDE = -90;
-const MAX_LONGITUDE = 180;
-const MIN_LONGITUDE = -180;
-
-/**
- * simple getting unique values of an array
- *
- * @param {array} values
- * @returns {array} unique values
- */
-export function unique(values) {
-  const results = [];
-  const uniqueSet = new Set(values);
-  uniqueSet.forEach(v => {
-    if (notNullorUndefined(v)) {
-      results.push(v);
-    }
-  });
-  return results;
-}
-
-export function uniqueSortedByFrequency(values) {
-  const frequency = values.reduce((res, value) => {
-    if (notNullorUndefined(value)) {
-      if (!res[value]) {
-        res[value] = 0;
-      }
-      res[value] += 1;
-    }
-    return res;
-  }, {});
-
-  return Object.keys(frequency).sort((a, b) => frequency[b] - frequency[a]);
-}
-
-/* eslint-disable max-statements */
-/**
- * return center of map from given points
- * @param {array} layers
- * @returns {object} coordinates of map center, empty if not found
- */
-export function findMapBounds(layers) {
-  // find bounds in formatted layerData
-  // take ALL layers into account when finding map bounds
-  const availableLayerBounds = layers.reduce((res, l) => {
-    if (l.meta && l.meta.bounds) {
-      res.push(l.meta.bounds);
-    }
-    return res;
-  }, []);
-  // return null if no layer is available
-  if (availableLayerBounds.length === 0) {
-    return null;
-  }
-  // merge bounds in each layer
-  const newBounds = availableLayerBounds.reduce(
-    (res, b) => {
-      return [
-        Math.min(res[0], b[0]),
-        Math.min(res[1], b[1]),
-        Math.max(res[2], b[2]),
-        Math.max(res[3], b[3])
-      ];
-    },
-    [MAX_LONGITUDE, MAX_LATITUDE, MIN_LONGITUDE, MIN_LATITUDE]
-  );
-  return newBounds;
-}
-/* eslint-enable max-statements */
-
-export function getLatLngBounds(points, idx, limit) {
-  const lats = points
-    .map(d => Array.isArray(d) && d[idx])
-    .filter(Number.isFinite)
-    .sort(numberSort);
-
-  if (!lats.length) {
-    return null;
-  }
-
-  // clamp to limit
-  return [Math.max(lats[0], limit[0]), Math.min(lats[lats.length - 1], limit[1])];
-}
-
-export function clamp([min, max], val) {
-  return val <= min ? min : val >= max ? max : val;
-}
-
-export function getSampleData(data, sampleSize = 500, getValue = d => d) {
-  const sampleStep = Math.max(Math.floor(data.length / sampleSize), 1);
-  const output = [];
-  for (let i = 0; i < data.length; i += sampleStep) {
-    output.push(getValue(data[i]));
-  }
-
-  return output;
-}
-
-/**
- * Convert different time format to unix milliseconds
- * @type {typeof import('./data-utils').timeToUnixMilli}
- */
-export function timeToUnixMilli(value, format) {
-  if (notNullorUndefined(value)) {
-    return typeof value === 'string'
-      ? moment.utc(value, format).valueOf()
-      : format === 'x'
-      ? value * 1000
-      : value;
-  }
-  return null;
-}
-
-/**
- *
- * @type {typeof import('./data-utils').maybeToDate}
- */
-export function maybeToDate(isTime, fieldIdx, format, dc, d) {
-  if (isTime) {
-    return timeToUnixMilli(dc.valueAt(d.index, fieldIdx), format);
-  }
-
-  return dc.valueAt(d.index, fieldIdx);
-}
-
-/**
- * whether null or undefined
- * @type {typeof import('./data-utils').notNullorUndefined}
- */
-export function notNullorUndefined(d) {
-  return d !== undefined && d !== null && d !== 'null';
-}
-
-/**
- * Whether d is a number, this filtered out NaN as well
- * @type {typeof import('./data-utils').notNullorUndefined}
- */
-export function isNumber(d) {
-  return Number.isFinite(d);
-}
-/**
- * whether null or undefined
- */
-export function isPlainObject(obj) {
-  return obj === Object(obj) && typeof obj !== 'function' && !Array.isArray(obj);
-}
-
-/**
- * @type {typeof import('./data-utils').numberSort}
- */
-export function numberSort(a, b) {
-  return a - b;
-}
-
-/**
- * @type {typeof import('./data-utils').getSortingFunction}
- */
-export function getSortingFunction(fieldType) {
-  switch (fieldType) {
-    case ALL_FIELD_TYPES.real:
-    case ALL_FIELD_TYPES.integer:
-    case ALL_FIELD_TYPES.timestamp:
-      return numberSort;
-    default:
-      return undefined;
-  }
-}
-
-/**
- * round number with exact number of decimals
- * return as a string
- * @type {typeof import('./data-utils').preciseRound}
- */
-export function preciseRound(num, decimals) {
-  const t = Math.pow(10, decimals);
-  return (
-    Math.round(
-      num * t + (decimals > 0 ? 1 : 0) * (Math.sign(num) * (10 / Math.pow(100, decimals)))
-    ) / t
-  ).toFixed(decimals);
-}
-
-/**
- * get number of decimals to round to for slider from step
- * @param {number} step
- * @returns {number} - number of decimal
- */
-export function getRoundingDecimalFromStep(step) {
-  if (isNaN(step)) {
-    assert('step is not a number');
-    assert(step);
-  }
-
-  const splitZero = step.toString().split('.');
-  if (splitZero.length === 1) {
-    return 0;
-  }
-  return splitZero[1].length;
-}
-
-/**
- * Use in slider, given a number and an array of numbers, return the nears number from the array
- * @type {typeof import('./data-utils').snapToMarks}
- * @param value
- * @param marks
- */
-export function snapToMarks(value, marks) {
-  // always use bin x0
-  const i = bisectLeft(marks, value);
-  if (i === 0) {
-    return marks[i];
-  } else if (i === marks.length) {
-    return marks[i - 1];
-  }
-  const idx = marks[i] - value < value - marks[i - 1] ? i : i - 1;
-  return marks[idx];
-}
-
-/**
- * If marks is provided, snap to marks, if not normalize to step
- * @type {typeof import('./data-utils').normalizeSliderValue}
- * @param val
- * @param minValue
- * @param step
- * @param marks
- */
-export function normalizeSliderValue(val, minValue, step, marks) {
-  if (marks && marks.length) {
-    return snapToMarks(val, marks);
-  }
-
-  return roundValToStep(minValue, step, val);
-}
-
-/**
- * round the value to step for the slider
- * @type {typeof import('./data-utils').roundValToStep}
- * @param minValue
- * @param step
- * @param val
- * @returns - rounded number
- */
-export function roundValToStep(minValue, step, val) {
-  if (!isNumber(step) || !isNumber(minValue)) {
-    return val;
-  }
-
-  const decimal = getRoundingDecimalFromStep(step);
-  const steps = Math.floor((val - minValue) / step);
-  let remain = val - (steps * step + minValue);
-
-  // has to round because javascript turns 0.1 into 0.9999999999999987
-  remain = Number(preciseRound(remain, 8));
-
-  let closest;
-  if (remain === 0) {
-    closest = val;
-  } else if (remain < step / 2) {
-    closest = steps * step + minValue;
-  } else {
-    closest = (steps + 1) * step + minValue;
-  }
-
-  // precise round return a string rounded to the defined decimal
-  const rounded = preciseRound(closest, decimal);
-
-  return Number(rounded);
-}
-
-/**
- * Get the value format based on field and format options
- * Used in render tooltip value
- * @type {typeof import('./data-utils').defaultFormatter}
- */
-export const defaultFormatter = v => (notNullorUndefined(v) ? String(v) : '');
-
-export const FIELD_DISPLAY_FORMAT = {
-  [ALL_FIELD_TYPES.string]: defaultFormatter,
-  [ALL_FIELD_TYPES.timestamp]: defaultFormatter,
-  [ALL_FIELD_TYPES.integer]: defaultFormatter,
-  [ALL_FIELD_TYPES.real]: defaultFormatter,
-  [ALL_FIELD_TYPES.boolean]: defaultFormatter,
-  [ALL_FIELD_TYPES.date]: defaultFormatter,
-  [ALL_FIELD_TYPES.geojson]: d =>
-    typeof d === 'string'
-      ? d
-      : isPlainObject(d)
-      ? JSON.stringify(d)
-      : Array.isArray(d)
-      ? `[${String(d)}]`
-      : ''
-};
-
-/**
- * Parse field value and type and return a string representation
- * @type {typeof import('./data-utils').parseFieldValue}
- */
-export const parseFieldValue = (value, type) => {
-  if (!notNullorUndefined(value)) {
-    return '';
-  }
-
-  return FIELD_DISPLAY_FORMAT[type] ? FIELD_DISPLAY_FORMAT[type](value) : String(value);
-};
-
-const arrayMoveMutate = (array, from, to) => {
-  array.splice(to < 0 ? array.length + to : to, 0, array.splice(from, 1)[0]);
-};
-
-/**
- *
- * @param {*} array
- * @param {*} from
- * @param {*} to
- */
-export const arrayMove = (array, from, to) => {
-  array = array.slice();
-  arrayMoveMutate(array, from, to);
-  return array;
-};
-
-/**
- * Get the value format based on field and format options
- * Used in render tooltip value
- * @type {typeof import('./data-utils').getFormatter}
- * @param format
- * @param field
- */
-export function getFormatter(format, field) {
-  if (!format) {
-    return defaultFormatter;
-  }
-  const tooltipFormat = Object.values(TOOLTIP_FORMATS).find(f => f[TOOLTIP_KEY] === format);
-
-  if (tooltipFormat) {
-    return applyDefaultFormat(tooltipFormat);
-  } else if (typeof format === 'string' && field) {
-    return applyCustomFormat(format, field);
-  }
-
-  return defaultFormatter;
-}
-
-export function applyDefaultFormat(tooltipFormat) {
-  if (!tooltipFormat || !tooltipFormat.format) {
-    return defaultFormatter;
-  }
-
-  switch (tooltipFormat.type) {
-    case TOOLTIP_FORMAT_TYPES.DECIMAL:
-      return d3Format(tooltipFormat.format);
-    case TOOLTIP_FORMAT_TYPES.DATE:
-    case TOOLTIP_FORMAT_TYPES.DATE_TIME:
-      return datetimeFormatter(null)(tooltipFormat.format);
-    case TOOLTIP_FORMAT_TYPES.PERCENTAGE:
-      return v => `${d3Format(TOOLTIP_FORMATS.DECIMAL_DECIMAL_FIXED_2.format)(v)}%`;
-    case TOOLTIP_FORMAT_TYPES.BOOLEAN:
-      return getBooleanFormatter(tooltipFormat.format);
-    default:
-      return defaultFormatter;
-  }
-}
-
-export function getBooleanFormatter(format) {
-  switch (format) {
-    case '01':
-      return v => (v ? '1' : '0');
-    case 'yn':
-      return v => (v ? 'yes' : 'no');
-    default:
-      return defaultFormatter;
-  }
-}
-// Allow user to specify custom tooltip format via config
-export function applyCustomFormat(format, field) {
-  switch (field.type) {
-    case ALL_FIELD_TYPES.real:
-    case ALL_FIELD_TYPES.integer:
-      return d3Format(format);
-    case ALL_FIELD_TYPES.date:
-    case ALL_FIELD_TYPES.timestamp:
-      return datetimeFormatter(null)(format);
-    default:
-      return v => v;
-  }
-}
-
-/**
- * Format epoch milliseconds with a format string
- * @type {typeof import('./data-utils').datetimeFormatter} timezone
- */
-export function datetimeFormatter(timezone) {
-  return timezone
-    ? format => ts =>
-        moment
-          .utc(ts)
-          .tz(timezone)
-          .format(format)
-    : format => ts => moment.utc(ts).format(format);
-}
-
-export function uniqueSortedPairByFrequency(values) {
-  const frequencies = values.reduce((res, value) => {
-    if (notNullorUndefined(value)) {
-      if (!res[value]) {
-        res[value] = {
-          key: value,
-          frequency: 1
-        };
-      }
-      res[value].frequency += 1;
-    }
-    return res;
-  }, {});
-
-  return Object.values(frequencies)
-    .sort((a, b) => b.frequency - a.frequency)
-    .map(({key: [left, right]}) => ({left, right}));
-}

package/src/utils/dataset-utils.d.ts

@@ -1,10 +0,0 @@
-import {Datasets, Field} from '../reducers/vis-state-updaters';
-import {ProtoDataset} from '../actions';
-import {RGBColor} from 'reducers/types';
-
-export function createNewDataEntry(data: ProtoDataset, datasets?: Datasets): Datasets;
-
-export function getNewDatasetColor(ds: Datasets): RGBColor;
-
-export const datasetColorMaker: Generator<RGBColor>;
-export function findDefaultColorField(dataset: Dataset): false | Field;

package/src/utils/dataset-utils.js

@@ -1,255 +0,0 @@
-// Copyright (c) 2022 Uber Technologies, Inc.
-//
-// Permission is hereby granted, free of charge, to any person obtaining a copy
-// of this software and associated documentation files (the "Software"), to deal
-// in the Software without restriction, including without limitation the rights
-// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-// copies of the Software, and to permit persons to whom the Software is
-// furnished to do so, subject to the following conditions:
-//
-// The above copyright notice and this permission notice shall be included in
-// all copies or substantial portions of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-// THE SOFTWARE.
-
-import {hexToRgb} from './color-utils';
-import uniq from 'lodash/uniq';
-import {ALL_FIELD_TYPES} from 'constants/default-settings';
-import {validateInputData} from 'processors/data-processor';
-import KeplerTable from './table-utils/kepler-table';
-
-// apply a color for each dataset
-// to use as label colors
-const datasetColors = [
-  '#8F2FBF',
-  '#005CFF',
-  '#C06C84',
-  '#F8B195',
-  '#547A82',
-  '#3EACA8',
-  '#A2D4AB'
-].map(hexToRgb);
-
-/**
- * Random color generator
- * @return {Generator<import('reducers/types').RGBColor>}
- */
-function* generateColor() {
-  let index = 0;
-  while (index < datasetColors.length + 1) {
-    if (index === datasetColors.length) {
-      index = 0;
-    }
-    yield datasetColors[index++];
-  }
-}
-
-export const datasetColorMaker = generateColor();
-
-/** @type {typeof import('./dataset-utils').getNewDatasetColor} */
-export function getNewDatasetColor(datasets) {
-  const presetColors = datasetColors.map(String);
-  const usedColors = uniq(Object.values(datasets).map(d => String(d.color))).filter(c =>
-    presetColors.includes(c)
-  );
-
-  if (usedColors.length === presetColors.length) {
-    // if we already depleted the pool of color
-    return datasetColorMaker.next().value;
-  }
-
-  let color = datasetColorMaker.next().value;
-  while (usedColors.includes(String(color))) {
-    color = datasetColorMaker.next().value;
-  }
-
-  return color;
-}
-
-/**
- * Take datasets payload from addDataToMap, create datasets entry save to visState
- * @type {typeof import('./dataset-utils').createNewDataEntry}
- */
-export function createNewDataEntry({info, data, ...opts}, datasets = {}) {
-  const validatedData = validateInputData(data);
-  if (!validatedData) {
-    return {};
-  }
-
-  info = info || {};
-  const color = info.color || getNewDatasetColor(datasets);
-
-  const keplerTable = new KeplerTable({info, data: validatedData, color, ...opts});
-  return {
-    [keplerTable.id]: keplerTable
-  };
-}
-
-/**
- * Field name prefixes and suffixes which should not be considered
- * as metrics. Fields will still be included if a 'metric word'
- * is found on the field name, however.
- */
-const EXCLUDED_DEFAULT_FIELDS = [
-  // Serial numbers and identification numbers
-  '_id',
-  'id',
-  'index',
-  'uuid',
-  'guid',
-  'uid',
-  'gid',
-  'serial',
-  // Geographic IDs are unlikely to be interesting to color
-  'zip',
-  'code',
-  'post',
-  'region',
-  'fips',
-  'cbgs',
-  'h3',
-  's2',
-  // Geographic coords (but not z/elevation/altitude
-  // since that might be a metric)
-  'lat',
-  'lon',
-  'lng',
-  'latitude',
-  'longitude',
-  '_x',
-  '_y'
-];
-
-/**
- * Prefixes and suffixes that indicate a field is a metric.
- *
- * Note that these are in order of preference, first being
- * most preferred.
- */
-const METRIC_DEFAULT_FIELDS = [
-  'metric',
-  'value',
-  'sum',
-  'count',
-  'unique',
-  'mean',
-  'mode',
-  'median',
-  'max',
-  'min',
-  'deviation',
-  'variance',
-  'p99',
-  'p95',
-  'p75',
-  'p50',
-  'p25',
-  'p05',
-  // Abbreviations are less preferred
-  'cnt',
-  'val'
-];
-
-/**
- * Choose a field to use as the default color field of a layer.
- *
- * The heuristic is:
- *
- * First, exclude fields that are on the exclusion list and don't
- * have names that suggest they contain metrics. Also exclude
- * field names that are blank.
- *
- * Next, look for a field that is of real type and contains one
- * of the preferred names (in order of the preferred names).
- *
- * Next, look for a field that is of integer type and contains
- * one of the preferred names (in order of the preferred names).
- *
- * Next, look for the first field that is of real type (in order
- * of field index).
- *
- * Next, look for the first field that is of integer type (in
- * order of field index).
- *
- * It's possible no field will be chosen (i.e. because all fields
- * are strings.)
- *
- * @param dataset
- */
-export function findDefaultColorField({fields, fieldPairs = []}) {
-  const fieldsWithoutExcluded = fields.filter(field => {
-    if (field.type !== ALL_FIELD_TYPES.real && field.type !== ALL_FIELD_TYPES.integer) {
-      // Only select numeric fields.
-      return false;
-    }
-    if (
-      fieldPairs.find(
-        pair => pair.pair.lat.value === field.name || pair.pair.lng.value === field.name
-      )
-    ) {
-      // Do not permit lat, lon fields
-      return false;
-    }
-
-    const normalizedFieldName = field.name.toLowerCase();
-    if (normalizedFieldName === '') {
-      // Special case excluded name when the name is blank.
-      return false;
-    }
-    const hasExcluded = EXCLUDED_DEFAULT_FIELDS.find(
-      f => normalizedFieldName.startsWith(f) || normalizedFieldName.endsWith(f)
-    );
-    const hasInclusion = METRIC_DEFAULT_FIELDS.find(
-      f => normalizedFieldName.startsWith(f) || normalizedFieldName.endsWith(f)
-    );
-    return !hasExcluded || hasInclusion;
-  });
-
-  const sortedFields = fieldsWithoutExcluded.sort((left, right) => {
-    const normalizedLeft = left.name.toLowerCase();
-    const normalizedRight = right.name.toLowerCase();
-    const leftHasInclusion = METRIC_DEFAULT_FIELDS.findIndex(
-      f => normalizedLeft.startsWith(f) || normalizedLeft.endsWith(f)
-    );
-    const rightHasInclusion = METRIC_DEFAULT_FIELDS.findIndex(
-      f => normalizedRight.startsWith(f) || normalizedRight.endsWith(f)
-    );
-    if (leftHasInclusion !== rightHasInclusion) {
-      if (leftHasInclusion === -1) {
-        // Elements that do not have the inclusion list should go after those that do.
-        return 1;
-      } else if (rightHasInclusion === -1) {
-        // Elements that do have the inclusion list should go before those that don't.
-        return -1;
-      }
-      // Compare based on order in the inclusion list
-      return leftHasInclusion - rightHasInclusion;
-    }
-
-    // Compare based on type
-    if (left.type !== right.type) {
-      if (left.type === ALL_FIELD_TYPES.real) {
-        return -1;
-      }
-      // left is an integer and right is not
-      // and reals come before integers
-      return 1;
-    }
-
-    // Finally, order based on the order in the datasets columns
-    return left.index - right.index;
-  });
-
-  if (sortedFields.length) {
-    // There was a best match
-    return sortedFields[0];
-  }
-  // No matches
-  return null;
-}