@carto/kepler.gl 0.0.77 → 0.0.78-0

package/src/utils/data-utils.ts

@@ -0,0 +1,459 @@
+// Copyright (c) 2022 Uber Technologies, Inc.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+import assert from 'assert';
+import {ALL_FIELD_TYPES} from 'constants/default-settings';
+import {TOOLTIP_FORMATS, TOOLTIP_FORMAT_TYPES, TOOLTIP_KEY} from 'constants/tooltip';
+import {format as d3Format} from 'd3-format';
+import {bisectLeft} from 'd3-array';
+import moment from 'moment-timezone';
+
+import {Millisecond} from 'reducers/types';
+import {Layer} from 'layers';
+import {Bounds} from 'reducers/map-state-updaters';
+import {DataContainerInterface} from './table-utils/data-container-interface';
+import {Field} from './table-utils/kepler-table';
+
+export type FieldFormatter = (value: any) => string;
+
+const MAX_LATITUDE = 90;
+const MIN_LATITUDE = -90;
+const MAX_LONGITUDE = 180;
+const MIN_LONGITUDE = -180;
+
+/**
+ * simple getting unique values of an array
+ *
+ * @param values
+ * @returns unique values
+ */
+export function unique<T>(values: T[]) {
+  const results: T[] = [];
+  const uniqueSet = new Set(values);
+  uniqueSet.forEach(v => {
+    if (notNullorUndefined(v)) {
+      results.push(v);
+    }
+  });
+  return results;
+}
+
+export function uniqueSortedByFrequency<T>(values: T[]) {
+  const frequency = values.reduce((res, value) => {
+    if (notNullorUndefined(value)) {
+      if (!res[value]) {
+        res[value] = 0;
+      }
+      res[value] += 1;
+    }
+    return res;
+  }, {});
+
+  return Object.keys(frequency).sort((a, b) => frequency[b] - frequency[a]);
+}
+
+/* eslint-disable max-statements */
+/**
+ * return center of map from given points
+ * @param layers
+ * @returns coordinates of map center, empty if not found
+ */
+export function findMapBounds(layers: Layer[]): Bounds | null {
+  // find bounds in formatted layerData
+  // take ALL layers into account when finding map bounds
+  const availableLayerBounds: Bounds | [] = layers.reduce((res, l) => {
+    // @ts-expect-error
+    if (l.meta && l.meta.bounds) {
+      // @ts-expect-error
+      res.push(l.meta.bounds);
+    }
+    return res;
+  }, []);
+  // return null if no layer is available
+  if (availableLayerBounds.length === 0) {
+    return null;
+  }
+  // merge bounds in each layer
+  const newBounds = (availableLayerBounds as Bounds).reduce(
+    (res, b) => {
+      return [
+        Math.min(res[0], b[0]),
+        Math.min(res[1], b[1]),
+        Math.max(res[2], b[2]),
+        Math.max(res[3], b[3])
+      ];
+    },
+    [MAX_LONGITUDE, MAX_LATITUDE, MIN_LONGITUDE, MIN_LATITUDE]
+  );
+  // @ts-expect-error
+  return newBounds;
+}
+
+export function getLatLngBounds(
+  points: number[][],
+  idx: number,
+  limit: [number, number]
+): [number, number] | null {
+  const lats = points
+    .map(d => Number(Array.isArray(d)) && d[idx])
+    .filter(Number.isFinite)
+    .sort(numberSort);
+
+  if (!lats.length) {
+    return null;
+  }
+
+  // clamp to limit
+  return [Math.max(lats[0], limit[0]), Math.min(lats[lats.length - 1], limit[1])];
+}
+
+export function clamp([min, max]: [number, number], val: number = 0): number {
+  return val <= min ? min : val >= max ? max : val;
+}
+
+export function getSampleData(data, sampleSize = 500, getValue = d => d) {
+  const sampleStep = Math.max(Math.floor(data.length / sampleSize), 1);
+  const output: any[] = [];
+  for (let i = 0; i < data.length; i += sampleStep) {
+    output.push(getValue(data[i]));
+  }
+
+  return output;
+}
+
+/**
+ * Convert different time format to unix milliseconds
+ */
+export function timeToUnixMilli(value: string | number, format: string): Millisecond | null {
+  if (notNullorUndefined(value)) {
+    return typeof value === 'string'
+      ? moment.utc(value, format).valueOf()
+      : format === 'x'
+      ? value * 1000
+      : value;
+  }
+  return null;
+}
+
+export function maybeToDate(
+  isTime: boolean,
+  fieldIdx: number,
+  format: string,
+  dc: DataContainerInterface,
+  d: {index: number}
+) {
+  if (isTime) {
+    return timeToUnixMilli(dc.valueAt(d.index, fieldIdx), format);
+  }
+
+  return dc.valueAt(d.index, fieldIdx);
+}
+
+/**
+ * whether null or undefined
+ */
+export function notNullorUndefined<T extends NonNullable<any>>(d: T | null | undefined): d is T {
+  return d !== undefined && d !== null;
+}
+
+/**
+ * Whether d is a number, this filtered out NaN as well
+ */
+export function isNumber(d: any): boolean {
+  return Number.isFinite(d);
+}
+/**
+ * whether null or undefined
+ */
+export function isPlainObject(obj: any): boolean {
+  return obj === Object(obj) && typeof obj !== 'function' && !Array.isArray(obj);
+}
+
+export function numberSort(a: number, b: number): number {
+  return a - b;
+}
+
+export function getSortingFunction(fieldType: string): typeof numberSort | undefined {
+  switch (fieldType) {
+    case ALL_FIELD_TYPES.real:
+    case ALL_FIELD_TYPES.integer:
+    case ALL_FIELD_TYPES.timestamp:
+      return numberSort;
+    default:
+      return undefined;
+  }
+}
+
+/**
+ * round number with exact number of decimals
+ * return as a string
+ */
+export function preciseRound(num: number, decimals: number): string {
+  const t = Math.pow(10, decimals);
+  return (
+    Math.round(
+      num * t + (decimals > 0 ? 1 : 0) * (Math.sign(num) * (10 / Math.pow(100, decimals)))
+    ) / t
+  ).toFixed(decimals);
+}
+
+/**
+ * get number of decimals to round to for slider from step
+ * @param step
+ * @returns- number of decimal
+ */
+export function getRoundingDecimalFromStep(step: number): number {
+  if (isNaN(step)) {
+    assert('step is not a number');
+    assert(step);
+  }
+
+  const splitZero = step.toString().split('.');
+  if (splitZero.length === 1) {
+    return 0;
+  }
+  return splitZero[1].length;
+}
+
+/**
+ * Use in slider, given a number and an array of numbers, return the nears number from the array
+ * @param value
+ * @param marks
+ */
+export function snapToMarks(value: number, marks: number[]): number {
+  // always use bin x0
+  const i = bisectLeft(marks, value);
+  if (i === 0) {
+    return marks[i];
+  } else if (i === marks.length) {
+    return marks[i - 1];
+  }
+  const idx = marks[i] - value < value - marks[i - 1] ? i : i - 1;
+  return marks[idx];
+}
+
+/**
+ * If marks is provided, snap to marks, if not normalize to step
+ * @param val
+ * @param minValue
+ * @param step
+ * @param marks
+ */
+export function normalizeSliderValue(
+  val: number,
+  minValue: number,
+  step: number,
+  marks?: number[]
+): number {
+  if (marks && marks.length) {
+    return snapToMarks(val, marks);
+  }
+
+  return roundValToStep(minValue, step, val);
+}
+
+/**
+ * round the value to step for the slider
+ * @param minValue
+ * @param step
+ * @param val
+ * @returns - rounded number
+ */
+export function roundValToStep(minValue: number, step: number, val: number): number {
+  if (!isNumber(step) || !isNumber(minValue)) {
+    return val;
+  }
+
+  const decimal = getRoundingDecimalFromStep(step);
+  const steps = Math.floor((val - minValue) / step);
+  let remain = val - (steps * step + minValue);
+
+  // has to round because javascript turns 0.1 into 0.9999999999999987
+  remain = Number(preciseRound(remain, 8));
+
+  let closest: number;
+  if (remain === 0) {
+    closest = val;
+  } else if (remain < step / 2) {
+    closest = steps * step + minValue;
+  } else {
+    closest = (steps + 1) * step + minValue;
+  }
+
+  // precise round return a string rounded to the defined decimal
+  const rounded = preciseRound(closest, decimal);
+
+  return Number(rounded);
+}
+
+/**
+ * Get the value format based on field and format options
+ * Used in render tooltip value
+ */
+export const defaultFormatter: FieldFormatter = v => (notNullorUndefined(v) ? String(v) : '');
+
+export const FIELD_DISPLAY_FORMAT: {
+  [key: string]: FieldFormatter;
+} = {
+  [ALL_FIELD_TYPES.string]: defaultFormatter,
+  [ALL_FIELD_TYPES.timestamp]: defaultFormatter,
+  [ALL_FIELD_TYPES.integer]: defaultFormatter,
+  [ALL_FIELD_TYPES.real]: defaultFormatter,
+  [ALL_FIELD_TYPES.boolean]: defaultFormatter,
+  [ALL_FIELD_TYPES.date]: defaultFormatter,
+  [ALL_FIELD_TYPES.geojson]: d =>
+    typeof d === 'string'
+      ? d
+      : isPlainObject(d)
+      ? JSON.stringify(d)
+      : Array.isArray(d)
+      ? `[${String(d)}]`
+      : ''
+};
+
+/**
+ * Parse field value and type and return a string representation
+ */
+export const parseFieldValue = (value: any, type: string): string => {
+  if (!notNullorUndefined(value)) {
+    return '';
+  }
+
+  return FIELD_DISPLAY_FORMAT[type] ? FIELD_DISPLAY_FORMAT[type](value) : String(value);
+};
+
+const arrayMoveMutate = <T>(array: T[], from: number, to: number) => {
+  array.splice(to < 0 ? array.length + to : to, 0, array.splice(from, 1)[0]);
+};
+
+/**
+ *
+ * @param array
+ * @param from
+ * @param to
+ */
+export const arrayMove = <T>(array: T[], from: number, to: number) => {
+  array = array.slice();
+  arrayMoveMutate(array, from, to);
+  return array;
+};
+
+/**
+ * Get the value format based on field and format options
+ * Used in render tooltip value
+ * @param format
+ * @param field
+ */
+export function getFormatter(
+  format: string | Record<string, string>,
+  field?: Field
+): FieldFormatter {
+  if (!format) {
+    return defaultFormatter;
+  }
+  const tooltipFormat = Object.values(TOOLTIP_FORMATS).find(f => f[TOOLTIP_KEY] === format);
+
+  if (tooltipFormat) {
+    return applyDefaultFormat(tooltipFormat);
+  } else if (typeof format === 'string' && field) {
+    return applyCustomFormat(format, field);
+  }
+
+  return defaultFormatter;
+}
+
+export function applyDefaultFormat(tooltipFormat) {
+  if (!tooltipFormat || !tooltipFormat.format) {
+    return defaultFormatter;
+  }
+
+  switch (tooltipFormat.type) {
+    case TOOLTIP_FORMAT_TYPES.DECIMAL:
+      return d3Format(tooltipFormat.format);
+    case TOOLTIP_FORMAT_TYPES.DATE:
+    case TOOLTIP_FORMAT_TYPES.DATE_TIME:
+      return datetimeFormatter(null)(tooltipFormat.format);
+    case TOOLTIP_FORMAT_TYPES.PERCENTAGE:
+      return v => `${d3Format(TOOLTIP_FORMATS.DECIMAL_DECIMAL_FIXED_2.format)(v)}%`;
+    case TOOLTIP_FORMAT_TYPES.BOOLEAN:
+      return getBooleanFormatter(tooltipFormat.format);
+    default:
+      return defaultFormatter;
+  }
+}
+
+export function getBooleanFormatter(format: string): FieldFormatter {
+  switch (format) {
+    case '01':
+      return (v: Boolean) => (v ? '1' : '0');
+    case 'yn':
+      return (v: Boolean) => (v ? 'yes' : 'no');
+    default:
+      return defaultFormatter;
+  }
+}
+// Allow user to specify custom tooltip format via config
+export function applyCustomFormat(format, field): FieldFormatter {
+  switch (field.type) {
+    case ALL_FIELD_TYPES.real:
+    case ALL_FIELD_TYPES.integer:
+      return d3Format(format);
+    case ALL_FIELD_TYPES.date:
+    case ALL_FIELD_TYPES.timestamp:
+      return datetimeFormatter(null)(format);
+    default:
+      return v => v;
+  }
+}
+
+/**
+ * Format epoch milliseconds with a format string
+ * @type timezone
+ */
+export function datetimeFormatter(
+  timezone?: string | null
+): (format?: string) => (ts: number) => string {
+  return timezone
+    ? format => ts =>
+        moment
+          .utc(ts)
+          .tz(timezone)
+          .format(format)
+    : format => ts => moment.utc(ts).format(format);
+}
+
+export function uniqueSortedPairByFrequency(values) {
+  const frequencies = values.reduce((res, value) => {
+    if (notNullorUndefined(value)) {
+      if (!res[value]) {
+        res[value] = {
+          key: value,
+          frequency: 1
+        };
+      }
+      res[value].frequency += 1;
+    }
+    return res;
+  }, {});
+
+  return Object.values(frequencies)
+    .sort((a, b) => b.frequency - a.frequency)
+    .map(({key: [left, right]}) => ({left, right}));
+}

package/src/utils/dataset-utils.ts

@@ -0,0 +1,261 @@
+// Copyright (c) 2022 Uber Technologies, Inc.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+import {hexToRgb} from './color-utils';
+import uniq from 'lodash.uniq';
+import {ALL_FIELD_TYPES} from 'constants/default-settings';
+import {validateInputData} from 'processors/data-processor';
+import KeplerTable, {Field} from './table-utils/kepler-table';
+
+import {Datasets} from '../reducers/vis-state-updaters';
+import {ProtoDataset} from '../actions';
+import {RGBColor} from 'reducers/types';
+
+// apply a color for each dataset
+// to use as label colors
+const datasetColors = [
+  '#8F2FBF',
+  '#005CFF',
+  '#C06C84',
+  '#F8B195',
+  '#547A82',
+  '#3EACA8',
+  '#A2D4AB'
+].map(hexToRgb);
+
+/**
+ * Random color generator
+ */
+function* generateColor(): Generator<RGBColor> {
+  let index = 0;
+  while (index < datasetColors.length + 1) {
+    if (index === datasetColors.length) {
+      index = 0;
+    }
+    yield datasetColors[index++];
+  }
+}
+
+export const datasetColorMaker = generateColor();
+
+/** @type {typeof import('./dataset-utils').getNewDatasetColor} */
+export function getNewDatasetColor(datasets: Datasets): RGBColor {
+  const presetColors = datasetColors.map(String);
+  const usedColors = uniq(Object.values(datasets).map(d => String(d.color))).filter(c =>
+    presetColors.includes(c)
+  );
+
+  if (usedColors.length === presetColors.length) {
+    // if we already depleted the pool of color
+    return datasetColorMaker.next().value;
+  }
+
+  let color = datasetColorMaker.next().value;
+  while (usedColors.includes(String(color))) {
+    color = datasetColorMaker.next().value;
+  }
+
+  return color;
+}
+
+/**
+ * Take datasets payload from addDataToMap, create datasets entry save to visState
+ */
+export function createNewDataEntry(
+  {info, data, ...opts}: ProtoDataset,
+  datasets: Datasets = {}
+): Datasets {
+  const validatedData = validateInputData(data);
+  if (!validatedData) {
+    return {};
+  }
+
+  info = info || {};
+  const color = info.color || getNewDatasetColor(datasets);
+
+  const keplerTable = new KeplerTable({info, data: validatedData, color, ...opts});
+  return {
+    [keplerTable.id]: keplerTable
+  };
+}
+
+/**
+ * Field name prefixes and suffixes which should not be considered
+ * as metrics. Fields will still be included if a 'metric word'
+ * is found on the field name, however.
+ */
+const EXCLUDED_DEFAULT_FIELDS = [
+  // Serial numbers and identification numbers
+  '_id',
+  'id',
+  'index',
+  'uuid',
+  'guid',
+  'uid',
+  'gid',
+  'serial',
+  // Geographic IDs are unlikely to be interesting to color
+  'zip',
+  'code',
+  'post',
+  'region',
+  'fips',
+  'cbgs',
+  'h3',
+  's2',
+  // Geographic coords (but not z/elevation/altitude
+  // since that might be a metric)
+  'lat',
+  'lon',
+  'lng',
+  'latitude',
+  'longitude',
+  '_x',
+  '_y'
+];
+
+/**
+ * Prefixes and suffixes that indicate a field is a metric.
+ *
+ * Note that these are in order of preference, first being
+ * most preferred.
+ */
+const METRIC_DEFAULT_FIELDS = [
+  'metric',
+  'value',
+  'sum',
+  'count',
+  'unique',
+  'mean',
+  'mode',
+  'median',
+  'max',
+  'min',
+  'deviation',
+  'variance',
+  'p99',
+  'p95',
+  'p75',
+  'p50',
+  'p25',
+  'p05',
+  // Abbreviations are less preferred
+  'cnt',
+  'val'
+];
+
+/**
+ * Choose a field to use as the default color field of a layer.
+ *
+ * The heuristic is:
+ *
+ * First, exclude fields that are on the exclusion list and don't
+ * have names that suggest they contain metrics. Also exclude
+ * field names that are blank.
+ *
+ * Next, look for a field that is of real type and contains one
+ * of the preferred names (in order of the preferred names).
+ *
+ * Next, look for a field that is of integer type and contains
+ * one of the preferred names (in order of the preferred names).
+ *
+ * Next, look for the first field that is of real type (in order
+ * of field index).
+ *
+ * Next, look for the first field that is of integer type (in
+ * order of field index).
+ *
+ * It's possible no field will be chosen (i.e. because all fields
+ * are strings.)
+ *
+ * @param dataset
+ */
+export function findDefaultColorField({fields, fieldPairs = []}: KeplerTable): null | Field {
+  const fieldsWithoutExcluded = fields.filter(field => {
+    if (field.type !== ALL_FIELD_TYPES.real && field.type !== ALL_FIELD_TYPES.integer) {
+      // Only select numeric fields.
+      return false;
+    }
+    if (
+      fieldPairs.find(
+        pair => pair.pair.lat.value === field.name || pair.pair.lng.value === field.name
+      )
+    ) {
+      // Do not permit lat, lon fields
+      return false;
+    }
+
+    const normalizedFieldName = field.name.toLowerCase();
+    if (normalizedFieldName === '') {
+      // Special case excluded name when the name is blank.
+      return false;
+    }
+    const hasExcluded = EXCLUDED_DEFAULT_FIELDS.find(
+      f => normalizedFieldName.startsWith(f) || normalizedFieldName.endsWith(f)
+    );
+    const hasInclusion = METRIC_DEFAULT_FIELDS.find(
+      f => normalizedFieldName.startsWith(f) || normalizedFieldName.endsWith(f)
+    );
+    return !hasExcluded || hasInclusion;
+  });
+
+  const sortedFields = fieldsWithoutExcluded.sort((left, right) => {
+    const normalizedLeft = left.name.toLowerCase();
+    const normalizedRight = right.name.toLowerCase();
+    const leftHasInclusion = METRIC_DEFAULT_FIELDS.findIndex(
+      f => normalizedLeft.startsWith(f) || normalizedLeft.endsWith(f)
+    );
+    const rightHasInclusion = METRIC_DEFAULT_FIELDS.findIndex(
+      f => normalizedRight.startsWith(f) || normalizedRight.endsWith(f)
+    );
+    if (leftHasInclusion !== rightHasInclusion) {
+      if (leftHasInclusion === -1) {
+        // Elements that do not have the inclusion list should go after those that do.
+        return 1;
+      } else if (rightHasInclusion === -1) {
+        // Elements that do have the inclusion list should go before those that don't.
+        return -1;
+      }
+      // Compare based on order in the inclusion list
+      return leftHasInclusion - rightHasInclusion;
+    }
+
+    // Compare based on type
+    if (left.type !== right.type) {
+      if (left.type === ALL_FIELD_TYPES.real) {
+        return -1;
+      }
+      // left is an integer and right is not
+      // and reals come before integers
+      return 1;
+    }
+
+    // Finally, order based on the order in the datasets columns
+    // @ts-expect-error
+    return left.index - right.index;
+  });
+
+  if (sortedFields.length) {
+    // There was a best match
+    return sortedFields[0];
+  }
+  // No matches
+  return null;
+}