kepler.gl 3.1.0 → 3.1.1

package/src/utils/src/dataset-utils.ts

@@ -1,13 +1,18 @@
 // SPDX-License-Identifier: MIT
 // Copyright contributors to the kepler.gl project
 
+import * as arrow from 'apache-arrow';
 import {
   ALL_FIELD_TYPES,
   FIELD_OPTS,
   TOOLTIP_FORMATS,
   TOOLTIP_FORMAT_TYPES
 } from '@kepler.gl/constants';
-import {getSampleForTypeAnalyze, getFieldsFromData} from '@kepler.gl/common-utils';
+import {
+  getSampleForTypeAnalyze,
+  getSampleForTypeAnalyzeArrow,
+  getFieldsFromData
+} from '@kepler.gl/common-utils';
 import {Analyzer} from 'type-analyzer';
 import assert from 'assert';
 
@@ -260,21 +265,25 @@ export function validateInputData(data: ProtoDataset['data']): ProcessorResult {
       return false;
     }
 
-    if (!fields.every(field => field.analyzerType)) {
-      assert('field missing analyzerType');
+    if (!f.analyzerType) {
+      assert(`field ${i} missing analyzerType`);
       return false;
     }
 
     // check time format is correct based on first 10 not empty element
     if (f.type === ALL_FIELD_TYPES.timestamp) {
-      const sample = findNonEmptyRowsAtField(rows, i, 10).map(r => ({ts: r[i]}));
+      const sample = (
+        cols ? findNonEmptyRowsAtFieldArrow(cols, i, 10) : findNonEmptyRowsAtField(rows, i, 10)
+      ).map(r => ({ts: r[i]}));
       const analyzedType = Analyzer.computeColMeta(sample)[0];
       return analyzedType && analyzedType.category === 'TIME' && analyzedType.format === f.format;
     }
 
     // check existing string field is H3 type
     if (f.type === ALL_FIELD_TYPES.string) {
-      const sample = findNonEmptyRowsAtField(rows, i, 10).map(r => r[i]);
+      const sample = (
+        cols ? findNonEmptyRowsAtFieldArrow(cols, i, 10) : findNonEmptyRowsAtField(rows, i, 10)
+      ).map(r => r[i]);
       return sample.every(item => !h3IsValid(item));
     }
 
@@ -287,10 +296,15 @@ export function validateInputData(data: ProtoDataset['data']): ProcessorResult {
 
   // if any field has missing type, recalculate it for everyone
   // because we simply lost faith in humanity
-  const sampleData = getSampleForTypeAnalyze({
-    fields: fields.map(f => f.name),
-    rows
-  });
+  const sampleData = cols
+    ? getSampleForTypeAnalyzeArrow(
+        cols,
+        fields.map(f => f.name)
+      )
+    : getSampleForTypeAnalyze({
+        fields: fields.map(f => f.name),
+        rows
+      });
   const fieldOrder = fields.map(f => f.name);
   const meta = getFieldsFromData(sampleData, fieldOrder);
   const updatedFields = fields.map((f, i) => ({
@@ -315,6 +329,24 @@ function findNonEmptyRowsAtField(rows: unknown[][], fieldIdx: number, total: num
   return sample;
 }
 
+function findNonEmptyRowsAtFieldArrow(
+  cols: arrow.Vector[],
+  fieldIdx: number,
+  total: number
+): any[] {
+  const sample: any[] = [];
+  const numRows = cols[fieldIdx].length;
+  let i = 0;
+  while (sample.length < total && i < numRows) {
+    if (notNullorUndefined(cols[fieldIdx].get(i))) {
+      const row = cols.map(col => col.get(i));
+      sample.push(row);
+    }
+    i++;
+  }
+  return sample;
+}
+
 const TIME_DISPLAY = '2020-05-11 14:00';
 
 export const addTimeLabel = (formats: TimeLabelFormat[]) =>

package/src/utils/src/indexed-data-container.ts

@@ -123,7 +123,7 @@ export class IndexedDataContainer implements DataContainerInterface {
   }
 
   mapIndex<T>(
-    func: ({index: number}, dc: DataContainerInterface) => T,
+    func: ({index}: {index: number}, dc: DataContainerInterface) => T,
     options: RangeOptions = {}
   ): T[] {
     const {start = 0, end = this.numRows()} = options;

package/src/utils/src/map-style-utils/mapbox-utils.ts

@@ -45,9 +45,9 @@ export function isStyleUsingOpenStreetMapTiles(mapStyle: any) {
  * @returns transformed url
  */
 export const transformRequest = (
-  mapboxKey: string
+  _mapboxKey: string
 ): ((url: string, resourceType: string) => {url: string}) => {
-  return (url: string, resourceType: string) => {
+  return (url: string, _resourceType: string) => {
     /*
     if (isMapboxURL(url)) {
       // ! TODO - use MapBox directly?

package/src/utils/src/plot.ts

@@ -172,7 +172,7 @@ export function getTimeBins(
 ): TimeBins {
   let bins = filter.timeBins || {};
 
-  filter.dataId.forEach((dataId, dataIdIdx) => {
+  filter.dataId.forEach(dataId => {
     // reuse bins if filterData did not change
     if (bins[dataId] && bins[dataId][interval]) {
       return;
@@ -396,7 +396,7 @@ export const getAggregationOptiosnBasedOnField = field => {
 function getDelta(
   bins: LineDatum[],
   y: number,
-  interval: PlotType['interval']
+  _interval: PlotType['interval']
 ): Partial<LineDatum> & {delta: 'last'; pct: number | null} {
   // if (WOW[interval]) return getWow(bins, y, interval);
   const lastBin = bins[bins.length - 1];
@@ -441,7 +441,7 @@ export function getLineChart(datasets: Datasets, filter: Filter): LineChart {
   const getYValue = getValueAggrFunc(yAxis, aggregation, dataset);
 
   const init: LineDatum[] = [];
-  const series = (bins || []).reduce((accu, bin, i) => {
+  const series = (bins || []).reduce((accu, bin) => {
     const y = getYValue(bin);
     const delta = getDelta(accu, y, interval);
     accu.push({
@@ -588,7 +588,7 @@ export function updateTimeFilterPlotType(
   filter: TimeRangeFilter,
   plotType: TimeRangeFilter['plotType'],
   datasets: Datasets,
-  dataId?: string
+  _dataId?: string
 ): TimeRangeFilter {
   let nextFilter = filter;
   let nextPlotType = plotType;
@@ -655,7 +655,7 @@ export function updateRangeFilterPlotType(
   filter: RangeFilter,
   plotType: RangeFilter['plotType'],
   datasets: Datasets,
-  dataId?: string
+  _dataId?: string
 ): RangeFilter {
   const nextFilter = {
     ...filter,