@carto/api-client 0.5.0-alpha.7 → 0.5.0-alpha.8

package/package.json

@@ -8,7 +8,7 @@
   "homepage": "https://github.com/CartoDB/carto-api-client#readme",
   "author": "Don McCurdy <donmccurdy@carto.com>",
   "packageManager": "yarn@4.3.1",
-  "version": "0.5.0-alpha.7",
+  "version": "0.5.0-alpha.8",
   "license": "MIT",
   "type": "module",
   "sideEffects": false,
@@ -40,6 +40,7 @@
     "test": "vitest run --typecheck",
     "test:watch": "vitest watch --typecheck",
     "coverage": "vitest run --coverage.enabled --coverage.all false",
+    "benchmark": "node scripts/benchmark.js",
     "lint": "eslint .",
     "format": "prettier \"**/*.{cjs,html,js,json,md,ts}\" --write",
     "format:check": "prettier \"**/*.{cjs,html,js,json,md,ts}\" --check",
@@ -86,6 +87,8 @@
     "@luma.gl/core": "~9.1.0",
     "@luma.gl/engine": "~9.1.0",
     "@luma.gl/shadertools": "~9.1.0",
+    "@turf/buffer": "^7.2.0",
+    "@turf/random": "^7.2.0",
     "@types/json-schema": "^7.0.15",
     "@types/react": "^18.3.18",
     "@types/semver": "^7.5.8",
@@ -101,6 +104,7 @@
     "rimraf": "^6.0.1",
     "semver": "^7.7.1",
     "thenby": "^1.3.4",
+    "tinybench": "^3.1.1",
     "tsup": "^8.3.6",
     "typescript": "~5.7.3",
     "typescript-eslint": "^8.24.0",

package/src/sources/h3-tileset-source.ts

@@ -6,9 +6,7 @@ import {getTileFormat} from '../utils/getTileFormat.js';
 import {
   WidgetTilesetSource,
   WidgetTilesetSourceResult,
-  WidgetTilesetWorkerSource,
 } from '../widget-sources/index.js';
-import {isModuleWorkerSupported} from '../workers/utils.js';
 import {baseSource} from './base-source.js';
 import type {
   SourceOptions,
@@ -28,15 +26,10 @@ export const h3TilesetSource = async function (
   const {tableName, spatialDataColumn = 'h3'} = options;
   const urlParameters: UrlParameters = {name: tableName};
 
-  const WidgetSourceClass =
-    options.widgetSourceWorker !== false && isModuleWorkerSupported()
-      ? WidgetTilesetWorkerSource
-      : WidgetTilesetSource;
-
   return baseSource<UrlParameters>('tileset', options, urlParameters).then(
     (result) => ({
       ...(result as TilejsonResult),
-      widgetSource: new WidgetSourceClass({
+      widgetSource: new WidgetTilesetSource({
         ...options,
         tileFormat: getTileFormat(result as TilejsonResult),
         spatialDataColumn,

package/src/sources/quadbin-tileset-source.ts

@@ -6,9 +6,7 @@ import {getTileFormat} from '../utils/getTileFormat.js';
 import {
   WidgetTilesetSource,
   WidgetTilesetSourceResult,
-  WidgetTilesetWorkerSource,
 } from '../widget-sources/index.js';
-import {isModuleWorkerSupported} from '../workers/utils.js';
 import {baseSource} from './base-source.js';
 import type {
   SourceOptions,
@@ -28,15 +26,10 @@ export const quadbinTilesetSource = async function (
   const {tableName, spatialDataColumn = 'quadbin'} = options;
   const urlParameters: UrlParameters = {name: tableName};
 
-  const WidgetSourceClass =
-    options.widgetSourceWorker !== false && isModuleWorkerSupported()
-      ? WidgetTilesetWorkerSource
-      : WidgetTilesetSource;
-
   return baseSource<UrlParameters>('tileset', options, urlParameters).then(
     (result) => ({
       ...(result as TilejsonResult),
-      widgetSource: new WidgetSourceClass({
+      widgetSource: new WidgetTilesetSource({
         ...options,
         tileFormat: getTileFormat(result as TilejsonResult),
         spatialDataColumn,

package/src/sources/vector-tileset-source.ts

@@ -7,9 +7,7 @@ import {getTileFormat} from '../utils/getTileFormat.js';
 import {
   WidgetTilesetSource,
   WidgetTilesetSourceResult,
-  WidgetTilesetWorkerSource,
 } from '../widget-sources/index.js';
-import {isModuleWorkerSupported} from '../workers/utils.js';
 import {baseSource} from './base-source.js';
 import type {
   SourceOptions,
@@ -29,15 +27,10 @@ export const vectorTilesetSource = async function (
   const {tableName, spatialDataColumn = DEFAULT_GEO_COLUMN} = options;
   const urlParameters: UrlParameters = {name: tableName};
 
-  const WidgetSourceClass =
-    options.widgetSourceWorker !== false && isModuleWorkerSupported()
-      ? WidgetTilesetWorkerSource
-      : WidgetTilesetSource;
-
   return baseSource<UrlParameters>('tileset', options, urlParameters).then(
     (result) => ({
       ...(result as TilejsonResult),
-      widgetSource: new WidgetSourceClass({
+      widgetSource: new WidgetTilesetSource({
         ...options,
         tileFormat: getTileFormat(result as TilejsonResult),
         spatialDataColumn,

package/src/widget-sources/index.ts

@@ -3,5 +3,4 @@ export * from './widget-query-source.js';
 export * from './widget-remote-source.js';
 export * from './widget-table-source.js';
 export * from './widget-tileset-source.js';
-export * from './widget-tileset-worker-source.js';
 export * from './types.js';

package/src/widget-sources/widget-source.ts

@@ -50,6 +50,17 @@ export abstract class WidgetSource<Props extends WidgetSourceProps> {
     this.props = {...WidgetSource.defaultProps, ...props};
   }
 
+  /**
+   * Destroys the widget source and releases allocated resources.
+   *
+   * For remote sources (tables, queries) this has no effect, but for local
+   * sources (tilesets, rasters) these resources will affect performance
+   * and stability if many (10+) sources are created and not released.
+   */
+  destroy() {
+    // no-op in most cases, but required for worker sources.
+  }
+
   protected _getSpatialFiltersResolution(
     source: Omit<ModelSource, 'type' | 'data'>,
     spatialFilter?: SpatialFilter,

package/src/widget-sources/widget-tileset-source-impl.ts

@@ -0,0 +1,417 @@
+/* eslint-disable @typescript-eslint/require-await */
+import {
+  CategoryRequestOptions,
+  CategoryResponse,
+  FeaturesResponse,
+  FormulaRequestOptions,
+  FormulaResponse,
+  HistogramRequestOptions,
+  HistogramResponse,
+  RangeRequestOptions,
+  RangeResponse,
+  ScatterRequestOptions,
+  ScatterResponse,
+  TableRequestOptions,
+  TableResponse,
+  TimeSeriesRequestOptions,
+  TimeSeriesResponse,
+} from './types.js';
+import {InvalidColumnError, assert, getApplicableFilters} from '../utils.js';
+import {Filter, SpatialFilter, Tile} from '../types.js';
+import {
+  TileFeatureExtractOptions,
+  applyFilters,
+  geojsonFeatures,
+  tileFeatures,
+} from '../filters/index.js';
+import {
+  aggregationFunctions,
+  applySorting,
+  groupValuesByColumn,
+  groupValuesByDateColumn,
+  histogram,
+  scatterPlot,
+} from '../operations/index.js';
+import {FeatureData} from '../types-internal.js';
+import {FeatureCollection} from 'geojson';
+import {WidgetSource} from './widget-source.js';
+import {booleanEqual} from '@turf/boolean-equal';
+import type {WidgetTilesetSourceProps} from './widget-tileset-source.js';
+
+// TODO(cleanup): Parameter defaults in source functions and widget API calls are
+// currently duplicated and possibly inconsistent. Consider consolidating and
+// operating on Required<T> objects. See:
+// https://github.com/CartoDB/carto-api-client/issues/39
+
+/**
+ * Local (in-memory) implementation of tileset widget calculations. This class
+ * may be instantiated by {@link WidgetTilesetSource} in a Web Worker when
+ * supported, or on the main thread.
+ */
+export class WidgetTilesetSourceImpl extends WidgetSource<WidgetTilesetSourceProps> {
+  private _tiles: Tile[] = [];
+  private _features: FeatureData[] = [];
+  private _tileFeatureExtractOptions: TileFeatureExtractOptions = {};
+  private _tileFeatureExtractPreviousInputs: {spatialFilter?: SpatialFilter} =
+    {};
+
+  /**
+   * Loads features as a list of tiles (typically provided by deck.gl).
+   * After tiles are loaded, {@link extractTileFeatures} must be called
+   * before computing statistics on the tiles.
+   */
+  loadTiles(tiles: unknown[]) {
+    this._tiles = tiles as Tile[];
+    this._features.length = 0;
+  }
+
+  /** Configures options used to extract features from tiles. */
+  setTileFeatureExtractOptions(options: TileFeatureExtractOptions) {
+    this._tileFeatureExtractOptions = options;
+    this._features.length = 0;
+  }
+
+  protected _extractTileFeatures(spatialFilter: SpatialFilter) {
+    // When spatial filter has not changed, don't redo extraction. If tiles or
+    // tile extract options change, features will have been cleared already.
+    const prevInputs = this._tileFeatureExtractPreviousInputs;
+    if (
+      this._features.length &&
+      prevInputs.spatialFilter &&
+      booleanEqual(prevInputs.spatialFilter, spatialFilter)
+    ) {
+      return;
+    }
+
+    this._features = tileFeatures({
+      tiles: this._tiles,
+      tileFormat: this.props.tileFormat,
+      ...this._tileFeatureExtractOptions,
+
+      spatialFilter,
+      spatialDataColumn: this.props.spatialDataColumn,
+      spatialDataType: this.props.spatialDataType,
+    });
+
+    prevInputs.spatialFilter = spatialFilter;
+  }
+
+  /**
+   * Loads features as GeoJSON (used for testing).
+   * @experimental
+   * @internal Not for public use. Spatial filters in other method calls will be ignored.
+   */
+  loadGeoJSON({
+    geojson,
+    spatialFilter,
+  }: {
+    geojson: FeatureCollection;
+    spatialFilter: SpatialFilter;
+  }) {
+    this._features = geojsonFeatures({
+      geojson,
+      spatialFilter,
+      ...this._tileFeatureExtractOptions,
+    });
+    this._tileFeatureExtractPreviousInputs.spatialFilter = spatialFilter;
+  }
+
+  override async getFeatures(): Promise<FeaturesResponse> {
+    throw new Error('getFeatures not supported for tilesets');
+  }
+
+  async getFormula({
+    column = '*',
+    operation = 'count',
+    joinOperation,
+    filters,
+    filterOwner,
+    spatialFilter,
+  }: FormulaRequestOptions): Promise<FormulaResponse> {
+    if (operation === 'custom') {
+      throw new Error('Custom aggregation not supported for tilesets');
+    }
+
+    // Column is required except when operation is 'count'.
+    if ((column && column !== '*') || operation !== 'count') {
+      assertColumn(this._features, column);
+    }
+
+    const filteredFeatures = this._getFilteredFeatures(
+      spatialFilter,
+      filters,
+      filterOwner
+    );
+
+    if (filteredFeatures.length === 0 && operation !== 'count') {
+      return {value: null};
+    }
+
+    const targetOperation = aggregationFunctions[operation];
+    return {
+      value: targetOperation(filteredFeatures, column, joinOperation),
+    };
+  }
+
+  override async getHistogram({
+    operation = 'count',
+    ticks,
+    column,
+    joinOperation,
+    filters,
+    filterOwner,
+    spatialFilter,
+  }: HistogramRequestOptions): Promise<HistogramResponse> {
+    const filteredFeatures = this._getFilteredFeatures(
+      spatialFilter,
+      filters,
+      filterOwner
+    );
+
+    if (!this._features.length) {
+      return [];
+    }
+
+    assertColumn(this._features, column);
+
+    return histogram({
+      data: filteredFeatures,
+      valuesColumns: normalizeColumns(column),
+      joinOperation,
+      ticks,
+      operation,
+    });
+  }
+
+  override async getCategories({
+    column,
+    operation = 'count',
+    operationColumn,
+    joinOperation,
+    filters,
+    filterOwner,
+    spatialFilter,
+  }: CategoryRequestOptions): Promise<CategoryResponse> {
+    const filteredFeatures = this._getFilteredFeatures(
+      spatialFilter,
+      filters,
+      filterOwner
+    );
+
+    if (!filteredFeatures.length) {
+      return [];
+    }
+
+    assertColumn(this._features, column, operationColumn as string);
+
+    const groups = groupValuesByColumn({
+      data: filteredFeatures,
+      valuesColumns: normalizeColumns(operationColumn || column),
+      joinOperation,
+      keysColumn: column,
+      operation,
+    });
+
+    return groups || [];
+  }
+
+  override async getScatter({
+    xAxisColumn,
+    yAxisColumn,
+    xAxisJoinOperation,
+    yAxisJoinOperation,
+    filters,
+    filterOwner,
+    spatialFilter,
+  }: ScatterRequestOptions): Promise<ScatterResponse> {
+    const filteredFeatures = this._getFilteredFeatures(
+      spatialFilter,
+      filters,
+      filterOwner
+    );
+
+    if (!filteredFeatures.length) {
+      return [];
+    }
+
+    assertColumn(this._features, xAxisColumn, yAxisColumn);
+
+    return scatterPlot({
+      data: filteredFeatures,
+      xAxisColumns: normalizeColumns(xAxisColumn),
+      xAxisJoinOperation,
+      yAxisColumns: normalizeColumns(yAxisColumn),
+      yAxisJoinOperation,
+    });
+  }
+
+  override async getTable({
+    columns,
+    searchFilterColumn,
+    searchFilterText,
+    sortBy,
+    sortDirection,
+    sortByColumnType,
+    offset = 0,
+    limit = 10,
+    filters,
+    filterOwner,
+    spatialFilter,
+  }: TableRequestOptions): Promise<TableResponse> {
+    // Filter.
+    let filteredFeatures = this._getFilteredFeatures(
+      spatialFilter,
+      filters,
+      filterOwner
+    );
+
+    if (!filteredFeatures.length) {
+      return {rows: [], totalCount: 0};
+    }
+
+    // Search.
+    if (searchFilterColumn && searchFilterText) {
+      filteredFeatures = filteredFeatures.filter(
+        (row) =>
+          row[searchFilterColumn] &&
+          String(row[searchFilterColumn] as unknown)
+            .toLowerCase()
+            .includes(String(searchFilterText).toLowerCase())
+      );
+    }
+
+    // Sort.
+    let rows = applySorting(filteredFeatures, {
+      sortBy,
+      sortByDirection: sortDirection,
+      sortByColumnType,
+    });
+    const totalCount = rows.length;
+
+    // Offset and limit.
+    rows = rows.slice(
+      Math.min(offset, totalCount),
+      Math.min(offset + limit, totalCount)
+    );
+
+    // Select columns.
+    rows = rows.map((srcRow: FeatureData) => {
+      const dstRow: FeatureData = {};
+      for (const column of columns) {
+        dstRow[column] = srcRow[column];
+      }
+      return dstRow;
+    });
+
+    return {rows, totalCount} as TableResponse;
+  }
+
+  override async getTimeSeries({
+    column,
+    stepSize,
+    operation,
+    operationColumn,
+    joinOperation,
+    filters,
+    filterOwner,
+    spatialFilter,
+  }: TimeSeriesRequestOptions): Promise<TimeSeriesResponse> {
+    const filteredFeatures = this._getFilteredFeatures(
+      spatialFilter,
+      filters,
+      filterOwner
+    );
+
+    if (!filteredFeatures.length) {
+      return {rows: []};
+    }
+
+    assertColumn(this._features, column, operationColumn as string);
+
+    const rows =
+      groupValuesByDateColumn({
+        data: filteredFeatures,
+        valuesColumns: normalizeColumns(operationColumn || column),
+        keysColumn: column,
+        groupType: stepSize,
+        operation,
+        joinOperation,
+      }) || [];
+
+    return {rows};
+  }
+
+  override async getRange({
+    column,
+    filters,
+    filterOwner,
+    spatialFilter,
+  }: RangeRequestOptions): Promise<RangeResponse> {
+    assertColumn(this._features, column);
+
+    const filteredFeatures = this._getFilteredFeatures(
+      spatialFilter,
+      filters,
+      filterOwner
+    );
+
+    if (!this._features.length) {
+      // TODO: Is this the only nullable response in the Widgets API? If so,
+      // can we do something more consistent?
+      return null;
+    }
+
+    return {
+      min: aggregationFunctions.min(filteredFeatures, column),
+      max: aggregationFunctions.max(filteredFeatures, column),
+    };
+  }
+
+  /****************************************************************************
+   * INTERNAL
+   */
+
+  private _getFilteredFeatures(
+    spatialFilter?: SpatialFilter,
+    filters?: Record<string, Filter>,
+    filterOwner?: string
+  ): FeatureData[] {
+    assert(spatialFilter, 'spatialFilter required for tilesets');
+    this._extractTileFeatures(spatialFilter);
+    return applyFilters(
+      this._features,
+      getApplicableFilters(filterOwner, filters || this.props.filters),
+      this.props.filtersLogicalOperator || 'and'
+    );
+  }
+}
+
+function assertColumn(
+  features: FeatureData[],
+  ...columnArgs: string[] | string[][]
+) {
+  // TODO(cleanup): Can drop support for multiple column shapes here?
+
+  // Due to the multiple column shape, we normalise it as an array with normalizeColumns
+  const columns = Array.from(new Set(columnArgs.map(normalizeColumns).flat()));
+
+  const featureKeys = Object.keys(features[0]);
+
+  const invalidColumns = columns.filter(
+    (column) => !featureKeys.includes(column)
+  );
+
+  if (invalidColumns.length) {
+    throw new InvalidColumnError(
+      `Missing column(s): ${invalidColumns.join(', ')}`
+    );
+  }
+}
+
+function normalizeColumns(columns: string | string[]): string[] {
+  return Array.isArray(columns)
+    ? columns
+    : typeof columns === 'string'
+      ? [columns]
+      : [];
+}