@carto/api-client 0.5.7-alpha.3 → 0.5.7-alpha.6

package/package.json

@@ -8,7 +8,7 @@
   "homepage": "https://github.com/CartoDB/carto-api-client#readme",
   "author": "Don McCurdy <donmccurdy@carto.com>",
   "packageManager": "yarn@4.3.1",
-  "version": "0.5.7-alpha.3",
+  "version": "0.5.7-alpha.6",
   "license": "MIT",
   "publishConfig": {
     "access": "public"
@@ -126,7 +126,7 @@
     "typescript": "~5.8.2",
     "typescript-eslint": "^8.26.1",
     "vite": "^6.2.2",
-    "vitest": "3.1.2"
+    "vitest": "3.1.4"
   },
   "resolutions": {
     "@carto/api-client": "portal:./",

package/src/filters/tileFeaturesRaster.ts

@@ -1,5 +1,6 @@
 import {
   cellToChildren as _cellToChildren,
+  cellToBoundary,
   cellToTile,
   geometryToCells,
   getResolution,
@@ -11,6 +12,9 @@ import type {
   RasterMetadataBand,
   SpatialDataType,
 } from '../sources/types.js';
+import booleanWithin from '@turf/boolean-within';
+import intersect from '@turf/intersect';
+import {feature, featureCollection} from '@turf/helpers';
 
 export type TileFeaturesRasterOptions = {
   tiles: RasterTile[];
@@ -43,23 +47,34 @@ export function tileFeaturesRaster({
   const tileBlockSize = tiles[0].data!.blockSize;
   const cellResolution = tileResolution + BigInt(Math.log2(tileBlockSize));
 
-  // Compute covering cells for the spatial filter, at same resolution as the
-  // raster pixels, to be used as a mask.
-  const spatialFilterCells = new Set(
-    geometryToCells(options.spatialFilter, cellResolution)
-  );
-
   const data = new Map<bigint, FeatureData>();
 
   for (const tile of tiles as Required<RasterTile>[]) {
     const parent = tile.index.q;
 
-    const children = cellToChildrenSorted(parent, cellResolution);
+    // If tile is partially overlapping with the spatial filter, compute a quadbin covering for the
+    // spatial filter + tile intersection, at cell resolution. If tile is fully inside or outside
+    // the spatial filter, computing a covering can be skipped. Avoid computing a quadbin covering
+    // for the _entire_ spatial filter, which may contain far too many cells (e.g. at zoom=3 and
+    // resolution=14, we have ~18,000,000 cells in the viewport.)
+    const tilePolygon = cellToBoundary(parent);
+    const tileFilter = intersect(
+      featureCollection([feature(tilePolygon), feature(options.spatialFilter)])
+    );
+    const needsFilter = tileFilter
+      ? !booleanWithin(tilePolygon, options.spatialFilter)
+      : false;
+    const tileFilterCells = needsFilter
+      ? new Set(geometryToCells(tileFilter!.geometry, cellResolution))
+      : null;
+    const tileSortedCells = cellToChildrenSorted(parent, cellResolution);
 
     // For each pixel/cell within the spatial filter, create a FeatureData.
     // Order is row-major, starting from NW and ending at SE.
-    for (let i = 0; i < children.length; i++) {
-      if (!spatialFilterCells.has(children[i])) continue;
+    for (let i = 0; i < tileSortedCells.length; i++) {
+      if (needsFilter && !tileFilterCells!.has(tileSortedCells[i])) {
+        continue;
+      }
 
       const cellData: FeatureData = {};
       let cellDataExists = false;
@@ -75,7 +90,7 @@ export function tileFeaturesRaster({
       }
 
       if (cellDataExists) {
-        data.set(children[i], cellData);
+        data.set(tileSortedCells[i], cellData);
       }
     }
   }

package/src/index.ts

@@ -28,4 +28,5 @@ export * from './filters/index.js';
 export * from './operations/index.js';
 export * from './utils/makeIntervalComplete.js';
 export * from './utils/transformToTileCoords.js';
+export * from './utils/CellSet.js';
 export * from './fetch-map/source.js';

package/src/operations/groupBy.ts

@@ -1,5 +1,4 @@
 import {aggregationFunctions, aggregate} from './aggregation.js';
-import {OTHERS_CATEGORY_NAME} from '../widget-sources/constants.js';
 import type {AggregationType} from '../types.js';
 import type {FeatureData} from '../types-internal.js';
 
@@ -16,14 +15,12 @@ export function groupValuesByColumn({
   joinOperation,
   keysColumn,
   operation,
-  othersThreshold,
 }: {
   data: FeatureData[];
   valuesColumns?: string[];
   joinOperation?: AggregationType;
   keysColumn: string;
   operation: AggregationType;
-  othersThreshold?: number;
 }): GroupByFeature | null {
   if (Array.isArray(data) && data.length === 0) {
     return null;
@@ -51,27 +48,12 @@ export function groupValuesByColumn({
   const targetOperation =
     aggregationFunctions[operation as Exclude<AggregationType, 'custom'>];
 
-  if (!targetOperation) {
-    return [];
+  if (targetOperation) {
+    return Array.from(groups).map(([name, value]) => ({
+      name,
+      value: targetOperation(value),
+    }));
   }
 
-  const allCategories = Array.from(groups).map(([name, value]) => ({
-    name,
-    value: targetOperation(value),
-  }));
-
-  allCategories.sort((a, b) => b.value - a.value);
-
-  if (othersThreshold && allCategories.length > othersThreshold) {
-    const otherNames = allCategories
-      .map((entry) => entry.name)
-      .slice(othersThreshold);
-    const otherValue = otherNames.flatMap((name) => groups.get(name));
-    allCategories.push({
-      name: OTHERS_CATEGORY_NAME,
-      value: targetOperation(otherValue),
-    });
-  }
-
-  return allCategories;
+  return [];
 }

package/src/utils/CellSet.ts

@@ -0,0 +1,90 @@
+/** Flags 'empty' values in a Uint32Array index. */
+const EMPTY_U32 = 2 ** 32 - 1;
+
+/**
+ * Custom Set-like interface optimized for BigUint64 cell IDs. Unlike Set,
+ * limited in most JavaScript runtimes to ~16M entries, this implementation
+ * can support up to `n = 2^32 - 1` (4 billion) entries, with lookups in
+ * amortized O(1) time.
+ */
+export class CellSet {
+  /** List of cells stored by the set. Stored by reference, without copying. */
+  private cells: bigint[];
+
+  /** DataView representing a single cell ID. Pre-allocated to reduce memory during queries. */
+  private cellView = new DataView(new ArrayBuffer(8));
+
+  /** Hash table, mapping a hash index (computed) to an index in the 'cells' array. */
+  private hashTable: Uint32Array;
+
+  constructor(cells: bigint[]) {
+    this.cells = cells;
+
+    // Pre-allocate hash table for queries.
+    this.hashTable = new Uint32Array(hashBuckets(cells.length)).fill(EMPTY_U32);
+    for (let cellIndex = 0; cellIndex < cells.length; cellIndex++) {
+      this.hashTable[this.hashLookup(cells[cellIndex])] = cellIndex;
+    }
+  }
+
+  has(cell: bigint): boolean {
+    const hashIndex = this.hashLookup(cell);
+    return this.hashTable[hashIndex] !== EMPTY_U32;
+  }
+
+  private hashLookup(cell: bigint): number {
+    // Hash implementation operates on 32-bit chunks, so write the cell ID
+    // into a pre-allocated DataView for easier iteration.
+    this.cellView.setBigUint64(0, cell);
+    const hashval = hash(this.cellView);
+    const hashmod = this.hashTable.length - 1;
+    let bucket = hashval & hashmod;
+
+    // Find the first bucket in the hash table where either (a) no cell
+    // is yet stored, or (b) the stored cell and the query cell are equal.
+    for (let probe = 0; probe <= hashmod; probe++) {
+      const cellIndex = this.hashTable[bucket];
+
+      if (cellIndex === EMPTY_U32 || cell === this.cells[cellIndex]) {
+        return bucket;
+      }
+
+      bucket = (bucket + probe + 1) & hashmod; // Hash collision; quadratic probing.
+    }
+
+    throw new Error('Hash table full.'); // Unreachable.
+  }
+}
+
+/**
+ * MurmurHash2
+ *
+ * References:
+ * - https://github.com/mikolalysenko/murmurhash-js/blob/f19136e9f9c17f8cddc216ca3d44ec7c5c502f60/murmurhash2_gc.js#L14
+ * - https://github.com/zeux/meshoptimizer/blob/e47e1be6d3d9513153188216455bdbed40a206ef/src/indexgenerator.cpp#L12
+ */
+function hash(view: DataView, h = 0): number {
+  const m = 0x5bd1e995;
+  const r = 24;
+
+  for (let i = 0, il = view.byteLength / 4; i < il; i++) {
+    let k = view.getUint32(i * 4);
+
+    k = Math.imul(k, m) >>> 0;
+    k = (k ^ (k >> r)) >>> 0;
+    k = Math.imul(k, m) >>> 0;
+
+    h = Math.imul(h, m) >>> 0;
+    h = (h ^ k) >>> 0;
+  }
+
+  return h;
+}
+
+function hashBuckets(initialCount: number) {
+  let buckets = 1;
+  while (buckets < initialCount + initialCount / 4) {
+    buckets *= 2;
+  }
+  return buckets;
+}

package/src/widget-sources/index.ts

@@ -5,4 +5,3 @@ export * from './widget-remote-source.js';
 export * from './widget-table-source.js';
 export * from './widget-tileset-source.js';
 export * from './types.js';
-export * from './constants.js';

package/src/widget-sources/types.ts

@@ -59,8 +59,6 @@ export interface CategoryRequestOptions extends BaseRequestOptions {
   operationColumn?: string;
   /** Local only. */
   joinOperation?: 'count' | 'avg' | 'min' | 'max' | 'sum';
-  /** Calculate `_carto_others` category for all categories after first N (N is threshold). */
-  othersThreshold?: number;
 }
 
 /**
@@ -208,7 +206,7 @@ export type FeaturesResponse = {rows: Record<string, unknown>[]};
 export type FormulaResponse = {value: number | null};
 
 /** Response from {@link WidgetRemoteSource#getCategories}. */
-export type CategoryResponse = {name: string | null; value: number}[];
+export type CategoryResponse = {name: string; value: number}[];
 
 /** Response from {@link WidgetRemoteSource#getRange}. */
 export type RangeResponse = {min: number; max: number} | null;

package/src/widget-sources/widget-remote-source.ts

@@ -74,8 +74,7 @@ export abstract class WidgetRemoteSource<
       spatialFiltersMode,
       ...params
     } = options;
-    const {column, operation, operationColumn, operationExp, othersThreshold} =
-      params;
+    const {column, operation, operationColumn, operationExp} = params;
 
     if (operation === AggregationTypes.Custom) {
       assert(operationExp, 'operationExp is required for custom operation');
@@ -95,7 +94,6 @@ export abstract class WidgetRemoteSource<
         operation,
         operationExp,
         operationColumn: operationColumn || column,
-        othersThreshold,
       },
       opts: {signal, headers: this.props.headers},
     }).then((res: CategoriesModelResponse) => normalizeObjectKeys(res.rows));

package/src/widget-sources/widget-tileset-source-impl.ts

@@ -190,7 +190,6 @@ export class WidgetTilesetSourceImpl extends WidgetSource<WidgetTilesetSourcePro
     filters,
     filterOwner,
     spatialFilter,
-    othersThreshold,
   }: CategoryRequestOptions): Promise<CategoryResponse> {
     const filteredFeatures = this._getFilteredFeatures(
       spatialFilter,
@@ -210,7 +209,6 @@ export class WidgetTilesetSourceImpl extends WidgetSource<WidgetTilesetSourcePro
       joinOperation,
       keysColumn: column,
       operation,
-      othersThreshold,
     });
 
     return groups || [];

package/src/widget-sources/constants.ts

@@ -1,6 +0,0 @@
-/**
- * Name of the category that represents the "Others" category.
- *
- * See `WidgetSource.getCategories` for more information.
- */
-export const OTHERS_CATEGORY_NAME = '_carto_others';