@carto/api-client 0.5.7-alpha.5 → 0.5.7

package/package.json

@@ -8,7 +8,7 @@
   "homepage": "https://github.com/CartoDB/carto-api-client#readme",
   "author": "Don McCurdy <donmccurdy@carto.com>",
   "packageManager": "yarn@4.3.1",
-  "version": "0.5.7-alpha.5",
+  "version": "0.5.7",
   "license": "MIT",
   "publishConfig": {
     "access": "public"
@@ -131,6 +131,5 @@
   "resolutions": {
     "@carto/api-client": "portal:./",
     "rollup": "^4.20.0"
-  },
-  "stableVersion": "0.5.4"
+  }
 }

package/src/filters/tileFeatures.ts

@@ -19,7 +19,7 @@ export type TileFeatures = {
   tileFormat: TileFormat;
   spatialDataType: SpatialDataType;
   spatialDataColumn?: string;
-  spatialFilter: SpatialFilter;
+  spatialFilter?: SpatialFilter;
   uniqueIdProperty?: string;
   rasterMetadata?: RasterMetadata;
   storeGeometry?: boolean;

package/src/filters/tileFeaturesGeometries.ts

@@ -1,22 +1,15 @@
-import bboxPolygon from '@turf/bbox-polygon';
 import intersects from '@turf/boolean-intersects';
-import booleanWithin from '@turf/boolean-within';
-import intersect from '@turf/intersect';
-import {transformToTileCoords} from '../utils/transformToTileCoords.js';
 import {transformTileCoordsToWGS84} from '../utils/transformTileCoordsToWGS84.js';
 import {TileFormat} from '../constants.js';
 import type {
   BBox,
-  Feature,
   Geometry,
   LineString,
-  MultiPolygon,
   Point,
   Polygon,
   Position,
 } from 'geojson';
 import type {SpatialFilter, Tile} from '../types.js';
-import {featureCollection} from '@turf/helpers';
 import type {FeatureData} from '../types-internal.js';
 import type {
   BinaryAttribute,
@@ -27,6 +20,7 @@ import type {
   BinaryPolygonFeature,
   TypedArrayConstructor,
 } from '@loaders.gl/schema';
+import {intersectTileGeometry} from './tileIntersection.js';
 
 export const FEATURE_GEOM_PROPERTY = '__geomValue';
 
@@ -51,7 +45,7 @@ export function tileFeaturesGeometries({
 }: {
   tiles: Tile[];
   tileFormat?: TileFormat;
-  spatialFilter: SpatialFilter;
+  spatialFilter?: SpatialFilter;
   uniqueIdProperty?: string;
   options?: GeometryExtractOptions;
 }): FeatureData[] {
@@ -64,65 +58,50 @@ export function tileFeaturesGeometries({
       continue;
     }
 
-    const bbox = [
+    const tileBbox = [
       tile.bbox.west,
       tile.bbox.south,
       tile.bbox.east,
       tile.bbox.north,
     ] as BBox;
-    const bboxToGeom = bboxPolygon(bbox);
-    const tileIsFullyVisible = booleanWithin(bboxToGeom, spatialFilter);
-
-    // Clip the geometry to intersect with the tile
-    const spatialFilterFeature: Feature<Polygon | MultiPolygon> = {
-      type: 'Feature',
-      geometry: spatialFilter,
-      properties: {},
-    };
-    const clippedGeometryToIntersect = intersect(
-      featureCollection([bboxToGeom, spatialFilterFeature])
+
+    const intersection = intersectTileGeometry(
+      tileBbox,
+      tileFormat,
+      spatialFilter
     );
 
-    if (!clippedGeometryToIntersect) {
-      continue;
-    }
+    if (intersection === false) continue;
 
-    // We assume that MVT tileFormat uses local coordinates so we transform the geometry to intersect to tile coordinates [0..1],
-    // while in the case of 'geojson' or binary, the geometries are already in WGS84
-    const transformedGeometryToIntersect =
-      tileFormat === TileFormat.MVT
-        ? transformToTileCoords(clippedGeometryToIntersect.geometry, bbox)
-        : clippedGeometryToIntersect.geometry;
+    const transformedSpatialFilter =
+      intersection === true ? undefined : intersection;
 
     calculateFeatures({
       map,
-      tileIsFullyVisible,
-      geometryIntersection: transformedGeometryToIntersect,
+      spatialFilter: transformedSpatialFilter,
       data: tile.data.points!,
       type: 'Point',
-      bbox,
+      bbox: tileBbox,
       tileFormat,
       uniqueIdProperty,
       options,
     });
     calculateFeatures({
       map,
-      tileIsFullyVisible,
-      geometryIntersection: transformedGeometryToIntersect,
+      spatialFilter: transformedSpatialFilter,
       data: tile.data.lines!,
       type: 'LineString',
-      bbox,
+      bbox: tileBbox,
       tileFormat,
       uniqueIdProperty,
       options,
     });
     calculateFeatures({
       map,
-      tileIsFullyVisible,
-      geometryIntersection: transformedGeometryToIntersect,
+      spatialFilter: transformedSpatialFilter,
       data: tile.data.polygons!,
       type: 'Polygon',
-      bbox,
+      bbox: tileBbox,
       tileFormat,
       uniqueIdProperty,
       options,
@@ -141,7 +120,7 @@ function processTileFeatureProperties({
   tileFormat,
   uniqueIdProperty,
   storeGeometry,
-  geometryIntersection,
+  spatialFilter,
 }: {
   map: TileMap;
   data: BinaryFeature;
@@ -152,7 +131,7 @@ function processTileFeatureProperties({
   tileFormat?: TileFormat;
   uniqueIdProperty?: string;
   storeGeometry: boolean;
-  geometryIntersection?: Geometry;
+  spatialFilter?: Geometry;
 }) {
   const tileProps = getPropertiesFromTile(data, startIndex);
   const uniquePropertyValue = getUniquePropertyValue(
@@ -167,7 +146,7 @@ function processTileFeatureProperties({
   let geometry: Geometry | null = null;
 
   // Only calculate geometry if necessary
-  if (storeGeometry || geometryIntersection) {
+  if (storeGeometry || spatialFilter) {
     const {positions} = data;
     const ringCoordinates = getRingCoordinatesFor(
       startIndex,
@@ -178,11 +157,7 @@ function processTileFeatureProperties({
   }
 
   // If intersection is required, check before proceeding
-  if (
-    geometry &&
-    geometryIntersection &&
-    !intersects(geometry, geometryIntersection)
-  ) {
+  if (geometry && spatialFilter && !intersects(geometry, spatialFilter)) {
     return;
   }
 
@@ -201,7 +176,7 @@ function processTileFeatureProperties({
 function addIntersectedFeaturesInTile({
   map,
   data,
-  geometryIntersection,
+  spatialFilter,
   type,
   bbox,
   tileFormat,
@@ -210,7 +185,7 @@ function addIntersectedFeaturesInTile({
 }: {
   map: TileMap;
   data: BinaryFeature;
-  geometryIntersection: Geometry;
+  spatialFilter: Geometry;
   type: BinaryGeometryType;
   bbox: BBox;
   tileFormat?: TileFormat;
@@ -233,7 +208,7 @@ function addIntersectedFeaturesInTile({
       tileFormat,
       uniqueIdProperty,
       storeGeometry,
-      geometryIntersection,
+      spatialFilter,
     });
   }
 }
@@ -350,8 +325,7 @@ function getRingCoordinatesFor(
 
 function calculateFeatures({
   map,
-  tileIsFullyVisible,
-  geometryIntersection,
+  spatialFilter,
   data,
   type,
   bbox,
@@ -360,8 +334,7 @@ function calculateFeatures({
   options,
 }: {
   map: TileMap;
-  tileIsFullyVisible: boolean;
-  geometryIntersection: SpatialFilter;
+  spatialFilter?: SpatialFilter;
   data: BinaryFeature;
   type: BinaryGeometryType;
   bbox: BBox;
@@ -373,7 +346,7 @@ function calculateFeatures({
     return;
   }
 
-  if (tileIsFullyVisible) {
+  if (!spatialFilter) {
     addAllFeaturesInTile({
       map,
       data,
@@ -387,7 +360,7 @@ function calculateFeatures({
     addIntersectedFeaturesInTile({
       map,
       data,
-      geometryIntersection,
+      spatialFilter,
       type,
       bbox,
       tileFormat,

package/src/filters/tileFeaturesRaster.ts

@@ -1,7 +1,6 @@
 import {
   cellToChildren as _cellToChildren,
   cellToTile,
-  geometryToCells,
   getResolution,
 } from 'quadbin';
 import type {RasterTile, SpatialFilter, Tile} from '../types.js';
@@ -11,11 +10,11 @@ import type {
   RasterMetadataBand,
   SpatialDataType,
 } from '../sources/types.js';
-import {CellSet} from '../utils/CellSet.js';
+import {intersectTileRaster} from './tileIntersection.js';
 
 export type TileFeaturesRasterOptions = {
   tiles: RasterTile[];
-  spatialFilter: SpatialFilter;
+  spatialFilter?: SpatialFilter;
   spatialDataColumn: string;
   spatialDataType: SpatialDataType;
   rasterMetadata: RasterMetadata;
@@ -44,23 +43,27 @@ export function tileFeaturesRaster({
   const tileBlockSize = tiles[0].data!.blockSize;
   const cellResolution = tileResolution + BigInt(Math.log2(tileBlockSize));
 
-  // Compute covering cells for the spatial filter, at same resolution as the
-  // raster pixels, to be used as a mask.
-  const spatialFilterCells = new CellSet(
-    geometryToCells(options.spatialFilter, cellResolution)
-  );
-
   const data = new Map<bigint, FeatureData>();
 
   for (const tile of tiles as Required<RasterTile>[]) {
     const parent = tile.index.q;
 
-    const children = cellToChildrenSorted(parent, cellResolution);
+    const intersection = intersectTileRaster(
+      parent,
+      cellResolution,
+      options.spatialFilter
+    );
+
+    if (intersection === false) continue;
+
+    const tileSortedCells = cellToChildrenSorted(parent, cellResolution);
 
     // For each pixel/cell within the spatial filter, create a FeatureData.
     // Order is row-major, starting from NW and ending at SE.
-    for (let i = 0; i < children.length; i++) {
-      if (!spatialFilterCells.has(children[i])) continue;
+    for (let i = 0; i < tileSortedCells.length; i++) {
+      if (intersection !== true && !intersection.has(tileSortedCells[i])) {
+        continue;
+      }
 
       const cellData: FeatureData = {};
       let cellDataExists = false;
@@ -76,7 +79,7 @@ export function tileFeaturesRaster({
       }
 
       if (cellDataExists) {
-        data.set(children[i], cellData);
+        data.set(tileSortedCells[i], cellData);
       }
     }
   }

package/src/filters/tileFeaturesSpatialIndex.ts

@@ -1,15 +1,15 @@
 import {SpatialIndex} from '../constants.js';
-import {getResolution as quadbinGetResolution, geometryToCells} from 'quadbin';
-import bboxClip from '@turf/bbox-clip';
+import {getResolution as quadbinGetResolution} from 'quadbin';
 import type {SpatialFilter, SpatialIndexTile} from '../types.js';
-import type {BBox, Feature} from 'geojson';
-import {getResolution as h3GetResolution, polygonToCells} from 'h3-js';
+import type {Feature} from 'geojson';
+import {getResolution as h3GetResolution} from 'h3-js';
 import type {FeatureData} from '../types-internal.js';
 import type {SpatialDataType} from '../sources/types.js';
+import {intersectTileH3, intersectTileQuadbin} from './tileIntersection.js';
 
 export type TileFeaturesSpatialIndexOptions = {
   tiles: SpatialIndexTile[];
-  spatialFilter: SpatialFilter;
+  spatialFilter?: SpatialFilter;
   spatialDataColumn: string;
   spatialDataType: SpatialDataType;
 };
@@ -22,30 +22,41 @@ export function tileFeaturesSpatialIndex({
 }: TileFeaturesSpatialIndexOptions): FeatureData[] {
   const map = new Map();
   const spatialIndex = getSpatialIndex(spatialDataType);
-  const resolution = getResolution(tiles, spatialIndex);
+  const cellResolution = getResolution(tiles, spatialIndex);
   const spatialIndexIDName = spatialDataColumn
     ? spatialDataColumn
     : spatialIndex;
 
-  if (!resolution) {
+  if (!cellResolution) {
     return [];
   }
-  const cells = getCellsCoverGeometry(spatialFilter, spatialIndex, resolution);
 
-  if (!cells?.length) {
-    return [];
-  }
+  let intersection: undefined | boolean | Set<bigint | string>;
 
-  // We transform cells to Set to improve the performace
-  const cellsSet = new Set<bigint | string>(cells);
+  // Compute H3 intersection globally, Quadbin intersection per-tile. See tileIntersection.ts.
+  if (spatialIndex === SpatialIndex.H3) {
+    intersection = intersectTileH3(cellResolution as number, spatialFilter);
+  }
 
   for (const tile of tiles) {
     if (tile.isVisible === false || !tile.data) {
       continue;
     }
 
+    if (spatialIndex === SpatialIndex.QUADBIN) {
+      const parent = getTileIndex(tile, spatialIndex);
+      intersection = intersectTileQuadbin(
+        parent as bigint,
+        cellResolution as bigint,
+        spatialFilter
+      );
+    }
+
+    if (!intersection) continue;
+
     tile.data.forEach((d: Feature) => {
-      if (cellsSet.has(d.id as bigint | string)) {
+      // @ts-expect-error Mixed types for cell indices.
+      if (intersection === true || intersection.has(d.id as bigint | string)) {
         map.set(d.id, {...d.properties, [spatialIndexIDName]: d.id});
       }
     });
@@ -53,10 +64,21 @@ export function tileFeaturesSpatialIndex({
   return Array.from(map.values());
 }
 
+function getTileIndex(
+  tile: SpatialIndexTile,
+  spatialIndex: SpatialIndex
+): bigint | string {
+  if (spatialIndex === SpatialIndex.QUADBIN) {
+    // @ts-expect-error Missing types for quadbin tile indices.
+    return tile.index.q;
+  }
+  return tile.id;
+}
+
 function getResolution(
   tiles: SpatialIndexTile[],
   spatialIndex: SpatialIndex
-): number | undefined {
+): bigint | number | undefined {
   const data = tiles.find((tile) => tile.data?.length)?.data;
 
   if (!data) {
@@ -72,41 +94,6 @@ function getResolution(
   }
 }
 
-const bboxWest: BBox = [-180, -90, 0, 90];
-const bboxEast: BBox = [0, -90, 180, 90];
-
-function getCellsCoverGeometry(
-  geometry: SpatialFilter,
-  spatialIndex: SpatialIndex,
-  resolution: number
-) {
-  if (spatialIndex === SpatialIndex.QUADBIN) {
-    // @ts-expect-error TODO: Probably ought to be stricter about number vs. bigint types in this file.
-    return geometryToCells(geometry, resolution);
-  }
-
-  if (spatialIndex === SpatialIndex.H3) {
-    // The current H3 polyfill algorithm can't deal with polygon segments of greater than 180 degrees longitude
-    // so we clip the geometry to be sure that none of them is greater than 180 degrees
-    // https://github.com/uber/h3-js/issues/24#issuecomment-431893796
-    return polygonToCells(
-      bboxClip(geometry, bboxWest).geometry.coordinates as
-        | number[][]
-        | number[][][],
-      resolution,
-      true
-    ).concat(
-      polygonToCells(
-        bboxClip(geometry, bboxEast).geometry.coordinates as
-          | number[][]
-          | number[][][],
-        resolution,
-        true
-      )
-    );
-  }
-}
-
 function getSpatialIndex(spatialDataType: SpatialDataType): SpatialIndex {
   switch (spatialDataType) {
     case 'h3':

package/src/filters/tileIntersection.ts

@@ -0,0 +1,155 @@
+import type {BBox} from 'geojson';
+import type {SpatialFilter} from '../types.js';
+import bboxPolygon from '@turf/bbox-polygon';
+import booleanWithin from '@turf/boolean-within';
+import intersect from '@turf/intersect';
+import {feature, featureCollection} from '@turf/helpers';
+import {TileFormat} from '../constants.js';
+import {transformToTileCoords} from '../utils/transformToTileCoords.js';
+import {
+  cellToBoundary as quadbinCellToBoundary,
+  geometryToCells as quadbinGeometryToCells,
+} from 'quadbin';
+import {polygonToCells as h3PolygonToCells} from 'h3-js';
+import bboxClip from '@turf/bbox-clip';
+
+// Computes intersections between spatial filters and tiles in various formats,
+// for pre-filtering each tile's features before performing widget calculations.
+//
+// Return types:
+// - true: All features in tile are within spatial filter.
+// - false: No features in tile are within spatial filter; skip tile.
+// - SpatialFilter/Set: Requires a more detailed per-feature check for each
+//    feature in the tile. Provides a clipped spatial filter local to the tile
+//    or, for spatial indexes, a set of indices ("covering set").
+//
+// Computing a covering set for spatial indexes may be very expensive for large
+// spatial filters and small cell resolutions. For example, a viewport at z=3
+// would contain ~18,000,000 raster cells at resolution=14. To avoid ever
+// creating a covering set of this size, compute per-tile (not global) coverings
+// when possible. For tiles fully inside or outside the spatial filter, creating
+// a covering set can be skipped.
+//
+// H3 is currently a special case where coverage must be computed for the entire
+// spatial filter; per-tile coverage would return a different result, because
+// H3 child cells are not fully contained within their parents.
+
+///////////////////////////////////////////////////////////////////////////////
+// GEOMETRY
+
+/** @internal */
+export function intersectTileGeometry(
+  tileBbox: BBox,
+  tileFormat?: TileFormat,
+  spatialFilter?: SpatialFilter
+): boolean | SpatialFilter {
+  const tilePolygon = bboxPolygon(tileBbox);
+
+  if (!spatialFilter || booleanWithin(tilePolygon, spatialFilter)) {
+    return true;
+  }
+
+  const clippedSpatialFilter = intersect(
+    featureCollection([tilePolygon, feature(spatialFilter)])
+  );
+
+  if (!clippedSpatialFilter) {
+    return false;
+  }
+
+  // Transform into local coordinates [0..1]. We assume MVT tiles use local
+  // coordinates but geojson or binary features are already WGS84.
+  return tileFormat === TileFormat.MVT
+    ? transformToTileCoords(clippedSpatialFilter.geometry, tileBbox)
+    : clippedSpatialFilter.geometry;
+}
+
+///////////////////////////////////////////////////////////////////////////////
+// RASTER
+
+/** @internal */
+export function intersectTileRaster(
+  parent: bigint,
+  cellResolution: bigint,
+  spatialFilter?: SpatialFilter
+) {
+  return intersectTileQuadbin(parent, cellResolution, spatialFilter);
+}
+
+///////////////////////////////////////////////////////////////////////////////
+// QUADBIN
+
+/** @internal */
+export function intersectTileQuadbin(
+  parent: bigint,
+  cellResolution: bigint,
+  spatialFilter?: SpatialFilter
+): boolean | Set<bigint> {
+  const tilePolygon = quadbinCellToBoundary(parent);
+
+  if (!spatialFilter || booleanWithin(tilePolygon, spatialFilter)) {
+    return true;
+  }
+
+  const clippedSpatialFilter = intersect(
+    featureCollection([feature(tilePolygon), feature(spatialFilter)])
+  );
+
+  if (!clippedSpatialFilter) {
+    return false;
+  }
+
+  const cells = quadbinGeometryToCells(
+    clippedSpatialFilter.geometry,
+    cellResolution
+  );
+
+  return new Set(cells);
+}
+
+///////////////////////////////////////////////////////////////////////////////
+// H3
+
+const BBOX_WEST: BBox = [-180, -90, 0, 90];
+const BBOX_EAST: BBox = [0, -90, 180, 90];
+
+/** @internal */
+export function intersectTileH3(
+  cellResolution: number,
+  spatialFilter?: SpatialFilter
+): true | Set<string> {
+  if (!spatialFilter) {
+    return true;
+  }
+
+  // Unlike quadbin and raster tiles, H3 children do not align to the parent's
+  // boundaries. Per-tile coverage with `h3PolygonToCells` would return only
+  // cells with _centers_ inside the clipped polygon — fewer cells. So for H3
+  // we compute the coverage set for the entire spatial filter (more expensive).
+  // In the future this could be replaced with `polygonToCellsExperimental`,
+  // which can compute child cells in different modes.
+
+  const spatialFilterFeature = feature(spatialFilter);
+
+  // The current H3 polyfill algorithm can't deal with polygon segments of greater than 180 degrees longitude
+  // so we clip the geometry to be sure that none of them is greater than 180 degrees
+  // https://github.com/uber/h3-js/issues/24#issuecomment-431893796
+
+  const cellsWest = h3PolygonToCells(
+    bboxClip(spatialFilterFeature, BBOX_WEST).geometry.coordinates as
+      | number[][]
+      | number[][][],
+    cellResolution,
+    true
+  );
+
+  const cellsEast = h3PolygonToCells(
+    bboxClip(spatialFilterFeature, BBOX_EAST).geometry.coordinates as
+      | number[][]
+      | number[][][],
+    cellResolution,
+    true
+  );
+
+  return new Set(cellsWest.concat(cellsEast));
+}

package/src/widget-sources/widget-tileset-source-impl.ts

@@ -74,14 +74,13 @@ export class WidgetTilesetSourceImpl extends WidgetSource<WidgetTilesetSourcePro
     this._features.length = 0;
   }
 
-  protected _extractTileFeatures(spatialFilter: SpatialFilter) {
+  protected _extractTileFeatures(spatialFilter?: SpatialFilter) {
     // When spatial filter has not changed, don't redo extraction. If tiles or
     // tile extract options change, features will have been cleared already.
     const prevInputs = this._tileFeatureExtractPreviousInputs;
     if (
       this._features.length &&
-      prevInputs.spatialFilter &&
-      booleanEqual(prevInputs.spatialFilter, spatialFilter)
+      spatialFilterEquals(prevInputs.spatialFilter, spatialFilter)
     ) {
       return;
     }
@@ -383,7 +382,6 @@ export class WidgetTilesetSourceImpl extends WidgetSource<WidgetTilesetSourcePro
     filters?: Record<string, Filter>,
     filterOwner?: string
   ): FeatureData[] {
-    assert(spatialFilter, 'spatialFilter required for tilesets');
     this._extractTileFeatures(spatialFilter);
     return applyFilters(
       this._features,
@@ -422,3 +420,9 @@ function normalizeColumns(columns: string | string[]): string[] {
       ? [columns]
       : [];
 }
+
+function spatialFilterEquals(a?: SpatialFilter, b?: SpatialFilter) {
+  if (a === b) return true;
+  if (!a || !b) return false;
+  return booleanEqual(a, b);
+}