@carto/api-client 0.5.7-alpha-optional-spatial-filter.1 → 0.5.7-alpha-others-orderby.1

package/package.json

@@ -8,7 +8,7 @@
   "homepage": "https://github.com/CartoDB/carto-api-client#readme",
   "author": "Don McCurdy <donmccurdy@carto.com>",
   "packageManager": "yarn@4.3.1",
-  "version": "0.5.7-alpha-optional-spatial-filter.1",
+  "version": "0.5.7-alpha-others-orderby.1",
   "license": "MIT",
   "publishConfig": {
     "access": "public"

package/src/filters/tileFeatures.ts

@@ -19,7 +19,7 @@ export type TileFeatures = {
   tileFormat: TileFormat;
   spatialDataType: SpatialDataType;
   spatialDataColumn?: string;
-  spatialFilter?: SpatialFilter;
+  spatialFilter: SpatialFilter;
   uniqueIdProperty?: string;
   rasterMetadata?: RasterMetadata;
   storeGeometry?: boolean;

package/src/filters/tileFeaturesGeometries.ts

@@ -1,15 +1,22 @@
+import bboxPolygon from '@turf/bbox-polygon';
 import intersects from '@turf/boolean-intersects';
+import booleanWithin from '@turf/boolean-within';
+import intersect from '@turf/intersect';
+import {transformToTileCoords} from '../utils/transformToTileCoords.js';
 import {transformTileCoordsToWGS84} from '../utils/transformTileCoordsToWGS84.js';
 import {TileFormat} from '../constants.js';
 import type {
   BBox,
+  Feature,
   Geometry,
   LineString,
+  MultiPolygon,
   Point,
   Polygon,
   Position,
 } from 'geojson';
 import type {SpatialFilter, Tile} from '../types.js';
+import {featureCollection} from '@turf/helpers';
 import type {FeatureData} from '../types-internal.js';
 import type {
   BinaryAttribute,
@@ -20,7 +27,6 @@ import type {
   BinaryPolygonFeature,
   TypedArrayConstructor,
 } from '@loaders.gl/schema';
-import {intersectTileGeometry} from './tileIntersection.js';
 
 export const FEATURE_GEOM_PROPERTY = '__geomValue';
 
@@ -45,7 +51,7 @@ export function tileFeaturesGeometries({
 }: {
   tiles: Tile[];
   tileFormat?: TileFormat;
-  spatialFilter?: SpatialFilter;
+  spatialFilter: SpatialFilter;
   uniqueIdProperty?: string;
   options?: GeometryExtractOptions;
 }): FeatureData[] {
@@ -58,50 +64,65 @@ export function tileFeaturesGeometries({
       continue;
     }
 
-    const tileBbox = [
+    const bbox = [
       tile.bbox.west,
       tile.bbox.south,
       tile.bbox.east,
       tile.bbox.north,
     ] as BBox;
-
-    const intersection = intersectTileGeometry(
-      tileBbox,
-      tileFormat,
-      spatialFilter
+    const bboxToGeom = bboxPolygon(bbox);
+    const tileIsFullyVisible = booleanWithin(bboxToGeom, spatialFilter);
+
+    // Clip the geometry to intersect with the tile
+    const spatialFilterFeature: Feature<Polygon | MultiPolygon> = {
+      type: 'Feature',
+      geometry: spatialFilter,
+      properties: {},
+    };
+    const clippedGeometryToIntersect = intersect(
+      featureCollection([bboxToGeom, spatialFilterFeature])
     );
 
-    if (intersection === false) continue;
+    if (!clippedGeometryToIntersect) {
+      continue;
+    }
 
-    const transformedSpatialFilter =
-      intersection === true ? undefined : intersection;
+    // We assume that MVT tileFormat uses local coordinates so we transform the geometry to intersect to tile coordinates [0..1],
+    // while in the case of 'geojson' or binary, the geometries are already in WGS84
+    const transformedGeometryToIntersect =
+      tileFormat === TileFormat.MVT
+        ? transformToTileCoords(clippedGeometryToIntersect.geometry, bbox)
+        : clippedGeometryToIntersect.geometry;
 
     calculateFeatures({
       map,
-      spatialFilter: transformedSpatialFilter,
+      tileIsFullyVisible,
+      geometryIntersection: transformedGeometryToIntersect,
       data: tile.data.points!,
       type: 'Point',
-      bbox: tileBbox,
+      bbox,
       tileFormat,
       uniqueIdProperty,
       options,
     });
     calculateFeatures({
       map,
-      spatialFilter: transformedSpatialFilter,
+      tileIsFullyVisible,
+      geometryIntersection: transformedGeometryToIntersect,
       data: tile.data.lines!,
       type: 'LineString',
-      bbox: tileBbox,
+      bbox,
       tileFormat,
       uniqueIdProperty,
       options,
     });
     calculateFeatures({
       map,
-      spatialFilter: transformedSpatialFilter,
+      tileIsFullyVisible,
+      geometryIntersection: transformedGeometryToIntersect,
       data: tile.data.polygons!,
       type: 'Polygon',
-      bbox: tileBbox,
+      bbox,
       tileFormat,
       uniqueIdProperty,
       options,
@@ -120,7 +141,7 @@ function processTileFeatureProperties({
   tileFormat,
   uniqueIdProperty,
   storeGeometry,
-  spatialFilter,
+  geometryIntersection,
 }: {
   map: TileMap;
   data: BinaryFeature;
@@ -131,7 +152,7 @@ function processTileFeatureProperties({
   tileFormat?: TileFormat;
   uniqueIdProperty?: string;
   storeGeometry: boolean;
-  spatialFilter?: Geometry;
+  geometryIntersection?: Geometry;
 }) {
   const tileProps = getPropertiesFromTile(data, startIndex);
   const uniquePropertyValue = getUniquePropertyValue(
@@ -146,7 +167,7 @@ function processTileFeatureProperties({
   let geometry: Geometry | null = null;
 
   // Only calculate geometry if necessary
-  if (storeGeometry || spatialFilter) {
+  if (storeGeometry || geometryIntersection) {
     const {positions} = data;
     const ringCoordinates = getRingCoordinatesFor(
       startIndex,
@@ -157,7 +178,11 @@ function processTileFeatureProperties({
   }
 
   // If intersection is required, check before proceeding
-  if (geometry && spatialFilter && !intersects(geometry, spatialFilter)) {
+  if (
+    geometry &&
+    geometryIntersection &&
+    !intersects(geometry, geometryIntersection)
+  ) {
     return;
   }
 
@@ -176,7 +201,7 @@ function processTileFeatureProperties({
 function addIntersectedFeaturesInTile({
   map,
   data,
-  spatialFilter,
+  geometryIntersection,
   type,
   bbox,
   tileFormat,
@@ -185,7 +210,7 @@ function addIntersectedFeaturesInTile({
 }: {
   map: TileMap;
   data: BinaryFeature;
-  spatialFilter: Geometry;
+  geometryIntersection: Geometry;
   type: BinaryGeometryType;
   bbox: BBox;
   tileFormat?: TileFormat;
@@ -208,7 +233,7 @@ function addIntersectedFeaturesInTile({
       tileFormat,
       uniqueIdProperty,
       storeGeometry,
-      spatialFilter,
+      geometryIntersection,
     });
   }
 }
@@ -325,7 +350,8 @@ function getRingCoordinatesFor(
 
 function calculateFeatures({
   map,
-  spatialFilter,
+  tileIsFullyVisible,
+  geometryIntersection,
   data,
   type,
   bbox,
@@ -334,7 +360,8 @@ function calculateFeatures({
   options,
 }: {
   map: TileMap;
-  spatialFilter?: SpatialFilter;
+  tileIsFullyVisible: boolean;
+  geometryIntersection: SpatialFilter;
   data: BinaryFeature;
   type: BinaryGeometryType;
   bbox: BBox;
@@ -346,7 +373,7 @@ function calculateFeatures({
     return;
   }
 
-  if (!spatialFilter) {
+  if (tileIsFullyVisible) {
     addAllFeaturesInTile({
       map,
       data,
@@ -360,7 +387,7 @@ function calculateFeatures({
     addIntersectedFeaturesInTile({
       map,
       data,
-      spatialFilter,
+      geometryIntersection,
       type,
       bbox,
       tileFormat,

package/src/filters/tileFeaturesRaster.ts

@@ -1,6 +1,8 @@
 import {
   cellToChildren as _cellToChildren,
+  cellToBoundary,
   cellToTile,
+  geometryToCells,
   getResolution,
 } from 'quadbin';
 import type {RasterTile, SpatialFilter, Tile} from '../types.js';
@@ -10,11 +12,13 @@ import type {
   RasterMetadataBand,
   SpatialDataType,
 } from '../sources/types.js';
-import {intersectTileRaster} from './tileIntersection.js';
+import booleanWithin from '@turf/boolean-within';
+import intersect from '@turf/intersect';
+import {feature, featureCollection} from '@turf/helpers';
 
 export type TileFeaturesRasterOptions = {
   tiles: RasterTile[];
-  spatialFilter?: SpatialFilter;
+  spatialFilter: SpatialFilter;
   spatialDataColumn: string;
   spatialDataType: SpatialDataType;
   rasterMetadata: RasterMetadata;
@@ -48,20 +52,27 @@ export function tileFeaturesRaster({
   for (const tile of tiles as Required<RasterTile>[]) {
     const parent = tile.index.q;
 
-    const intersection = intersectTileRaster(
-      parent,
-      cellResolution,
-      options.spatialFilter
+    // If tile is partially overlapping with the spatial filter, compute a quadbin covering for the
+    // spatial filter + tile intersection, at cell resolution. If tile is fully inside or outside
+    // the spatial filter, computing a covering can be skipped. Avoid computing a quadbin covering
+    // for the _entire_ spatial filter, which may contain far too many cells (e.g. at zoom=3 and
+    // resolution=14, we have ~18,000,000 cells in the viewport.)
+    const tilePolygon = cellToBoundary(parent);
+    const tileFilter = intersect(
+      featureCollection([feature(tilePolygon), feature(options.spatialFilter)])
     );
-
-    if (intersection === false) continue;
-
+    const needsFilter = tileFilter
+      ? !booleanWithin(tilePolygon, options.spatialFilter)
+      : false;
+    const tileFilterCells = needsFilter
+      ? new Set(geometryToCells(tileFilter!.geometry, cellResolution))
+      : null;
     const tileSortedCells = cellToChildrenSorted(parent, cellResolution);
 
     // For each pixel/cell within the spatial filter, create a FeatureData.
     // Order is row-major, starting from NW and ending at SE.
     for (let i = 0; i < tileSortedCells.length; i++) {
-      if (intersection !== true && !intersection.has(tileSortedCells[i])) {
+      if (needsFilter && !tileFilterCells!.has(tileSortedCells[i])) {
         continue;
       }
 

package/src/filters/tileFeaturesSpatialIndex.ts

@@ -1,15 +1,15 @@
 import {SpatialIndex} from '../constants.js';
-import {getResolution as quadbinGetResolution} from 'quadbin';
+import {getResolution as quadbinGetResolution, geometryToCells} from 'quadbin';
+import bboxClip from '@turf/bbox-clip';
 import type {SpatialFilter, SpatialIndexTile} from '../types.js';
-import type {Feature} from 'geojson';
-import {getResolution as h3GetResolution} from 'h3-js';
+import type {BBox, Feature} from 'geojson';
+import {getResolution as h3GetResolution, polygonToCells} from 'h3-js';
 import type {FeatureData} from '../types-internal.js';
 import type {SpatialDataType} from '../sources/types.js';
-import {intersectTileH3, intersectTileQuadbin} from './tileIntersection.js';
 
 export type TileFeaturesSpatialIndexOptions = {
   tiles: SpatialIndexTile[];
-  spatialFilter?: SpatialFilter;
+  spatialFilter: SpatialFilter;
   spatialDataColumn: string;
   spatialDataType: SpatialDataType;
 };
@@ -22,41 +22,30 @@ export function tileFeaturesSpatialIndex({
 }: TileFeaturesSpatialIndexOptions): FeatureData[] {
   const map = new Map();
   const spatialIndex = getSpatialIndex(spatialDataType);
-  const cellResolution = getResolution(tiles, spatialIndex);
+  const resolution = getResolution(tiles, spatialIndex);
   const spatialIndexIDName = spatialDataColumn
     ? spatialDataColumn
     : spatialIndex;
 
-  if (!cellResolution) {
+  if (!resolution) {
     return [];
   }
+  const cells = getCellsCoverGeometry(spatialFilter, spatialIndex, resolution);
 
-  let intersection: undefined | boolean | Set<bigint | string>;
-
-  // Compute H3 intersection globally, Quadbin intersection per-tile. See tileIntersection.ts.
-  if (spatialIndex === SpatialIndex.H3) {
-    intersection = intersectTileH3(cellResolution as number, spatialFilter);
+  if (!cells?.length) {
+    return [];
   }
 
+  // We transform cells to Set to improve the performace
+  const cellsSet = new Set<bigint | string>(cells);
+
   for (const tile of tiles) {
     if (tile.isVisible === false || !tile.data) {
       continue;
     }
 
-    if (spatialIndex === SpatialIndex.QUADBIN) {
-      const parent = getTileIndex(tile, spatialIndex);
-      intersection = intersectTileQuadbin(
-        parent as bigint,
-        cellResolution as bigint,
-        spatialFilter
-      );
-    }
-
-    if (!intersection) continue;
-
     tile.data.forEach((d: Feature) => {
-      // @ts-expect-error Mixed types for cell indices.
-      if (intersection === true || intersection.has(d.id as bigint | string)) {
+      if (cellsSet.has(d.id as bigint | string)) {
         map.set(d.id, {...d.properties, [spatialIndexIDName]: d.id});
       }
     });
@@ -64,21 +53,10 @@ export function tileFeaturesSpatialIndex({
   return Array.from(map.values());
 }
 
-function getTileIndex(
-  tile: SpatialIndexTile,
-  spatialIndex: SpatialIndex
-): bigint | string {
-  if (spatialIndex === SpatialIndex.QUADBIN) {
-    // @ts-expect-error Missing types for quadbin tile indices.
-    return tile.index.q;
-  }
-  return tile.id;
-}
-
 function getResolution(
   tiles: SpatialIndexTile[],
   spatialIndex: SpatialIndex
-): bigint | number | undefined {
+): number | undefined {
   const data = tiles.find((tile) => tile.data?.length)?.data;
 
   if (!data) {
@@ -94,6 +72,41 @@ function getResolution(
   }
 }
 
+const bboxWest: BBox = [-180, -90, 0, 90];
+const bboxEast: BBox = [0, -90, 180, 90];
+
+function getCellsCoverGeometry(
+  geometry: SpatialFilter,
+  spatialIndex: SpatialIndex,
+  resolution: number
+) {
+  if (spatialIndex === SpatialIndex.QUADBIN) {
+    // @ts-expect-error TODO: Probably ought to be stricter about number vs. bigint types in this file.
+    return geometryToCells(geometry, resolution);
+  }
+
+  if (spatialIndex === SpatialIndex.H3) {
+    // The current H3 polyfill algorithm can't deal with polygon segments of greater than 180 degrees longitude
+    // so we clip the geometry to be sure that none of them is greater than 180 degrees
+    // https://github.com/uber/h3-js/issues/24#issuecomment-431893796
+    return polygonToCells(
+      bboxClip(geometry, bboxWest).geometry.coordinates as
+        | number[][]
+        | number[][][],
+      resolution,
+      true
+    ).concat(
+      polygonToCells(
+        bboxClip(geometry, bboxEast).geometry.coordinates as
+          | number[][]
+          | number[][][],
+        resolution,
+        true
+      )
+    );
+  }
+}
+
 function getSpatialIndex(spatialDataType: SpatialDataType): SpatialIndex {
   switch (spatialDataType) {
     case 'h3':

package/src/operations/groupBy.ts

@@ -1,12 +1,12 @@
 import {aggregationFunctions, aggregate} from './aggregation.js';
+import {OTHERS_CATEGORY_NAME} from '../widget-sources/constants.js';
 import type {AggregationType} from '../types.js';
 import type {FeatureData} from '../types-internal.js';
-
-/** @privateRemarks Source: @carto/react-core */
-export type GroupByFeature = {
-  name: string;
-  value: number;
-}[];
+import type {
+  CategoryOrderBy,
+  CategoryResponseEntry,
+  CategoryResponseRaw,
+} from '../widget-sources/types.js';
 
 /** @privateRemarks Source: @carto/react-core */
 export function groupValuesByColumn({
@@ -15,15 +15,19 @@ export function groupValuesByColumn({
   joinOperation,
   keysColumn,
   operation,
+  othersThreshold,
+  orderBy = 'frequency_desc',
 }: {
   data: FeatureData[];
   valuesColumns?: string[];
   joinOperation?: AggregationType;
   keysColumn: string;
   operation: AggregationType;
-}): GroupByFeature | null {
+  othersThreshold?: number;
+  orderBy?: CategoryOrderBy;
+}): CategoryResponseRaw | null {
   if (Array.isArray(data) && data.length === 0) {
-    return null;
+    return {rows: null};
   }
   const groups = data.reduce((accumulator, item) => {
     const group = item[keysColumn];
@@ -48,12 +52,57 @@ export function groupValuesByColumn({
   const targetOperation =
     aggregationFunctions[operation as Exclude<AggregationType, 'custom'>];
 
-  if (targetOperation) {
-    return Array.from(groups).map(([name, value]) => ({
+  if (!targetOperation) {
+    return {rows: []};
+  }
+
+  const allCategories = Array.from(groups)
+    .map(([name, value]) => ({
       name,
       value: targetOperation(value),
-    }));
+    }))
+    .sort(getSorter(orderBy));
+
+  if (othersThreshold && allCategories.length > othersThreshold) {
+    const otherValue = allCategories
+      .slice(othersThreshold)
+      .flatMap(({name}) => groups.get(name));
+    allCategories.push({
+      name: OTHERS_CATEGORY_NAME,
+      value: targetOperation(otherValue),
+    });
+    return {
+      rows: allCategories,
+      metadata: {
+        others: targetOperation(otherValue),
+      },
+    };
   }
 
-  return [];
+  return {
+    rows: allCategories,
+  };
+}
+
+export function getSorter(
+  orderBy: CategoryOrderBy
+): (a: CategoryResponseEntry, b: CategoryResponseEntry) => number {
+  switch (orderBy) {
+    case 'frequency_asc':
+      // 'value ASC, name ASC'
+      return (a, b) => a.value - b.value || localeCompare(a.name, b.name);
+    case 'frequency_desc':
+      // 'value DESC, name ASC'
+      return (a, b) => b.value - a.value || localeCompare(a.name, b.name);
+    case 'alphabetical_asc':
+      // 'name ASC, value DESC'
+      return (a, b) => localeCompare(a.name, b.name) || b.value - a.value;
+    case 'alphabetical_desc':
+      // 'name DESC, value DESC'
+      return (a, b) => localeCompare(b.name, a.name) || b.value - a.value;
+  }
+}
+
+function localeCompare(a?: string | null, b?: string | null) {
+  return (a ?? 'null').localeCompare(b ?? 'null');
 }

package/src/operations/groupByDate.ts

@@ -1,7 +1,12 @@
 import type {AggregationType, GroupDateType} from '../types.js';
 import {getUTCMonday} from '../utils/dateUtils.js';
 import {aggregate, aggregationFunctions} from './aggregation.js';
-import type {GroupByFeature} from './groupBy.js';
+
+/** @privateRemarks Source: @carto/react-core */
+export type GroupByFeature = {
+  name: string;
+  value: number;
+}[];
 
 const GROUP_KEY_FN_MAPPING: Record<GroupDateType, (date: Date) => number> = {
   year: (date: Date) => Date.UTC(date.getUTCFullYear()),

package/src/widget-sources/constants.ts

@@ -0,0 +1,6 @@
+/**
+ * Name of the category that represents the "Others" category.
+ *
+ * See `WidgetSource.getCategories` for more information.
+ */
+export const OTHERS_CATEGORY_NAME = '_carto_others';

package/src/widget-sources/index.ts

@@ -5,3 +5,4 @@ export * from './widget-remote-source.js';
 export * from './widget-table-source.js';
 export * from './widget-tileset-source.js';
 export * from './types.js';
+export * from './constants.js';

package/src/widget-sources/types.ts

@@ -31,6 +31,12 @@ export interface BaseRequestOptions {
   filterOwner?: string;
 }
 
+export type CategoryOrderBy =
+  | 'frequency_asc' // sort by aggregate value ascending, then by name ascending
+  | 'frequency_desc' // sort by aggregate value descending, then by name ascending
+  | 'alphabetical_asc' // sort by category name ascending, then by value descending
+  | 'alphabetical_desc'; // sort by category name descending, then by value descending
+
 /**
  * Examples:
  *   * population by state
@@ -59,6 +65,15 @@ export interface CategoryRequestOptions extends BaseRequestOptions {
   operationColumn?: string;
   /** Local only. */
   joinOperation?: 'count' | 'avg' | 'min' | 'max' | 'sum';
+  /** Calculate `_carto_others` category for all categories after first N (N is threshold). */
+  othersThreshold?: number;
+  /**
+   * Order categories by frequency or alphabetically.
+   * @default 'frequency_desc'
+   */
+  orderBy?: CategoryOrderBy;
+  /** Return raw result (CategoryResponseRaw). */
+  rawResult?: boolean;
 }
 
 /**
@@ -205,8 +220,17 @@ export type FeaturesResponse = {rows: Record<string, unknown>[]};
 /** Response from {@link WidgetRemoteSource#getFormula}. */
 export type FormulaResponse = {value: number | null};
 
+/** Entry in the category widget response, see  {@link WidgetRemoteSource#getCategories}. */
+export type CategoryResponseEntry = {name: string | null; value: number};
 /** Response from {@link WidgetRemoteSource#getCategories}. */
-export type CategoryResponse = {name: string; value: number}[];
+export type CategoryResponse = CategoryResponseEntry[];
+
+export type CategoryResponseRaw = {
+  rows: CategoryResponseEntry[] | null;
+  metadata?: {
+    others?: number;
+  };
+};
 
 /** Response from {@link WidgetRemoteSource#getRange}. */
 export type RangeResponse = {min: number; max: number} | null;

package/src/widget-sources/widget-remote-source.ts

@@ -23,6 +23,7 @@ import {WidgetSource, type WidgetSourceProps} from './widget-source.js';
 import type {Filters} from '../types.js';
 import {AggregationTypes, ApiVersion} from '../constants.js';
 import {getApplicableFilters} from '../filters.js';
+import {OTHERS_CATEGORY_NAME} from './constants.js';
 
 export type WidgetRemoteSourceProps = WidgetSourceProps;
 
@@ -72,17 +73,23 @@ export abstract class WidgetRemoteSource<
       filterOwner,
       spatialFilter,
       spatialFiltersMode,
+      rawResult,
       ...params
     } = options;
-    const {column, operation, operationColumn, operationExp} = params;
+    const {
+      column,
+      operation,
+      operationColumn,
+      operationExp,
+      othersThreshold,
+      orderBy,
+    } = params;
 
     if (operation === AggregationTypes.Custom) {
       assert(operationExp, 'operationExp is required for custom operation');
     }
 
-    type CategoriesModelResponse = {rows: {name: string; value: number}[]};
-
-    return executeModel({
+    const result = await executeModel({
       model: 'category',
       source: {
         ...this.getModelSource(filters, filterOwner),
@@ -94,9 +101,25 @@ export abstract class WidgetRemoteSource<
         operation,
         operationExp,
         operationColumn: operationColumn || column,
+        othersThreshold,
+        orderBy,
       },
       opts: {signal, headers: this.props.headers},
-    }).then((res: CategoriesModelResponse) => normalizeObjectKeys(res.rows));
+    });
+
+    const normalizedRows = normalizeObjectKeys(result.rows || []);
+    if (rawResult) {
+      return result as unknown as CategoryResponse;
+    }
+
+    if (!othersThreshold) {
+      return normalizedRows;
+    }
+
+    return [
+      ...normalizedRows,
+      {name: OTHERS_CATEGORY_NAME, value: result?.metadata?.others as number},
+    ];
   }
 
   async getFeatures(