@carto/api-client 0.4.3 → 0.5.0-alpha.0

package/src/filters/tileFeaturesGeometries.ts

@@ -0,0 +1,444 @@
+import bboxPolygon from '@turf/bbox-polygon';
+import intersects from '@turf/boolean-intersects';
+import booleanWithin from '@turf/boolean-within';
+import intersect from '@turf/intersect';
+import {transformToTileCoords} from '../utils/transformToTileCoords.js';
+import {transformTileCoordsToWGS84} from '../utils/transformTileCoordsToWGS84.js';
+import {TileFormat} from '../constants.js';
+import {
+  BBox,
+  Feature,
+  Geometry,
+  LineString,
+  MultiPolygon,
+  Point,
+  Polygon,
+  Position,
+} from 'geojson';
+import {SpatialFilter, Tile} from '../types.js';
+import {TileFeatureExtractOptions} from './tileFeatures.js';
+import {featureCollection} from '@turf/helpers';
+import {FeatureData} from '../types-internal.js';
+import {
+  BinaryAttribute,
+  BinaryFeature,
+  BinaryGeometryType,
+  BinaryPointFeature,
+  TypedArrayConstructor,
+} from '@loaders.gl/schema';
+
+export const FEATURE_GEOM_PROPERTY = '__geomValue';
+
+type TileMap = Map<unknown, unknown>;
+
+type TileDataInternal = {
+  uniqueId: string | number | undefined;
+  properties: any;
+  numericProps: Record<string, number>;
+};
+
+export function tileFeaturesGeometries({
+  tiles,
+  tileFormat,
+  spatialFilter,
+  uniqueIdProperty,
+  options,
+}: {
+  tiles: Tile[];
+  tileFormat?: TileFormat;
+  spatialFilter: SpatialFilter;
+  uniqueIdProperty?: string;
+  options?: {storeGeometry?: boolean};
+}): FeatureData[] {
+  const map = new Map();
+
+  for (const tile of tiles) {
+    // Discard if it's not a visible tile (only check false value, not undefined)
+    // or tile has not data
+    if (tile.isVisible === false || !tile.data) {
+      continue;
+    }
+
+    const bbox = [
+      tile.bbox.west,
+      tile.bbox.south,
+      tile.bbox.east,
+      tile.bbox.north,
+    ] as BBox;
+    const bboxToGeom = bboxPolygon(bbox);
+    const tileIsFullyVisible = booleanWithin(bboxToGeom, spatialFilter);
+
+    // Clip the geometry to intersect with the tile
+    const spatialFilterFeature: Feature<Polygon | MultiPolygon> = {
+      type: 'Feature',
+      geometry: spatialFilter,
+      properties: {},
+    };
+    const clippedGeometryToIntersect = intersect(
+      featureCollection([bboxToGeom, spatialFilterFeature])
+    );
+
+    if (!clippedGeometryToIntersect) {
+      continue;
+    }
+
+    // We assume that MVT tileFormat uses local coordinates so we transform the geometry to intersect to tile coordinates [0..1],
+    // while in the case of 'geojson' or binary, the geometries are already in WGS84
+    const transformedGeometryToIntersect =
+      tileFormat === TileFormat.MVT
+        ? transformToTileCoords(clippedGeometryToIntersect.geometry, bbox)
+        : clippedGeometryToIntersect.geometry;
+
+    createIndicesForPoints(tile.data.points!);
+
+    calculateFeatures({
+      map,
+      tileIsFullyVisible,
+      geometryIntersection: transformedGeometryToIntersect,
+      data: tile.data.points!,
+      type: 'Point',
+      bbox,
+      tileFormat,
+      uniqueIdProperty,
+      options,
+    });
+    calculateFeatures({
+      map,
+      tileIsFullyVisible,
+      geometryIntersection: transformedGeometryToIntersect,
+      data: tile.data.lines!,
+      type: 'LineString',
+      bbox,
+      tileFormat,
+      uniqueIdProperty,
+      options,
+    });
+    calculateFeatures({
+      map,
+      tileIsFullyVisible,
+      geometryIntersection: transformedGeometryToIntersect,
+      data: tile.data.polygons!,
+      type: 'Polygon',
+      bbox,
+      tileFormat,
+      uniqueIdProperty,
+      options,
+    });
+  }
+  return Array.from(map.values());
+}
+
+function processTileFeatureProperties({
+  map,
+  data,
+  startIndex,
+  endIndex,
+  type,
+  bbox,
+  tileFormat,
+  uniqueIdProperty,
+  storeGeometry,
+  geometryIntersection,
+}: {
+  map: TileMap;
+  data: BinaryFeature;
+  startIndex: number;
+  endIndex: number;
+  type: BinaryGeometryType;
+  bbox: BBox;
+  tileFormat?: TileFormat;
+  uniqueIdProperty?: string;
+  storeGeometry: boolean;
+  geometryIntersection?: Geometry;
+}) {
+  const tileProps = getPropertiesFromTile(data, startIndex);
+  const uniquePropertyValue = getUniquePropertyValue(
+    tileProps,
+    uniqueIdProperty,
+    map
+  );
+
+  if (!uniquePropertyValue || map.has(uniquePropertyValue)) {
+    return;
+  }
+  let geometry: Geometry | null = null;
+
+  // Only calculate geometry if necessary
+  if (storeGeometry || geometryIntersection) {
+    const {positions} = data;
+    const ringCoordinates = getRingCoordinatesFor(
+      startIndex,
+      endIndex,
+      positions
+    );
+    geometry = getFeatureByType(ringCoordinates, type);
+  }
+
+  // If intersection is required, check before proceeding
+  if (
+    geometry &&
+    geometryIntersection &&
+    !intersects(geometry, geometryIntersection)
+  ) {
+    return;
+  }
+
+  const properties = parseProperties(tileProps);
+
+  // Only save geometry if necessary
+  if (storeGeometry && geometry) {
+    properties[FEATURE_GEOM_PROPERTY] =
+      tileFormat === TileFormat.MVT
+        ? transformTileCoordsToWGS84(geometry, bbox)
+        : geometry;
+  }
+  map.set(uniquePropertyValue, properties);
+}
+
+function addIntersectedFeaturesInTile({
+  map,
+  data,
+  geometryIntersection,
+  type,
+  bbox,
+  tileFormat,
+  uniqueIdProperty,
+  options,
+}: {
+  map: TileMap;
+  data: BinaryFeature;
+  geometryIntersection: Geometry;
+  type: BinaryGeometryType;
+  bbox: BBox;
+  tileFormat?: TileFormat;
+  uniqueIdProperty?: string;
+  options?: TileFeatureExtractOptions;
+}) {
+  const indices = getIndices(data);
+  const storeGeometry = options?.storeGeometry || false;
+
+  for (let i = 0; i < indices.length - 1; i++) {
+    const startIndex = indices[i];
+    const endIndex = indices[i + 1];
+    processTileFeatureProperties({
+      map,
+      data,
+      startIndex,
+      endIndex,
+      type,
+      bbox,
+      tileFormat,
+      uniqueIdProperty,
+      storeGeometry,
+      geometryIntersection,
+    });
+  }
+}
+
+function getIndices(data: BinaryFeature) {
+  let indices: BinaryAttribute;
+  switch (data.type) {
+    case 'Point':
+      // @ts-expect-error Missing or changed types?
+      indices = data.pointIndices;
+      break;
+    case 'LineString':
+      indices = data.pathIndices;
+      break;
+    case 'Polygon':
+      indices = data.primitivePolygonIndices;
+      break;
+    default:
+      throw new Error(`Unexpected type, "${(data as BinaryFeature).type}"`);
+  }
+  return indices.value;
+}
+
+function getFeatureId(data: BinaryFeature, startIndex: number) {
+  return data.featureIds.value[startIndex];
+}
+
+function getPropertiesFromTile(data: BinaryFeature, startIndex: number) {
+  const featureId = getFeatureId(data, startIndex);
+  const {properties, numericProps, fields} = data;
+  const result: TileDataInternal = {
+    uniqueId: (fields?.[featureId] as {id: string | number})?.id,
+    properties: properties[featureId],
+    numericProps: {},
+  };
+
+  for (const key in numericProps) {
+    result.numericProps[key] = numericProps[key].value[startIndex];
+  }
+
+  return result;
+}
+
+function parseProperties(tileProps: TileDataInternal) {
+  const {properties, numericProps} = tileProps;
+  return Object.assign({}, properties, numericProps);
+}
+
+function getUniquePropertyValue(
+  tileProps: TileDataInternal,
+  uniqueIdProperty: string | undefined,
+  map: TileMap
+) {
+  if (uniqueIdProperty) {
+    return getValueFromTileProps(tileProps, uniqueIdProperty);
+  }
+
+  if (tileProps.uniqueId) {
+    return tileProps.uniqueId;
+  }
+
+  const artificialId = map.size + 1; // a counter, assumed as a valid new id
+  return (
+    getValueFromTileProps(tileProps, 'cartodb_id') ||
+    getValueFromTileProps(tileProps, 'geoid') ||
+    artificialId
+  );
+}
+
+function getValueFromTileProps(
+  tileProps: TileDataInternal,
+  propertyName: string
+) {
+  const {properties, numericProps} = tileProps;
+  return numericProps[propertyName] || properties[propertyName];
+}
+
+function getFeatureByType(
+  coordinates: Position[],
+  type: BinaryGeometryType
+): Polygon | LineString | Point {
+  switch (type) {
+    case 'Polygon':
+      return {type: 'Polygon', coordinates: [coordinates]};
+    case 'LineString':
+      return {type: 'LineString', coordinates};
+    case 'Point':
+      return {type: 'Point', coordinates: coordinates[0]};
+    default:
+      throw new Error('Invalid geometry type');
+  }
+}
+
+function getRingCoordinatesFor(
+  startIndex: number,
+  endIndex: number,
+  positions: BinaryAttribute
+) {
+  const ringCoordinates = [];
+
+  for (let j = startIndex; j < endIndex; j++) {
+    ringCoordinates.push(
+      Array.from(
+        positions.value.subarray(j * positions.size, (j + 1) * positions.size)
+      )
+    );
+  }
+
+  return ringCoordinates;
+}
+
+function calculateFeatures({
+  map,
+  tileIsFullyVisible,
+  geometryIntersection,
+  data,
+  type,
+  bbox,
+  tileFormat,
+  uniqueIdProperty,
+  options,
+}: {
+  map: TileMap;
+  tileIsFullyVisible: boolean;
+  geometryIntersection: SpatialFilter;
+  data: BinaryFeature;
+  type: BinaryGeometryType;
+  bbox: BBox;
+  tileFormat?: TileFormat;
+  uniqueIdProperty?: string;
+  options?: TileFeatureExtractOptions;
+}) {
+  if (!data?.properties.length) {
+    return;
+  }
+
+  if (tileIsFullyVisible) {
+    addAllFeaturesInTile({
+      map,
+      data,
+      type,
+      bbox,
+      tileFormat,
+      uniqueIdProperty,
+      options,
+    });
+  } else {
+    addIntersectedFeaturesInTile({
+      map,
+      data,
+      geometryIntersection,
+      type,
+      bbox,
+      tileFormat,
+      uniqueIdProperty,
+      options,
+    });
+  }
+}
+
+function addAllFeaturesInTile({
+  map,
+  data,
+  type,
+  bbox,
+  tileFormat,
+  uniqueIdProperty,
+  options,
+}: {
+  map: TileMap;
+  data: BinaryFeature;
+  type: BinaryGeometryType;
+  bbox: BBox;
+  tileFormat?: TileFormat;
+  uniqueIdProperty?: string;
+  options?: TileFeatureExtractOptions;
+}) {
+  const indices = getIndices(data);
+  const storeGeometry = options?.storeGeometry || false;
+  for (let i = 0; i < indices.length - 1; i++) {
+    const startIndex = indices[i];
+    const endIndex = indices[i + 1];
+    processTileFeatureProperties({
+      map,
+      data,
+      startIndex,
+      endIndex,
+      type,
+      bbox,
+      tileFormat,
+      uniqueIdProperty,
+      storeGeometry,
+    });
+  }
+}
+
+function createIndicesForPoints(data: BinaryPointFeature) {
+  const featureIds = data.featureIds.value;
+  const lastFeatureId = featureIds[featureIds.length - 1];
+  const PointIndicesArray = featureIds.constructor as TypedArrayConstructor;
+
+  const pointIndices: BinaryAttribute = {
+    value: new PointIndicesArray(featureIds.length + 1),
+    size: 1,
+  };
+  pointIndices.value.set(featureIds);
+  pointIndices.value.set([lastFeatureId + 1], featureIds.length);
+
+  // @ts-expect-error Missing or changed types?
+  data.pointIndices = pointIndices;
+}

package/src/filters/tileFeaturesSpatialIndex.ts

@@ -0,0 +1,119 @@
+import {SpatialIndex} from '../constants.js';
+import {getResolution as quadbinGetResolution, geometryToCells} from 'quadbin';
+import bboxClip from '@turf/bbox-clip';
+import {SpatialFilter, SpatialIndexTile} from '../types.js';
+import {BBox, Feature} from 'geojson';
+import {getResolution as h3GetResolution, polygonToCells} from 'h3-js';
+import {FeatureData} from '../types-internal.js';
+import {SpatialDataType} from '../sources/types.js';
+
+export type TileFeaturesSpatialIndexOptions = {
+  tiles: SpatialIndexTile[];
+  spatialFilter: SpatialFilter;
+  spatialDataColumn: string;
+  spatialDataType: SpatialDataType;
+};
+
+export function tileFeaturesSpatialIndex({
+  tiles,
+  spatialFilter,
+  spatialDataColumn,
+  spatialDataType,
+}: TileFeaturesSpatialIndexOptions): FeatureData[] {
+  const map = new Map();
+  const spatialIndex = getSpatialIndex(spatialDataType);
+  const resolution = getResolution(tiles, spatialIndex);
+  const spatialIndexIDName = spatialDataColumn
+    ? spatialDataColumn
+    : spatialIndex;
+
+  if (!resolution) {
+    return [];
+  }
+  const cells = getCellsCoverGeometry(spatialFilter, spatialIndex, resolution);
+
+  if (!cells?.length) {
+    return [];
+  }
+
+  // We transform cells to Set to improve the performace
+  const cellsSet = new Set<bigint | string>(cells);
+
+  for (const tile of tiles) {
+    if (tile.isVisible === false || !tile.data) {
+      continue;
+    }
+
+    tile.data.forEach((d: Feature) => {
+      if (cellsSet.has(d.id as bigint | string)) {
+        map.set(d.id, {...d.properties, [spatialIndexIDName]: d.id});
+      }
+    });
+  }
+  return Array.from(map.values());
+}
+
+function getResolution(
+  tiles: SpatialIndexTile[],
+  spatialIndex: SpatialIndex
+): number | undefined {
+  const data = tiles.find((tile) => tile.data?.length)?.data;
+
+  if (!data) {
+    return;
+  }
+
+  if (spatialIndex === SpatialIndex.QUADBIN) {
+    return Number(quadbinGetResolution(data[0].id));
+  }
+
+  if (spatialIndex === SpatialIndex.H3) {
+    return h3GetResolution(data[0].id);
+  }
+}
+
+const bboxWest: BBox = [-180, -90, 0, 90];
+const bboxEast: BBox = [0, -90, 180, 90];
+
+function getCellsCoverGeometry(
+  geometry: SpatialFilter,
+  spatialIndex: SpatialIndex,
+  resolution: number
+) {
+  if (spatialIndex === SpatialIndex.QUADBIN) {
+    // @ts-expect-error TODO: Probably ought to be stricter about number vs. bigint types in this file.
+    return geometryToCells(geometry, resolution);
+  }
+
+  if (spatialIndex === SpatialIndex.H3) {
+    // The current H3 polyfill algorithm can't deal with polygon segments of greater than 180 degrees longitude
+    // so we clip the geometry to be sure that none of them is greater than 180 degrees
+    // https://github.com/uber/h3-js/issues/24#issuecomment-431893796
+    return polygonToCells(
+      bboxClip(geometry, bboxWest).geometry.coordinates as
+        | number[][]
+        | number[][][],
+      resolution,
+      true
+    ).concat(
+      polygonToCells(
+        bboxClip(geometry, bboxEast).geometry.coordinates as
+          | number[][]
+          | number[][][],
+        resolution,
+        true
+      )
+    );
+  }
+}
+
+function getSpatialIndex(spatialDataType: SpatialDataType): SpatialIndex {
+  switch (spatialDataType) {
+    case 'h3':
+      return SpatialIndex.H3;
+    case 'quadbin':
+      return SpatialIndex.QUADBIN;
+    default:
+      throw new Error('Unexpected spatial data type');
+  }
+}

package/src/index.ts

@@ -16,3 +16,9 @@ export {
   query,
   requestWithParameters,
 } from './api/index.js';
+
+// For unit testing only.
+export * from './filters/index.js';
+export * from './operations/index.js';
+export * from './utils/makeIntervalComplete.js';
+export * from './utils/transformToTileCoords.js';

package/src/models/index.ts

@@ -1,3 +1,3 @@
 export {executeModel} from './model.js';
-export type {Model} from './model.js';
+export type {Model, ModelSource} from './model.js';
 export type {ModelRequestOptions} from './common.js';

package/src/models/model.ts

@@ -7,9 +7,10 @@ import {
   SpatialFilter,
 } from '../types.js';
 import {$TODO} from '../types-internal.js';
-import {assert} from '../utils.js';
+import {assert, isPureObject} from '../utils.js';
 import {ModelRequestOptions, makeCall} from './common.js';
 import {ApiVersion} from '../constants.js';
+import {SpatialDataType, SpatialFilterPolyfillMode} from '../sources/types.js';
 
 /** @internalRemarks Source: @carto/react-api */
 const AVAILABLE_MODELS = [
@@ -35,9 +36,14 @@ export interface ModelSource {
   data: string;
   filters?: Record<string, Filter>;
   filtersLogicalOperator?: FilterLogicalOperator;
-  geoColumn?: string;
   spatialFilter?: SpatialFilter;
   queryParameters?: QueryParameters;
+  spatialDataColumn?: string;
+  spatialDataType?: SpatialDataType;
+  spatialFiltersResolution?: number;
+  spatialFiltersMode?: SpatialFilterPolyfillMode;
+  /** original resolution of the spatial index data as stored in the DW */
+  dataResolution?: number;
 }
 
 const {V3} = ApiVersion;
@@ -79,50 +85,51 @@ export function executeModel(props: {
     data,
     filters,
     filtersLogicalOperator = 'and',
-    geoColumn = DEFAULT_GEO_COLUMN,
+    spatialDataType = 'geo',
+    spatialFiltersMode = 'intersects',
+    spatialFiltersResolution = 0,
   } = source;
 
-  const queryParameters = source.queryParameters
-    ? JSON.stringify(source.queryParameters)
-    : '';
-
-  const queryParams: Record<string, string> = {
+  const queryParams: Record<string, unknown> = {
     type,
     client: clientId,
     source: data,
-    params: JSON.stringify(params),
-    queryParameters,
-    filters: JSON.stringify(filters),
+    params,
+    queryParameters: source.queryParameters || '',
+    filters,
     filtersLogicalOperator,
   };
 
+  const spatialDataColumn = source.spatialDataColumn || DEFAULT_GEO_COLUMN;
+
   // Picking Model API requires 'spatialDataColumn'.
   if (model === 'pick') {
-    queryParams.spatialDataColumn = geoColumn;
+    queryParams.spatialDataColumn = spatialDataColumn;
   }
 
-  // API supports multiple filters, we apply it only to geoColumn
+  // API supports multiple filters, we apply it only to spatialDataColumn
   const spatialFilters = source.spatialFilter
-    ? {[geoColumn]: source.spatialFilter}
+    ? {[spatialDataColumn]: source.spatialFilter}
     : undefined;
 
   if (spatialFilters) {
-    queryParams.spatialFilters = JSON.stringify(spatialFilters);
+    queryParams.spatialFilters = spatialFilters;
+    queryParams.spatialDataColumn = spatialDataColumn;
+    queryParams.spatialDataType = spatialDataType;
+  }
+
+  if (spatialDataType !== 'geo') {
+    if (spatialFiltersResolution > 0) {
+      queryParams.spatialFiltersResolution = spatialFiltersResolution;
+    }
+    queryParams.spatialFiltersMode = spatialFiltersMode;
   }
 
   const urlWithSearchParams =
-    url + '?' + new URLSearchParams(queryParams).toString();
+    url + '?' + objectToURLSearchParams(queryParams).toString();
   const isGet = urlWithSearchParams.length <= REQUEST_GET_MAX_URL_LENGTH;
   if (isGet) {
     url = urlWithSearchParams;
-  } else {
-    // undo the JSON.stringify, @TODO find a better pattern
-    queryParams.params = params as $TODO;
-    queryParams.filters = filters as $TODO;
-    queryParams.queryParameters = source.queryParameters as $TODO;
-    if (spatialFilters) {
-      queryParams.spatialFilters = spatialFilters as $TODO;
-    }
   }
   return makeCall({
     url,
@@ -134,3 +141,19 @@ export function executeModel(props: {
     },
   });
 }
+
+function objectToURLSearchParams(object: Record<string, unknown>) {
+  const params = new URLSearchParams();
+  for (const key in object) {
+    if (isPureObject(object[key])) {
+      params.append(key, JSON.stringify(object[key]));
+    } else if (Array.isArray(object[key])) {
+      params.append(key, JSON.stringify(object[key]));
+    } else if (object[key] === null) {
+      params.append(key, 'null');
+    } else if (object[key] !== undefined) {
+      params.append(key, String(object[key]));
+    }
+  }
+  return params;
+}