@deck.gl/aggregation-layers 9.0.35 → 9.1.0-beta.1

package/package.json

@@ -3,7 +3,7 @@
   "description": "deck.gl layers that aggregate the input data into alternative representations",
   "license": "MIT",
   "type": "module",
-  "version": "9.0.35",
+  "version": "9.1.0-beta.1",
   "publishConfig": {
     "access": "public"
   },
@@ -38,16 +38,17 @@
     "prepublishOnly": "npm run build-bundle && npm run build-bundle -- --env=dev"
   },
   "dependencies": {
-    "@luma.gl/constants": "~9.0.27",
-    "@luma.gl/shadertools": "~9.0.27",
-    "@math.gl/web-mercator": "^4.0.0",
+    "@luma.gl/constants": "^9.1.0-beta.11",
+    "@luma.gl/shadertools": "^9.1.0-beta.11",
+    "@math.gl/core": "^4.1.0",
+    "@math.gl/web-mercator": "^4.1.0",
     "d3-hexbin": "^0.2.1"
   },
   "peerDependencies": {
-    "@deck.gl/core": "^9.0.0",
-    "@deck.gl/layers": "^9.0.0",
-    "@luma.gl/core": "~9.0.0",
-    "@luma.gl/engine": "~9.0.0"
+    "@deck.gl/core": "9.0.0-alpha.0",
+    "@deck.gl/layers": "9.0.0-alpha.0",
+    "@luma.gl/core": "^9.1.0-beta.11",
+    "@luma.gl/engine": "^9.1.0-beta.11"
   },
-  "gitHead": "8e8fc65d8fa9b9bf7e6d82a7e89d37c6d0126aae"
+  "gitHead": "a86fb154cf323eacc3f56ef087f636fb9f6c30b6"
 }

package/src/common/aggregation-layer.ts

@@ -0,0 +1,107 @@
+// deck.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import {
+  CompositeLayer,
+  LayerDataSource,
+  LayerContext,
+  UpdateParameters,
+  CompositeLayerProps,
+  Attribute,
+  AttributeManager
+} from '@deck.gl/core';
+import {Aggregator} from './aggregator/aggregator';
+
+export type AggregationLayerProps<DataT> = CompositeLayerProps & {
+  data: LayerDataSource<DataT>;
+};
+
+export default abstract class AggregationLayer<
+  DataT,
+  ExtraPropsT extends {} = {}
+> extends CompositeLayer<Required<AggregationLayer<DataT>> & ExtraPropsT> {
+  static layerName = 'AggregationLayer';
+
+  state!: {
+    aggregatorType: string;
+    aggregator: Aggregator;
+  };
+
+  /** Allow this layer to participates in the draw cycle */
+  get isDrawable() {
+    return true;
+  }
+
+  abstract getAggregatorType(): string;
+  /** Called to create an Aggregator instance */
+  abstract createAggregator(type: string): Aggregator;
+  /** Called when some attributes change, a chance to mark Aggregator as dirty */
+  abstract onAttributeChange(id: string): void;
+
+  initializeState(): void {
+    this.getAttributeManager()!.remove(['instancePickingColors']);
+  }
+
+  // Extend Layer.updateState to update the Aggregator instance
+  // returns true if aggregator is changed
+  updateState(params: UpdateParameters<this>): boolean {
+    super.updateState(params);
+
+    const aggregatorType = this.getAggregatorType();
+    if (params.changeFlags.extensionsChanged || this.state.aggregatorType !== aggregatorType) {
+      this.state.aggregator?.destroy();
+      const aggregator = this.createAggregator(aggregatorType);
+      aggregator.setProps({
+        attributes: this.getAttributeManager()?.attributes
+      });
+      this.setState({aggregator, aggregatorType});
+      return true;
+    }
+    return false;
+  }
+
+  // Override Layer.finalizeState to dispose the Aggregator instance
+  finalizeState(context: LayerContext) {
+    super.finalizeState(context);
+    this.state.aggregator.destroy();
+  }
+
+  // Override Layer.updateAttributes to update the aggregator
+  protected updateAttributes(changedAttributes: {[id: string]: Attribute}) {
+    const {aggregator} = this.state;
+    aggregator.setProps({
+      attributes: changedAttributes
+    });
+
+    for (const id in changedAttributes) {
+      this.onAttributeChange(id);
+    }
+
+    // In aggregator.update() the aggregator allocates the buffers to store its output
+    // These buffers will be exposed by aggregator.getResults() and passed to the sublayers
+    // Therefore update() must be called before renderLayers()
+    // CPUAggregator's output is populated right here in update()
+    // GPUAggregator's output is pre-allocated and populated in preDraw(), see comments below
+    aggregator.update();
+  }
+
+  draw({shaderModuleProps}) {
+    // GPU aggregation needs `shaderModuleProps` for projection/filter uniforms which are only accessible at draw time
+    // GPUAggregator's Buffers are pre-allocated during `update()` and passed down to the sublayer attributes in renderLayers()
+    // Although the Buffers have been bound to the sublayer's Model, their content are not populated yet
+    // GPUAggregator.preDraw() is called in the draw cycle here right before Buffers are used by sublayer.draw()
+    const {aggregator} = this.state;
+    // @ts-expect-error only used by GPU aggregators
+    aggregator.setProps({shaderModuleProps});
+    aggregator.preDraw();
+  }
+
+  // override CompositeLayer._getAttributeManager to create AttributeManager instance
+  _getAttributeManager() {
+    return new AttributeManager(this.context.device, {
+      id: this.props.id,
+      stats: this.context.stats
+    });
+  }
+}

package/src/common/aggregator/aggregator.ts

@@ -0,0 +1,88 @@
+// deck.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {Attribute, BinaryAttribute} from '@deck.gl/core';
+
+/** Method used to reduce a list of values to one number */
+export type AggregationOperation = 'SUM' | 'MEAN' | 'MIN' | 'MAX' | 'COUNT';
+
+/** Baseline inputs to an Aggregator */
+export type AggregationProps = {
+  /** Number of data points */
+  pointCount: number;
+  /** The input data */
+  attributes: {[id: string]: Attribute};
+  /** How to aggregate the values inside a bin, defined for each channel */
+  operations: AggregationOperation[];
+  /** Additional options to control bin sorting, e.g. bin size */
+  binOptions: Record<string, number | number[]>;
+  /** Callback after a channel is updated */
+  onUpdate?: (event: {channel: number}) => void;
+};
+
+/** Descriptor of an aggregated bin */
+export type AggregatedBin = {
+  /** The unique identifier of the bin */
+  id: number[];
+  /** Aggregated values by channel */
+  value: number[];
+  /** Count of data points in this bin */
+  count: number;
+  /** Indices of data points in this bin. Only available if using CPU aggregation. */
+  pointIndices?: number[];
+};
+
+/**
+ * The Aggregator interface describes a class that performs aggregation.
+ *
+ * _Aggregation_ is a 2-step process:
+ * 1. Sort: Group a collection of _data points_ by some property into _bins_.
+ * 2. Aggregate: for each _bin_, calculate a numeric output (_result_) from some metrics (_values_) from all its members.
+ *    Multiple results can be obtained independently (_channels_).
+ *
+ * An implementation of the _Aggregator_ interface takes the following inputs:
+ * - The number of data points
+ * - The group that each data point belongs to, by mapping each data point to a _binId_ (integer or array of integers)
+ * - The values to aggregate, by mapping each data point in each channel to one _value_ (number)
+ * - The method (_operation_) to reduce a list of values to one number, such as SUM
+ *
+ * And yields the following outputs:
+ * - A list of _binId_ that data points get sorted into
+ * - The aggregated values (_result_) as a list of numbers, comprised of one number per bin per channel
+ * - The [min, max] among all aggregated values (_domain_) for each channel
+ *
+ */
+export interface Aggregator {
+  /** Update aggregation props */
+  setProps(props: Partial<AggregationProps>): void;
+
+  /** Flags a channel to need update
+   * @param {number} channel - mark the given channel as dirty. If not provided, all channels will be updated.
+   */
+  setNeedsUpdate(channel?: number): void;
+
+  /** Called after props are set and before results are accessed */
+  update(): void;
+
+  /** Called before layer is drawn to screen. */
+  preDraw(): void;
+
+  /** Dispose all allocated resources */
+  destroy(): void;
+
+  /** Get the number of bins */
+  get binCount(): number;
+
+  /** Returns an accessor to the bins. */
+  getBins(): BinaryAttribute | null;
+
+  /** Returns an accessor to the output for a given channel. */
+  getResult(channel: number): BinaryAttribute | null;
+
+  /** Returns the [min, max] of aggregated values for a given channel. */
+  getResultDomain(channel: number): [min: number, max: number];
+
+  /** Returns the information for a given bin. */
+  getBin(index: number): AggregatedBin | null;
+}

package/src/common/aggregator/cpu-aggregator/aggregate.ts

@@ -0,0 +1,101 @@
+// deck.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {Bin} from './cpu-aggregator';
+import type {AggregationOperation} from '../aggregator';
+
+/** A reducer function that takes a list of data points and outputs one measurement */
+export type AggregationFunc = (
+  /** Indices of the points */
+  pointIndices: number[],
+  /** Accessor to the value for each point */
+  getValue: (index: number) => number
+) => number;
+
+const count: AggregationFunc = pointIndices => {
+  return pointIndices.length;
+};
+
+const sum: AggregationFunc = (pointIndices, getValue) => {
+  let result = 0;
+  for (const i of pointIndices) {
+    result += getValue(i);
+  }
+  return result;
+};
+
+const mean: AggregationFunc = (pointIndices, getValue) => {
+  if (pointIndices.length === 0) {
+    return NaN;
+  }
+  return sum(pointIndices, getValue) / pointIndices.length;
+};
+
+const min: AggregationFunc = (pointIndices, getValue) => {
+  let result = Infinity;
+  for (const i of pointIndices) {
+    const value = getValue(i);
+    if (value < result) {
+      result = value;
+    }
+  }
+  return result;
+};
+
+const max: AggregationFunc = (pointIndices, getValue) => {
+  let result = -Infinity;
+  for (const i of pointIndices) {
+    const value = getValue(i);
+    if (value > result) {
+      result = value;
+    }
+  }
+  return result;
+};
+
+export const BUILT_IN_OPERATIONS: Record<AggregationOperation, AggregationFunc> = {
+  COUNT: count,
+  SUM: sum,
+  MEAN: mean,
+  MIN: min,
+  MAX: max
+} as const;
+
+/**
+ * Performs the aggregation step. See interface Aggregator comments.
+ * @returns Floa32Array of aggregated values, one for each bin, and the [min,max] of the values
+ */
+export function aggregate({
+  bins,
+  getValue,
+  operation,
+  target
+}: {
+  /** Data points sorted by bins */
+  bins: Bin[];
+  /** Given the index of a data point, returns its value */
+  getValue: (index: number) => number;
+  /** Method used to reduce a list of values to one number */
+  operation: AggregationFunc;
+  /** Array to write the output into */
+  target?: Float32Array | null;
+}): {
+  value: Float32Array;
+  domain: [min: number, max: number];
+} {
+  if (!target || target.length < bins.length) {
+    target = new Float32Array(bins.length);
+  }
+  let min = Infinity;
+  let max = -Infinity;
+
+  for (let j = 0; j < bins.length; j++) {
+    const {points} = bins[j];
+    target[j] = operation(points, getValue);
+    if (target[j] < min) min = target[j];
+    if (target[j] > max) max = target[j];
+  }
+
+  return {value: target, domain: [min, max]};
+}

package/src/common/aggregator/cpu-aggregator/cpu-aggregator.ts

@@ -0,0 +1,203 @@
+// deck.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {Aggregator, AggregationProps, AggregatedBin} from '../aggregator';
+import {_deepEqual as deepEqual, BinaryAttribute} from '@deck.gl/core';
+import {sortBins, packBinIds} from './sort-bins';
+import {aggregate, AggregationFunc, BUILT_IN_OPERATIONS} from './aggregate';
+import {VertexAccessor, evaluateVertexAccessor} from './vertex-accessor';
+
+/** Options used to construct a new CPUAggregator */
+export type CPUAggregatorProps = {
+  /** Size of bin IDs */
+  dimensions: number;
+  /** Accessor to map each data point to a bin ID.
+   * Bin ID should be an array with [dimensions] elements; or null if the data point should be skipped
+   */
+  getBin: VertexAccessor<number[] | null, any>;
+  /** Accessor to map each data point to a weight value, defined per channel */
+  getValue: VertexAccessor<number>[];
+} & Partial<CPUAggregationProps>;
+
+/** Props used to run CPU aggregation, can be changed at any time */
+type CPUAggregationProps = AggregationProps & {
+  /** Custom callback to aggregate points, overrides the built-in operations */
+  customOperations: (AggregationFunc | null | undefined)[];
+};
+
+export type Bin = {
+  id: number[];
+  index: number;
+  /** list of data point indices */
+  points: number[];
+};
+
+/** An Aggregator implementation that calculates aggregation on the CPU */
+export class CPUAggregator implements Aggregator {
+  readonly dimensions: number;
+  readonly channelCount: number;
+
+  props: CPUAggregatorProps & CPUAggregationProps;
+
+  /** Dirty flag
+   * If true, redo sorting
+   * If array, redo aggregation on the specified channel
+   */
+  protected needsUpdate: boolean[] | boolean;
+
+  protected bins: Bin[] = [];
+  protected binIds: (BinaryAttribute & {value: Float32Array}) | null = null;
+  protected results: (BinaryAttribute & {
+    value: Float32Array;
+    domain: [min: number, max: number];
+  })[] = [];
+
+  constructor(props: CPUAggregatorProps) {
+    this.dimensions = props.dimensions;
+    this.channelCount = props.getValue.length;
+    this.props = {
+      ...props,
+      binOptions: {},
+      pointCount: 0,
+      operations: [],
+      customOperations: [],
+      attributes: {}
+    };
+    this.needsUpdate = true;
+    this.setProps(props);
+  }
+
+  destroy() {}
+
+  get binCount() {
+    return this.bins.length;
+  }
+
+  /** Update aggregation props */
+  setProps(props: Partial<CPUAggregationProps>) {
+    const oldProps = this.props;
+
+    if (props.binOptions) {
+      if (!deepEqual(props.binOptions, oldProps.binOptions, 2)) {
+        this.setNeedsUpdate();
+      }
+    }
+    if (props.operations) {
+      for (let channel = 0; channel < this.channelCount; channel++) {
+        if (props.operations[channel] !== oldProps.operations[channel]) {
+          this.setNeedsUpdate(channel);
+        }
+      }
+    }
+    if (props.customOperations) {
+      for (let channel = 0; channel < this.channelCount; channel++) {
+        if (
+          Boolean(props.customOperations[channel]) !== Boolean(oldProps.customOperations[channel])
+        ) {
+          this.setNeedsUpdate(channel);
+        }
+      }
+    }
+    if (props.pointCount !== undefined && props.pointCount !== oldProps.pointCount) {
+      this.setNeedsUpdate();
+    }
+    if (props.attributes) {
+      props.attributes = {...oldProps.attributes, ...props.attributes};
+    }
+    Object.assign(this.props, props);
+  }
+
+  /** Flags a channel to need update
+   * This is called internally by setProps() if certain props change
+   * Users of this class still need to manually set the dirty flag sometimes, because even if no props changed
+   * the underlying buffers could have been updated and require rerunning the aggregation
+   * @param {number} channel - mark the given channel as dirty. If not provided, all channels will be updated.
+   */
+  setNeedsUpdate(channel?: number): void {
+    if (channel === undefined) {
+      this.needsUpdate = true;
+    } else if (this.needsUpdate !== true) {
+      this.needsUpdate = this.needsUpdate || [];
+      this.needsUpdate[channel] = true;
+    }
+  }
+
+  /** Run aggregation */
+  update() {
+    if (this.needsUpdate === true) {
+      this.bins = sortBins({
+        pointCount: this.props.pointCount,
+        getBinId: evaluateVertexAccessor(
+          this.props.getBin,
+          this.props.attributes,
+          this.props.binOptions
+        )
+      });
+      const value = packBinIds({
+        bins: this.bins,
+        dimensions: this.dimensions,
+        // Reuse allocated typed array
+        target: this.binIds?.value
+      });
+      this.binIds = {value, type: 'float32', size: this.dimensions};
+    }
+    for (let channel = 0; channel < this.channelCount; channel++) {
+      if (this.needsUpdate === true || this.needsUpdate[channel]) {
+        const operation =
+          this.props.customOperations[channel] ||
+          BUILT_IN_OPERATIONS[this.props.operations[channel]];
+        const {value, domain} = aggregate({
+          bins: this.bins,
+          getValue: evaluateVertexAccessor(
+            this.props.getValue[channel],
+            this.props.attributes,
+            undefined
+          ),
+          operation,
+          // Reuse allocated typed array
+          target: this.results[channel]?.value
+        });
+        this.results[channel] = {value, domain, type: 'float32', size: 1};
+        this.props.onUpdate?.({channel});
+      }
+    }
+    this.needsUpdate = false;
+  }
+
+  preDraw() {}
+
+  /** Returns an accessor to the bins. */
+  getBins(): BinaryAttribute | null {
+    return this.binIds;
+  }
+
+  /** Returns an accessor to the output for a given channel. */
+  getResult(channel: number): BinaryAttribute | null {
+    return this.results[channel];
+  }
+
+  /** Returns the [min, max] of aggregated values for a given channel. */
+  getResultDomain(channel: number): [min: number, max: number] {
+    return this.results[channel]?.domain ?? [Infinity, -Infinity];
+  }
+
+  /** Returns the information for a given bin. */
+  getBin(index: number): AggregatedBin | null {
+    const bin = this.bins[index];
+    if (!bin) {
+      return null;
+    }
+    const value = new Array(this.channelCount);
+    for (let i = 0; i < value.length; i++) {
+      const result = this.results[i];
+      value[i] = result?.value[index];
+    }
+    return {
+      id: bin.id,
+      value,
+      count: bin.points.length,
+      pointIndices: bin.points
+    };
+  }
+}

package/src/common/aggregator/cpu-aggregator/sort-bins.ts

@@ -0,0 +1,63 @@
+// deck.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {Bin} from './cpu-aggregator';
+
+/** Group data points into bins */
+export function sortBins({
+  pointCount,
+  getBinId
+}: {
+  pointCount: number;
+  getBinId: (index: number) => number[] | null;
+}): Bin[] {
+  const binsById: Map<string, Bin> = new Map();
+
+  for (let i = 0; i < pointCount; i++) {
+    const id = getBinId(i);
+    if (id === null) {
+      continue;
+    }
+    let bin = binsById.get(String(id));
+    if (bin) {
+      bin.points.push(i);
+    } else {
+      bin = {
+        id,
+        index: binsById.size,
+        points: [i]
+      };
+      binsById.set(String(id), bin);
+    }
+  }
+  return Array.from(binsById.values());
+}
+
+/** Pack bin ids into a typed array */
+export function packBinIds({
+  bins,
+  dimensions,
+  target
+}: {
+  bins: Bin[];
+  /** Size of bin IDs */
+  dimensions: number;
+  /** Array to write output into */
+  target?: Float32Array | null;
+}): Float32Array {
+  const targetLength = bins.length * dimensions;
+  if (!target || target.length < targetLength) {
+    target = new Float32Array(targetLength);
+  }
+
+  for (let i = 0; i < bins.length; i++) {
+    const {id} = bins[i];
+    if (Array.isArray(id)) {
+      target.set(id, i * dimensions);
+    } else {
+      target[i] = id;
+    }
+  }
+  return target;
+}

package/src/common/aggregator/cpu-aggregator/vertex-accessor.ts

@@ -0,0 +1,86 @@
+// deck.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {Attribute} from '@deck.gl/core';
+import type {TypedArray} from '@luma.gl/core';
+
+/** This is designed to mirror a vertex shader function
+ * For each vertex, calculates a value from attribtes, vertex index and options (uniforms)
+ */
+export type VertexAccessor<ValueT, OptionsT = undefined> = {
+  /** Attribute ids that provide input to getValue, used to index into the attributes map.
+   * For example `['position', 'size']`
+   */
+  sources?: string[];
+  /** Called for each data point to retrieve a value during update. */
+  getValue: (
+    /** Attributes at the vertex index */
+    data: any,
+    /** Vertex index */
+    index: number,
+    /** Shared options across all vertices */
+    options: OptionsT
+  ) => ValueT;
+};
+
+/** Evaluate a VertexAccessor with a set of attributes */
+export function evaluateVertexAccessor<ValueT, OptionsT>(
+  accessor: VertexAccessor<ValueT, OptionsT>,
+  attributes: Record<string, Attribute>,
+  options: OptionsT
+): (vertexIndex: number) => ValueT {
+  const vertexReaders: {[id: string]: (i: number) => number | number[]} = {};
+  for (const id of accessor.sources || []) {
+    const attribute = attributes[id];
+    if (attribute) {
+      vertexReaders[id] = getVertexReader(attribute);
+    } else {
+      throw new Error(`Cannot find attribute ${id}`);
+    }
+  }
+  const data: {[id: string]: number | number[]} = {};
+
+  return (vertexIndex: number) => {
+    for (const id in vertexReaders) {
+      data[id] = vertexReaders[id](vertexIndex);
+    }
+    return accessor.getValue(data, vertexIndex, options);
+  };
+}
+
+/** Read value out of a deck.gl Attribute by vertex */
+function getVertexReader(attribute: Attribute): (vertexIndex: number) => number | number[] {
+  const value = attribute.value as TypedArray;
+  const {offset = 0, stride, size} = attribute.getAccessor();
+  const bytesPerElement = value.BYTES_PER_ELEMENT;
+  const elementOffset = offset / bytesPerElement;
+  const elementStride = stride ? stride / bytesPerElement : size;
+
+  if (size === 1) {
+    // Size 1, returns (i: number) => number
+    if (attribute.isConstant) {
+      return () => value[0];
+    }
+    return (vertexIndex: number) => {
+      const i = elementOffset + elementStride * vertexIndex;
+      return value[i];
+    };
+  }
+
+  // Size >1, returns (i: number) => number[]
+  let result: number[];
+  if (attribute.isConstant) {
+    result = Array.from(value);
+    return () => result;
+  }
+
+  result = new Array(size);
+  return (vertexIndex: number) => {
+    const i = elementOffset + elementStride * vertexIndex;
+    for (let j = 0; j < size; j++) {
+      result[j] = value[i + j];
+    }
+    return result;
+  };
+}