@deck.gl/aggregation-layers 9.0.35 → 9.1.0-beta.1

package/src/common/aggregator/gpu-aggregator/webgl-bin-sorter.ts

@@ -0,0 +1,258 @@
+// deck.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import {Model, ModelProps} from '@luma.gl/engine';
+import {createRenderTarget} from './utils';
+
+import type {Device, Framebuffer, Texture} from '@luma.gl/core';
+import type {WebGLAggregatorProps} from './webgl-aggregator';
+import type {AggregationOperation} from '../aggregator';
+import {BinSorterProps, binSorterUniforms} from './bin-sorter-uniforms';
+
+const COLOR_CHANNELS = [0x1, 0x2, 0x4, 0x8]; // GPU color mask RED, GREEN, BLUE, ALPHA
+const MAX_FLOAT32 = 3e38;
+const EMPTY_MASKS = {SUM: 0, MEAN: 0, MIN: 0, MAX: 0, COUNT: 0};
+
+export const TEXTURE_WIDTH = 1024;
+
+/**
+ * This class manages the resources for performing the first step of aggregation
+ * Sort a list of data points into a number of bins
+ */
+export class WebGLBinSorter {
+  device: Device;
+  model: Model;
+
+  /**
+   * A packed texture in which each pixel represents a bin.
+   * The index of the pixel in the memory layout is the bin index.
+   * Alpha value is the count of data points that fall into this bin
+   * R,G,B values are the aggregated values of each channel:
+   *   - Sum of all data points if operation is 'SUM', or 'MEAN'
+   *   - Min of all data points if operation is 'MIN'
+   *   - Max of all data points if operation is 'MAX'
+   */
+  private binsFBO: Framebuffer | null = null;
+
+  constructor(device: Device, props: WebGLAggregatorProps) {
+    this.device = device;
+    this.model = createModel(device, props);
+  }
+
+  get texture(): Texture | null {
+    return this.binsFBO ? this.binsFBO.colorAttachments[0].texture : null;
+  }
+
+  destroy() {
+    this.model.destroy();
+    this.binsFBO?.colorAttachments[0].texture.destroy();
+    this.binsFBO?.destroy();
+  }
+
+  getBinValues(index: number): Float32Array | null {
+    if (!this.binsFBO) {
+      return null;
+    }
+    const x = index % TEXTURE_WIDTH;
+    const y = Math.floor(index / TEXTURE_WIDTH);
+    const buffer = this.device.readPixelsToArrayWebGL(this.binsFBO, {
+      sourceX: x,
+      sourceY: y,
+      sourceWidth: 1,
+      sourceHeight: 1
+    }).buffer;
+    return new Float32Array(buffer);
+  }
+
+  setDimensions(binCount: number, binIdRange: [number, number][]) {
+    const width = TEXTURE_WIDTH;
+    const height = Math.ceil(binCount / width);
+
+    // Only destroy existing texture if it is not large enough
+    if (!this.binsFBO) {
+      this.binsFBO = createRenderTarget(this.device, width, height);
+    } else if (this.binsFBO.height < height) {
+      this.binsFBO.resize({width, height});
+    }
+
+    const binSorterProps: BinSorterProps = {
+      binIdRange: [
+        binIdRange[0][0],
+        binIdRange[0][1],
+        binIdRange[1]?.[0] || 0,
+        binIdRange[1]?.[1] || 0
+      ],
+      targetSize: [this.binsFBO.width, this.binsFBO.height]
+    };
+    this.model.shaderInputs.setProps({binSorter: binSorterProps});
+  }
+
+  setModelProps(
+    props: Pick<ModelProps, 'vertexCount' | 'uniforms' | 'attributes' | 'constantAttributes'> & {
+      shaderModuleProps?: Record<string, any>;
+    }
+  ) {
+    const model = this.model;
+    if (props.attributes) {
+      model.setAttributes(props.attributes);
+    }
+    if (props.constantAttributes) {
+      model.setConstantAttributes(props.constantAttributes);
+    }
+    if (props.vertexCount !== undefined) {
+      model.setVertexCount(props.vertexCount);
+    }
+    if (props.shaderModuleProps) {
+      model.shaderInputs.setProps(props.shaderModuleProps);
+    }
+  }
+
+  /** Update aggregation */
+  update(
+    /** The aggregation operation for each channel. Use null to skip update. */
+    operations: (AggregationOperation | null)[]
+  ) {
+    if (!this.binsFBO) {
+      return;
+    }
+    const masks = getMaskByOperation(operations);
+    this._updateBins('SUM', masks.SUM + masks.MEAN);
+    this._updateBins('MIN', masks.MIN);
+    this._updateBins('MAX', masks.MAX);
+  }
+
+  /** Recalculate aggregation on the given channels using the given operation */
+  private _updateBins(
+    operation: AggregationOperation,
+    /** GPU bit mask of one or more channels that should be updated */
+    colorMask: number
+  ) {
+    if (colorMask === 0) {
+      return;
+    }
+    colorMask |= COLOR_CHANNELS[3]; // Also renders to the alpha channel (point count)
+
+    const model = this.model;
+    const target = this.binsFBO!;
+
+    const initialValue = operation === 'MAX' ? -MAX_FLOAT32 : operation === 'MIN' ? MAX_FLOAT32 : 0;
+    const renderPass = this.device.beginRenderPass({
+      id: `gpu-aggregation-${operation}`,
+      framebuffer: target,
+      parameters: {
+        viewport: [0, 0, target.width, target.height],
+        colorMask
+      },
+      clearColor: [initialValue, initialValue, initialValue, 0],
+      clearDepth: false,
+      clearStencil: false
+    });
+    model.setParameters({
+      blend: true,
+      blendColorSrcFactor: 'one',
+      blendColorDstFactor: 'one',
+      blendAlphaSrcFactor: 'one',
+      blendAlphaDstFactor: 'one',
+      blendColorOperation: operation === 'MAX' ? 'max' : operation === 'MIN' ? 'min' : 'add',
+      blendAlphaOperation: 'add'
+    });
+    model.draw(renderPass);
+    renderPass.end();
+  }
+}
+
+/** Convert a [channel -> operation] map to a [operation -> GPU color mask] map */
+function getMaskByOperation(
+  operations: (AggregationOperation | null)[]
+): Record<AggregationOperation, number> {
+  const result: Record<AggregationOperation, number> = {...EMPTY_MASKS};
+  for (let channel = 0; channel < operations.length; channel++) {
+    const op = operations[channel];
+    if (op) {
+      result[op] += COLOR_CHANNELS[channel];
+    }
+  }
+  return result;
+}
+
+function createModel(device: Device, props: WebGLAggregatorProps): Model {
+  let userVs = props.vs;
+
+  if (props.dimensions === 2) {
+    // If user provides 2d bin IDs, convert them to 1d indices for data packing
+    userVs += /* glsl */ `
+void getBin(out int binId) {
+  ivec2 binId2;
+  getBin(binId2);
+  if (binId2.x < binSorter.binIdRange.x || binId2.x >= binSorter.binIdRange.y) {
+    binId = -1;
+  } else {
+    binId = (binId2.y - binSorter.binIdRange.z) * (binSorter.binIdRange.y - binSorter.binIdRange.x) + binId2.x;
+  }
+}
+`;
+  }
+
+  const vs = `\
+#version 300 es
+#define SHADER_NAME gpu-aggregation-sort-bins-vertex
+
+${userVs}
+
+out vec3 v_Value;
+
+void main() {
+  int binIndex;
+  getBin(binIndex);
+  binIndex = binIndex - binSorter.binIdRange.x;
+  if (binIndex < 0) {
+    gl_Position = vec4(0.);
+    return;
+  }
+  int row = binIndex / binSorter.targetSize.x;
+  int col = binIndex - row * binSorter.targetSize.x;
+  vec2 position = (vec2(col, row) + 0.5) / vec2(binSorter.targetSize) * 2.0 - 1.0;
+  gl_Position = vec4(position, 0.0, 1.0);
+  gl_PointSize = 1.0;
+
+#if NUM_CHANNELS == 3
+  getValue(v_Value);
+#elif NUM_CHANNELS == 2
+  getValue(v_Value.xy);
+#else
+  getValue(v_Value.x);
+#endif
+}
+`;
+  const fs = /* glsl */ `\
+#version 300 es
+#define SHADER_NAME gpu-aggregation-sort-bins-fragment
+
+precision highp float;
+
+in vec3 v_Value;
+out vec4 fragColor;
+
+void main() {
+  fragColor.xyz = v_Value;
+
+  #ifdef MODULE_GEOMETRY
+  geometry.uv = vec2(0.);
+  DECKGL_FILTER_COLOR(fragColor, geometry);
+  #endif
+
+  fragColor.w = 1.0;
+}
+`;
+  const model = new Model(device, {
+    bufferLayout: props.bufferLayout,
+    modules: [...(props.modules || []), binSorterUniforms],
+    defines: {...props.defines, NON_INSTANCED_MODEL: 1, NUM_CHANNELS: props.channelCount},
+    isInstanced: false,
+    vs,
+    fs,
+    topology: 'point-list'
+  });
+  return model;
+}

package/src/common/aggregator/index.ts

@@ -0,0 +1,10 @@
+// deck.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+export {CPUAggregator} from './cpu-aggregator/cpu-aggregator';
+export {WebGLAggregator} from './gpu-aggregator/webgl-aggregator';
+
+export type {Aggregator, AggregatedBin, AggregationOperation, AggregationProps} from './aggregator';
+export type {CPUAggregatorProps} from './cpu-aggregator/cpu-aggregator';
+export type {WebGLAggregatorProps} from './gpu-aggregator/webgl-aggregator';

package/src/{types.ts → common/types.ts}

@@ -1,11 +1,17 @@
+// deck.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
 /** Type assigned to e.g. getColorValue/getElevationValue props in aggregation layers (e.g. CPUGridLayer, HexagonLayer)*/
-export type AggregateAccessor<DataT = any> = (args: {
+export type AggregateAccessor<DataT = any> = (
   /** a list of objects whose positions fall inside this cell. */
-  objects: DataT[];
+  objects: DataT[],
   objectInfo: {
     /** the indices of `objects` in the original data. */
     indices: number[];
     /** the value of the `data` prop */
     data: any;
-  };
-}) => number;
+  }
+) => number;
+
+export type ScaleType = 'linear' | 'quantize' | 'quantile' | 'ordinal';

package/src/common/utils/bounds-utils.ts

@@ -0,0 +1,34 @@
+// deck.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+/** Utility to estimate binIdRange as expected by AggregatorProps */
+export function getBinIdRange({
+  dataBounds,
+  getBinId,
+  padding = 0
+}: {
+  /** Bounds of the input data */
+  dataBounds: [min: number[], max: number[]];
+  /** Given a data point, returns the bin id that it belongs to */
+  getBinId: (p: number[]) => number[];
+  /** Add a border around the result to avoid clipping */
+  padding?: number;
+}): [number, number][] {
+  const corners = [
+    dataBounds[0],
+    dataBounds[1],
+    [dataBounds[0][0], dataBounds[1][1]],
+    [dataBounds[1][0], dataBounds[0][1]]
+  ].map(p => getBinId(p));
+
+  const minX = Math.min(...corners.map(p => p[0])) - padding;
+  const minY = Math.min(...corners.map(p => p[1])) - padding;
+  const maxX = Math.max(...corners.map(p => p[0])) + padding + 1;
+  const maxY = Math.max(...corners.map(p => p[1])) + padding + 1;
+
+  return [
+    [minX, maxX],
+    [minY, maxY]
+  ];
+}

package/src/common/utils/color-utils.ts

@@ -0,0 +1,86 @@
+// deck.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {Color} from '@deck.gl/core';
+import type {Device, Texture} from '@luma.gl/core';
+import type {NumericArray, TypedArray, TypedArrayConstructor} from '@math.gl/types';
+import type {ScaleType} from '../types';
+
+export const defaultColorRange: Color[] = [
+  [255, 255, 178],
+  [254, 217, 118],
+  [254, 178, 76],
+  [253, 141, 60],
+  [240, 59, 32],
+  [189, 0, 38]
+];
+
+// Converts a colorRange array to a flat array with 4 components per color
+export function colorRangeToFlatArray(
+  colorRange: Color[] | NumericArray,
+  normalize = false,
+  ArrayType: TypedArrayConstructor = Float32Array
+): TypedArray {
+  let flatArray: TypedArray;
+
+  if (Number.isFinite(colorRange[0])) {
+    // its already a flat array.
+    flatArray = new ArrayType(colorRange as NumericArray);
+  } else {
+    // flatten it
+    flatArray = new ArrayType(colorRange.length * 4);
+    let index = 0;
+
+    for (let i = 0; i < colorRange.length; i++) {
+      const color = colorRange[i];
+      flatArray[index++] = color[0];
+      flatArray[index++] = color[1];
+      flatArray[index++] = color[2];
+      flatArray[index++] = Number.isFinite(color[3]) ? color[3] : 255;
+    }
+  }
+
+  if (normalize) {
+    for (let i = 0; i < flatArray.length; i++) {
+      flatArray[i] /= 255;
+    }
+  }
+  return flatArray;
+}
+
+export const COLOR_RANGE_FILTER: Record<ScaleType, 'linear' | 'nearest'> = {
+  linear: 'linear',
+  quantile: 'nearest',
+  quantize: 'nearest',
+  ordinal: 'nearest'
+} as const;
+
+export function updateColorRangeTexture(texture: Texture, type: ScaleType) {
+  texture.setSampler({
+    minFilter: COLOR_RANGE_FILTER[type],
+    magFilter: COLOR_RANGE_FILTER[type]
+  });
+}
+
+export function createColorRangeTexture(
+  device: Device,
+  colorRange: Color[] | NumericArray,
+  type: ScaleType = 'linear'
+): Texture {
+  const colors = colorRangeToFlatArray(colorRange, false, Uint8Array);
+
+  return device.createTexture({
+    format: 'rgba8unorm',
+    mipmaps: false,
+    sampler: {
+      minFilter: COLOR_RANGE_FILTER[type],
+      magFilter: COLOR_RANGE_FILTER[type],
+      addressModeU: 'clamp-to-edge',
+      addressModeV: 'clamp-to-edge'
+    },
+    data: colors,
+    width: colors.length / 4,
+    height: 1
+  });
+}

package/src/common/utils/prop-utils.ts

@@ -0,0 +1,13 @@
+// deck.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+export function filterProps(props, filterKeys) {
+  const filteredProps = {};
+  for (const key in props) {
+    if (!filterKeys.includes(key)) {
+      filteredProps[key] = props[key];
+    }
+  }
+  return filteredProps;
+}

package/src/common/utils/scale-utils.ts

@@ -0,0 +1,236 @@
+// deck.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {BinaryAttribute} from '@deck.gl/core';
+import type {ScaleType} from '../types';
+
+type ScaleProps = {
+  scaleType: ScaleType;
+  /** Trim the lower end of the domain by this percentile. Set to `0` to disable. */
+  lowerPercentile: number;
+  /** Trim the upper end of the domain by this percentile. Set to `100` to disable. */
+  upperPercentile: number;
+};
+
+/** Applies a scale to BinaryAttribute */
+export class AttributeWithScale {
+  /** Input values accessor. Has either a `value` (CPU aggregation) or a `buffer` (GPU aggregation) */
+  private readonly input: BinaryAttribute;
+  private readonly inputLength: number;
+
+  private props: ScaleProps = {
+    scaleType: 'linear',
+    lowerPercentile: 0,
+    upperPercentile: 100
+  };
+
+  // cached calculations
+  private _percentile?: {attribute: BinaryAttribute; domain: number[]};
+  private _ordinal?: {attribute: BinaryAttribute; domain: number[]};
+
+  /** Output values accessor */
+  attribute: BinaryAttribute;
+  /** [min, max] of attribute values, or null if unknown */
+  domain: [number, number] | null = null;
+  /** Valid domain if lower/upper percentile are defined */
+  cutoff: [number, number] | null = null;
+
+  constructor(input: BinaryAttribute, inputLength: number) {
+    this.input = input;
+    this.inputLength = inputLength;
+    // No processing is needed with the default scale
+    this.attribute = input;
+  }
+
+  private getScalePercentile() {
+    if (!this._percentile) {
+      const value = getAttributeValue(this.input, this.inputLength);
+      this._percentile = applyScaleQuantile(value);
+    }
+    return this._percentile;
+  }
+
+  private getScaleOrdinal() {
+    if (!this._ordinal) {
+      const value = getAttributeValue(this.input, this.inputLength);
+      this._ordinal = applyScaleOrdinal(value);
+    }
+    return this._ordinal;
+  }
+
+  /** Returns the [lowerCutoff, upperCutoff] of scaled values, or null if not applicable */
+  private getCutoff({
+    scaleType,
+    lowerPercentile,
+    upperPercentile
+  }: ScaleProps): [number, number] | null {
+    if (scaleType === 'quantile') {
+      return [lowerPercentile, upperPercentile - 1];
+    }
+
+    if (lowerPercentile > 0 || upperPercentile < 100) {
+      const {domain: thresholds} = this.getScalePercentile();
+      let lowValue = thresholds[Math.floor(lowerPercentile) - 1] ?? -Infinity;
+      let highValue = thresholds[Math.floor(upperPercentile) - 1] ?? Infinity;
+
+      if (scaleType === 'ordinal') {
+        const {domain: sortedUniqueValues} = this.getScaleOrdinal();
+        lowValue = sortedUniqueValues.findIndex(x => x >= lowValue);
+        highValue = sortedUniqueValues.findIndex(x => x > highValue) - 1;
+        if (highValue === -2) {
+          highValue = sortedUniqueValues.length - 1;
+        }
+      }
+      return [lowValue, highValue];
+    }
+
+    return null;
+  }
+
+  update(props: ScaleProps) {
+    const oldProps = this.props;
+
+    if (props.scaleType !== oldProps.scaleType) {
+      switch (props.scaleType) {
+        case 'quantile': {
+          const {attribute} = this.getScalePercentile();
+          this.attribute = attribute;
+          this.domain = [0, 99];
+          break;
+        }
+        case 'ordinal': {
+          const {attribute, domain} = this.getScaleOrdinal();
+          this.attribute = attribute;
+          this.domain = [0, domain.length - 1];
+          break;
+        }
+
+        default:
+          this.attribute = this.input;
+          this.domain = null;
+      }
+    }
+    if (
+      props.scaleType !== oldProps.scaleType ||
+      props.lowerPercentile !== oldProps.lowerPercentile ||
+      props.upperPercentile !== oldProps.upperPercentile
+    ) {
+      this.cutoff = this.getCutoff(props);
+    }
+    this.props = props;
+    return this;
+  }
+}
+
+/**
+ * Transform an array of values to ordinal indices
+ */
+export function applyScaleOrdinal(values: Float32Array): {
+  attribute: BinaryAttribute;
+  domain: number[];
+} {
+  const uniqueValues = new Set<number>();
+  for (const x of values) {
+    if (Number.isFinite(x)) {
+      uniqueValues.add(x);
+    }
+  }
+  const sortedUniqueValues = Array.from(uniqueValues).sort();
+  const domainMap = new Map();
+  for (let i = 0; i < sortedUniqueValues.length; i++) {
+    domainMap.set(sortedUniqueValues[i], i);
+  }
+
+  return {
+    attribute: {
+      value: values.map(x => (Number.isFinite(x) ? domainMap.get(x) : NaN)),
+      type: 'float32',
+      size: 1
+    },
+    domain: sortedUniqueValues
+  };
+}
+
+/**
+ * Transform an array of values to percentiles
+ */
+export function applyScaleQuantile(
+  values: Float32Array,
+  rangeLength = 100
+): {
+  attribute: BinaryAttribute;
+  domain: number[];
+} {
+  const sortedValues = Array.from(values).filter(Number.isFinite).sort(ascending);
+  let i = 0;
+  const n = Math.max(1, rangeLength);
+  const thresholds: number[] = new Array(n - 1);
+  while (++i < n) {
+    thresholds[i - 1] = threshold(sortedValues, i / n);
+  }
+  return {
+    attribute: {
+      value: values.map(x => (Number.isFinite(x) ? bisectRight(thresholds, x) : NaN)),
+      type: 'float32',
+      size: 1
+    },
+    domain: thresholds
+  };
+}
+
+function getAttributeValue(attribute: BinaryAttribute, length: number): Float32Array {
+  const elementStride = (attribute.stride ?? 4) / 4;
+  const elementOffset = (attribute.offset ?? 0) / 4;
+  let value = attribute.value as Float32Array;
+  if (!value) {
+    const bytes = attribute.buffer?.readSyncWebGL(0, elementStride * 4 * length);
+    if (bytes) {
+      value = new Float32Array(bytes.buffer);
+      attribute.value = value;
+    }
+  }
+
+  if (elementStride === 1) {
+    return value.subarray(0, length);
+  }
+  const result = new Float32Array(length);
+  for (let i = 0; i < length; i++) {
+    result[i] = value[i * elementStride + elementOffset];
+  }
+  return result;
+}
+
+function ascending(a: number, b: number): number {
+  return a - b;
+}
+
+function threshold(domain: number[], fraction: number): number {
+  const domainLength = domain.length;
+  if (fraction <= 0 || domainLength < 2) {
+    return domain[0];
+  }
+  if (fraction >= 1) {
+    return domain[domainLength - 1];
+  }
+
+  const domainFraction = (domainLength - 1) * fraction;
+  const lowIndex = Math.floor(domainFraction);
+  const low = domain[lowIndex];
+  const high = domain[lowIndex + 1];
+  return low + (high - low) * (domainFraction - lowIndex);
+}
+
+function bisectRight(a: number[], x: number): number {
+  let lo = 0;
+  let hi = a.length;
+  while (lo < hi) {
+    const mid = (lo + hi) >>> 1;
+    if (a[mid] > x) {
+      hi = mid;
+    } else {
+      lo = mid + 1;
+    }
+  }
+  return lo;
+}

package/src/contour-layer/bin-options-uniforms.ts

@@ -0,0 +1,26 @@
+// deck.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {ShaderModule} from '@luma.gl/shadertools';
+
+const uniformBlock = /* glsl */ `\
+uniform binOptionsUniforms {
+  vec2 cellOriginCommon;
+  vec2 cellSizeCommon;
+} binOptions;
+`;
+
+export type BinOptions = {
+  cellOriginCommon: [number, number];
+  cellSizeCommon: [number, number];
+};
+
+export const binOptionsUniforms = {
+  name: 'binOptions',
+  vs: uniformBlock,
+  uniformTypes: {
+    cellOriginCommon: 'vec2<f32>',
+    cellSizeCommon: 'vec2<f32>'
+  }
+} as const satisfies ShaderModule<BinOptions>;