@luma.gl/engine 9.1.0-alpha.2 → 9.1.0-beta.1

package/src/modules/picking/README.md

@@ -0,0 +1,88 @@
+# Shader Module: Picking
+
+Provides support for color-coding-based picking and highlighting. In particular, supports picking a specific instance in an instanced draw call and highlighting an instance based on its picking color, and correspondingly, supports picking and highlighting groups of primitives with the same picking color in non-instanced draw-calls
+
+Color based picking lets the application draw a primitive with a color that can later be used to index this specific primitive.
+
+Highlighting allows application to specify a picking color corresponding to an object that need to be highlighted and the highlight color to be used.
+
+## Usage
+
+In your vertex shader, your inform the picking module what object we are currently rendering by supplying a picking color, perhaps from an attribute.
+
+```
+in vec3 aPickingColor;
+main() {
+  picking_setColor(aPickingColor);
+  ...
+}
+```
+
+In your fragment shader, you simply apply (call) the `picking_filterColor` filter function at the very end of the shader. This will return the normal color, or the highlight color, or the picking color, as appropriate.
+
+```
+main() {
+  fragColor = ...
+  fragColor = picking_filterColor(color);
+}
+```
+
+If highlighting is not needed, you simply apply (call) the `picking_filterPickingColor` filter function at the very end of the shader. This will return the normal color or the picking color, as appropriate.
+
+```
+main() {
+  fragColor = ...
+  fragColor = picking_filterPickingColor(fragColor);
+}
+```
+
+If additional filters need to be applied on the non-picking color (vertex or highlight color) you can use above functions in following order.
+
+```
+main() {
+  fragColor = ...
+  fragColor = picking_filterHighlightColor(fragColor);
+   ... apply any filters on fragColor ...
+ fragColor = picking_filterPickingColor(fragColor);
+}
+```
+
+## JavaScript Functions
+
+### getUniforms
+
+`getUniforms` takes an object with key/value pairs, returns an object with key/value pairs representing the uniforms that the `picking` module shaders need.
+
+`getUniforms(opts)`
+opts can contain following keys:
+
+- `pickingSelectedColorValid` (_boolean_) - When true current instance picking color is ignored, hence no instance is highlighted.
+- `pickingSelectedColor` (_array_) - Picking color of the currently selected instance.
+- `pickingHighlightColor` (_array_)- Color used to highlight the currently selected instance.
+- `pickingActive`=`false` (_boolean_) - When true, renders the picking colors instead of the normal colors. Normally only used with an off-screen framebuffer during picking. Default value is `false`.
+
+Note that the selected item will be rendered using `pickingHighlightColor`, if blending is enabled for the draw, alpha channel can be used to control the blending result.
+
+## Vertex Shader Functions
+
+### `void picking_setPickingColor(vec3)`
+
+Sets the color that will be returned by the fragment shader if color based picking is enabled. Typically set from a `pickingColor` uniform or a `pickingColors` attribute (e.g. when using instanced rendering, to identify the actual instance that was picked).
+
+## Fragment Shader Functions
+
+### picking_filterPickingColor
+
+If picking active, returns the current vertex's picking color set by `picking_setPickingColor`, otherwise returns its argument unmodified.
+
+`vec4 picking_filterPickingColor(vec4 color)`
+
+### picking_filterHighlightColor
+
+Returns picking highlight color if the pixel belongs to currently selected model, otherwise returns its argument unmodified.
+
+`vec4 picking_filterHighlightColor(vec4 color)`
+
+## Remarks
+
+- It is strongly recommended that `picking_filterPickingColor` is called last in a fragment shader, as the picking color (returned when picking is enabled) must not be modified in any way (and alpha must remain 1) or picking results will not be correct.

package/src/modules/picking/color-picking.ts

@@ -0,0 +1,190 @@
+// luma.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {ShaderModule} from '@luma.gl/shadertools';
+
+import type {PickingProps, PickingUniforms, PickingBindings} from './picking-uniforms';
+import {pickingUniforms, GLSL_UNIFORMS, WGSL_UNIFORMS} from './picking-uniforms';
+
+const source = /* wgsl */ `\
+${WGSL_UNIFORMS}
+`;
+
+const vs = /* glsl */ `\
+${GLSL_UNIFORMS}
+out vec4 picking_vRGBcolor_Avalid;
+
+// Normalize unsigned byte color to 0-1 range
+vec3 picking_normalizeColor(vec3 color) {
+  return picking.useFloatColors > 0.5 ? color : color / 255.0;
+}
+
+// Normalize unsigned byte color to 0-1 range
+vec4 picking_normalizeColor(vec4 color) {
+  return picking.useFloatColors > 0.5 ? color : color / 255.0;
+}
+
+bool picking_isColorZero(vec3 color) {
+  return dot(color, vec3(1.0)) < 0.00001;
+}
+
+bool picking_isColorValid(vec3 color) {
+  return dot(color, vec3(1.0)) > 0.00001;
+}
+
+// Check if this vertex is highlighted 
+bool isVertexHighlighted(vec3 vertexColor) {
+  vec3 highlightedObjectColor = picking_normalizeColor(picking.highlightedObjectColor);
+  return
+    bool(picking.isHighlightActive) && picking_isColorZero(abs(vertexColor - highlightedObjectColor));
+}
+
+// Set the current picking color
+void picking_setPickingColor(vec3 pickingColor) {
+  pickingColor = picking_normalizeColor(pickingColor);
+
+  if (bool(picking.isActive)) {
+    // Use alpha as the validity flag. If pickingColor is [0, 0, 0] fragment is non-pickable
+    picking_vRGBcolor_Avalid.a = float(picking_isColorValid(pickingColor));
+
+    if (!bool(picking.isAttribute)) {
+      // Stores the picking color so that the fragment shader can render it during picking
+      picking_vRGBcolor_Avalid.rgb = pickingColor;
+    }
+  } else {
+    // Do the comparison with selected item color in vertex shader as it should mean fewer compares
+    picking_vRGBcolor_Avalid.a = float(isVertexHighlighted(pickingColor));
+  }
+}
+
+void picking_setObjectIndex(uint objectIndex) {
+  if (bool(picking.isActive)) {
+    uint index = objectIndex;
+    if (picking.indexMode == PICKING_INDEX_MODE_INSTANCE) {
+      index = uint(gl_InstanceID);
+    }
+    picking_vRGBcolor_Avalid.r = float(index % 255) / 255.0;
+    picking_vRGBcolor_Avalid.g = float((index / 255) % 255) / 255.0;
+    picking_vRGBcolor_Avalid.b = float((index / 255 / 255) %255) / 255.0;
+  }
+}
+
+void picking_setPickingAttribute(float value) {
+  if (bool(picking.isAttribute)) {
+    picking_vRGBcolor_Avalid.r = value;
+  }
+}
+
+void picking_setPickingAttribute(vec2 value) {
+  if (bool(picking.isAttribute)) {
+    picking_vRGBcolor_Avalid.rg = value;
+  }
+}
+
+void picking_setPickingAttribute(vec3 value) {
+  if (bool(picking.isAttribute)) {
+    picking_vRGBcolor_Avalid.rgb = value;
+  }
+}
+`;
+
+const fs = /* glsl */ `\
+${GLSL_UNIFORMS}
+
+in vec4 picking_vRGBcolor_Avalid;
+
+/*
+ * Returns highlight color if this item is selected.
+ */
+vec4 picking_filterHighlightColor(vec4 color) {
+  // If we are still picking, we don't highlight
+  if (picking.isActive > 0.5) {
+    return color;
+  }
+
+  bool selected = bool(picking_vRGBcolor_Avalid.a);
+
+  if (selected) {
+    // Blend in highlight color based on its alpha value
+    float highLightAlpha = picking.highlightColor.a;
+    float blendedAlpha = highLightAlpha + color.a * (1.0 - highLightAlpha);
+    float highLightRatio = highLightAlpha / blendedAlpha;
+
+    vec3 blendedRGB = mix(color.rgb, picking.highlightColor.rgb, highLightRatio);
+    return vec4(blendedRGB, blendedAlpha);
+  } else {
+    return color;
+  }
+}
+
+/*
+ * Returns picking color if picking enabled else unmodified argument.
+ */
+vec4 picking_filterPickingColor(vec4 color) {
+  if (bool(picking.isActive)) {
+    if (picking_vRGBcolor_Avalid.a == 0.0) {
+      discard;
+    }
+    return picking_vRGBcolor_Avalid;
+  }
+  return color;
+}
+
+/*
+ * Returns picking color if picking is enabled if not
+ * highlight color if this item is selected, otherwise unmodified argument.
+ */
+vec4 picking_filterColor(vec4 color) {
+  vec4 highlightColor = picking_filterHighlightColor(color);
+  return picking_filterPickingColor(highlightColor);
+}
+`;
+
+/**
+ * Provides support for color-coding-based picking and highlighting.
+ * In particular, supports picking a specific instance in an instanced
+ * draw call and highlighting an instance based on its picking color,
+ * and correspondingly, supports picking and highlighting groups of
+ * primitives with the same picking color in non-instanced draw-calls
+ */
+export const picking = {
+  ...pickingUniforms,
+  name: 'picking',
+  source,
+  vs,
+  fs
+} as const satisfies ShaderModule<PickingProps, PickingUniforms, PickingBindings>;
+
+// function getUniforms(opts: PickingProps = {}, prevUniforms?: PickingUniforms): PickingUniforms {
+//   const uniforms = {} as PickingUniforms;
+
+//   if (opts.highlightedObjectColor === undefined) {
+//     // Unless highlightedObjectColor explicitly null or set, do not update state
+//   } else if (opts.highlightedObjectColor === null) {
+//     uniforms.isHighlightActive = false;
+//   } else {
+//     uniforms.isHighlightActive = true;
+//     const highlightedObjectColor = opts.highlightedObjectColor.slice(0, 3);
+//     uniforms.highlightedObjectColor = highlightedObjectColor;
+//   }
+
+//   if (opts.highlightColor) {
+//     const color = Array.from(opts.highlightColor, x => x / 255);
+//     if (!Number.isFinite(color[3])) {
+//       color[3] = 1;
+//     }
+//     uniforms.highlightColor = color;
+//   }
+
+//   if (opts.isActive !== undefined) {
+//     uniforms.isActive = Boolean(opts.isActive);
+//     uniforms.isAttribute = Boolean(opts.isAttribute);
+//   }
+
+//   if (opts.useFloatColors !== undefined) {
+//     uniforms.useFloatColors = Boolean(opts.useFloatColors);
+//   }
+
+//   return uniforms;
+// }

package/src/modules/picking/index-picking.ts

@@ -0,0 +1,156 @@
+// luma.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {ShaderModule} from '@luma.gl/shadertools';
+
+import type {PickingBindings, PickingProps, PickingUniforms} from './picking-uniforms';
+import {pickingUniforms, GLSL_UNIFORMS, WGSL_UNIFORMS, INVALID_INDEX} from './picking-uniforms';
+
+// SHADERS
+
+const source = /* wgsl */ `\
+${WGSL_UNIFORMS}
+
+const INDEX_PICKING_MODE_INSTANCE = 0;
+const INDEX_PICKING_MODE_CUSTOM = 1;
+const INDEX_PICKING_INVALID_INDEX = ${INVALID_INDEX}; // 2^32 - 1
+
+struct indexPickingFragmentInputs = {
+  objectIndex: int32;
+};
+
+let indexPickingFragmentInputs: indexPickingFragmentInputs;
+
+/**
+ * Vertex shaders should call this function to set the object index.
+ * If using instance or vertex mode, argument will be ignored, 0 can be supplied.
+ */
+fn picking_setObjectIndex(objectIndex: int32) {
+  switch (picking.indexMode) {
+    case INDEX_PICKING_MODE_INSTANCE, default: {
+      picking_objectIndex = instance_index;
+    };
+    case INDEX_PICKING_MODE_CUSTOM: {
+      picking_objectIndex = objectIndex;
+    };
+  }
+}
+
+`;
+
+const vs = /* glsl */ `\
+${GLSL_UNIFORMS}
+
+const int INDEX_PICKING_MODE_INSTANCE = 0;
+const int INDEX_PICKING_MODE_CUSTOM = 1;
+
+const int INDEX_PICKING_INVALID_INDEX = ${INVALID_INDEX}; // 2^32 - 1
+
+flat out int picking_objectIndex;
+
+/**
+ * Vertex shaders should call this function to set the object index.
+ * If using instance or vertex mode, argument will be ignored, 0 can be supplied.
+ */
+void picking_setObjectIndex(int objectIndex) {
+  switch (picking.indexMode) {
+    case INDEX_PICKING_MODE_INSTANCE:
+      picking_objectIndex = gl_InstanceID;
+      break;
+    case INDEX_PICKING_MODE_CUSTOM:
+      picking_objectIndex = objectIndex;
+      break;
+  }
+}
+`;
+
+const fs = /* glsl */ `\
+${GLSL_UNIFORMS}
+
+const int INDEX_PICKING_INVALID_INDEX = ${INVALID_INDEX}; // 2^32 - 1
+
+flat in int picking_objectIndex;
+
+/**
+ * Check if this vertex is highlighted (part of the selected batch and object)
+ */ 
+bool picking_isFragmentHighlighted() {
+  return 
+    bool(picking.isHighlightActive) &&
+    picking.highlightedBatchIndex == picking.batchIndex &&
+    picking.highlightedObjectIndex == picking_objectIndex
+    ;
+}
+
+/**
+ * Returns highlight color if this item is selected.
+ */
+vec4 picking_filterHighlightColor(vec4 color) {
+  // If we are still picking, we don't highlight
+  if (bool(picking.isActive)) {
+    return color;
+  }
+
+  // If we are not highlighted, return color as is
+  if (!picking_isFragmentHighlighted()) {
+    return color;
+  }
+   
+  // Blend in highlight color based on its alpha value
+  float highLightAlpha = picking.highlightColor.a;
+  float blendedAlpha = highLightAlpha + color.a * (1.0 - highLightAlpha);
+  float highLightRatio = highLightAlpha / blendedAlpha;
+
+  vec3 blendedRGB = mix(color.rgb, picking.highlightColor.rgb, highLightRatio);
+  return vec4(blendedRGB, blendedAlpha);
+}
+
+/*
+ * Returns picking color if picking enabled else unmodified argument.
+ */
+ivec4 picking_getPickingColor() {
+  // Assumes that colorAttachment0 is rg32int
+  // TODO? - we could render indices into a second color attachment and not mess with fragColor
+  return ivec4(picking_objectIndex, picking.batchIndex, 0u, 0u);  
+}
+
+vec4 picking_filterPickingColor(vec4 color) {
+  if (bool(picking.isActive)) {
+    if (picking_objectIndex == INDEX_PICKING_INVALID_INDEX) {
+      discard;
+    }
+  }
+  return color;
+}
+
+/*
+ * Returns picking color if picking is enabled if not
+ * highlight color if this item is selected, otherwise unmodified argument.
+ */
+vec4 picking_filterColor(vec4 color) {
+  vec4 outColor = color;
+  outColor = picking_filterHighlightColor(outColor);
+  outColor = picking_filterPickingColor(outColor);
+  return outColor;
+}
+`;
+
+/**
+ * Provides support for color-based picking and highlighting.
+ *
+ * In particular, supports picking a specific instance in an instanced
+ * draw call and highlighting an instance based on its picking color,
+ * and correspondingly, supports picking and highlighting groups of
+ * primitives with the same picking color in non-instanced draw-calls
+ *
+ * @note Color based picking has the significant advantage in that it can be added to any
+ * existing shader without requiring any additional picking logic.
+ */
+export const picking = {
+  ...pickingUniforms,
+  name: 'picking',
+  source,
+  vs,
+  fs
+} as const satisfies ShaderModule<PickingProps, PickingUniforms, PickingBindings>;

package/src/modules/picking/legacy-picking-manager.ts

@@ -0,0 +1,99 @@
+// luma.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import {Device, Framebuffer} from '@luma.gl/core';
+import {picking} from '@luma.gl/shadertools';
+import {ShaderInputs} from '../../shader-inputs';
+import {NumberArray3} from '@math.gl/types';
+
+/**
+ * Helper class for using the legacy picking module
+ */
+export class LegacyPickingManager {
+  device: Device;
+  framebuffer: Framebuffer | null = null;
+  shaderInputs: ShaderInputs<{picking: typeof picking.props}>;
+
+  constructor(device: Device, shaderInputs: ShaderInputs) {
+    this.device = device;
+    this.shaderInputs = shaderInputs as ShaderInputs<{picking: typeof picking.props}>;
+  }
+
+  destroy() {
+    this.framebuffer?.destroy();
+  }
+
+  getFramebuffer() {
+    if (!this.framebuffer) {
+      this.framebuffer = this.device.createFramebuffer({
+        colorAttachments: ['rgba8unorm'],
+        depthStencilAttachment: 'depth24plus'
+      });
+    }
+    return this.framebuffer;
+  }
+
+  /** Clear highlighted / picked object */
+  clearPickState() {
+    this.shaderInputs.setProps({picking: {highlightedObjectColor: null}});
+  }
+
+  /** Prepare for rendering picking colors */
+  beginRenderPass() {
+    const framebuffer = this.getFramebuffer();
+    framebuffer.resize(this.device.getCanvasContext().getPixelSize());
+
+    this.shaderInputs.setProps({picking: {isActive: true}});
+
+    const pickingPass = this.device.beginRenderPass({
+      framebuffer,
+      clearColor: [0, 0, 0, 0],
+      clearDepth: 1
+    });
+
+    return pickingPass;
+  }
+
+  updatePickState(mousePosition: [number, number]) {
+    const framebuffer = this.getFramebuffer();
+
+    // use the center pixel location in device pixel range
+    const [pickX, pickY] = this.getPickPosition(mousePosition);
+
+    // Read back
+    const color255 = this.device.readPixelsToArrayWebGL(framebuffer, {
+      sourceX: pickX,
+      sourceY: pickY,
+      sourceWidth: 1,
+      sourceHeight: 1
+    });
+    // console.log(color255);
+
+    // Check if we have
+    let highlightedObjectColor: NumberArray3 | null = [...color255].map(
+      x => x / 255
+    ) as NumberArray3;
+    const isHighlightActive =
+      highlightedObjectColor[0] + highlightedObjectColor[1] + highlightedObjectColor[2] > 0;
+
+    if (!isHighlightActive) {
+      highlightedObjectColor = null;
+    }
+
+    this.shaderInputs.setProps({
+      picking: {isActive: false, highlightedObjectColor}
+    });
+  }
+
+  /**
+   * Get pick position in device pixel range
+   * use the center pixel location in device pixel range
+   */
+  getPickPosition(mousePosition: number[]): [number, number] {
+    const devicePixels = this.device.getCanvasContext().cssToDevicePixels(mousePosition);
+    const pickX = devicePixels.x + Math.floor(devicePixels.width / 2);
+    const pickY = devicePixels.y + Math.floor(devicePixels.height / 2);
+    return [pickX, pickY];
+  }
+}

package/src/modules/picking/picking-manager.ts

@@ -0,0 +1,137 @@
+// luma.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import {Device, Framebuffer} from '@luma.gl/core';
+import {ShaderInputs} from '../../shader-inputs';
+import {pickingUniforms, INVALID_INDEX} from './picking-uniforms';
+// import {picking} from './color-picking';
+
+/** Information about picked object */
+export type PickInfo = {
+  batchIndex: number | null;
+  objectIndex: number | null;
+};
+
+export type PickingManagerProps = {
+  /** Shader Inputs from models to pick */
+  shaderInputs?: ShaderInputs<{picking: typeof pickingUniforms.props}>;
+  /** Callback */
+  onObjectPicked?: (info: PickInfo) => void;
+};
+
+/**
+ * Helper class for using the new picking module
+ * @todo Port to WebGPU
+ * @todo Support multiple models
+ * @todo Switching picking module
+ */
+export class PickingManager {
+  device: Device;
+  props: Required<PickingManagerProps>;
+  /** Info from latest pick operation */
+  pickInfo: PickInfo = {batchIndex: null, objectIndex: null};
+  /** Framebuffer used for picking */
+  framebuffer: Framebuffer | null = null;
+
+  static defaultProps: Required<PickingManagerProps> = {
+    shaderInputs: undefined!,
+    onObjectPicked: () => {}
+  };
+
+  constructor(device: Device, props: PickingManagerProps) {
+    this.device = device;
+    this.props = {...PickingManager.defaultProps, ...props};
+  }
+
+  destroy() {
+    this.framebuffer?.destroy();
+  }
+
+  // TODO - Ask for a cached framebuffer? a Framebuffer factory?
+  getFramebuffer() {
+    if (!this.framebuffer) {
+      this.framebuffer = this.device.createFramebuffer({
+        colorAttachments: ['rgba8unorm', 'rg32sint'],
+        depthStencilAttachment: 'depth24plus'
+      });
+    }
+    return this.framebuffer;
+  }
+
+  /** Clear highlighted / picked object */
+  clearPickState() {
+    this.props.shaderInputs.setProps({picking: {highlightedObjectIndex: null}});
+  }
+
+  /** Prepare for rendering picking colors */
+  beginRenderPass() {
+    const framebuffer = this.getFramebuffer();
+    framebuffer.resize(this.device.getDefaultCanvasContext().getPixelSize());
+
+    this.props.shaderInputs?.setProps({picking: {isActive: true}});
+
+    const pickingPass = this.device.beginRenderPass({
+      framebuffer,
+      clearColors: [new Float32Array([0, 0, 0, 0]), new Int32Array([-1, -1, 0, 0])],
+      clearDepth: 1
+    });
+
+    return pickingPass;
+  }
+
+  getPickInfo(mousePosition: [number, number]): PickInfo | null {
+    const framebuffer = this.getFramebuffer();
+
+    // use the center pixel location in device pixel range
+    const [pickX, pickY] = this.getPickPosition(mousePosition);
+
+    // Read back
+    const pixelData = this.device.readPixelsToArrayWebGL(framebuffer, {
+      sourceX: pickX,
+      sourceY: pickY,
+      sourceWidth: 1,
+      sourceHeight: 1,
+      sourceAttachment: 1
+    });
+    if (!pixelData) {
+      return null;
+    }
+
+    const pickInfo: PickInfo = {
+      objectIndex: pixelData[0] === INVALID_INDEX ? null : pixelData[0],
+      batchIndex: pixelData[1] === INVALID_INDEX ? null : pixelData[1]
+    };
+
+    // Call callback if picked object has changed
+    if (
+      pickInfo.objectIndex !== this.pickInfo.objectIndex ||
+      pickInfo.batchIndex !== this.pickInfo.batchIndex
+    ) {
+      this.pickInfo = pickInfo;
+      this.props.onObjectPicked(pickInfo);
+      // console.log(`Object ${pickInfo.objectIndex} in batch ${pickInfo.batchIndex} was picked`)
+    }
+
+    this.props.shaderInputs?.setProps({
+      picking: {
+        isActive: false,
+        highlightedBatchIndex: pickInfo.batchIndex,
+        highlightedObjectIndex: pickInfo.objectIndex
+      }
+    });
+
+    return this.pickInfo;
+  }
+
+  /**
+   * Get pick position in device pixel range
+   * use the center pixel location in device pixel range
+   */
+  getPickPosition(mousePosition: number[]): [number, number] {
+    const devicePixels = this.device.getDefaultCanvasContext().cssToDevicePixels(mousePosition);
+    const pickX = devicePixels.x + Math.floor(devicePixels.width / 2);
+    const pickY = devicePixels.y + Math.floor(devicePixels.height / 2);
+    return [pickX, pickY];
+  }
+}