@luma.gl/engine 9.2.6 → 9.3.0-alpha.10

package/src/modules/picking/picking-manager.ts

@@ -2,10 +2,14 @@
 // SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
-import {Device, Framebuffer} from '@luma.gl/core';
+import {Buffer, Device, Framebuffer, Texture} from '@luma.gl/core';
 import {ShaderInputs} from '../../shader-inputs';
 import {pickingUniforms, INVALID_INDEX} from './picking-uniforms';
-// import {picking} from './color-picking';
+
+const INDEX_PICKING_ATTACHMENT_INDEX = 1;
+const INDEX_PICKING_CLEAR_COLOR = new Int32Array([INVALID_INDEX, INVALID_INDEX, 0, 0]);
+const COLOR_PICKING_MAX_OBJECT_INDEX = 16777214;
+const COLOR_PICKING_MAX_BATCH_INDEX = 254;
 
 /** Information about picked object */
 export type PickInfo = {
@@ -13,22 +17,81 @@ export type PickInfo = {
   objectIndex: number | null;
 };
 
+export type PickingMode = 'auto' | 'index' | 'color';
+export type ResolvedPickingMode = Exclude<PickingMode, 'auto'>;
+/** @deprecated Use `PickingMode`. */
+export type PickingBackend = PickingMode;
+/** @deprecated Use `ResolvedPickingMode`. */
+export type ResolvedPickingBackend = ResolvedPickingMode;
+
 export type PickingManagerProps = {
-  /** Shader Inputs from models to pick */
+  /** Shader inputs from models to pick */
   shaderInputs?: ShaderInputs<{picking: typeof pickingUniforms.props}>;
   /** Callback */
   onObjectPicked?: (info: PickInfo) => void;
+  /** Select a picking mode. Defaults to `color`. Use `auto` to prefer `index` when supported. */
+  mode?: PickingMode;
+  /** @deprecated Use `mode`. */
+  backend?: PickingBackend;
 };
 
+export function resolvePickingMode(
+  deviceType: Device['type'],
+  mode: PickingMode = 'color',
+  indexPickingSupported: boolean = deviceType === 'webgpu'
+): ResolvedPickingMode {
+  if (mode === 'auto') {
+    return indexPickingSupported ? 'index' : 'color';
+  }
+
+  if (mode === 'index' && !indexPickingSupported) {
+    throw new Error(
+      `Picking mode "${mode}" requires WebGPU or a WebGL device that supports renderable rg32sint textures.`
+    );
+  }
+
+  return mode;
+}
+
+export function supportsIndexPicking(device: Device): boolean {
+  return (
+    device.type === 'webgpu' ||
+    (device.type === 'webgl' && device.isTextureFormatRenderable('rg32sint'))
+  );
+}
+
+/** @deprecated Use `resolvePickingMode`. */
+export const resolvePickingBackend = resolvePickingMode;
+
+export function decodeIndexPickInfo(pixelData: Int32Array): PickInfo {
+  return {
+    objectIndex: pixelData[0] === INVALID_INDEX ? null : pixelData[0],
+    batchIndex: pixelData[1] === INVALID_INDEX ? null : pixelData[1]
+  };
+}
+
+export function decodeColorPickInfo(pixelData: Uint8Array): PickInfo {
+  const encodedObjectIndex = pixelData[0] + pixelData[1] * 256 + pixelData[2] * 65536;
+  if (encodedObjectIndex === 0) {
+    return {objectIndex: null, batchIndex: null};
+  }
+
+  const batchIndex = pixelData[3] > 0 ? pixelData[3] - 1 : 0;
+  return {
+    objectIndex: encodedObjectIndex - 1,
+    batchIndex
+  };
+}
+
 /**
- * Helper class for using the new picking module
- * @todo Port to WebGPU
+ * Helper class for using object picking with backend-specific readback.
  * @todo Support multiple models
  * @todo Switching picking module
  */
 export class PickingManager {
   device: Device;
   props: Required<PickingManagerProps>;
+  mode: ResolvedPickingMode;
   /** Info from latest pick operation */
   pickInfo: PickInfo = {batchIndex: null, objectIndex: null};
   /** Framebuffer used for picking */
@@ -36,12 +99,22 @@ export class PickingManager {
 
   static defaultProps: Required<PickingManagerProps> = {
     shaderInputs: undefined!,
-    onObjectPicked: () => {}
+    onObjectPicked: () => {},
+    mode: 'color',
+    backend: 'color'
   };
 
   constructor(device: Device, props: PickingManagerProps) {
     this.device = device;
     this.props = {...PickingManager.defaultProps, ...props};
+    const requestedMode = props.mode ?? props.backend ?? PickingManager.defaultProps.mode;
+    this.props.mode = requestedMode;
+    this.props.backend = requestedMode;
+    this.mode = resolvePickingMode(
+      this.device.type,
+      requestedMode,
+      supportsIndexPicking(this.device)
+    );
   }
 
   destroy() {
@@ -51,17 +124,15 @@ export class PickingManager {
   // TODO - Ask for a cached framebuffer? a Framebuffer factory?
   getFramebuffer() {
     if (!this.framebuffer) {
-      this.framebuffer = this.device.createFramebuffer({
-        colorAttachments: ['rgba8unorm', 'rg32sint'],
-        depthStencilAttachment: 'depth24plus'
-      });
+      this.framebuffer =
+        this.mode === 'index' ? this.createIndexFramebuffer() : this.createColorFramebuffer();
     }
     return this.framebuffer;
   }
 
   /** Clear highlighted / picked object */
   clearPickState() {
-    this.props.shaderInputs.setProps({picking: {highlightedObjectIndex: null}});
+    this.setPickingProps({highlightedBatchIndex: null, highlightedObjectIndex: null});
   }
 
   /** Prepare for rendering picking colors */
@@ -69,56 +140,38 @@ export class PickingManager {
     const framebuffer = this.getFramebuffer();
     framebuffer.resize(this.device.getDefaultCanvasContext().getDevicePixelSize());
 
-    this.props.shaderInputs?.setProps({picking: {isActive: true}});
-
-    const pickingPass = this.device.beginRenderPass({
-      framebuffer,
-      clearColors: [new Float32Array([0, 0, 0, 0]), new Int32Array([-1, -1, 0, 0])],
-      clearDepth: 1
-    });
+    this.setPickingProps({isActive: true});
 
-    return pickingPass;
+    return this.mode === 'index'
+      ? this.device.beginRenderPass({
+          framebuffer,
+          clearColors: [new Float32Array([0, 0, 0, 0]), INDEX_PICKING_CLEAR_COLOR],
+          clearDepth: 1
+        })
+      : this.device.beginRenderPass({
+          framebuffer,
+          clearColor: [0, 0, 0, 0],
+          clearDepth: 1
+        });
   }
 
   async updatePickInfo(mousePosition: [number, number]): Promise<PickInfo | null> {
     const framebuffer = this.getFramebuffer();
-
-    // use the center pixel location in device pixel range
-    const [pickX, pickY] = this.getPickPosition(mousePosition);
-
-    // Read back
-    const pixelData = this.device.readPixelsToArrayWebGL(framebuffer, {
-      sourceX: pickX,
-      sourceY: pickY,
-      sourceWidth: 1,
-      sourceHeight: 1,
-      sourceAttachment: 1
-    });
-    if (!pixelData) {
+    const pickPosition = this.getPickPosition(mousePosition);
+    const pickInfo = await this.readPickInfo(framebuffer, pickPosition);
+    if (!pickInfo) {
       return null;
     }
 
-    const pickInfo: PickInfo = {
-      objectIndex: pixelData[0] === INVALID_INDEX ? null : pixelData[0],
-      batchIndex: pixelData[1] === INVALID_INDEX ? null : pixelData[1]
-    };
-
-    // Call callback if picked object has changed
-    if (
-      pickInfo.objectIndex !== this.pickInfo.objectIndex ||
-      pickInfo.batchIndex !== this.pickInfo.batchIndex
-    ) {
+    if (this.hasPickInfoChanged(pickInfo)) {
       this.pickInfo = pickInfo;
       this.props.onObjectPicked(pickInfo);
-      // console.log(`Object ${pickInfo.objectIndex} in batch ${pickInfo.batchIndex} was picked`)
     }
 
-    this.props.shaderInputs?.setProps({
-      picking: {
-        isActive: false,
-        highlightedBatchIndex: pickInfo.batchIndex,
-        highlightedObjectIndex: pickInfo.objectIndex
-      }
+    this.setPickingProps({
+      isActive: false,
+      highlightedBatchIndex: pickInfo.batchIndex,
+      highlightedObjectIndex: pickInfo.objectIndex
     });
 
     return this.pickInfo;
@@ -129,9 +182,158 @@ export class PickingManager {
    * use the center pixel location in device pixel range
    */
   getPickPosition(mousePosition: [number, number]): [number, number] {
-    const devicePixels = this.device.getDefaultCanvasContext().cssToDevicePixels(mousePosition);
+    const yInvert = this.device.type !== 'webgpu';
+    const devicePixels = this.device
+      .getDefaultCanvasContext()
+      .cssToDevicePixels(mousePosition, yInvert);
     const pickX = devicePixels.x + Math.floor(devicePixels.width / 2);
     const pickY = devicePixels.y + Math.floor(devicePixels.height / 2);
     return [pickX, pickY];
   }
+
+  protected createIndexFramebuffer(): Framebuffer {
+    const colorTexture = this.device.createTexture({
+      format: 'rgba8unorm',
+      width: 1,
+      height: 1,
+      usage: Texture.RENDER_ATTACHMENT
+    });
+    const pickingTexture = this.device.createTexture({
+      format: 'rg32sint',
+      width: 1,
+      height: 1,
+      usage: Texture.RENDER_ATTACHMENT | Texture.COPY_SRC
+    });
+
+    return this.device.createFramebuffer({
+      colorAttachments: [colorTexture, pickingTexture],
+      depthStencilAttachment: 'depth24plus'
+    });
+  }
+
+  protected createColorFramebuffer(): Framebuffer {
+    const pickingTexture = this.device.createTexture({
+      format: 'rgba8unorm',
+      width: 1,
+      height: 1,
+      usage: Texture.RENDER_ATTACHMENT | Texture.COPY_SRC
+    });
+
+    return this.device.createFramebuffer({
+      colorAttachments: [pickingTexture],
+      depthStencilAttachment: 'depth24plus'
+    });
+  }
+
+  protected setPickingProps(props: Partial<typeof pickingUniforms.props>): void {
+    this.props.shaderInputs?.setProps({picking: props});
+  }
+
+  protected async readPickInfo(
+    framebuffer: Framebuffer,
+    pickPosition: [number, number]
+  ): Promise<PickInfo | null> {
+    return this.mode === 'index'
+      ? this.readIndexPickInfo(framebuffer, pickPosition)
+      : this.readColorPickInfo(framebuffer, pickPosition);
+  }
+
+  protected async readIndexPickInfo(
+    framebuffer: Framebuffer,
+    [pickX, pickY]: [number, number]
+  ): Promise<PickInfo | null> {
+    if (this.device.type === 'webgpu') {
+      const pickTexture = framebuffer.colorAttachments[INDEX_PICKING_ATTACHMENT_INDEX]?.texture;
+      if (!pickTexture) {
+        return null;
+      }
+
+      const layout = pickTexture.computeMemoryLayout({width: 1, height: 1});
+      const readBuffer = this.device.createBuffer({
+        byteLength: layout.byteLength,
+        usage: Buffer.COPY_DST | Buffer.MAP_READ
+      });
+      try {
+        pickTexture.readBuffer(
+          {
+            x: pickX,
+            y: pickY,
+            width: 1,
+            height: 1
+          },
+          readBuffer
+        );
+        const pickDataView = await readBuffer.readAsync(0, layout.byteLength);
+        return decodeIndexPickInfo(new Int32Array(pickDataView.buffer, pickDataView.byteOffset, 2));
+      } finally {
+        readBuffer.destroy();
+      }
+    }
+
+    const pixelData = this.device.readPixelsToArrayWebGL(framebuffer, {
+      sourceX: pickX,
+      sourceY: pickY,
+      sourceWidth: 1,
+      sourceHeight: 1,
+      sourceAttachment: INDEX_PICKING_ATTACHMENT_INDEX
+    });
+
+    return pixelData
+      ? decodeIndexPickInfo(new Int32Array(pixelData.buffer, pixelData.byteOffset, 2))
+      : null;
+  }
+
+  protected async readColorPickInfo(
+    framebuffer: Framebuffer,
+    [pickX, pickY]: [number, number]
+  ): Promise<PickInfo | null> {
+    if (this.device.type === 'webgpu') {
+      const pickTexture = framebuffer.colorAttachments[0]?.texture;
+      if (!pickTexture) {
+        return null;
+      }
+
+      const layout = pickTexture.computeMemoryLayout({width: 1, height: 1});
+      const readBuffer = this.device.createBuffer({
+        byteLength: layout.byteLength,
+        usage: Buffer.COPY_DST | Buffer.MAP_READ
+      });
+      try {
+        pickTexture.readBuffer(
+          {
+            x: pickX,
+            y: pickY,
+            width: 1,
+            height: 1
+          },
+          readBuffer
+        );
+        const pickDataView = await readBuffer.readAsync(0, layout.byteLength);
+        return decodeColorPickInfo(new Uint8Array(pickDataView.buffer, pickDataView.byteOffset, 4));
+      } finally {
+        readBuffer.destroy();
+      }
+    }
+
+    const pixelData = this.device.readPixelsToArrayWebGL(framebuffer, {
+      sourceX: pickX,
+      sourceY: pickY,
+      sourceWidth: 1,
+      sourceHeight: 1,
+      sourceAttachment: 0
+    });
+
+    return pixelData
+      ? decodeColorPickInfo(new Uint8Array(pixelData.buffer, pixelData.byteOffset, 4))
+      : null;
+  }
+
+  protected hasPickInfoChanged(pickInfo: PickInfo): boolean {
+    return (
+      pickInfo.objectIndex !== this.pickInfo.objectIndex ||
+      pickInfo.batchIndex !== this.pickInfo.batchIndex
+    );
+  }
 }
+
+export {COLOR_PICKING_MAX_BATCH_INDEX, COLOR_PICKING_MAX_OBJECT_INDEX};

package/src/modules/picking/picking-uniforms.ts

@@ -9,6 +9,7 @@ import type {ShaderModule} from '@luma.gl/shadertools';
 const DEFAULT_HIGHLIGHT_COLOR: NumberArray4 = [0, 1, 1, 1];
 
 export const INVALID_INDEX = -1;
+export type PickingPayloadMode = 'instance' | 'attribute';
 
 /**
  * Props for the picking module, which depending on mode renders picking colors or highlighted item.
@@ -19,14 +20,14 @@ export const INVALID_INDEX = -1;
 export type PickingProps = {
   /** Are we picking? I.e. rendering picking colors? */
   isActive?: boolean;
-  /** Whether to use instance_index (built-in) or a custom application supplied index (usually from an attribute) */
-  indexMode?: 'instance' | 'custom';
-  /** Batch index (used when rendering multiple models to identify which model was picked), defaults to 0 */
+  /** Whether the payload is sourced from the builtin instance index or a custom integer attribute */
+  indexMode?: PickingPayloadMode;
+  /** Identifier of the batch currently being rendered */
   batchIndex?: number;
 
-  /** Index of the highlighted batch, defaults to 0 */
+  /** Identifier of the highlighted batch */
   highlightedBatchIndex?: number | null;
-  /** Set an index to highlight that item, or `null` to explicitly clear **/
+  /** Set the highlighted object index, or `null` to explicitly clear **/
   highlightedObjectIndex?: number | null;
   /** Color of visual highlight of "selected" item () */
   highlightColor?: NumberArray4;
@@ -43,18 +44,18 @@ export type PickingUniforms = {
    * When false, renders normal colors, with the exception of selected object which is rendered with highlight
    */
   isActive: boolean;
-  /** Set to true when picking an attribute value instead of object index */
+  /** Whether the current payload comes from instance_index or a custom integer attribute */
   indexMode: 0 | 1;
-  /** Index of batch currently being rendered */
+  /** Identifier of the batch currently being rendered */
   batchIndex: number;
 
   /** Do we have a highlighted item? */
   isHighlightActive: boolean;
   /** Color of visual highlight of "selected" item. Note: RGBA components must in the range 0-1 */
   highlightColor: NumberArray4;
-  /** Indicates which batch to visually highlight an item in (defaults to 0) */
+  /** Indicates which batch to visually highlight an item in */
   highlightedBatchIndex: number;
-  /** Indicates which index in the batch to highlight an item in */
+  /** Indicates which object index in the batch to highlight */
   highlightedObjectIndex: number;
 };
 
@@ -77,7 +78,7 @@ export const GLSL_UNIFORMS = /* glsl */ `\
 precision highp float;
 precision highp int;
 
-uniform pickingUniforms {
+layout(std140) uniform pickingUniforms {
   int isActive;
   int indexMode;
   int batchIndex;
@@ -91,15 +92,17 @@ uniform pickingUniforms {
 
 export const WGSL_UNIFORMS = /* wgsl */ `\
 struct pickingUniforms {
-  isActive: int32;
-  indexMode: int32;
-  batchIndex: int32;
-
-  isHighlightActive: int32;
-  highlightedBatchIndex: int32;
-  highlightedObjectIndex: int32;
-  highlightColor: vec4<f32>;
-} picking;
+  isActive: i32,
+  indexMode: i32,
+  batchIndex: i32,
+
+  isHighlightActive: i32,
+  highlightedBatchIndex: i32,
+  highlightedObjectIndex: i32,
+  highlightColor: vec4<f32>,
+};
+
+@group(0) @binding(auto) var<uniform> picking: pickingUniforms;
 `;
 
 function getUniforms(props: PickingProps = {}, prevUniforms?: PickingUniforms): PickingUniforms {
@@ -114,7 +117,7 @@ function getUniforms(props: PickingProps = {}, prevUniforms?: PickingUniforms):
     case 'instance':
       uniforms.indexMode = 0;
       break;
-    case 'custom':
+    case 'attribute':
       uniforms.indexMode = 1;
       break;
     case undefined:
@@ -122,9 +125,13 @@ function getUniforms(props: PickingProps = {}, prevUniforms?: PickingUniforms):
       break;
   }
 
+  if (typeof props.batchIndex === 'number') {
+    uniforms.batchIndex = props.batchIndex;
+  }
+
   switch (props.highlightedObjectIndex) {
     case undefined:
-      // Unless highlightedObjectColor explicitly null or set, do not update state
+      // Unless highlighted payload explicitly null or set, do not update state
       break;
     case null:
       // Clear highlight
@@ -136,8 +143,16 @@ function getUniforms(props: PickingProps = {}, prevUniforms?: PickingUniforms):
       uniforms.highlightedObjectIndex = props.highlightedObjectIndex;
   }
 
-  if (typeof props.highlightedBatchIndex === 'number') {
-    uniforms.highlightedBatchIndex = props.highlightedBatchIndex;
+  switch (props.highlightedBatchIndex) {
+    case undefined:
+      break;
+    case null:
+      uniforms.isHighlightActive = false;
+      uniforms.highlightedBatchIndex = INVALID_INDEX;
+      break;
+    default:
+      uniforms.isHighlightActive = true;
+      uniforms.highlightedBatchIndex = props.highlightedBatchIndex;
   }
 
   if (props.highlightColor) {
@@ -169,7 +184,7 @@ export const pickingUniforms = {
     isActive: false,
     indexMode: 0,
     batchIndex: 0,
-    isHighlightActive: true,
+    isHighlightActive: false,
     highlightedBatchIndex: INVALID_INDEX,
     highlightedObjectIndex: INVALID_INDEX,
     highlightColor: DEFAULT_HIGHLIGHT_COLOR

package/src/modules/picking/picking.ts

@@ -0,0 +1,22 @@
+// luma.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {ShaderModule} from '@luma.gl/shadertools';
+
+import type {PickingBindings, PickingProps, PickingUniforms} from './picking-uniforms';
+import {pickingUniforms} from './picking-uniforms';
+import {picking as colorPicking} from './color-picking';
+import {picking as indexPicking} from './index-picking';
+
+/**
+ * Unified object-picking shader module.
+ * Uses color picking on GLSL/WebGL paths and index picking on WGSL/WebGPU paths.
+ */
+export const picking = {
+  ...pickingUniforms,
+  name: 'picking',
+  source: indexPicking.source,
+  vs: colorPicking.vs,
+  fs: colorPicking.fs
+} as const satisfies ShaderModule<PickingProps, PickingUniforms, PickingBindings>;

package/src/passes/get-fragment-shader.ts

@@ -36,22 +36,16 @@ export function getFragmentShaderForRenderPass(options: {
 /** Get a filtering WGSL fragment shader */
 function getFilterShaderWGSL(func: string) {
   return /* wgsl */ `\
-// Binding 0:1 is reserved for shader passes
-// @group(0) @binding(0) var<uniform> brightnessContrast : brightnessContrastUniforms;
-@group(0) @binding(1) var texture: texture_2d<f32>;
-@group(0) @binding(2) var sampler: sampler;
-
-struct FragmentInputs {
-  @location(0) fragUV: vec2f,
-  @location(1) fragPosition: vec4f,
-  @location(2) fragCoordinate: vec4f
-};
+@group(0) @binding(0) var sourceTexture: texture_2d<f32>;
+@group(0) @binding(2) var sourceTextureSampler: sampler;
 
 @fragment
 fn fragmentMain(inputs: FragmentInputs) -> @location(0) vec4f {
-  let texSize = textureDimensions(texture, 0);
-  var fragColor = textureSample(texture, sampler, fragUV);
-  fragColor = ${func}(gl_FragColor, texSize, texCoord);
+  let texCoord = inputs.coordinate;
+  let texSize = vec2f(textureDimensions(sourceTexture));
+
+  var fragColor = textureSample(sourceTexture, sourceTextureSampler, texCoord);
+  fragColor = ${func}(fragColor, texSize, texCoord);
   return fragColor;
 }
 `;
@@ -60,23 +54,14 @@ fn fragmentMain(inputs: FragmentInputs) -> @location(0) vec4f {
 /** Get a sampling WGSL fragment shader */
 function getSamplerShaderWGSL(func: string) {
   return /* wgsl */ `\
-// Binding 0:1 is reserved for shader passes
-@group(0) @binding(0) var<uniform> brightnessContrast : brightnessContrastUniforms;
-@group(0) @binding(1) var texture: texture_2d<f32>;
-@group(0) @binding(2) var sampler: sampler;
-
-struct FragmentInputs = {
-  @location(0) fragUV: vec2f,
-  @location(1) fragPosition: vec4f,
-  @location(2) fragCoordinate: vec4f
-};
+@group(0) @binding(0) var sourceTexture: texture_2d<f32>;
+@group(0) @binding(2) var sourceTextureSampler: sampler;
 
 @fragment
 fn fragmentMain(inputs: FragmentInputs) -> @location(0) vec4f {
-  let texSize = textureDimensions(texture, 0);
-  var fragColor = textureSample(texture, sampler, fragUV);
-  fragColor = ${func}(gl_FragColor, texSize, texCoord);
-  return fragColor;
+  let texCoord = inputs.coordinate;
+  let texSize = vec2f(textureDimensions(sourceTexture));
+  return ${func}(sourceTexture, sourceTextureSampler, texSize, texCoord);
 }
 `;
 }