@luma.gl/shaders 0.0.0-stage → 10.0.0-alpha.11

package/src/modules/engine/gpu-animation/gpu-animation.ts

@@ -0,0 +1,244 @@
+// luma.gl
+// SPDX-License-Identifier: MIT
+// SPDX-FileCopyrightText: Copyright (c) vis.gl contributors
+
+import type {ShaderBinding as Binding} from '../../../types';
+
+import type {ShaderModule} from '../../../lib/shader-module/shader-module';
+
+const source = /* wgsl */ `
+#ifdef HAS_GPU_CROWD_ANIMATION
+@group(0) @binding(auto) var<storage, read> gpuAnimationFrames: array<vec4f>;
+
+fn readGPUAnimationFrame(frame: u32, offset: u32, frameStride: u32) -> vec4f {
+  return gpuAnimationFrames[frame * frameStride + offset];
+}
+
+fn sampleGPUAnimationFrame(
+  frames: vec4f,
+  blend: vec4f,
+  offset: u32,
+  frameStride: u32
+) -> vec4f {
+  let first = mix(
+    readGPUAnimationFrame(u32(frames.x), offset, frameStride),
+    readGPUAnimationFrame(u32(frames.y), offset, frameStride),
+    frames.z
+  );
+  if (blend.w <= 0.0) {
+    return first;
+  }
+  let second = mix(
+    readGPUAnimationFrame(u32(blend.x), offset, frameStride),
+    readGPUAnimationFrame(u32(blend.y), offset, frameStride),
+    blend.z
+  );
+  return mix(first, second, blend.w);
+}
+
+fn sampleGPUAnimationMatrix(
+  frames: vec4f,
+  blend: vec4f,
+  firstColumn: u32,
+  frameStride: u32
+) -> mat4x4f {
+  return mat4x4f(
+    sampleGPUAnimationFrame(frames, blend, firstColumn, frameStride),
+    sampleGPUAnimationFrame(frames, blend, firstColumn + 1u, frameStride),
+    sampleGPUAnimationFrame(frames, blend, firstColumn + 2u, frameStride),
+    sampleGPUAnimationFrame(frames, blend, firstColumn + 3u, frameStride)
+  );
+}
+
+fn getGPUAnimatedSkinMatrix(
+  weights: vec4f,
+  joints: vec4u,
+  frames: vec4f,
+  blend: vec4f,
+  frameStride: u32
+) -> mat4x4f {
+  return weights.x * sampleGPUAnimationMatrix(frames, blend, 4u + joints.x * 4u, frameStride)
+       + weights.y * sampleGPUAnimationMatrix(frames, blend, 4u + joints.y * 4u, frameStride)
+       + weights.z * sampleGPUAnimationMatrix(frames, blend, 4u + joints.z * 4u, frameStride)
+       + weights.w * sampleGPUAnimationMatrix(frames, blend, 4u + joints.w * 4u, frameStride);
+}
+#endif
+
+#ifdef HAS_INSTANCED_MORPH
+@group(0) @binding(auto) var<storage, read> gpuMorphTargets: array<vec4f>;
+
+#ifndef HAS_GPU_CROWD_ANIMATION
+@group(0) @binding(auto) var<storage, read> gpuMorphWeights: array<vec4f>;
+#endif
+
+fn getGPUCrowdMorphWeight(
+  instanceIndex: u32,
+  targetIndex: u32,
+  targetCount: u32,
+  jointsPerInstance: u32,
+  frames: vec4f,
+  blend: vec4f,
+  frameStride: u32
+) -> f32 {
+#ifdef HAS_GPU_CROWD_ANIMATION
+  let offset = 4u + jointsPerInstance * 4u + targetIndex;
+  return sampleGPUAnimationFrame(frames, blend, offset, frameStride).x;
+#else
+  let packedCount = (targetCount + 3u) / 4u;
+  let packedWeights = gpuMorphWeights[instanceIndex * packedCount + targetIndex / 4u];
+  return packedWeights[targetIndex % 4u];
+#endif
+}
+
+fn getGPUCrowdMorphDelta(
+  instanceIndex: u32,
+  vertexIndex: u32,
+  attributeIndex: u32,
+  vertexCount: u32,
+  targetCount: u32,
+  jointsPerInstance: u32,
+  frames: vec4f,
+  blend: vec4f,
+  frameStride: u32
+) -> vec3f {
+  var result = vec3f(0.0);
+  for (var targetIndex = 0u; targetIndex < targetCount; targetIndex++) {
+    let weight = getGPUCrowdMorphWeight(
+      instanceIndex,
+      targetIndex,
+      targetCount,
+      jointsPerInstance,
+      frames,
+      blend,
+      frameStride
+    );
+    let offset = (targetIndex * 3u + attributeIndex) * vertexCount + vertexIndex;
+    result += gpuMorphTargets[offset].xyz * weight;
+  }
+  return result;
+}
+#endif
+`;
+
+const vs = /* glsl */ `
+#ifdef HAS_GPU_CROWD_ANIMATION
+uniform highp sampler2D gpuAnimationFrames;
+
+vec4 sampleGPUAnimationFrame(vec4 frames, vec4 blend, int offset) {
+  vec4 first = mix(
+    texelFetch(gpuAnimationFrames, ivec2(offset, int(frames.x)), 0),
+    texelFetch(gpuAnimationFrames, ivec2(offset, int(frames.y)), 0),
+    frames.z
+  );
+  if (blend.w <= 0.0) {
+    return first;
+  }
+  vec4 second = mix(
+    texelFetch(gpuAnimationFrames, ivec2(offset, int(blend.x)), 0),
+    texelFetch(gpuAnimationFrames, ivec2(offset, int(blend.y)), 0),
+    blend.z
+  );
+  return mix(first, second, blend.w);
+}
+
+mat4 sampleGPUAnimationMatrix(vec4 frames, vec4 blend, int firstColumn) {
+  return mat4(
+    sampleGPUAnimationFrame(frames, blend, firstColumn),
+    sampleGPUAnimationFrame(frames, blend, firstColumn + 1),
+    sampleGPUAnimationFrame(frames, blend, firstColumn + 2),
+    sampleGPUAnimationFrame(frames, blend, firstColumn + 3)
+  );
+}
+
+mat4 getGPUAnimatedSkinMatrix(vec4 weights, uvec4 joints, vec4 frames, vec4 blend) {
+  return weights.x * sampleGPUAnimationMatrix(frames, blend, 4 + int(joints.x) * 4)
+       + weights.y * sampleGPUAnimationMatrix(frames, blend, 4 + int(joints.y) * 4)
+       + weights.z * sampleGPUAnimationMatrix(frames, blend, 4 + int(joints.z) * 4)
+       + weights.w * sampleGPUAnimationMatrix(frames, blend, 4 + int(joints.w) * 4);
+}
+#endif
+
+#ifdef HAS_INSTANCED_MORPH
+uniform highp sampler2D gpuMorphTargets;
+
+#ifndef HAS_GPU_CROWD_ANIMATION
+uniform highp sampler2D gpuMorphWeights;
+#endif
+
+float getGPUCrowdMorphWeight(
+  uint instanceIndex,
+  uint targetIndex,
+  uint jointsPerInstance,
+  vec4 frames,
+  vec4 blend
+) {
+#ifdef HAS_GPU_CROWD_ANIMATION
+  int offset = 4 + int(jointsPerInstance) * 4 + int(targetIndex);
+  return sampleGPUAnimationFrame(frames, blend, offset).x;
+#else
+  vec4 packedWeights = texelFetch(
+    gpuMorphWeights,
+    ivec2(int(targetIndex / 4u), int(instanceIndex)),
+    0
+  );
+  return packedWeights[int(targetIndex % 4u)];
+#endif
+}
+
+vec3 getGPUCrowdMorphDelta(
+  uint instanceIndex,
+  uint vertexIndex,
+  uint attributeIndex,
+  uint targetCount,
+  uint jointsPerInstance,
+  vec4 frames,
+  vec4 blend
+) {
+  vec3 result = vec3(0.0);
+  for (uint targetIndex = 0u; targetIndex < targetCount; targetIndex++) {
+    float weight = getGPUCrowdMorphWeight(
+      instanceIndex,
+      targetIndex,
+      jointsPerInstance,
+      frames,
+      blend
+    );
+    result += texelFetch(
+      gpuMorphTargets,
+      ivec2(int(vertexIndex), int(targetIndex * 3u + attributeIndex)),
+      0
+    ).xyz * weight;
+  }
+  return result;
+}
+#endif
+`;
+
+/** GPU-resident baked animation frames and immutable morph-target resources. */
+export type GPUAnimationProps = {
+  gpuAnimationFrames?: Binding;
+  gpuMorphTargets?: Binding;
+  gpuMorphWeights?: Binding;
+};
+
+/** Backend-native resources consumed only by the vertex stage. */
+export type GPUAnimationBindings = GPUAnimationProps;
+
+/** Portable baked clip sampling and independent per-instance morph deformation. */
+export const gpuAnimation = {
+  name: 'gpuAnimation',
+  props: {} as GPUAnimationProps,
+  uniforms: {},
+  bindings: {} as GPUAnimationBindings,
+  source,
+  vs,
+  fs: '',
+  bindingLayout: [
+    {name: 'gpuAnimationFrames', group: 0, visibility: 1},
+    {name: 'gpuMorphTargets', group: 0, visibility: 1},
+    {name: 'gpuMorphWeights', group: 0, visibility: 1}
+  ],
+  getUniforms(props: GPUAnimationProps = {}): GPUAnimationBindings {
+    return props;
+  }
+} as const satisfies ShaderModule<GPUAnimationProps, {}, GPUAnimationBindings>;

package/src/modules/engine/picking/README.md

@@ -0,0 +1,88 @@
+# Shader Module: Picking
+
+Provides support for color-coding-based picking and highlighting. In particular, supports picking a specific instance in an instanced draw call and highlighting an instance based on its picking color, and correspondingly, supports picking and highlighting groups of primitives with the same picking color in non-instanced draw-calls
+
+Color based picking lets the application draw a primitive with a color that can later be used to index this specific primitive.
+
+Highlighting allows application to specify a picking color corresponding to an object that need to be highlighted and the highlight color to be used.
+
+## Usage
+
+In your vertex shader, your inform the picking module what object we are currently rendering by supplying a picking color, perhaps from an attribute.
+
+```
+in vec3 aPickingColor;
+main() {
+  picking_setColor(aPickingColor);
+  ...
+}
+```
+
+In your fragment shader, you simply apply (call) the `picking_filterColor` filter function at the very end of the shader. This will return the normal color, or the highlight color, or the picking color, as appropriate.
+
+```
+main() {
+  fragColor = ...
+  fragColor = picking_filterColor(color);
+}
+```
+
+If highlighting is not needed, you simply apply (call) the `picking_filterPickingColor` filter function at the very end of the shader. This will return the normal color or the picking color, as appropriate.
+
+```
+main() {
+  fragColor = ...
+  fragColor = picking_filterPickingColor(fragColor);
+}
+```
+
+If additional filters need to be applied on the non-picking color (vertex or highlight color) you can use above functions in following order.
+
+```
+main() {
+  fragColor = ...
+  fragColor = picking_filterHighlightColor(fragColor);
+   ... apply any filters on fragColor ...
+ fragColor = picking_filterPickingColor(fragColor);
+}
+```
+
+## JavaScript Functions
+
+### getUniforms
+
+`getUniforms` takes an object with key/value pairs, returns an object with key/value pairs representing the uniforms that the `picking` module shaders need.
+
+`getUniforms(opts)`
+opts can contain following keys:
+
+- `pickingSelectedColorValid` (_boolean_) - When true current instance picking color is ignored, hence no instance is highlighted.
+- `pickingSelectedColor` (_array_) - Picking color of the currently selected instance.
+- `pickingHighlightColor` (_array_)- Color used to highlight the currently selected instance.
+- `pickingActive`=`false` (_boolean_) - When true, renders the picking colors instead of the normal colors. Normally only used with an off-screen framebuffer during picking. Default value is `false`.
+
+Note that the selected item will be rendered using `pickingHighlightColor`, if blending is enabled for the draw, alpha channel can be used to control the blending result.
+
+## Vertex Shader Functions
+
+### `void picking_setPickingColor(vec3)`
+
+Sets the color that will be returned by the fragment shader if color based picking is enabled. Typically set from a `pickingColor` uniform or a `pickingColors` attribute (e.g. when using instanced rendering, to identify the actual instance that was picked).
+
+## Fragment Shader Functions
+
+### picking_filterPickingColor
+
+If picking active, returns the current vertex's picking color set by `picking_setPickingColor`, otherwise returns its argument unmodified.
+
+`vec4 picking_filterPickingColor(vec4 color)`
+
+### picking_filterHighlightColor
+
+Returns picking highlight color if the pixel belongs to currently selected model, otherwise returns its argument unmodified.
+
+`vec4 picking_filterHighlightColor(vec4 color)`
+
+## Remarks
+
+- It is strongly recommended that `picking_filterPickingColor` is called last in a fragment shader, as the picking color (returned when picking is enabled) must not be modified in any way (and alpha must remain 1) or picking results will not be correct.

package/src/modules/engine/picking/picking.ts

@@ -0,0 +1,252 @@
+// luma.gl
+// SPDX-License-Identifier: MIT
+// SPDX-FileCopyrightText: Copyright (c) vis.gl contributors
+
+import {ShaderModule} from '../../../lib/shader-module/shader-module';
+import type {NumberArray3, NumberArray4} from '@math.gl/core';
+import {normalizeByteColor4, resolveUseByteColors} from '../../../lib/color/normalize-byte-colors';
+
+// cyan color
+const DEFAULT_HIGHLIGHT_COLOR: NumberArray4 = [0, 1, 1, 1];
+
+/**
+ * Props for the picking module, which depending on mode renders picking colors or highlighted item.
+ * When active, renders picking colors, assumed to be rendered to off-screen "picking" buffer.
+ * When inactive, renders normal colors, with the exception of selected object which is rendered with highlight
+ */
+export type PickingProps = {
+  /** Are we picking? I.e. rendering picking colors? */
+  isActive?: boolean;
+  /** Set to true when picking an attribute value instead of object index */
+  isAttribute?: boolean;
+  /** Set to a picking color to visually highlight that item, or `null` to explicitly clear **/
+  highlightedObjectColor?: NumberArray3 | null;
+  /** Color of visual highlight of "selected" item */
+  highlightColor?: NumberArray3 | NumberArray4;
+  /** Interpret highlight colors as byte-style 0-255 values. */
+  useByteColors?: boolean;
+};
+
+/**
+ * Uniforms for the picking module, which renders picking colors and highlighted item.
+ * When active, renders picking colors, assumed to be rendered to off-screen "picking" buffer.
+ * When inactive, renders normal colors, with the exception of selected object which is rendered with highlight
+ */
+export type PickingUniforms = {
+  /**
+   * When true, renders picking colors. Set when rendering to off-screen "picking" buffer.
+   * When false, renders normal colors, with the exception of selected object which is rendered with highlight
+   */
+  isActive?: boolean;
+  /** Set to true when picking an attribute value instead of object index */
+  isAttribute?: boolean;
+  /** Interpret highlight colors as byte-style 0-255 values. */
+  useByteColors?: boolean;
+  /** Do we have a highlighted item? */
+  isHighlightActive?: boolean;
+  /** Set to a picking color to visually highlight that item */
+  highlightedObjectColor?: NumberArray3;
+  /** Color of visual highlight of "selected" item */
+  highlightColor?: NumberArray4;
+};
+
+const vs = /* glsl */ `\
+layout(std140) uniform pickingUniforms {
+  float isActive;
+  float isAttribute;
+  float isHighlightActive;
+  float useByteColors;
+  vec3 highlightedObjectColor;
+  vec4 highlightColor;
+} picking;
+
+out vec4 picking_vRGBcolor_Avalid;
+
+// Normalize unsigned byte color to 0-1 range
+vec3 picking_normalizeColor(vec3 color) {
+  return picking.useByteColors > 0.5 ? color / 255.0 : color;
+}
+
+// Normalize unsigned byte color to 0-1 range
+vec4 picking_normalizeColor(vec4 color) {
+  return picking.useByteColors > 0.5 ? color / 255.0 : color;
+}
+
+bool picking_isColorZero(vec3 color) {
+  return dot(color, vec3(1.0)) < 0.00001;
+}
+
+bool picking_isColorValid(vec3 color) {
+  return dot(color, vec3(1.0)) > 0.00001;
+}
+
+// Check if this vertex is highlighted 
+bool isVertexHighlighted(vec3 vertexColor) {
+  vec3 highlightedObjectColor = picking_normalizeColor(picking.highlightedObjectColor);
+  return
+    bool(picking.isHighlightActive) && picking_isColorZero(abs(vertexColor - highlightedObjectColor));
+}
+
+// Set the current picking color
+void picking_setPickingColor(vec3 pickingColor) {
+  pickingColor = picking_normalizeColor(pickingColor);
+
+  if (bool(picking.isActive)) {
+    // Use alpha as the validity flag. If pickingColor is [0, 0, 0] fragment is non-pickable
+    picking_vRGBcolor_Avalid.a = float(picking_isColorValid(pickingColor));
+
+    if (!bool(picking.isAttribute)) {
+      // Stores the picking color so that the fragment shader can render it during picking
+      picking_vRGBcolor_Avalid.rgb = pickingColor;
+    }
+  } else {
+    // Do the comparison with selected item color in vertex shader as it should mean fewer compares
+    picking_vRGBcolor_Avalid.a = float(isVertexHighlighted(pickingColor));
+  }
+}
+
+void picking_setPickingAttribute(float value) {
+  if (bool(picking.isAttribute)) {
+    picking_vRGBcolor_Avalid.r = value;
+  }
+}
+
+void picking_setPickingAttribute(vec2 value) {
+  if (bool(picking.isAttribute)) {
+    picking_vRGBcolor_Avalid.rg = value;
+  }
+}
+
+void picking_setPickingAttribute(vec3 value) {
+  if (bool(picking.isAttribute)) {
+    picking_vRGBcolor_Avalid.rgb = value;
+  }
+}
+`;
+
+const fs = /* glsl */ `\
+layout(std140) uniform pickingUniforms {
+  float isActive;
+  float isAttribute;
+  float isHighlightActive;
+  float useByteColors;
+  vec3 highlightedObjectColor;
+  vec4 highlightColor;
+} picking;
+
+in vec4 picking_vRGBcolor_Avalid;
+
+/*
+ * Returns highlight color if this item is selected.
+ */
+vec4 picking_filterHighlightColor(vec4 color) {
+  // If we are still picking, we don't highlight
+  if (picking.isActive > 0.5) {
+    return color;
+  }
+
+  bool selected = bool(picking_vRGBcolor_Avalid.a);
+
+  if (selected) {
+    // Blend in highlight color based on its alpha value
+    float highLightAlpha = picking.highlightColor.a;
+    float blendedAlpha = highLightAlpha + color.a * (1.0 - highLightAlpha);
+    float highLightRatio = highLightAlpha / blendedAlpha;
+
+    vec3 blendedRGB = mix(color.rgb, picking.highlightColor.rgb, highLightRatio);
+    return vec4(blendedRGB, blendedAlpha);
+  } else {
+    return color;
+  }
+}
+
+/*
+ * Returns picking color if picking enabled else unmodified argument.
+ */
+vec4 picking_filterPickingColor(vec4 color) {
+  if (bool(picking.isActive)) {
+    if (picking_vRGBcolor_Avalid.a == 0.0) {
+      discard;
+    }
+    return picking_vRGBcolor_Avalid;
+  }
+  return color;
+}
+
+/*
+ * Returns picking color if picking is enabled if not
+ * highlight color if this item is selected, otherwise unmodified argument.
+ */
+vec4 picking_filterColor(vec4 color) {
+  vec4 highlightColor = picking_filterHighlightColor(color);
+  return picking_filterPickingColor(highlightColor);
+}
+`;
+
+/**
+ * Deprecated legacy picking module retained for compatibility with existing
+ * shaders users such as deck.gl. Keep the shader contract stable.
+ *
+ * Provides support for color-coding-based picking and highlighting.
+ * In particular, supports picking a specific instance in an instanced
+ * draw call and highlighting an instance based on its picking color,
+ * and correspondingly, supports picking and highlighting groups of
+ * primitives with the same picking color in non-instanced draw-calls
+ */
+export const picking = {
+  props: {} as PickingProps,
+  uniforms: {} as PickingUniforms,
+
+  name: 'picking',
+
+  uniformTypes: {
+    isActive: 'f32',
+    isAttribute: 'f32',
+    isHighlightActive: 'f32',
+    useByteColors: 'f32',
+    highlightedObjectColor: 'vec3<f32>',
+    highlightColor: 'vec4<f32>'
+  },
+  defaultUniforms: {
+    isActive: false,
+    isAttribute: false,
+    isHighlightActive: false,
+    useByteColors: true,
+    highlightedObjectColor: [0, 0, 0],
+    highlightColor: DEFAULT_HIGHLIGHT_COLOR
+  },
+
+  vs,
+  fs,
+  getUniforms
+} as const satisfies ShaderModule<PickingProps, PickingUniforms>;
+
+function getUniforms(opts: PickingProps = {}, prevUniforms?: PickingUniforms): PickingUniforms {
+  const uniforms = {} as PickingUniforms;
+  const useByteColors = resolveUseByteColors(opts.useByteColors, true);
+
+  if (opts.highlightedObjectColor === undefined) {
+    // Unless highlightedObjectColor explicitly null or set, do not update state
+  } else if (opts.highlightedObjectColor === null) {
+    uniforms.isHighlightActive = false;
+  } else {
+    uniforms.isHighlightActive = true;
+    const highlightedObjectColor = opts.highlightedObjectColor.slice(0, 3) as NumberArray3;
+    uniforms.highlightedObjectColor = highlightedObjectColor;
+  }
+
+  if (opts.highlightColor) {
+    uniforms.highlightColor = normalizeByteColor4(opts.highlightColor, useByteColors);
+  }
+
+  if (opts.isActive !== undefined) {
+    uniforms.isActive = Boolean(opts.isActive);
+    uniforms.isAttribute = Boolean(opts.isAttribute);
+  }
+
+  if (opts.useByteColors !== undefined) {
+    uniforms.useByteColors = Boolean(opts.useByteColors);
+  }
+
+  return uniforms;
+}