@aics/vole-core 3.13.0 → 3.14.0

package/README.md

@@ -1,6 +1,9 @@
 # Vol-E core
 
-This is a WebGL canvas-based volume viewer. It can display multichannel volume data with high channel counts, and is optimized for OME-Zarr files. With OME-Zarr, the viewer can prefetch and cache Zarr chunks in browser memory for optimized performance.
+![NPM Version](https://img.shields.io/npm/v/%40aics%2Fvole-core)
+![NPM Last Update](https://img.shields.io/npm/last-update/%40aics%2Fvole-core)
+
+**Vol-E core** is a WebGL canvas-based volume viewer. It can display multichannel volume data with high channel counts. The viewer is optimized for OME-Zarr files, and can prefetch and cache Zarr chunks in browser memory for performance.
 
 The Vol-E core package exposes several key modules:
 
@@ -61,7 +64,7 @@ loader.loadVolumeData(volume);
 
 ## React example
 
-See [vole-app](https://github.com/allen-cell-animated/vole-app) for a complete application that wraps View3D in a React component.
+See [vole-app](https://github.com/allen-cell-animated/vole-app) for a complete application that wraps Vol-E core in a React component.
 
 ## Acknowledgements
 

package/es/FusedChannelData.js

@@ -1,11 +1,13 @@
 import { Color, DataTexture, RedFormat, UnsignedByteType, ClampToEdgeWrapping, Scene, OrthographicCamera, WebGLRenderTarget, RGBAFormat, ShaderMaterial, Mesh, PlaneGeometry, OneFactor, CustomBlending, MaxEquation, LinearFilter, Vector2 } from "three";
 import { renderToBufferVertShader } from "./constants/basicShaders.js";
 /* babel-plugin-inline-import './constants/shaders/fuseUI.frag' */
-const fuseShaderSrcUI = "precision highp float;\nprecision highp int;\nprecision highp usampler2D;\nprecision highp sampler3D;\n\n// the lut texture is a 256x1 rgba texture for each channel\nuniform sampler2D lutSampler;\n\nuniform vec2 lutMinMax;\n\n// src texture is the raw volume intensity data\nuniform usampler2D srcTexture;\n\nvoid main()\n{\n    ivec2 vUv = ivec2(int(gl_FragCoord.x), int(gl_FragCoord.y));\n    uint intensity = texelFetch(srcTexture, vUv, 0).r;\n    float ilookup = float(float(intensity) - lutMinMax.x) / float(lutMinMax.y - lutMinMax.x);\n    // apply lut to intensity:\n    vec4 pix = texture(lutSampler, vec2(ilookup, 0.5));\n    gl_FragColor = vec4(pix.xyz*pix.w, pix.w);\n}\n";
+const fuseShaderSrcUI = "precision highp float;\nprecision highp int;\nprecision highp usampler2D;\nprecision highp sampler3D;\n\n// the lut texture is a 256x1 rgba texture for each channel\nuniform sampler2D lutSampler;\n\nuniform vec2 lutMinMax;\nuniform uint highlightedId;\n\n// src texture is the raw volume intensity data\nuniform usampler2D srcTexture;\n\nvoid main()\n{\n    ivec2 vUv = ivec2(int(gl_FragCoord.x), int(gl_FragCoord.y));\n    uint intensity = texelFetch(srcTexture, vUv, 0).r;\n    if (intensity == (highlightedId)) {\n        gl_FragColor = vec4(1.0, 1.0, 1.0, 1.0);\n        return;\n    }\n    float ilookup = float(float(intensity) - lutMinMax.x) / float(lutMinMax.y - lutMinMax.x);\n    // apply lut to intensity:\n    vec4 pix = texture(lutSampler, vec2(ilookup, 0.5));\n    gl_FragColor = vec4(pix.xyz*pix.w, pix.w);\n}\n";
 /* babel-plugin-inline-import './constants/shaders/fuseF.frag' */
 const fuseShaderSrcF = "precision highp float;\nprecision highp int;\nprecision highp sampler2D;\nprecision highp sampler3D;\n\n// the lut texture is a 256x1 rgba texture for each channel\nuniform sampler2D lutSampler;\n\nuniform vec2 lutMinMax;\n\n// src texture is the raw volume intensity data\nuniform sampler2D srcTexture;\n\nvoid main()\n{\n    ivec2 vUv = ivec2(int(gl_FragCoord.x), int(gl_FragCoord.y));\n\n    // load from channel\n    float intensity = texelFetch(srcTexture, vUv, 0).r;\n\n    float ilookup = float(float(intensity) - lutMinMax.x) / float(lutMinMax.y - lutMinMax.x);\n    // apply lut to intensity:\n    vec4 pix = texture(lutSampler, vec2(ilookup, 0.5));\n    gl_FragColor = vec4(pix.xyz*pix.w, pix.w);\n}\n";
 /* babel-plugin-inline-import './constants/shaders/fuseI.frag' */
 const fuseShaderSrcI = "precision highp float;\nprecision highp int;\nprecision highp sampler2D;\nprecision highp sampler3D;\n\n// the lut texture is a 256x1 rgba texture for each channel\nuniform sampler2D lutSampler;\n\nuniform vec2 lutMinMax;\n\n// src texture is the raw volume intensity data\nuniform isampler2D srcTexture;\n\nvoid main()\n{\n    ivec2 vUv = ivec2(int(gl_FragCoord.x), int(gl_FragCoord.y));\n    int intensity = texelFetch(srcTexture, vUv, 0).r;\n    float ilookup = float(float(intensity) - lutMinMax.x) / float(lutMinMax.y - lutMinMax.x);\n    // apply lut to intensity:\n    vec4 pix = texture(lutSampler, vec2(ilookup, 0.5));\n    gl_FragColor = vec4(pix.xyz*pix.w, pix.w);\n}\n";
+/* babel-plugin-inline-import './constants/shaders/colorizeUI.frag' */
+const colorizeSrcUI = "precision highp float;\nprecision highp int;\nprecision highp usampler2D;\nprecision highp sampler3D;\n\nuniform sampler2D featureData;\n/** Min and max feature values that define the endpoints of the color map. Values\n * outside the range will be clamped to the nearest endpoint.\n */\nuniform float featureColorRampMin;\nuniform float featureColorRampMax;\nuniform sampler2D colorRamp;\nuniform usampler2D inRangeIds;\nuniform usampler2D outlierData;\n\nuniform vec3 outlineColor;\n\n/** MUST be synchronized with the DrawMode enum in ColorizeCanvas! */\nconst uint DRAW_MODE_HIDE = 0u;\nconst uint DRAW_MODE_COLOR = 1u;\n\nuniform vec3 outlierColor;\nuniform uint outlierDrawMode;\nuniform vec3 outOfRangeColor;\nuniform uint outOfRangeDrawMode;\n\nuniform uint highlightedId;\n\nuniform bool hideOutOfRange;\n\n// src texture is the raw volume intensity data\nuniform usampler2D srcTexture;\n\nuint getId(ivec2 uv) {\n    return texelFetch(srcTexture, uv, 0).r;\n}\n\nvec4 getFloatFromTex(sampler2D tex, int index) {\n  int width = textureSize(tex, 0).x;\n  ivec2 featurePos = ivec2(index % width, index / width);\n  return texelFetch(tex, featurePos, 0);\n}\nuvec4 getUintFromTex(usampler2D tex, int index) {\n  int width = textureSize(tex, 0).x;\n  ivec2 featurePos = ivec2(index % width, index / width);\n  return texelFetch(tex, featurePos, 0);\n}\nvec4 getColorRamp(float val) {\n  float width = float(textureSize(colorRamp, 0).x);\n  float range = (width - 1.0) / width;\n  float adjustedVal = (0.5 / width) + (val * range);\n  return texture(colorRamp, vec2(adjustedVal, 0.5));\n}\nvec4 getColorFromDrawMode(uint drawMode, vec3 defaultColor) {\n  const uint DRAW_MODE_HIDE = 0u;\n  vec3 backgroundColor = vec3(0.0, 0.0, 0.0);\n  if (drawMode == DRAW_MODE_HIDE) {\n    return vec4(backgroundColor, 0.0);\n  } else {\n    return vec4(defaultColor, 1.0);\n  }\n}\n\nfloat getFeatureVal(uint id) {\n  // Data buffer starts at 0, non-background segmentation IDs start at 1\n    return getFloatFromTex(featureData, int(id) - 1).r;\n}\nuint getOutlierVal(uint id) {\n  // Data buffer starts at 0, non-background segmentation IDs start at 1\n    return getUintFromTex(outlierData, int(id) - 1).r;\n}\nbool getIsInRange(uint id) {\n    return getUintFromTex(inRangeIds, int(id) - 1).r == 1u;\n}\nbool getIsOutlier(float featureVal, uint outlierVal) {\n    return isinf(featureVal) || outlierVal != 0u;\n}\n\nvec4 getObjectColor(ivec2 sUv, float opacity) {\n  // Get the segmentation id at this pixel\n  uint id = getId(sUv);\n\n  // A segmentation id of 0 represents background\n  if (id == 0u) {\n    return vec4(0,0,0,0);\n  }\n\n  // color the highlighted object\n  if (id == highlightedId) {\n    return vec4(outlineColor, 1.0);\n  }\n\n\n  float featureVal = getFeatureVal(id);\n  uint outlierVal = getOutlierVal(id);\n  float normFeatureVal = (featureVal - featureColorRampMin) / (featureColorRampMax - featureColorRampMin);\n\n  // Use the selected draw mode to handle out of range and outlier values;\n  // otherwise color with the color ramp as usual.\n  bool isInRange = getIsInRange(id);\n  bool isOutlier = getIsOutlier(featureVal, outlierVal);\n\n  // Features outside the filtered/thresholded range will all be treated the same (use `outOfRangeDrawColor`).\n  // Features inside the range can either be outliers or standard values, and are colored accordingly.\n  vec4 color;\n  if (isInRange) {\n    if (isOutlier) {\n      color = getColorFromDrawMode(outlierDrawMode, outlierColor);\n    } else {\n      color = getColorRamp(normFeatureVal);\n    }\n  } else {\n    color = getColorFromDrawMode(outOfRangeDrawMode, outOfRangeColor);\n  }\n  color.a *= opacity;\n  return color;\n}\n\nvoid main()\n{\n    ivec2 vUv = ivec2(int(gl_FragCoord.x), int(gl_FragCoord.y));\n    gl_FragColor = getObjectColor(vUv, 1.0);\n}\n";
 // This is the owner of the fused RGBA volume texture atlas, and the mask texture atlas.
 // This module is responsible for updating the fused texture, given the read-only volume channel data.
 export default class FusedChannelData {
@@ -52,6 +54,7 @@ export default class FusedChannelData {
     this.fuseMaterialF = this.setupFuseMaterial(fuseShaderSrcF);
     this.fuseMaterialUI = this.setupFuseMaterial(fuseShaderSrcUI);
     this.fuseMaterialI = this.setupFuseMaterial(fuseShaderSrcI);
+    this.fuseMaterialColorizeUI = this.setupFuseColorizeMaterial(colorizeSrcUI);
     this.fuseMaterialF.needsUpdate = true;
     this.fuseMaterialUI.needsUpdate = true;
     this.fuseMaterialI.needsUpdate = true;
@@ -60,6 +63,9 @@ export default class FusedChannelData {
   setupFuseMaterial(fragShaderSrc) {
     return new ShaderMaterial({
       uniforms: {
+        highlightedId: {
+          value: -1
+        },
         lutSampler: {
           value: null
         },
@@ -74,6 +80,56 @@ export default class FusedChannelData {
       ...this.fuseMaterialProps
     });
   }
+  setupFuseColorizeMaterial(fragShaderSrc) {
+    return new ShaderMaterial({
+      uniforms: {
+        highlightedId: {
+          value: -1
+        },
+        featureData: {
+          value: null
+        },
+        outlierData: {
+          value: null
+        },
+        inRangeIds: {
+          value: null
+        },
+        srcTexture: {
+          value: null
+        },
+        featureColorRampMin: {
+          value: 0
+        },
+        featureColorRampMax: {
+          value: 1
+        },
+        colorRamp: {
+          value: null
+        },
+        outlineColor: {
+          value: new Color(0xffffff)
+        },
+        outlierColor: {
+          value: new Color(0x444444)
+        },
+        outOfRangeColor: {
+          value: new Color(0x444444)
+        },
+        outlierDrawMode: {
+          value: 0
+        },
+        outOfRangeDrawMode: {
+          value: 0
+        },
+        hideOutOfRange: {
+          value: false
+        }
+      },
+      fragmentShader: fragShaderSrc,
+      ...this.fuseMaterialProps
+    });
+  }
   getFusedTexture() {
     return this.fuseRenderTarget.texture;
   }
@@ -81,14 +137,14 @@ export default class FusedChannelData {
     this.fuseScene.clear();
     this.maskTexture.dispose();
   }
-  getShader(dtype) {
+  getShader(dtype, isColorize) {
     switch (dtype) {
       case "float32":
         return this.fuseMaterialF;
       case "uint8":
       case "uint16":
       case "uint32":
-        return this.fuseMaterialUI;
+        return isColorize ? this.fuseMaterialColorizeUI : this.fuseMaterialUI;
       case "int8":
       case "int16":
       case "int32":
@@ -143,13 +199,30 @@ export default class FusedChannelData {
         if (!channels[chIndex].loaded) {
           continue;
         }
+        const isColorize = combination[i].feature !== undefined;
         // add a draw call per channel here.
         // must clone the material to keep a unique set of uniforms
-        const mat = this.getShader(channels[chIndex].dtype).clone();
-        mat.uniforms.lutSampler.value = channels[chIndex].lutTexture;
-        // the lut texture is spanning only the data range of the channel, not the datatype range
-        mat.uniforms.lutMinMax.value = new Vector2(channels[chIndex].rawMin, channels[chIndex].rawMax);
+        const mat = this.getShader(channels[chIndex].dtype, isColorize).clone();
         mat.uniforms.srcTexture.value = channels[chIndex].dataTexture;
+        mat.uniforms.highlightedId.value = combination[i].selectedID == undefined ? -1 : combination[i].selectedID;
+        if (isColorize) {
+          mat.uniforms.featureData.value = combination[i].feature?.idsToFeatureValue;
+          mat.uniforms.outlierData.value = combination[i].feature?.outlierData;
+          mat.uniforms.inRangeIds.value = combination[i].feature?.inRangeIds;
+          mat.uniforms.featureColorRampMin.value = combination[i].feature?.featureMin;
+          mat.uniforms.featureColorRampMax.value = combination[i].feature?.featureMax;
+          mat.uniforms.colorRamp.value = combination[i].feature?.featureValueToColor;
+          mat.uniforms.outlineColor.value = combination[i].feature?.outlineColor;
+          mat.uniforms.outlierColor.value = combination[i].feature?.outlierColor;
+          mat.uniforms.outOfRangeColor.value = combination[i].feature?.outOfRangeColor;
+          mat.uniforms.outlierDrawMode.value = combination[i].feature?.outlierDrawMode;
+          mat.uniforms.outOfRangeDrawMode.value = combination[i].feature?.outOfRangeDrawMode;
+          mat.uniforms.hideOutOfRange.value = combination[i].feature?.hideOutOfRange;
+        } else {
+          // the lut texture is spanning only the data range of the channel, not the datatype range
+          mat.uniforms.lutMinMax.value = new Vector2(channels[chIndex].rawMin, channels[chIndex].rawMax);
+          mat.uniforms.lutSampler.value = channels[chIndex].lutTexture;
+        }
         this.fuseScene.add(new Mesh(this.fuseGeometry, mat));
       }
     }

package/es/PickVolume.js

@@ -0,0 +1,244 @@
+import { BoxGeometry, Color, DataTexture, FloatType, Group, Matrix4, Mesh, NearestFilter, RGBAFormat, Scene, ShaderMaterial, Vector2, WebGLRenderTarget } from "three";
+import { pickVertexShaderSrc, pickFragmentShaderSrc, pickShaderUniforms } from "./constants/volumeRayMarchPickShader.js";
+import { VolumeRenderSettings, SettingsFlags } from "./VolumeRenderSettings.js";
+export default class PickVolume {
+  needRedraw = false;
+  channelToPick = 0;
+
+  /**
+   * Creates a new PickVolume.
+   * @param volume The volume that this renderer should render data from.
+   * @param settings Optional settings object. If set, updates the renderer with
+   * the given settings. Otherwise, uses the default VolumeRenderSettings.
+   */
+  constructor(volume, settings = new VolumeRenderSettings(volume)) {
+    this.volume = volume;
+    this.uniforms = pickShaderUniforms();
+    [this.geometry, this.geometryMesh] = this.createGeometry(this.uniforms);
+    this.geometryTransformNode = new Group();
+    this.geometryTransformNode.name = "PickVolumeContainerNode";
+    this.geometryTransformNode.add(this.geometryMesh);
+    this.scene = new Scene();
+    this.scene.name = "PickVolumeScene";
+    this.scene.add(this.geometryTransformNode);
+    this.emptyPositionTex = new DataTexture(new Uint8Array(Array(16).fill(0)), 2, 2);
+
+    // buffers:
+    this.pickBuffer = new WebGLRenderTarget(2, 2, {
+      count: 1,
+      minFilter: NearestFilter,
+      magFilter: NearestFilter,
+      format: RGBAFormat,
+      type: FloatType,
+      generateMipmaps: false
+    });
+    // Name our G-Buffer attachments for debugging
+    const OBJECTBUFFER = 0;
+    this.pickBuffer.textures[OBJECTBUFFER].name = "objectinfo";
+    this.settings = settings;
+    this.updateSettings(settings, SettingsFlags.ALL);
+    // TODO this is doing *more* redundant work! Fix?
+    this.updateVolumeDimensions();
+  }
+  setChannelToPick(channel) {
+    this.channelToPick = channel;
+  }
+  getPickBuffer() {
+    return this.pickBuffer;
+  }
+  updateVolumeDimensions() {
+    const {
+      normPhysicalSize,
+      normRegionSize
+    } = this.volume;
+    // Set offset
+    this.geometryMesh.position.copy(this.volume.getContentCenter().multiply(this.settings.scale));
+    // Set scale
+    const fullRegionScale = normPhysicalSize.clone().multiply(this.settings.scale);
+    this.geometryMesh.scale.copy(fullRegionScale).multiply(normRegionSize);
+    this.setUniform("volumeScale", normPhysicalSize);
+    this.settings && this.updateSettings(this.settings, SettingsFlags.ROI);
+
+    // Set atlas dimension uniforms
+    const {
+      atlasTileDims,
+      subregionSize
+    } = this.volume.imageInfo;
+    const atlasSize = new Vector2(subregionSize.x, subregionSize.y).multiply(atlasTileDims);
+    this.setUniform("ATLAS_DIMS", atlasTileDims);
+    this.setUniform("textureRes", atlasSize);
+    this.setUniform("SLICES", this.volume.imageInfo.volumeSize.z);
+
+    // (re)create channel data
+  }
+  viewpointMoved() {
+    this.needRedraw = true;
+  }
+  updateSettings(newSettings, dirtyFlags) {
+    if (dirtyFlags === undefined) {
+      dirtyFlags = SettingsFlags.ALL;
+    }
+    this.settings = newSettings;
+    if (dirtyFlags & SettingsFlags.VIEW) {
+      this.needRedraw = true;
+      this.geometryMesh.visible = this.settings.visible;
+      // Configure ortho
+      this.setUniform("orthoScale", this.settings.orthoScale);
+      this.setUniform("isOrtho", this.settings.isOrtho ? 1.0 : 0.0);
+      // Ortho line thickness
+      const axis = this.settings.viewAxis;
+      if (this.settings.isOrtho && axis) {
+        // TODO: Does this code do any relevant changes?
+        const maxVal = this.settings.bounds.bmax[axis];
+        const minVal = this.settings.bounds.bmin[axis];
+        const thicknessPct = maxVal - minVal;
+        this.setUniform("orthoThickness", thicknessPct);
+      } else {
+        this.setUniform("orthoThickness", 1.0);
+      }
+    }
+    if (dirtyFlags & SettingsFlags.VIEW || dirtyFlags & SettingsFlags.BOUNDING_BOX) {
+      // Update tick marks with either view or bounding box changes
+      //      this.setUniform("maxProject", this.settings.maxProjectMode ? 1 : 0);
+    }
+    if (dirtyFlags & SettingsFlags.TRANSFORM) {
+      this.needRedraw = true;
+      // Set rotation and translation
+      this.geometryTransformNode.position.copy(this.settings.translation);
+      this.geometryTransformNode.rotation.copy(this.settings.rotation);
+      // TODO this does some redundant work. Including a new call to this very function! Fix?
+      this.updateVolumeDimensions();
+      this.setUniform("flipVolume", this.settings.flipAxes);
+    }
+    if (dirtyFlags & SettingsFlags.MATERIAL) {
+      // nothing
+    }
+    if (dirtyFlags & SettingsFlags.CAMERA) {
+      // nothing
+    }
+    if (dirtyFlags & SettingsFlags.ROI) {
+      this.needRedraw = true;
+      // Normalize and set bounds
+      const bounds = this.settings.bounds;
+      const {
+        normRegionSize,
+        normRegionOffset
+      } = this.volume;
+      const offsetToCenter = normRegionSize.clone().divideScalar(2).add(normRegionOffset).subScalar(0.5);
+      const bmin = bounds.bmin.clone().sub(offsetToCenter).divide(normRegionSize).clampScalar(-0.5, 0.5);
+      const bmax = bounds.bmax.clone().sub(offsetToCenter).divide(normRegionSize).clampScalar(-0.5, 0.5);
+      this.setUniform("AABB_CLIP_MIN", bmin);
+      this.setUniform("AABB_CLIP_MAX", bmax);
+    }
+    if (dirtyFlags & SettingsFlags.SAMPLING) {
+      this.needRedraw = true;
+      const resolution = this.settings.resolution.clone();
+      const dpr = window.devicePixelRatio ? window.devicePixelRatio : 1.0;
+      const nx = Math.floor(resolution.x / dpr);
+      const ny = Math.floor(resolution.y / dpr);
+      this.setUniform("iResolution", new Vector2(nx, ny));
+      this.pickBuffer.setSize(nx, ny);
+    }
+    if (dirtyFlags & SettingsFlags.MASK_ALPHA) {
+      // nothing
+    }
+    if (dirtyFlags & SettingsFlags.MASK_DATA) {
+      // nothing
+    }
+  }
+
+  /**
+   * Creates the geometry mesh and material for rendering the volume.
+   * @param uniforms object containing uniforms to pass to the shader material.
+   * @returns the new geometry and geometry mesh.
+   */
+  createGeometry(pickUniforms) {
+    const geom = new BoxGeometry(1.0, 1.0, 1.0);
+
+    // shader,vtx and frag.
+
+    const threePickMaterial = new ShaderMaterial({
+      uniforms: pickUniforms,
+      vertexShader: pickVertexShaderSrc,
+      fragmentShader: pickFragmentShaderSrc,
+      depthTest: true,
+      depthWrite: false
+    });
+    const pickMesh = new Mesh(geom, threePickMaterial);
+    pickMesh.name = "PickVolume";
+    return [geom, pickMesh];
+  }
+  cleanup() {
+    // do i need to empty out the pickscene?
+    this.scene.clear(); // remove all children from the scene
+    this.geometryTransformNode.clear(); // remove all children from the transform node
+
+    this.geometry.dispose();
+    this.geometryMesh.material.dispose();
+  }
+  doRender(renderer, camera, depthTexture) {
+    if (!this.geometryMesh.visible) {
+      return;
+    }
+    if (!this.needRedraw) {
+      return;
+    }
+    this.needRedraw = false;
+    this.setUniform("iResolution", this.settings.resolution);
+    this.setUniform("textureRes", this.settings.resolution);
+    const depthTex = depthTexture ?? this.emptyPositionTex;
+    this.setUniform("textureDepth", depthTex);
+    this.setUniform("usingPositionTexture", depthTex.isDepthTexture ? 0 : 1);
+    this.setUniform("CLIP_NEAR", camera.near);
+    this.setUniform("CLIP_FAR", camera.far);
+
+    // this.channelData.gpuFuse(renderer);
+
+    // set up texture from segmentation channel!!!!
+    // we need to know the channel index for this.
+    // ...channel.dataTexture...
+    // TODO TODO TODO FIXME
+    this.setUniform("textureAtlas", this.volume.getChannel(this.channelToPick).dataTexture);
+    this.geometryTransformNode.updateMatrixWorld(true);
+    const mvm = new Matrix4();
+    mvm.multiplyMatrices(camera.matrixWorldInverse, this.geometryMesh.matrixWorld);
+    mvm.invert();
+    this.setUniform("inverseModelViewMatrix", mvm);
+    this.setUniform("inverseProjMatrix", camera.projectionMatrixInverse);
+    const VOLUME_LAYER = 0;
+    // draw into pick buffer...
+    camera.layers.set(VOLUME_LAYER);
+    renderer.setRenderTarget(this.pickBuffer);
+    renderer.autoClear = true;
+    const prevClearColor = new Color();
+    renderer.getClearColor(prevClearColor);
+    const prevClearAlpha = renderer.getClearAlpha();
+    renderer.setClearColor(0x000000, 0);
+    renderer.render(this.scene, camera);
+    renderer.autoClear = true;
+    renderer.setClearColor(prevClearColor, prevClearAlpha);
+    renderer.setRenderTarget(null);
+  }
+  get3dObject() {
+    return this.geometryTransformNode;
+  }
+
+  //////////////////////////////////////////
+  //////////////////////////////////////////
+
+  setUniform(name, value) {
+    if (!this.uniforms[name]) {
+      return;
+    }
+    this.uniforms[name].value = value;
+  }
+
+  // channelcolors is array of {rgbColor, lut} and channeldata is volume.channels
+  updateActiveChannels(_channelcolors, _channeldata) {
+    // TODO consider if we can use this as a way to assing this.channelToPick?
+    // (e.g. put some kind of flag in FuseChannel)
+  }
+  setRenderUpdateListener(_listener) {
+    return;
+  }
+}

package/es/RayMarchedAtlasVolume.js

@@ -149,21 +149,19 @@ export default class RayMarchedAtlasVolume {
    */
   createGeometry(uniforms) {
     const geom = new BoxGeometry(1.0, 1.0, 1.0);
-    const mesh = new Mesh(geom);
-    mesh.name = "Volume";
 
     // shader,vtx and frag.
-    const vtxsrc = rayMarchingVertexShaderSrc;
-    const fgmtsrc = rayMarchingFragmentShaderSrc;
+
     const threeMaterial = new ShaderMaterial({
       uniforms: uniforms,
-      vertexShader: vtxsrc,
-      fragmentShader: fgmtsrc,
+      vertexShader: rayMarchingVertexShaderSrc,
+      fragmentShader: rayMarchingFragmentShaderSrc,
       transparent: true,
       depthTest: true,
       depthWrite: false
     });
-    mesh.material = threeMaterial;
+    const mesh = new Mesh(geom, threeMaterial);
+    mesh.name = "Volume";
     return [geom, mesh];
   }
   createTickMarks() {

package/es/ThreeJsPanel.js

@@ -1,4 +1,4 @@
-import { AxesHelper, Vector3, Object3D, Mesh, BoxGeometry, MeshBasicMaterial, OrthographicCamera, PerspectiveCamera, NormalBlending, WebGLRenderer, Scene, DepthTexture, WebGLRenderTarget, NearestFilter, UnsignedByteType, RGBAFormat } from "three";
+import { AxesHelper, BoxGeometry, DepthTexture, Mesh, MeshBasicMaterial, Object3D, OrthographicCamera, PerspectiveCamera, NearestFilter, NormalBlending, RGBAFormat, Scene, UnsignedByteType, Vector2, Vector3, WebGLRenderer, WebGLRenderTarget } from "three";
 import TrackballControls from "./TrackballControls.js";
 import Timing from "./Timing.js";
 import scaleBarSVG from "./constants/scaleBarSVG.js";
@@ -630,4 +630,34 @@ export class ThreeJsPanel {
       this.controls.addEventListener("end", this.controlEndHandler);
     }
   }
+  hitTest(offsetX, offsetY, pickBuffer) {
+    if (!pickBuffer) {
+      return -1;
+    }
+    const size = new Vector2();
+    this.renderer.getSize(size);
+    // read from instance buffer pixel!
+    const x = offsetX;
+    const y = size.y - offsetY;
+
+    // if the pick buffer is a different size from our render canvas,
+    // then we have to transform the mouse event coordinates
+    const sx = Math.floor(x / size.x * pickBuffer.width);
+    const sy = Math.floor(y / size.y * pickBuffer.height);
+
+    // read from the instance buffer
+    const pixel = new Float32Array(4).fill(-1);
+    this.renderer.readRenderTargetPixels(pickBuffer, sx, sy, 1, 1, pixel);
+    // For future reference, Simularium stores the following: 
+    // (typeId), (instanceId), fragViewPos.z, fragPosDepth;
+
+    if (pixel[3] === -1 || pixel[3] === 0) {
+      return -1;
+    } else {
+      // look up the object from its instance.
+      // and round it off to nearest integer
+      const instance = Math.round(pixel[1]);
+      return instance;
+    }
+  }
 }

package/es/View3d.js

@@ -96,6 +96,9 @@ export class View3d {
   getDOMElement() {
     return this.canvas3d.containerdiv;
   }
+  getCanvasDOMElement() {
+    return this.canvas3d.renderer.domElement;
+  }
   getCameraState() {
     return this.canvas3d.getCameraState();
   }
@@ -106,9 +109,16 @@ export class View3d {
 
   /**
    * Force a redraw.
+   * @param synchronous If true, the redraw will be done synchronously. If false (default), the
+   * redraw will be done asynchronously via `requestAnimationFrame`. Redraws should be done async
+   * whenever possible for the best performance.
    */
-  redraw() {
-    this.canvas3d.redraw();
+  redraw(synchronous = false) {
+    if (synchronous) {
+      this.canvas3d.onAnimationLoop();
+    } else {
+      this.canvas3d.redraw();
+    }
   }
   unsetImage() {
     if (this.image) {
@@ -206,10 +216,11 @@ export class View3d {
   }
   setTime(volume, time, onChannelLoaded) {
     const timeClamped = Math.max(0, Math.min(time, volume.imageInfo.times - 1));
-    volume.updateRequiredData({
+    const loadPromise = volume.updateRequiredData({
       time: timeClamped
     }, onChannelLoaded);
     this.updateTimestepIndicator(volume);
+    return loadPromise;
   }
 
   /**
@@ -232,6 +243,18 @@ export class View3d {
     this.redraw();
   }
 
+  /**
+   * @description Set the necessary data to colorize a segmentation channel, or turn off colorization.
+   * @param volume The volume to set the colorize feature for
+   * @param channelIndex The channel that will be colorized. This only makes sense for segmentation volumes.
+   * @param featureInfo A collection of all parameters necessary to colorize the channel. Pass null to turn off colorization.
+   */
+  setChannelColorizeFeature(volume, channelIndex, featureInfo) {
+    this.image?.setChannelColorizeFeature(channelIndex, featureInfo);
+    this.image?.fuse();
+    this.redraw();
+  }
+
   /**
    * Set voxel dimensions - controls volume scaling. For example, the physical measurements of the voxels from a biological data set
    * @param {Object} volume
@@ -302,6 +325,7 @@ export class View3d {
     this.canvas3d.setControlHandlers(this.onStartControls.bind(this), this.onChangeControls.bind(this), this.onEndControls.bind(this));
     this.canvas3d.animateFuncs.push(this.preRender.bind(this));
     this.canvas3d.animateFuncs.push(img.onAnimate.bind(img));
+    this.canvas3d.animateFuncs.push(img.fillPickBuffer.bind(img));
     this.updatePerspectiveScaleBar(img.volume);
     this.updateTimestepIndicator(img.volume);
 
@@ -785,6 +809,45 @@ export class View3d {
   removeEventListeners() {
     window.removeEventListener("keydown", this.handleKeydown);
   }
+
+  /**
+   * @description Set the selected ID for a given channel.  This is used to change the appearance of the volume where that id is.
+   * @param volume the image to set the selected ID on
+   * @param channel the channel index where the selected ID is
+   * @param id the selected id
+   */
+  setSelectedID(volume, channel, id) {
+    const needRedraw = this.image?.setSelectedID(channel, id);
+    if (needRedraw) {
+      this.image?.fuse();
+      this.redraw();
+    }
+  }
+
+  /**
+   * @description Enable or disable picking on a volume.  If enabled, the channelIndex is used to determine which channel to pick.
+   * @param volume the image to enable picking on
+   * @param enabled set true to enable, false to disable
+   * @param channelIndex if enabled is set to true, pass the pickable channel index here
+   */
+  enablePicking(volume, enabled, channelIndex = 0) {
+    if (this.image) {
+      this.image.enablePicking(enabled, channelIndex);
+    }
+  }
+
+  /**
+   * @description This function is used to determine if a mouse event occurred over a volume object.
+   * @param offsetX mouse event x coordinate
+   * @param offsetY mouse event y coordinate
+   * @returns id of object that is under offsetX, offsetY. -1 if none
+   */
+  hitTest(offsetX, offsetY) {
+    if (!this.image) {
+      return -1;
+    }
+    return this.canvas3d.hitTest(offsetX, offsetY, this.image.getPickBuffer());
+  }
   setupGui(container) {
     const pane = new Pane({
       title: "Advanced Settings",

package/es/Volume.js

@@ -139,7 +139,7 @@ export default class Volume {
       }
     }
     if (shouldReload) {
-      this.loadNewData(onChannelLoaded);
+      await this.loadNewData(onChannelLoaded);
     }
   }
   async loadScaleLevelDims() {

package/es/VolumeCache.js

@@ -1,3 +1,9 @@
+export const isChunk = data => data.data !== undefined;
+const chunkSize = ({
+  data
+}) => Array.isArray(data) ? data.length : data.byteLength;
+const dataSize = data => data.byteLength ?? chunkSize(data);
+
 /** Default: 250MB. Should be large enough to be useful but safe for most any computer that can run the app */
 const CACHE_MAX_SIZE_DEFAULT = 250_000_000;
 export default class VolumeCache {
@@ -31,7 +37,7 @@ export default class VolumeCache {
    */
   removeEntryFromStore(entry) {
     this.entries.delete(entry.key);
-    this.currentSize -= entry.data.byteLength;
+    this.currentSize -= dataSize(entry.data);
   }
 
   /**
@@ -98,7 +104,8 @@ export default class VolumeCache {
    * @returns {boolean} a boolean indicating whether the insertion succeeded.
    */
   insert(key, data) {
-    if (data.byteLength > this.maxSize) {
+    const size = dataSize(data);
+    if (size > this.maxSize) {
       console.error("VolumeCache: attempt to insert a single entry larger than the cache");
       return false;
     }
@@ -120,7 +127,7 @@ export default class VolumeCache {
     };
     this.addEntryAsFirst(newEntry);
     this.entries.set(key, newEntry);
-    this.currentSize += data.byteLength;
+    this.currentSize += size;
 
     // Evict until size is within limit
     while (this.currentSize > this.maxSize) {