@aics/vole-core 3.14.0 → 3.15.0

package/es/Channel.js

@@ -14,6 +14,7 @@ export default class Channel {
     };
     this.rawMin = 0;
     this.rawMax = 255;
+    this.time = 0;
 
     // on gpu
     this.dataTexture = new DataTexture(new Uint8Array(), 0, 0);
@@ -147,7 +148,7 @@ export default class Channel {
 
   // give the channel fresh data and initialize from that data
   // data is formatted as a texture atlas where each tile is a z slice of the volume
-  setFromAtlas(bitsArray, w, h, dtype, rawMin, rawMax, subregionSize) {
+  setFromAtlas(bitsArray, w, h, dtype, rawMin, rawMax, subregionSize, time = 0) {
     this.dtype = dtype;
     this.imgData = {
       data: bitsArray,
@@ -157,6 +158,7 @@ export default class Channel {
     this.rebuildDataTexture(this.imgData.data, w, h);
     this.loaded = true;
     this.histogram = new Histogram(bitsArray);
+    this.time = time;
 
     // reuse old lut but auto-remap it to new data range
     this.setRawDataRange(rawMin, rawMax);
@@ -193,10 +195,11 @@ export default class Channel {
   }
 
   // give the channel fresh volume data and initialize from that data
-  setFromVolumeData(bitsArray, vx, vy, vz, ax, ay, rawMin, rawMax, dtype) {
+  setFromVolumeData(bitsArray, vx, vy, vz, ax, ay, rawMin, rawMax, dtype, time = 0) {
     this.dims = [vx, vy, vz];
     this.volumeData = bitsArray;
     this.dtype = dtype;
+    this.time = time;
     // TODO FIXME performance hit for shuffling the data and storing 2 versions of it (could do this in worker at least?)
     this.packToAtlas(vx, vy, vz, ax, ay);
     this.loaded = true;

package/es/FusedChannelData.js

@@ -7,7 +7,7 @@ const fuseShaderSrcF = "precision highp float;\nprecision highp int;\nprecision
 /* babel-plugin-inline-import './constants/shaders/fuseI.frag' */
 const fuseShaderSrcI = "precision highp float;\nprecision highp int;\nprecision highp sampler2D;\nprecision highp sampler3D;\n\n// the lut texture is a 256x1 rgba texture for each channel\nuniform sampler2D lutSampler;\n\nuniform vec2 lutMinMax;\n\n// src texture is the raw volume intensity data\nuniform isampler2D srcTexture;\n\nvoid main()\n{\n    ivec2 vUv = ivec2(int(gl_FragCoord.x), int(gl_FragCoord.y));\n    int intensity = texelFetch(srcTexture, vUv, 0).r;\n    float ilookup = float(float(intensity) - lutMinMax.x) / float(lutMinMax.y - lutMinMax.x);\n    // apply lut to intensity:\n    vec4 pix = texture(lutSampler, vec2(ilookup, 0.5));\n    gl_FragColor = vec4(pix.xyz*pix.w, pix.w);\n}\n";
 /* babel-plugin-inline-import './constants/shaders/colorizeUI.frag' */
-const colorizeSrcUI = "precision highp float;\nprecision highp int;\nprecision highp usampler2D;\nprecision highp sampler3D;\n\nuniform sampler2D featureData;\n/** Min and max feature values that define the endpoints of the color map. Values\n * outside the range will be clamped to the nearest endpoint.\n */\nuniform float featureColorRampMin;\nuniform float featureColorRampMax;\nuniform sampler2D colorRamp;\nuniform usampler2D inRangeIds;\nuniform usampler2D outlierData;\n\nuniform vec3 outlineColor;\n\n/** MUST be synchronized with the DrawMode enum in ColorizeCanvas! */\nconst uint DRAW_MODE_HIDE = 0u;\nconst uint DRAW_MODE_COLOR = 1u;\n\nuniform vec3 outlierColor;\nuniform uint outlierDrawMode;\nuniform vec3 outOfRangeColor;\nuniform uint outOfRangeDrawMode;\n\nuniform uint highlightedId;\n\nuniform bool hideOutOfRange;\n\n// src texture is the raw volume intensity data\nuniform usampler2D srcTexture;\n\nuint getId(ivec2 uv) {\n    return texelFetch(srcTexture, uv, 0).r;\n}\n\nvec4 getFloatFromTex(sampler2D tex, int index) {\n  int width = textureSize(tex, 0).x;\n  ivec2 featurePos = ivec2(index % width, index / width);\n  return texelFetch(tex, featurePos, 0);\n}\nuvec4 getUintFromTex(usampler2D tex, int index) {\n  int width = textureSize(tex, 0).x;\n  ivec2 featurePos = ivec2(index % width, index / width);\n  return texelFetch(tex, featurePos, 0);\n}\nvec4 getColorRamp(float val) {\n  float width = float(textureSize(colorRamp, 0).x);\n  float range = (width - 1.0) / width;\n  float adjustedVal = (0.5 / width) + (val * range);\n  return texture(colorRamp, vec2(adjustedVal, 0.5));\n}\nvec4 getColorFromDrawMode(uint drawMode, vec3 defaultColor) {\n  const uint DRAW_MODE_HIDE = 0u;\n  vec3 backgroundColor = vec3(0.0, 0.0, 0.0);\n  if (drawMode == DRAW_MODE_HIDE) {\n    return vec4(backgroundColor, 0.0);\n  } else {\n    return vec4(defaultColor, 1.0);\n  }\n}\n\nfloat getFeatureVal(uint id) {\n  // Data buffer starts at 0, non-background segmentation IDs start at 1\n    return getFloatFromTex(featureData, int(id) - 1).r;\n}\nuint getOutlierVal(uint id) {\n  // Data buffer starts at 0, non-background segmentation IDs start at 1\n    return getUintFromTex(outlierData, int(id) - 1).r;\n}\nbool getIsInRange(uint id) {\n    return getUintFromTex(inRangeIds, int(id) - 1).r == 1u;\n}\nbool getIsOutlier(float featureVal, uint outlierVal) {\n    return isinf(featureVal) || outlierVal != 0u;\n}\n\nvec4 getObjectColor(ivec2 sUv, float opacity) {\n  // Get the segmentation id at this pixel\n  uint id = getId(sUv);\n\n  // A segmentation id of 0 represents background\n  if (id == 0u) {\n    return vec4(0,0,0,0);\n  }\n\n  // color the highlighted object\n  if (id == highlightedId) {\n    return vec4(outlineColor, 1.0);\n  }\n\n\n  float featureVal = getFeatureVal(id);\n  uint outlierVal = getOutlierVal(id);\n  float normFeatureVal = (featureVal - featureColorRampMin) / (featureColorRampMax - featureColorRampMin);\n\n  // Use the selected draw mode to handle out of range and outlier values;\n  // otherwise color with the color ramp as usual.\n  bool isInRange = getIsInRange(id);\n  bool isOutlier = getIsOutlier(featureVal, outlierVal);\n\n  // Features outside the filtered/thresholded range will all be treated the same (use `outOfRangeDrawColor`).\n  // Features inside the range can either be outliers or standard values, and are colored accordingly.\n  vec4 color;\n  if (isInRange) {\n    if (isOutlier) {\n      color = getColorFromDrawMode(outlierDrawMode, outlierColor);\n    } else {\n      color = getColorRamp(normFeatureVal);\n    }\n  } else {\n    color = getColorFromDrawMode(outOfRangeDrawMode, outOfRangeColor);\n  }\n  color.a *= opacity;\n  return color;\n}\n\nvoid main()\n{\n    ivec2 vUv = ivec2(int(gl_FragCoord.x), int(gl_FragCoord.y));\n    gl_FragColor = getObjectColor(vUv, 1.0);\n}\n";
+const colorizeSrcUI = "precision highp float;\nprecision highp int;\nprecision highp usampler2D;\nprecision highp sampler3D;\n\nuniform sampler2D featureData;\n/** Min and max feature values that define the endpoints of the color map. Values\n * outside the range will be clamped to the nearest endpoint.\n */\nuniform float featureColorRampMin;\nuniform float featureColorRampMax;\nuniform sampler2D colorRamp;\nuniform usampler2D inRangeIds;\nuniform usampler2D outlierData;\n\n/**\n * LUT mapping from the segmentation ID (raw pixel value) to the\n * global ID (index in data buffers like `featureData` and `outlierData`).\n * \n * For a given segmentation ID `segId`, the global ID is given by:\n * `segIdToGlobalId[segId - segIdOffset] - 1`.\n*/\nuniform usampler2D segIdToGlobalId;\nuniform uint segIdOffset;\n\nuniform vec3 outlineColor;\n\n/** MUST be synchronized with the DrawMode enum in ColorizeCanvas! */\nconst uint DRAW_MODE_HIDE = 0u;\nconst uint DRAW_MODE_COLOR = 1u;\nconst uint BACKGROUND_ID = 0u;\nconst uint MISSING_DATA_ID = 0xFFFFFFFFu;\n\nuniform vec3 outlierColor;\nuniform uint outlierDrawMode;\nuniform vec3 outOfRangeColor;\nuniform uint outOfRangeDrawMode;\n\nuniform uint highlightedId;\n\nuniform bool hideOutOfRange;\n\n// src texture is the raw volume intensity data\nuniform usampler2D srcTexture;\n\nvec4 getFloatFromTex(sampler2D tex, int index) {\n  int width = textureSize(tex, 0).x;\n  ivec2 featurePos = ivec2(index % width, index / width);\n  return texelFetch(tex, featurePos, 0);\n}\nuvec4 getUintFromTex(usampler2D tex, int index) {\n  int width = textureSize(tex, 0).x;\n  ivec2 featurePos = ivec2(index % width, index / width);\n  return texelFetch(tex, featurePos, 0);\n}\nuint getId(ivec2 uv) {\n  uint rawId = texelFetch(srcTexture, uv, 0).r;\n  if (rawId == 0u) {\n    return BACKGROUND_ID;\n  }\n  uvec4 c = getUintFromTex(segIdToGlobalId, int(rawId - segIdOffset));\n  // Note: IDs are offset by `1` to reserve `0` for segmentations that don't\n  // have associated data. `1` MUST be subtracted from the ID when accessing\n  // data buffers.\n  uint globalId = c.r;\n  if (globalId == 0u) {\n    return MISSING_DATA_ID;\n  }\n  return globalId;\n}\nvec4 getColorRamp(float val) {\n  float width = float(textureSize(colorRamp, 0).x);\n  float range = (width - 1.0) / width;\n  float adjustedVal = (0.5 / width) + (val * range);\n  return texture(colorRamp, vec2(adjustedVal, 0.5));\n}\nvec4 getColorFromDrawMode(uint drawMode, vec3 defaultColor) {\n  const uint DRAW_MODE_HIDE = 0u;\n  vec3 backgroundColor = vec3(0.0, 0.0, 0.0);\n  if (drawMode == DRAW_MODE_HIDE) {\n    return vec4(backgroundColor, 0.0);\n  } else {\n    return vec4(defaultColor, 1.0);\n  }\n}\n\nfloat getFeatureVal(uint id) {\n  // Data buffer starts at 0, non-background segmentation IDs start at 1\n  return getFloatFromTex(featureData, int(id) - 1).r;\n}\nuint getOutlierVal(uint id) {\n  // Data buffer starts at 0, non-background segmentation IDs start at 1\n  return getUintFromTex(outlierData, int(id) - 1).r;\n}\nbool getIsInRange(uint id) {\n  return getUintFromTex(inRangeIds, int(id) - 1).r == 1u;\n}\nbool getIsOutlier(float featureVal, uint outlierVal) {\n  return isinf(featureVal) || outlierVal != 0u;\n}\n\nvec4 getObjectColor(ivec2 sUv, float opacity) {\n  // Get the segmentation id at this pixel\n  uint id = getId(sUv);\n\n  // A segmentation id of 0 represents background\n  if (id == BACKGROUND_ID) {\n    return vec4(0, 0, 0, 0);\n  }\n\n  // color the highlighted object. Note, `highlightedId` is a 0-based index\n  // (global ID w/o offset), while `id` is a 1-based index.\n  if (id - 1u == highlightedId) {\n    return vec4(outlineColor, 1.0);\n  }\n\n  float featureVal = getFeatureVal(id);\n  uint outlierVal = getOutlierVal(id);\n  float normFeatureVal = (featureVal - featureColorRampMin) / (featureColorRampMax - featureColorRampMin);\n\n  // Use the selected draw mode to handle out of range and outlier values;\n  // otherwise color with the color ramp as usual.\n  bool isInRange = getIsInRange(id);\n  bool isOutlier = getIsOutlier(featureVal, outlierVal);\n  bool isMissingData = (id == MISSING_DATA_ID);\n\n  // Features outside the filtered/thresholded range will all be treated the same (use `outOfRangeDrawColor`).\n  // Features inside the range can either be outliers or standard values, and are colored accordingly.\n  vec4 color;\n  if (isMissingData) { \n    // TODO: Add color controls for missing data\n    color = getColorFromDrawMode(outlierDrawMode, outlierColor);\n  } else if (isInRange) {\n    if (isOutlier) {\n      color = getColorFromDrawMode(outlierDrawMode, outlierColor);\n    } else {\n      color = getColorRamp(normFeatureVal);\n    }\n  } else {\n    color = getColorFromDrawMode(outOfRangeDrawMode, outOfRangeColor);\n  }\n  color.a *= opacity;\n  return color;\n}\n\nvoid main() {\n  ivec2 vUv = ivec2(int(gl_FragCoord.x), int(gl_FragCoord.y));\n  gl_FragColor = getObjectColor(vUv, 1.0);\n}\n";
 // This is the owner of the fused RGBA volume texture atlas, and the mask texture atlas.
 // This module is responsible for updating the fused texture, given the read-only volume channel data.
 export default class FusedChannelData {
@@ -124,6 +124,12 @@ export default class FusedChannelData {
         },
         hideOutOfRange: {
           value: false
+        },
+        segIdToGlobalId: {
+          value: new DataTexture()
+        },
+        segIdOffset: {
+          value: 0
         }
       },
       fragmentShader: fragShaderSrc,
@@ -205,19 +211,31 @@ export default class FusedChannelData {
         const mat = this.getShader(channels[chIndex].dtype, isColorize).clone();
         mat.uniforms.srcTexture.value = channels[chIndex].dataTexture;
         mat.uniforms.highlightedId.value = combination[i].selectedID == undefined ? -1 : combination[i].selectedID;
-        if (isColorize) {
-          mat.uniforms.featureData.value = combination[i].feature?.idsToFeatureValue;
-          mat.uniforms.outlierData.value = combination[i].feature?.outlierData;
-          mat.uniforms.inRangeIds.value = combination[i].feature?.inRangeIds;
-          mat.uniforms.featureColorRampMin.value = combination[i].feature?.featureMin;
-          mat.uniforms.featureColorRampMax.value = combination[i].feature?.featureMax;
-          mat.uniforms.colorRamp.value = combination[i].feature?.featureValueToColor;
-          mat.uniforms.outlineColor.value = combination[i].feature?.outlineColor;
-          mat.uniforms.outlierColor.value = combination[i].feature?.outlierColor;
-          mat.uniforms.outOfRangeColor.value = combination[i].feature?.outOfRangeColor;
-          mat.uniforms.outlierDrawMode.value = combination[i].feature?.outlierDrawMode;
-          mat.uniforms.outOfRangeDrawMode.value = combination[i].feature?.outOfRangeDrawMode;
-          mat.uniforms.hideOutOfRange.value = combination[i].feature?.hideOutOfRange;
+        const feature = combination[i].feature;
+        if (isColorize && feature) {
+          mat.uniforms.featureData.value = feature.idsToFeatureValue;
+          mat.uniforms.outlierData.value = feature.outlierData;
+          mat.uniforms.inRangeIds.value = feature.inRangeIds;
+          mat.uniforms.featureColorRampMin.value = feature.featureMin;
+          mat.uniforms.featureColorRampMax.value = feature.featureMax;
+          mat.uniforms.colorRamp.value = feature.featureValueToColor;
+          mat.uniforms.outlineColor.value = feature.outlineColor;
+          mat.uniforms.outlierColor.value = feature.outlierColor;
+          mat.uniforms.outOfRangeColor.value = feature.outOfRangeColor;
+          mat.uniforms.outlierDrawMode.value = feature.outlierDrawMode;
+          mat.uniforms.outOfRangeDrawMode.value = feature.outOfRangeDrawMode;
+          mat.uniforms.hideOutOfRange.value = feature.hideOutOfRange;
+          const time = channels[chIndex].time;
+          let globalIdLookupInfo = feature.frameToGlobalIdLookup.get(time);
+          if (!globalIdLookupInfo) {
+            console.warn(`FusedChannelData.gpuFuse: No global ID lookup info for time ${time} in channel ${chIndex}. A default lookup will be used, which may cause visual artifacts.`);
+            globalIdLookupInfo = {
+              texture: new DataTexture(Uint32Array[0]),
+              minSegId: 1
+            };
+          }
+          mat.uniforms.segIdToGlobalId.value = globalIdLookupInfo.texture;
+          mat.uniforms.segIdOffset.value = globalIdLookupInfo.minSegId;
         } else {
           // the lut texture is spanning only the data range of the channel, not the datatype range
           mat.uniforms.lutMinMax.value = new Vector2(channels[chIndex].rawMin, channels[chIndex].rawMax);

package/es/Volume.js

@@ -218,8 +218,8 @@ export default class Volume {
    * @param {number} atlaswidth
    * @param {number} atlasheight
    */
-  setChannelDataFromAtlas(channelIndex, atlasdata, atlaswidth, atlasheight, range, dtype = "uint8") {
-    this.channels[channelIndex].setFromAtlas(atlasdata, atlaswidth, atlasheight, dtype, range[0], range[1], this.imageInfo.subregionSize);
+  setChannelDataFromAtlas(channelIndex, atlasdata, atlaswidth, atlasheight, range, dtype = "uint8", time = 0) {
+    this.channels[channelIndex].setFromAtlas(atlasdata, atlaswidth, atlasheight, dtype, range[0], range[1], this.imageInfo.subregionSize, time);
     this.onChannelLoaded([channelIndex]);
   }
 
@@ -229,12 +229,12 @@ export default class Volume {
    * @param {number} channelIndex
    * @param {Uint8Array} volumeData
    */
-  setChannelDataFromVolume(channelIndex, volumeData, range, dtype = "uint8") {
+  setChannelDataFromVolume(channelIndex, volumeData, range, dtype = "uint8", time = 0) {
     const {
       subregionSize,
       atlasTileDims
     } = this.imageInfo;
-    this.channels[channelIndex].setFromVolumeData(volumeData, subregionSize.x, subregionSize.y, subregionSize.z, atlasTileDims.x * subregionSize.x, atlasTileDims.y * subregionSize.y, range[0], range[1], dtype);
+    this.channels[channelIndex].setFromVolumeData(volumeData, subregionSize.x, subregionSize.y, subregionSize.z, atlasTileDims.x * subregionSize.x, atlasTileDims.y * subregionSize.y, range[0], range[1], dtype, time);
     this.onChannelLoaded([channelIndex]);
   }
 

package/es/loaders/IVolumeLoader.js

@@ -114,10 +114,11 @@ export class ThreadableVolumeLoader {
         const dtype = dtypes[i];
         const data = dataArrays[i];
         const range = ranges[i];
+        const time = volume.loadSpec.time;
         if (atlasDims) {
-          volume.setChannelDataFromAtlas(channelIndex, data, atlasDims[0], atlasDims[1], range, dtype);
+          volume.setChannelDataFromAtlas(channelIndex, data, atlasDims[0], atlasDims[1], range, dtype, time);
         } else {
-          volume.setChannelDataFromVolume(channelIndex, data, range, dtype);
+          volume.setChannelDataFromVolume(channelIndex, data, range, dtype, time);
         }
         onChannelLoaded?.(volume, channelIndex);
       }

package/es/types/Channel.d.ts

@@ -25,6 +25,7 @@ export default class Channel {
     lutTexture: DataTexture;
     rawMin: number;
     rawMax: number;
+    time: number;
     constructor(name: string);
     combineLuts(rgbColor: [number, number, number] | number, out?: Uint8Array): Uint8Array;
     setRawDataRange(min: number, max: number): void;
@@ -33,9 +34,9 @@ export default class Channel {
     normalizeRaw(val: number): number;
     getIntensityFromAtlas(x: number, y: number, z: number): number;
     private rebuildDataTexture;
-    setFromAtlas(bitsArray: TypedArray<NumberType>, w: number, h: number, dtype: NumberType, rawMin: number, rawMax: number, subregionSize: Vector3): void;
+    setFromAtlas(bitsArray: TypedArray<NumberType>, w: number, h: number, dtype: NumberType, rawMin: number, rawMax: number, subregionSize: Vector3, time?: number): void;
     private unpackFromAtlas;
-    setFromVolumeData(bitsArray: TypedArray<NumberType>, vx: number, vy: number, vz: number, ax: number, ay: number, rawMin: number, rawMax: number, dtype: NumberType): void;
+    setFromVolumeData(bitsArray: TypedArray<NumberType>, vx: number, vy: number, vz: number, ax: number, ay: number, rawMin: number, rawMax: number, dtype: NumberType, time?: number): void;
     private packToAtlas;
     setLut(lut: Lut): void;
     setColorPalette(palette: Uint8Array): void;

package/es/types/Volume.d.ts

@@ -82,13 +82,13 @@ export default class Volume {
      * @param {number} atlaswidth
      * @param {number} atlasheight
      */
-    setChannelDataFromAtlas(channelIndex: number, atlasdata: TypedArray<NumberType>, atlaswidth: number, atlasheight: number, range: [number, number], dtype?: NumberType): void;
+    setChannelDataFromAtlas(channelIndex: number, atlasdata: TypedArray<NumberType>, atlaswidth: number, atlasheight: number, range: [number, number], dtype?: NumberType, time?: number): void;
     /**
      * Assign volume data as a 3d array ordered x,y,z. The xy size must be equal to tilewidth*tileheight from the imageInfo used to construct this Volume.  Assumes that the incoming data is consistent with the image's pre-existing imageInfo tile metadata.
      * @param {number} channelIndex
      * @param {Uint8Array} volumeData
      */
-    setChannelDataFromVolume(channelIndex: number, volumeData: TypedArray<NumberType>, range: [number, number], dtype?: NumberType): void;
+    setChannelDataFromVolume(channelIndex: number, volumeData: TypedArray<NumberType>, range: [number, number], dtype?: NumberType, time?: number): void;
     /**
      * Add a new channel ready to receive data from one of the setChannelDataFrom* calls.
      * Name and color will be defaulted if not provided. For now, leave imageInfo alone as the "original" data

package/es/types/types.d.ts

@@ -30,6 +30,29 @@ export declare const ARRAY_CONSTRUCTORS: {
 export interface ColorizeFeature {
     idsToFeatureValue: DataTexture;
     featureValueToColor: DataTexture;
+    /**
+     * Maps from a frame number to an info object used to look up the global ID
+     * from a given segmentation ID (raw pixel value) on that frame. The info
+     * object contains a texture and a minimum segmentation ID for that frame, the
+     * latter of which is used to minimize the memory footprint of the lookup
+     * table.
+     *
+     * For a frame at time `t`, the global ID of a segmentation `segId` is given
+     * by:
+     * ```
+     * lookup[t].texture.getAt(segId - lookup[t].minSegId) - 1
+     * ```
+     * The result is `-1` if there is no global ID for that segmentation ID on
+     * that frame.
+     *
+     * The global ID can be used directly as an index into the
+     * `idsToFeatureValue`, `inRangeIds`, and `outlierData` data textures to get
+     * values for that segmentation ID.
+     */
+    frameToGlobalIdLookup: Map<number, {
+        texture: DataTexture;
+        minSegId: number;
+    }>;
     inRangeIds: DataTexture;
     outlierData: DataTexture;
     featureMin: number;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aics/vole-core",
-  "version": "3.14.0",
+  "version": "3.15.0",
   "description": "volume renderer for 3d, 4d, or 5d imaging data with OME-Zarr support",
   "main": "es/index.js",
   "type": "module",