@cornerstonejs/core 1.28.2 → 1.29.0

package/src/enums/ImageQualityStatus.ts

@@ -0,0 +1,37 @@
+/**
+ * Status of a frame as it gets loaded.  This is ordered, with lower
+ * values being more lossy, and higher values being less lossy.
+ */
+enum ImageQualityStatus {
+  /**
+   *  Replicate is a duplicated image, from some larger distance
+   */
+  FAR_REPLICATE = 1,
+
+  // Skipping a value here and after the next replicate to allow for interpolation
+  // enum values.
+
+  /**
+   * Adjacent replicate is a duplicated image of a nearby image
+   */
+  ADJACENT_REPLICATE = 3,
+
+  /**
+   * Sub resolution images are encodings of smaller than full resolution
+   * images.  The encoding may or may not be lossy, but the lower resolution
+   * means it has lost information already compared to full resolution/lossless.
+   */
+  SUBRESOLUTION = 6,
+
+  /**
+   *  Lossy images, encoded with a lossy encoding, but full resolution or size.
+   */
+  LOSSY = 7,
+  /**
+   *  Full resolution means the image is full resolution/complete data/lossless
+   * (or at least as lossless as the image is going to get)
+   */
+  FULL_RESOLUTION = 8,
+}
+
+export default ImageQualityStatus;

package/src/enums/index.ts

@@ -11,6 +11,7 @@ import VOILUTFunctionType from './VOILUTFunctionType';
 import DynamicOperatorType from './DynamicOperatorType';
 import CalibrationTypes from './CalibrationTypes';
 import ViewportStatus from './ViewportStatus';
+import ImageQualityStatus from './ImageQualityStatus';
 import * as VideoViewport from './VideoViewport';
 
 export {
@@ -28,4 +29,5 @@ export {
   DynamicOperatorType,
   ViewportStatus,
   VideoViewport,
+  ImageQualityStatus,
 };

package/src/index.ts

@@ -49,7 +49,15 @@ import Settings from './Settings';
 import * as volumeLoader from './loaders/volumeLoader';
 import * as imageLoader from './loaders/imageLoader';
 import * as geometryLoader from './loaders/geometryLoader';
-import * as Types from './types';
+import ProgressiveRetrieveImages from './loaders/ProgressiveRetrieveImages';
+import type * as Types from './types';
+import {
+  IRetrieveConfiguration,
+  IImagesLoader,
+  RetrieveOptions,
+  RetrieveStage,
+  ImageLoadListener,
+} from './types';
 import * as utilities from './utilities';
 import { registerImageLoader } from './loaders/imageLoader'; // since it is used by CSWIL right now
 
@@ -60,7 +68,15 @@ import {
   addVolumesToViewports,
 } from './RenderingEngine/helpers';
 
-export type { Types };
+// Add new types here so that they can be imported singly as required.
+export type {
+  Types,
+  IRetrieveConfiguration,
+  RetrieveOptions,
+  RetrieveStage,
+  ImageLoadListener,
+  IImagesLoader,
+};
 
 export {
   // init
@@ -124,4 +140,5 @@ export {
   resetUseSharedArrayBuffer,
   // Geometry Loader
   geometryLoader,
+  ProgressiveRetrieveImages,
 };

package/src/loaders/ProgressiveRetrieveImages.ts

@@ -0,0 +1,380 @@
+import {
+  IRetrieveConfiguration,
+  IImagesLoader,
+  IImage,
+  RetrieveStage,
+  EventTypes,
+  ImageLoadListener,
+  RetrieveOptions,
+} from '../types';
+import singleRetrieveStages from './configuration/singleRetrieve';
+import sequentialRetrieveStages from './configuration/sequentialRetrieve';
+import interleavedRetrieveStages from './configuration/interleavedRetrieve';
+import { loadAndCacheImage } from './imageLoader';
+import { triggerEvent, ProgressiveIterator, decimate } from '../utilities';
+import imageLoadPoolManager from '../requestPool/imageLoadPoolManager';
+import { ImageQualityStatus, RequestType, Events } from '../enums';
+import cache from '../cache';
+import eventTarget from '../eventTarget';
+import { fillNearbyFrames } from './fillNearbyFrames';
+
+export {
+  sequentialRetrieveStages,
+  interleavedRetrieveStages,
+  singleRetrieveStages,
+};
+
+type StageStatus = {
+  stageId: string;
+  // startTime is the overall start of loading a given image id
+  startTime?: number;
+  // stageStartTime is the time to start loading this stage item
+  stageStartTime?: number;
+  totalImageCount: number;
+  imageLoadFailedCount: number;
+  imageLoadPendingCount: number;
+};
+
+/**
+ * A nearby request is a request that can be fulfilled by copying another image
+ */
+export type NearbyRequest = {
+  // The item id to fill
+  itemId: string;
+  linearId?: string;
+  // The new status of the filled image (will only fill if the existing status
+  // is less than this one)
+  imageQualityStatus: ImageQualityStatus;
+};
+
+export type ProgressiveRequest = {
+  imageId: string;
+  stage: RetrieveStage;
+  next?: ProgressiveRequest;
+  /**
+   * Nearby requests are a set of requests for filling nearby images which
+   * could be filled by using this image as a copied image to generate the
+   * nearby data as a low-resolution alternative image.
+   */
+  nearbyRequests?: NearbyRequest[];
+};
+
+/**
+ * A progressive loader is given some number of images to load,
+ * and calls a success or failure callback some number of times in some
+ * ordering, possibly calling back multiple times.
+ * This allows the progressive loader to be configured for different setups
+ * and to return render results for various images.
+ *
+ * When used by the a stack viewport, the progressive loader can return multiple
+ * representations to the viewport, replacing earlier/more lossy versions with better ones.
+ *
+ * When used by a streaming loader, the progressive loader can change the ordering
+ * of the rendering to retrieve high priority images first, and the lower priority
+ * images later to provide a complete final rendering.
+ *
+ * Requests are held in a queue, such that subsequent requests for a given
+ * image can be cancelled or ensured to be not initiated until the higher
+ * priority image sets have been completed.
+ *
+ * This loader is also used for the base streamimg image volume, configured with
+ * a minimal interleaved load order, combined with filling nearby volume slices
+ * on load, resulting in much faster initial apparent display.
+ *
+ * The loader will load images from existing cached images, cached volumes, and
+ * from other nearby images or one or more calls to back end services.
+ *
+ * @param imageIds - the set of images to load.  For a volume, these should be
+ *                   ordered from top to bottom.
+ * @param listener - has success and failure callbacks to listen for image deliver events, and may
+ *                   have a getTargetOptions to get information on the retrieve
+ * @param retrieveOptions - is a set of retrieve options to use
+ */
+export class ProgressiveRetrieveImages
+  implements IImagesLoader, IRetrieveConfiguration
+{
+  public static createProgressive = createProgressive;
+
+  public static interleavedRetrieveStages = {
+    stages: interleavedRetrieveStages,
+  };
+
+  public static singleRetrieveStages = {
+    stages: singleRetrieveStages,
+  };
+
+  public static sequentialRetrieveStages = {
+    stages: sequentialRetrieveStages,
+  };
+
+  stages: RetrieveStage[];
+  retrieveOptions: Record<string, RetrieveOptions>;
+
+  constructor(imageRetrieveConfiguration: IRetrieveConfiguration) {
+    this.stages = imageRetrieveConfiguration.stages || singleRetrieveStages;
+    this.retrieveOptions = imageRetrieveConfiguration.retrieveOptions || {};
+  }
+
+  public loadImages(imageIds: string[], listener: ImageLoadListener) {
+    const instance = new ProgressiveRetrieveImagesInstance(
+      this,
+      imageIds,
+      listener
+    );
+    return instance.loadImages();
+  }
+}
+
+class ProgressiveRetrieveImagesInstance {
+  imageIds: string[];
+  listener: ImageLoadListener;
+  stages: RetrieveStage[];
+  retrieveOptions: Record<string, RetrieveOptions>;
+  outstandingRequests = 0;
+
+  stageStatusMap = new Map<string, StageStatus>();
+  imageQualityStatusMap = new Map<string, ImageQualityStatus>();
+  displayedIterator = new ProgressiveIterator<void | IImage>('displayed');
+
+  constructor(configuration: IRetrieveConfiguration, imageIds, listener) {
+    this.stages = configuration.stages;
+    this.retrieveOptions = configuration.retrieveOptions;
+    this.imageIds = imageIds;
+    this.listener = listener;
+  }
+
+  public async loadImages() {
+    // The actual function is to just setup the interleave and add the
+    // requests, with all the actual work being handled by the nested functions
+    const interleaved = this.createStageRequests();
+    this.outstandingRequests = interleaved.length;
+    for (const request of interleaved) {
+      this.addRequest(request);
+    }
+    if (this.outstandingRequests === 0) {
+      return Promise.resolve(null);
+    }
+
+    return this.displayedIterator.getDonePromise();
+  }
+
+  protected sendRequest(request, options) {
+    const { imageId, next } = request;
+    const errorCallback = (reason, done) => {
+      this.listener.errorCallback(imageId, complete || !next, reason);
+      if (done) {
+        this.updateStageStatus(request.stage, reason);
+      }
+    };
+    const loadedPromise = (options.loader || loadAndCacheImage)(
+      imageId,
+      options
+    );
+    const uncompressedIterator = ProgressiveIterator.as(loadedPromise);
+    let complete = false;
+
+    uncompressedIterator
+      .forEach(async (image, done) => {
+        const oldStatus = this.imageQualityStatusMap.get(imageId);
+        if (!image) {
+          console.warn('No image retrieved', imageId);
+          return;
+        }
+        const { imageQualityStatus } = image;
+        complete ||= imageQualityStatus === ImageQualityStatus.FULL_RESOLUTION;
+        if (oldStatus !== undefined && oldStatus > imageQualityStatus) {
+          // We already have a better status, so don't update it
+          this.updateStageStatus(request.stage, null, true);
+          return;
+        }
+
+        this.listener.successCallback(imageId, image);
+        this.imageQualityStatusMap.set(imageId, imageQualityStatus);
+        this.displayedIterator.add(image);
+        if (done) {
+          this.updateStageStatus(request.stage);
+        }
+        fillNearbyFrames(
+          this.listener,
+          this.imageQualityStatusMap,
+          request,
+          image,
+          options
+        );
+      }, errorCallback)
+      .finally(() => {
+        if (!complete && next) {
+          if (cache.getImageLoadObject(imageId)) {
+            cache.removeImageLoadObject(imageId);
+          }
+          this.addRequest(next, options.streamingData);
+        } else {
+          if (!complete) {
+            this.listener.errorCallback(imageId, true, "Couldn't decode");
+          }
+          this.outstandingRequests--;
+          for (let skip = next; skip; skip = skip.next) {
+            this.updateStageStatus(skip.stage, null, true);
+          }
+        }
+        if (this.outstandingRequests <= 0) {
+          this.displayedIterator.resolve();
+        }
+      });
+    const doneLoad = uncompressedIterator.getDonePromise();
+    // Errors already handled above in the callback
+    return doneLoad.catch((e) => null);
+  }
+
+  /** Adds a rquest to the image load pool manager */
+  protected addRequest(request, streamingData = {}) {
+    const { imageId, stage } = request;
+    const baseOptions = this.listener.getLoaderImageOptions(imageId);
+    if (!baseOptions) {
+      // Image no longer of interest
+      return;
+    }
+    const { retrieveType = 'default' } = stage;
+    const { retrieveOptions: keyedRetrieveOptions } = this;
+    const retrieveOptions =
+      keyedRetrieveOptions[retrieveType] || keyedRetrieveOptions.default;
+    const options = {
+      ...baseOptions,
+      retrieveType,
+      retrieveOptions,
+      streamingData,
+    };
+    const priority = stage.priority ?? -5;
+    const requestType = stage.requestType || RequestType.Interaction;
+    const additionalDetails = { imageId };
+
+    imageLoadPoolManager.addRequest(
+      this.sendRequest.bind(this, request, options),
+      requestType,
+      additionalDetails,
+      priority
+    );
+  }
+
+  protected updateStageStatus(stage, failure?, skipped = false) {
+    const { id } = stage;
+    const stageStatus = this.stageStatusMap.get(id);
+    if (!stageStatus) {
+      return;
+    }
+    stageStatus.imageLoadPendingCount--;
+    if (failure) {
+      stageStatus.imageLoadFailedCount++;
+    } else if (!skipped) {
+      stageStatus.totalImageCount++;
+    }
+    if (!skipped && !stageStatus.stageStartTime) {
+      stageStatus.stageStartTime = Date.now();
+    }
+    if (!stageStatus.imageLoadPendingCount) {
+      const {
+        imageLoadFailedCount: numberOfFailures,
+        totalImageCount: numberOfImages,
+        stageStartTime = Date.now(),
+        startTime,
+      } = stageStatus;
+      const detail: EventTypes.ImageLoadStageEventDetail = {
+        stageId: id,
+        numberOfFailures,
+        numberOfImages,
+        stageDurationInMS: stageStartTime ? Date.now() - stageStartTime : null,
+        startDurationInMS: Date.now() - startTime,
+      };
+      triggerEvent(eventTarget, Events.IMAGE_RETRIEVAL_STAGE, detail);
+      this.stageStatusMap.delete(id);
+    }
+  }
+
+  /** Interleaves the values according to the stages definition */
+  protected createStageRequests() {
+    const interleaved = new Array<ProgressiveRequest>();
+    // Maps image id to the LAST progressive request - to allow tail append
+    const imageRequests = new Map<string, ProgressiveRequest>();
+
+    const addStageInstance = (stage, position) => {
+      const index =
+        position < 0
+          ? this.imageIds.length + position
+          : position < 1
+          ? Math.floor((this.imageIds.length - 1) * position)
+          : position;
+      const imageId = this.imageIds[index];
+      if (!imageId) {
+        throw new Error(`No value found to add to requests at ${position}`);
+      }
+      const request: ProgressiveRequest = {
+        imageId,
+        stage,
+        nearbyRequests: this.findNearbyRequests(index, stage),
+      };
+      this.addStageStatus(stage);
+      const existingRequest = imageRequests.get(imageId);
+      if (existingRequest) {
+        existingRequest.next = request;
+      } else {
+        interleaved.push(request);
+      }
+      imageRequests.set(imageId, request);
+    };
+
+    for (const stage of this.stages) {
+      const indices =
+        stage.positions ||
+        decimate(this.imageIds, stage.decimate || 1, stage.offset ?? 0);
+      indices.forEach((index) => addStageInstance(stage, index));
+    }
+    return interleaved;
+  }
+
+  /**
+   * Finds nearby requests to fulfill to show the merge information earlier.
+   * @param index - to use as the base value
+   * @param imageIds - set of image ids to request
+   * @param stage - to find information from
+   * @returns Array of nearby frames to fill when the main stage is done
+   */
+  protected findNearbyRequests(index: number, stage): NearbyRequest[] {
+    const nearby = new Array<NearbyRequest>();
+    if (!stage.nearbyFrames) {
+      return nearby;
+    }
+    for (const nearbyItem of stage.nearbyFrames) {
+      const nearbyIndex = index + nearbyItem.offset;
+      if (nearbyIndex < 0 || nearbyIndex >= this.imageIds.length) {
+        continue;
+      }
+      nearby.push({
+        itemId: this.imageIds[nearbyIndex],
+        imageQualityStatus: nearbyItem.imageQualityStatus,
+      });
+    }
+
+    return nearby;
+  }
+
+  protected addStageStatus(stage) {
+    const { id } = stage;
+    const stageStatus = this.stageStatusMap.get(id) || {
+      stageId: id,
+      startTime: Date.now(),
+      stageStartTime: null,
+      totalImageCount: 0,
+      imageLoadFailedCount: 0,
+      imageLoadPendingCount: 0,
+    };
+    stageStatus.imageLoadPendingCount++;
+    this.stageStatusMap.set(id, stageStatus);
+    return stageStatus;
+  }
+}
+
+export function createProgressive(configuration: IRetrieveConfiguration) {
+  return new ProgressiveRetrieveImages(configuration);
+}
+
+export default ProgressiveRetrieveImages;

package/src/loaders/configuration/interleavedRetrieve.ts

@@ -0,0 +1,96 @@
+import { RetrieveStage, NearbyFrames } from '../../types';
+import { RequestType, ImageQualityStatus } from '../../enums';
+
+// Defines some nearby frames to replicate to
+const nearbyFrames: NearbyFrames[] = [
+  {
+    offset: -1,
+    imageQualityStatus: ImageQualityStatus.ADJACENT_REPLICATE,
+  },
+  {
+    offset: +1,
+    imageQualityStatus: ImageQualityStatus.ADJACENT_REPLICATE,
+  },
+  { offset: +2, imageQualityStatus: ImageQualityStatus.FAR_REPLICATE },
+];
+
+/**
+ * This configuration is designed to interleave the data requests, using
+ * lossy/thumbnail requests when available, but falling back to full retrieve
+ * requests in an interleaved manner.
+ * The basic ordering is:
+ *   1. Retrieve middle image, first, last
+ *   2. Retrieve every 4th image, offset 1, lossy if available
+ *   3. Retrieve every 4th image, offset 3, lossy if available
+ *   4. Retrieve every 4th image, offset 0, full images
+ *   5. Retrieve every 4th image, offset 2, full images
+ *   6. Retrieve every 4th image, offsets 1 and 3, full images if not already done
+ */
+const interleavedRetrieveConfiguration: RetrieveStage[] = [
+  {
+    id: 'initialImages',
+    // Values between -1 and 1 are relative to size, so 0.5 is middle image
+    // and 0 is first image, -1 is last image
+    positions: [0.5, 0, -1],
+    retrieveType: 'default',
+    requestType: RequestType.Thumbnail,
+    priority: 5,
+    nearbyFrames,
+  },
+  {
+    id: 'quarterThumb',
+    decimate: 4,
+    offset: 3,
+    requestType: RequestType.Thumbnail,
+    retrieveType: 'multipleFast',
+    priority: 6,
+    nearbyFrames,
+  },
+  {
+    id: 'halfThumb',
+    decimate: 4,
+    offset: 1,
+    priority: 7,
+    requestType: RequestType.Thumbnail,
+    retrieveType: 'multipleFast',
+    nearbyFrames,
+  },
+  {
+    id: 'quarterFull',
+    decimate: 4,
+    offset: 2,
+    priority: 8,
+    requestType: RequestType.Thumbnail,
+    retrieveType: 'multipleFinal',
+  },
+  {
+    id: 'halfFull',
+    decimate: 4,
+    offset: 0,
+    priority: 9,
+    requestType: RequestType.Thumbnail,
+    retrieveType: 'multipleFinal',
+  },
+  {
+    id: 'threeQuarterFull',
+    decimate: 4,
+    offset: 1,
+    priority: 10,
+    requestType: RequestType.Thumbnail,
+    retrieveType: 'multipleFinal',
+  },
+  {
+    id: 'finalFull',
+    decimate: 4,
+    offset: 3,
+    priority: 11,
+    requestType: RequestType.Thumbnail,
+    retrieveType: 'multipleFinal',
+  },
+  {
+    // This goes back to basic retrieve to recover from retrieving against
+    // servers returning errors for any of the above requests.
+    id: 'errorRetrieve',
+  },
+];
+export default interleavedRetrieveConfiguration;

package/src/loaders/configuration/sequentialRetrieve.ts

@@ -0,0 +1,17 @@
+import type { RetrieveStage } from '../../types';
+
+/**
+ * This simply retrieves the images sequentially as provided.
+ */
+const sequentialRetrieveStages: RetrieveStage[] = [
+  {
+    id: 'lossySequential',
+    retrieveType: 'singleFast',
+  },
+  {
+    id: 'finalSequential',
+    retrieveType: 'singleFinal',
+  },
+];
+
+export default sequentialRetrieveStages;

package/src/loaders/configuration/singleRetrieve.ts

@@ -0,0 +1,18 @@
+import type { RetrieveStage } from '../../types';
+
+/**
+ * This simply retrieves the images sequentially as provided.
+ */
+const singleRetrieveStages: RetrieveStage[] = [
+  {
+    id: 'initialImages',
+    retrieveType: 'single',
+  },
+  // Shouldn't be necessary, but if the server returns an error for the above
+  // configuration, this will ensure the image is still fetched.
+  {
+    id: 'errorRetrieve',
+  },
+];
+
+export default singleRetrieveStages;

package/src/loaders/fillNearbyFrames.ts

@@ -0,0 +1,53 @@
+import { ImageLoadListener } from '../types';
+import { ImageQualityStatus } from '../enums';
+
+/** Actually fills the nearby frames from the given frame */
+export function fillNearbyFrames(
+  listener: ImageLoadListener,
+  imageQualityStatusMap: Map<string, ImageQualityStatus>,
+  request,
+  image,
+  options
+) {
+  if (!request?.nearbyRequests?.length) {
+    return;
+  }
+
+  const {
+    arrayBuffer,
+    offset: srcOffset,
+    type,
+    length: frameLength,
+  } = options.targetBuffer;
+  if (!arrayBuffer || srcOffset === undefined || !type) {
+    return;
+  }
+  const scalarData = new Float32Array(arrayBuffer);
+  const bytesPerPixel = scalarData.byteLength / scalarData.length;
+  const offset = options.targetBuffer.offset / bytesPerPixel; // in bytes
+
+  // since set is based on the underlying type,
+  // we need to divide the offset bytes by the byte type
+  const src = scalarData.slice(offset, offset + frameLength);
+
+  for (const nearbyItem of request.nearbyRequests) {
+    try {
+      const { itemId: targetId, imageQualityStatus } = nearbyItem;
+      const targetStatus = imageQualityStatusMap.get(targetId);
+      if (targetStatus !== undefined && targetStatus >= imageQualityStatus) {
+        continue;
+      }
+      const targetOptions = listener.getLoaderImageOptions(targetId);
+      const { offset: targetOffset } = targetOptions.targetBuffer as any;
+      scalarData.set(src, targetOffset / bytesPerPixel);
+      const nearbyImage = {
+        ...image,
+        imageQualityStatus,
+      };
+      listener.successCallback(targetId, nearbyImage);
+      imageQualityStatusMap.set(targetId, imageQualityStatus);
+    } catch (e) {
+      console.log("Couldn't fill nearby item ", nearbyItem.itemId, e);
+    }
+  }
+}

package/src/loaders/imageLoader.ts

@@ -239,7 +239,8 @@ export function cancelLoadAll(): void {
 
     Object.keys(requests).forEach((priority) => {
       const requestDetails = requests[priority].pop();
-      const { imageId, volumeId } = requestDetails.additionalDetails;
+      const additionalDetails = requestDetails.additionalDetails as any;
+      const { imageId, volumeId } = additionalDetails;
 
       let loadObject;
 

package/src/metaData.ts

@@ -66,11 +66,14 @@ export function removeAllProviders(): void {
  *
  * @param type -  The type of metadata requested from the metadata store
  * @param query - The query for the metadata store, often imageId
+ *        Some metadata providers support multi-valued strings, which are interpretted
+ *        as the provider chooses.
  *
  * @returns The metadata retrieved from the metadata store
  * @category MetaData
  */
-function getMetaData(type: string, query: string): any {
+function getMetaData(type: string, ...queries): any {
+  const query = queries.length === 1 ? queries[0] : queries;
   // Invoke each provider in priority order until one returns something
   for (let i = 0; i < providers.length; i++) {
     const result = providers[i].provider(type, query);

package/src/types/CPUIImageData.ts

@@ -1,4 +1,4 @@
-import { Point3, Scaling, Mat3, PixelDataTypedArray } from '../types';
+import type { Point3, Scaling, Mat3, PixelDataTypedArray } from '../types';
 import IImageCalibration from './IImageCalibration';
 
 type CPUImageData = {