@cornerstonejs/core 2.0.0-beta.3 → 2.0.0-beta.5

package/src/loaders/ProgressiveRetrieveImages.ts

@@ -0,0 +1,380 @@
+import {
+  IRetrieveConfiguration,
+  IImagesLoader,
+  IImage,
+  RetrieveStage,
+  EventTypes,
+  ImageLoadListener,
+  RetrieveOptions,
+} from '../types';
+import singleRetrieveStages from './configuration/singleRetrieve';
+import sequentialRetrieveStages from './configuration/sequentialRetrieve';
+import interleavedRetrieveStages from './configuration/interleavedRetrieve';
+import { loadAndCacheImage } from './imageLoader';
+import { triggerEvent, ProgressiveIterator, decimate } from '../utilities';
+import imageLoadPoolManager from '../requestPool/imageLoadPoolManager';
+import { ImageQualityStatus, RequestType, Events } from '../enums';
+import cache from '../cache';
+import eventTarget from '../eventTarget';
+import { fillNearbyFrames } from './fillNearbyFrames';
+
+export {
+  sequentialRetrieveStages,
+  interleavedRetrieveStages,
+  singleRetrieveStages,
+};
+
+type StageStatus = {
+  stageId: string;
+  // startTime is the overall start of loading a given image id
+  startTime?: number;
+  // stageStartTime is the time to start loading this stage item
+  stageStartTime?: number;
+  totalImageCount: number;
+  imageLoadFailedCount: number;
+  imageLoadPendingCount: number;
+};
+
+/**
+ * A nearby request is a request that can be fulfilled by copying another image
+ */
+export type NearbyRequest = {
+  // The item id to fill
+  itemId: string;
+  linearId?: string;
+  // The new status of the filled image (will only fill if the existing status
+  // is less than this one)
+  imageQualityStatus: ImageQualityStatus;
+};
+
+export type ProgressiveRequest = {
+  imageId: string;
+  stage: RetrieveStage;
+  next?: ProgressiveRequest;
+  /**
+   * Nearby requests are a set of requests for filling nearby images which
+   * could be filled by using this image as a copied image to generate the
+   * nearby data as a low-resolution alternative image.
+   */
+  nearbyRequests?: NearbyRequest[];
+};
+
+/**
+ * A progressive loader is given some number of images to load,
+ * and calls a success or failure callback some number of times in some
+ * ordering, possibly calling back multiple times.
+ * This allows the progressive loader to be configured for different setups
+ * and to return render results for various images.
+ *
+ * When used by the a stack viewport, the progressive loader can return multiple
+ * representations to the viewport, replacing earlier/more lossy versions with better ones.
+ *
+ * When used by a streaming loader, the progressive loader can change the ordering
+ * of the rendering to retrieve high priority images first, and the lower priority
+ * images later to provide a complete final rendering.
+ *
+ * Requests are held in a queue, such that subsequent requests for a given
+ * image can be cancelled or ensured to be not initiated until the higher
+ * priority image sets have been completed.
+ *
+ * This loader is also used for the base streamimg image volume, configured with
+ * a minimal interleaved load order, combined with filling nearby volume slices
+ * on load, resulting in much faster initial apparent display.
+ *
+ * The loader will load images from existing cached images, cached volumes, and
+ * from other nearby images or one or more calls to back end services.
+ *
+ * @param imageIds - the set of images to load.  For a volume, these should be
+ *                   ordered from top to bottom.
+ * @param listener - has success and failure callbacks to listen for image deliver events, and may
+ *                   have a getTargetOptions to get information on the retrieve
+ * @param retrieveOptions - is a set of retrieve options to use
+ */
+export class ProgressiveRetrieveImages
+  implements IImagesLoader, IRetrieveConfiguration
+{
+  public static createProgressive = createProgressive;
+
+  public static interleavedRetrieveStages = {
+    stages: interleavedRetrieveStages,
+  };
+
+  public static singleRetrieveStages = {
+    stages: singleRetrieveStages,
+  };
+
+  public static sequentialRetrieveStages = {
+    stages: sequentialRetrieveStages,
+  };
+
+  stages: RetrieveStage[];
+  retrieveOptions: Record<string, RetrieveOptions>;
+
+  constructor(imageRetrieveConfiguration: IRetrieveConfiguration) {
+    this.stages = imageRetrieveConfiguration.stages || singleRetrieveStages;
+    this.retrieveOptions = imageRetrieveConfiguration.retrieveOptions || {};
+  }
+
+  public loadImages(imageIds: string[], listener: ImageLoadListener) {
+    const instance = new ProgressiveRetrieveImagesInstance(
+      this,
+      imageIds,
+      listener
+    );
+    return instance.loadImages();
+  }
+}
+
+class ProgressiveRetrieveImagesInstance {
+  imageIds: string[];
+  listener: ImageLoadListener;
+  stages: RetrieveStage[];
+  retrieveOptions: Record<string, RetrieveOptions>;
+  outstandingRequests = 0;
+
+  stageStatusMap = new Map<string, StageStatus>();
+  imageQualityStatusMap = new Map<string, ImageQualityStatus>();
+  displayedIterator = new ProgressiveIterator<void | IImage>('displayed');
+
+  constructor(configuration: IRetrieveConfiguration, imageIds, listener) {
+    this.stages = configuration.stages;
+    this.retrieveOptions = configuration.retrieveOptions;
+    this.imageIds = imageIds;
+    this.listener = listener;
+  }
+
+  public async loadImages() {
+    // The actual function is to just setup the interleave and add the
+    // requests, with all the actual work being handled by the nested functions
+    const interleaved = this.createStageRequests();
+    this.outstandingRequests = interleaved.length;
+    for (const request of interleaved) {
+      this.addRequest(request);
+    }
+    if (this.outstandingRequests === 0) {
+      return Promise.resolve(null);
+    }
+
+    return this.displayedIterator.getDonePromise();
+  }
+
+  protected sendRequest(request, options) {
+    const { imageId, next } = request;
+    const errorCallback = (reason, done) => {
+      this.listener.errorCallback(imageId, complete || !next, reason);
+      if (done) {
+        this.updateStageStatus(request.stage, reason);
+      }
+    };
+    const loadedPromise = (options.loader || loadAndCacheImage)(
+      imageId,
+      options
+    );
+    const uncompressedIterator = ProgressiveIterator.as(loadedPromise);
+    let complete = false;
+
+    uncompressedIterator
+      .forEach(async (image, done) => {
+        const oldStatus = this.imageQualityStatusMap.get(imageId);
+        if (!image) {
+          console.warn('No image retrieved', imageId);
+          return;
+        }
+        const { imageQualityStatus } = image;
+        complete ||= imageQualityStatus === ImageQualityStatus.FULL_RESOLUTION;
+        if (oldStatus !== undefined && oldStatus > imageQualityStatus) {
+          // We already have a better status, so don't update it
+          this.updateStageStatus(request.stage, null, true);
+          return;
+        }
+
+        this.listener.successCallback(imageId, image);
+        this.imageQualityStatusMap.set(imageId, imageQualityStatus);
+        this.displayedIterator.add(image);
+        if (done) {
+          this.updateStageStatus(request.stage);
+        }
+        fillNearbyFrames(
+          this.listener,
+          this.imageQualityStatusMap,
+          request,
+          image,
+          options
+        );
+      }, errorCallback)
+      .finally(() => {
+        if (!complete && next) {
+          if (cache.getImageLoadObject(imageId)) {
+            cache.removeImageLoadObject(imageId);
+          }
+          this.addRequest(next, options.streamingData);
+        } else {
+          if (!complete) {
+            this.listener.errorCallback(imageId, true, "Couldn't decode");
+          }
+          this.outstandingRequests--;
+          for (let skip = next; skip; skip = skip.next) {
+            this.updateStageStatus(skip.stage, null, true);
+          }
+        }
+        if (this.outstandingRequests <= 0) {
+          this.displayedIterator.resolve();
+        }
+      });
+    const doneLoad = uncompressedIterator.getDonePromise();
+    // Errors already handled above in the callback
+    return doneLoad.catch((e) => null);
+  }
+
+  /** Adds a rquest to the image load pool manager */
+  protected addRequest(request, streamingData = {}) {
+    const { imageId, stage } = request;
+    const baseOptions = this.listener.getLoaderImageOptions(imageId);
+    if (!baseOptions) {
+      // Image no longer of interest
+      return;
+    }
+    const { retrieveType = 'default' } = stage;
+    const { retrieveOptions: keyedRetrieveOptions } = this;
+    const retrieveOptions =
+      keyedRetrieveOptions[retrieveType] || keyedRetrieveOptions.default;
+    const options = {
+      ...baseOptions,
+      retrieveType,
+      retrieveOptions,
+      streamingData,
+    };
+    const priority = stage.priority ?? -5;
+    const requestType = stage.requestType || RequestType.Interaction;
+    const additionalDetails = { imageId };
+
+    imageLoadPoolManager.addRequest(
+      this.sendRequest.bind(this, request, options),
+      requestType,
+      additionalDetails,
+      priority
+    );
+  }
+
+  protected updateStageStatus(stage, failure?, skipped = false) {
+    const { id } = stage;
+    const stageStatus = this.stageStatusMap.get(id);
+    if (!stageStatus) {
+      return;
+    }
+    stageStatus.imageLoadPendingCount--;
+    if (failure) {
+      stageStatus.imageLoadFailedCount++;
+    } else if (!skipped) {
+      stageStatus.totalImageCount++;
+    }
+    if (!skipped && !stageStatus.stageStartTime) {
+      stageStatus.stageStartTime = Date.now();
+    }
+    if (!stageStatus.imageLoadPendingCount) {
+      const {
+        imageLoadFailedCount: numberOfFailures,
+        totalImageCount: numberOfImages,
+        stageStartTime = Date.now(),
+        startTime,
+      } = stageStatus;
+      const detail: EventTypes.ImageLoadStageEventDetail = {
+        stageId: id,
+        numberOfFailures,
+        numberOfImages,
+        stageDurationInMS: stageStartTime ? Date.now() - stageStartTime : null,
+        startDurationInMS: Date.now() - startTime,
+      };
+      triggerEvent(eventTarget, Events.IMAGE_RETRIEVAL_STAGE, detail);
+      this.stageStatusMap.delete(id);
+    }
+  }
+
+  /** Interleaves the values according to the stages definition */
+  protected createStageRequests() {
+    const interleaved = new Array<ProgressiveRequest>();
+    // Maps image id to the LAST progressive request - to allow tail append
+    const imageRequests = new Map<string, ProgressiveRequest>();
+
+    const addStageInstance = (stage, position) => {
+      const index =
+        position < 0
+          ? this.imageIds.length + position
+          : position < 1
+          ? Math.floor((this.imageIds.length - 1) * position)
+          : position;
+      const imageId = this.imageIds[index];
+      if (!imageId) {
+        throw new Error(`No value found to add to requests at ${position}`);
+      }
+      const request: ProgressiveRequest = {
+        imageId,
+        stage,
+        nearbyRequests: this.findNearbyRequests(index, stage),
+      };
+      this.addStageStatus(stage);
+      const existingRequest = imageRequests.get(imageId);
+      if (existingRequest) {
+        existingRequest.next = request;
+      } else {
+        interleaved.push(request);
+      }
+      imageRequests.set(imageId, request);
+    };
+
+    for (const stage of this.stages) {
+      const indices =
+        stage.positions ||
+        decimate(this.imageIds, stage.decimate || 1, stage.offset ?? 0);
+      indices.forEach((index) => addStageInstance(stage, index));
+    }
+    return interleaved;
+  }
+
+  /**
+   * Finds nearby requests to fulfill to show the merge information earlier.
+   * @param index - to use as the base value
+   * @param imageIds - set of image ids to request
+   * @param stage - to find information from
+   * @returns Array of nearby frames to fill when the main stage is done
+   */
+  protected findNearbyRequests(index: number, stage): NearbyRequest[] {
+    const nearby = new Array<NearbyRequest>();
+    if (!stage.nearbyFrames) {
+      return nearby;
+    }
+    for (const nearbyItem of stage.nearbyFrames) {
+      const nearbyIndex = index + nearbyItem.offset;
+      if (nearbyIndex < 0 || nearbyIndex >= this.imageIds.length) {
+        continue;
+      }
+      nearby.push({
+        itemId: this.imageIds[nearbyIndex],
+        imageQualityStatus: nearbyItem.imageQualityStatus,
+      });
+    }
+
+    return nearby;
+  }
+
+  protected addStageStatus(stage) {
+    const { id } = stage;
+    const stageStatus = this.stageStatusMap.get(id) || {
+      stageId: id,
+      startTime: Date.now(),
+      stageStartTime: null,
+      totalImageCount: 0,
+      imageLoadFailedCount: 0,
+      imageLoadPendingCount: 0,
+    };
+    stageStatus.imageLoadPendingCount++;
+    this.stageStatusMap.set(id, stageStatus);
+    return stageStatus;
+  }
+}
+
+export function createProgressive(configuration: IRetrieveConfiguration) {
+  return new ProgressiveRetrieveImages(configuration);
+}
+
+export default ProgressiveRetrieveImages;

package/src/loaders/configuration/interleavedRetrieve.ts

@@ -0,0 +1,96 @@
+import { RetrieveStage, NearbyFrames } from '../../types';
+import { RequestType, ImageQualityStatus } from '../../enums';
+
+// Defines some nearby frames to replicate to
+const nearbyFrames: NearbyFrames[] = [
+  {
+    offset: -1,
+    imageQualityStatus: ImageQualityStatus.ADJACENT_REPLICATE,
+  },
+  {
+    offset: +1,
+    imageQualityStatus: ImageQualityStatus.ADJACENT_REPLICATE,
+  },
+  { offset: +2, imageQualityStatus: ImageQualityStatus.FAR_REPLICATE },
+];
+
+/**
+ * This configuration is designed to interleave the data requests, using
+ * lossy/thumbnail requests when available, but falling back to full retrieve
+ * requests in an interleaved manner.
+ * The basic ordering is:
+ *   1. Retrieve middle image, first, last
+ *   2. Retrieve every 4th image, offset 1, lossy if available
+ *   3. Retrieve every 4th image, offset 3, lossy if available
+ *   4. Retrieve every 4th image, offset 0, full images
+ *   5. Retrieve every 4th image, offset 2, full images
+ *   6. Retrieve every 4th image, offsets 1 and 3, full images if not already done
+ */
+const interleavedRetrieveConfiguration: RetrieveStage[] = [
+  {
+    id: 'initialImages',
+    // Values between -1 and 1 are relative to size, so 0.5 is middle image
+    // and 0 is first image, -1 is last image
+    positions: [0.5, 0, -1],
+    retrieveType: 'default',
+    requestType: RequestType.Thumbnail,
+    priority: 5,
+    nearbyFrames,
+  },
+  {
+    id: 'quarterThumb',
+    decimate: 4,
+    offset: 3,
+    requestType: RequestType.Thumbnail,
+    retrieveType: 'multipleFast',
+    priority: 6,
+    nearbyFrames,
+  },
+  {
+    id: 'halfThumb',
+    decimate: 4,
+    offset: 1,
+    priority: 7,
+    requestType: RequestType.Thumbnail,
+    retrieveType: 'multipleFast',
+    nearbyFrames,
+  },
+  {
+    id: 'quarterFull',
+    decimate: 4,
+    offset: 2,
+    priority: 8,
+    requestType: RequestType.Thumbnail,
+    retrieveType: 'multipleFinal',
+  },
+  {
+    id: 'halfFull',
+    decimate: 4,
+    offset: 0,
+    priority: 9,
+    requestType: RequestType.Thumbnail,
+    retrieveType: 'multipleFinal',
+  },
+  {
+    id: 'threeQuarterFull',
+    decimate: 4,
+    offset: 1,
+    priority: 10,
+    requestType: RequestType.Thumbnail,
+    retrieveType: 'multipleFinal',
+  },
+  {
+    id: 'finalFull',
+    decimate: 4,
+    offset: 3,
+    priority: 11,
+    requestType: RequestType.Thumbnail,
+    retrieveType: 'multipleFinal',
+  },
+  {
+    // This goes back to basic retrieve to recover from retrieving against
+    // servers returning errors for any of the above requests.
+    id: 'errorRetrieve',
+  },
+];
+export default interleavedRetrieveConfiguration;

package/src/loaders/configuration/sequentialRetrieve.ts

@@ -0,0 +1,17 @@
+import type { RetrieveStage } from '../../types';
+
+/**
+ * This simply retrieves the images sequentially as provided.
+ */
+const sequentialRetrieveStages: RetrieveStage[] = [
+  {
+    id: 'lossySequential',
+    retrieveType: 'singleFast',
+  },
+  {
+    id: 'finalSequential',
+    retrieveType: 'singleFinal',
+  },
+];
+
+export default sequentialRetrieveStages;

package/src/loaders/configuration/singleRetrieve.ts

@@ -0,0 +1,18 @@
+import type { RetrieveStage } from '../../types';
+
+/**
+ * This simply retrieves the images sequentially as provided.
+ */
+const singleRetrieveStages: RetrieveStage[] = [
+  {
+    id: 'initialImages',
+    retrieveType: 'single',
+  },
+  // Shouldn't be necessary, but if the server returns an error for the above
+  // configuration, this will ensure the image is still fetched.
+  {
+    id: 'errorRetrieve',
+  },
+];
+
+export default singleRetrieveStages;

package/src/loaders/fillNearbyFrames.ts

@@ -0,0 +1,53 @@
+import { ImageLoadListener } from '../types';
+import { ImageQualityStatus } from '../enums';
+
+/** Actually fills the nearby frames from the given frame */
+export function fillNearbyFrames(
+  listener: ImageLoadListener,
+  imageQualityStatusMap: Map<string, ImageQualityStatus>,
+  request,
+  image,
+  options
+) {
+  if (!request?.nearbyRequests?.length) {
+    return;
+  }
+
+  const {
+    arrayBuffer,
+    offset: srcOffset,
+    type,
+    length: frameLength,
+  } = options.targetBuffer;
+  if (!arrayBuffer || srcOffset === undefined || !type) {
+    return;
+  }
+  const scalarData = new Float32Array(arrayBuffer);
+  const bytesPerPixel = scalarData.byteLength / scalarData.length;
+  const offset = options.targetBuffer.offset / bytesPerPixel; // in bytes
+
+  // since set is based on the underlying type,
+  // we need to divide the offset bytes by the byte type
+  const src = scalarData.slice(offset, offset + frameLength);
+
+  for (const nearbyItem of request.nearbyRequests) {
+    try {
+      const { itemId: targetId, imageQualityStatus } = nearbyItem;
+      const targetStatus = imageQualityStatusMap.get(targetId);
+      if (targetStatus !== undefined && targetStatus >= imageQualityStatus) {
+        continue;
+      }
+      const targetOptions = listener.getLoaderImageOptions(targetId);
+      const { offset: targetOffset } = targetOptions.targetBuffer as any;
+      scalarData.set(src, targetOffset / bytesPerPixel);
+      const nearbyImage = {
+        ...image,
+        imageQualityStatus,
+      };
+      listener.successCallback(targetId, nearbyImage);
+      imageQualityStatusMap.set(targetId, imageQualityStatus);
+    } catch (e) {
+      console.log("Couldn't fill nearby item ", nearbyItem.itemId, e);
+    }
+  }
+}

package/src/loaders/imageLoader.ts

@@ -239,7 +239,8 @@ export function cancelLoadAll(): void {
 
     Object.keys(requests).forEach((priority) => {
       const requestDetails = requests[priority].pop();
-      const { imageId, volumeId } = requestDetails.additionalDetails;
+      const additionalDetails = requestDetails.additionalDetails as any;
+      const { imageId, volumeId } = additionalDetails;
 
       let loadObject;
 

package/src/loaders/volumeLoader.ts

@@ -3,7 +3,6 @@ import '@kitware/vtk.js/Rendering/Profiles/Volume';
 import vtkImageData from '@kitware/vtk.js/Common/DataModel/ImageData';
 import type { vtkImageData as vtkImageDataType } from '@kitware/vtk.js/Common/DataModel/ImageData';
 import vtkDataArray from '@kitware/vtk.js/Common/Core/DataArray';
-import cloneDeep from 'lodash.clonedeep';
 
 import { ImageVolume } from '../cache/classes/ImageVolume';
 import type * as Types from '../types';
@@ -330,7 +329,7 @@ export async function createAndCacheDerivedVolume(
 
   const derivedVolume = new ImageVolume({
     volumeId,
-    metadata: cloneDeep(metadata),
+    metadata: structuredClone(metadata),
     dimensions: [dimensions[0], dimensions[1], dimensions[2]],
     spacing,
     origin,
@@ -419,7 +418,7 @@ export function createLocalVolume(
 
   const derivedVolume = new ImageVolume({
     volumeId,
-    metadata: cloneDeep(metadata),
+    metadata: structuredClone(metadata),
     dimensions: [dimensions[0], dimensions[1], dimensions[2]],
     spacing,
     origin,

package/src/metaData.ts

@@ -66,11 +66,14 @@ export function removeAllProviders(): void {
  *
  * @param type -  The type of metadata requested from the metadata store
  * @param query - The query for the metadata store, often imageId
+ *        Some metadata providers support multi-valued strings, which are interpretted
+ *        as the provider chooses.
  *
  * @returns The metadata retrieved from the metadata store
  * @category MetaData
  */
-function getMetaData(type: string, query: string): any {
+function getMetaData(type: string, ...queries): any {
+  const query = queries.length === 1 ? queries[0] : queries;
   // Invoke each provider in priority order until one returns something
   for (let i = 0; i < providers.length; i++) {
     const result = providers[i].provider(type, query);

package/src/types/CPUIImageData.ts

@@ -1,4 +1,4 @@
-import { Point3, Scaling, Mat3, PixelDataTypedArray } from '../types';
+import type { Point3, Scaling, Mat3, PixelDataTypedArray } from '../types';
 import IImageCalibration from './IImageCalibration';
 
 type CPUImageData = {

package/src/types/EventTypes.ts

@@ -7,8 +7,8 @@ import type ICamera from './ICamera';
 import type IImage from './IImage';
 import type IImageVolume from './IImageVolume';
 import type { VOIRange } from './voi';
-import type VOILUTFunctionType from '../enums/VOILUTFunctionType';
-import type ViewportStatus from '../enums/ViewportStatus';
+import VOILUTFunctionType from '../enums/VOILUTFunctionType';
+import ViewportStatus from '../enums/ViewportStatus';
 import type DisplayArea from './displayArea';
 import IImageCalibration from './IImageCalibration';
 
@@ -129,6 +129,16 @@ type ImageLoadedEventDetail = {
   image: IImage;
 };
 
+export type ImageLoadStageEventDetail = {
+  stageId: string;
+  numberOfImages: number;
+  numberOfFailures: number;
+  // The duration of just this stage
+  stageDurationInMS: number;
+  // The overall duration
+  startDurationInMS: number;
+};
+
 /**
  * IMAGE_LOADED_FAILED Event's data
  */
@@ -245,22 +255,6 @@ type ImageSpacingCalibratedEventDetail = {
   worldToIndex: mat4;
 };
 
-/**
- * IMAGE_LOAD_PROGRESS Event's data. Note this is only for one image load and NOT volume load.
- */
-type ImageLoadProgressEventDetail = {
-  /** url we are loading from */
-  url: string;
-  /** loading image image id */
-  imageId: string;
-  /** the bytes browser receive */
-  loaded: number;
-  /** the total bytes settled by the header */
-  total: number;
-  /** loaded divided by total * 100 - shows the percentage of the image loaded */
-  percent: number;
-};
-
 /**
  * The STACK_VIEWPORT_NEW_STACK event's data, when a new stack is set on a StackViewport
  */
@@ -391,11 +385,6 @@ type PreStackNewImageEvent = CustomEventType<PreStackNewImageEventDetail>;
 type ImageSpacingCalibratedEvent =
   CustomEventType<ImageSpacingCalibratedEventDetail>;
 
-/**
- * IMAGE_LOAD_PROGRESS
- */
-type ImageLoadProgressEvent = CustomEventType<ImageLoadProgressEventDetail>;
-
 /**
  * STACK_VIEWPORT_NEW_STACK
  */
@@ -443,8 +432,6 @@ export type {
   PreStackNewImageEventDetail,
   ImageSpacingCalibratedEvent,
   ImageSpacingCalibratedEventDetail,
-  ImageLoadProgressEvent,
-  ImageLoadProgressEventDetail,
   VolumeNewImageEvent,
   VolumeNewImageEventDetail,
   StackViewportNewStackEvent,