@cornerstonejs/core 2.0.0-beta.1 → 2.0.0-beta.10

package/src/init.ts

@@ -6,36 +6,42 @@ let useSharedArrayBuffer = true;
 let sharedArrayBufferMode = SharedArrayBufferModes.TRUE;
 import { deepMerge } from './utilities';
 import { Cornerstone3DConfig } from './types';
+import CentralizedWebWorkerManager from './webWorkerManager/webWorkerManager';
+
 // TODO: move sharedArrayBuffer into config.
 // TODO: change config into a class with methods to better control get/set
 const defaultConfig: Cornerstone3DConfig = {
   gpuTier: undefined,
   detectGPUConfig: {},
+  isMobile: false, // is mobile device
   rendering: {
     useCPURendering: false,
     // GPU rendering options
     preferSizeOverAccuracy: false,
-    useNorm16Texture: false, // _hasNorm16TextureSupport(),
+    useNorm16Texture: false,
     strictZSpacingForVolumeViewport: true,
   },
   // cache
-  // ...
+  enableCacheOptimization: true,
 };
 
 let config: Cornerstone3DConfig = {
   gpuTier: undefined,
   detectGPUConfig: {},
+  isMobile: false, // is mobile device
   rendering: {
     useCPURendering: false,
     // GPU rendering options
     preferSizeOverAccuracy: false,
-    useNorm16Texture: false, // _hasNorm16TextureSupport(),
+    useNorm16Texture: false,
     strictZSpacingForVolumeViewport: true,
   },
   // cache
-  // ...
+  enableCacheOptimization: true,
 };
 
+let webWorkerManager = null;
+
 function _getGLContext(): RenderingContext {
   // Create canvas element. The canvas is not added to the
   // document itself, so it is never displayed in the
@@ -73,23 +79,33 @@ function hasSharedArrayBuffer() {
   }
 }
 
-// Todo: commenting this out until proper support for int16 textures
-// are added to browsers, current implementation is buggy
-// function _hasNorm16TextureSupport() {
-//   const gl = _getGLContext();
+function _hasNorm16TextureSupport() {
+  const gl = _getGLContext();
 
-//   if (gl) {
-//     const ext = (gl as WebGL2RenderingContext).getExtension(
-//       'EXT_texture_norm16'
-//     );
+  if (gl) {
+    const ext = (gl as WebGL2RenderingContext).getExtension(
+      'EXT_texture_norm16'
+    );
 
-//     if (ext) {
-//       return true;
-//     }
-//   }
+    if (ext) {
+      return true;
+    }
+  }
 
-//   return false;
-// }
+  return false;
+}
+
+function isIOS() {
+  if (/iPad|iPhone|iPod/.test(navigator.platform)) {
+    return true;
+  } else {
+    return (
+      navigator.maxTouchPoints &&
+      navigator.maxTouchPoints > 2 &&
+      /MacIntel/.test(navigator.platform)
+    );
+  }
+}
 
 /**
  * Initialize the cornerstone-core. If the browser has a webgl context and
@@ -100,7 +116,7 @@ function hasSharedArrayBuffer() {
  * @returns A promise that resolves to true cornerstone has been initialized successfully.
  * @category Initialization
  */
-async function init(configuration = {}): Promise<boolean> {
+async function init(configuration = config): Promise<boolean> {
   if (csRenderInitialized) {
     return csRenderInitialized;
   }
@@ -108,6 +124,22 @@ async function init(configuration = {}): Promise<boolean> {
   // merge configs
   config = deepMerge(defaultConfig, configuration);
 
+  if (isIOS()) {
+    // iOS devices don't have support for OES_texture_float_linear
+    // and thus we should use native data type if we are on iOS
+    config.rendering.useNorm16Texture = _hasNorm16TextureSupport();
+
+    if (!config.rendering.useNorm16Texture) {
+      if (configuration.rendering?.preferSizeOverAccuracy) {
+        config.rendering.preferSizeOverAccuracy = true;
+      } else {
+        console.log(
+          'norm16 texture not supported, you can turn on the preferSizeOverAccuracy flag to use native data type, but be aware of the inaccuracy of the rendering in high bits'
+        );
+      }
+    }
+  }
+
   // gpuTier
   const hasWebGLContext = _hasActiveWebGLContext();
   if (!hasWebGLContext) {
@@ -120,7 +152,7 @@ async function init(configuration = {}): Promise<boolean> {
       'CornerstoneRender: Using detect-gpu to get the GPU benchmark:',
       config.gpuTier
     );
-    if (config.gpuTier.tier < 1) {
+    if (config.gpuTier?.tier < 1) {
       console.log(
         'CornerstoneRender: GPU is not powerful enough, using CPU rendering'
       );
@@ -133,6 +165,11 @@ async function init(configuration = {}): Promise<boolean> {
   setUseSharedArrayBuffer(sharedArrayBufferMode);
 
   csRenderInitialized = true;
+
+  if (!webWorkerManager) {
+    webWorkerManager = new CentralizedWebWorkerManager();
+  }
+
   return csRenderInitialized;
 }
 
@@ -156,6 +193,18 @@ function setPreferSizeOverAccuracy(status: boolean): void {
   _updateRenderingPipelinesForAllViewports();
 }
 
+/**
+ * Only IPhone IOS cannot render float textures right now due to the lack of support for OES_texture_float_linear.
+ * So we should not use float textures on IOS devices.
+ */
+function canRenderFloatTextures(): boolean {
+  if (!isIOS()) {
+    return true;
+  }
+
+  return false;
+}
+
 /**
  * Resets the cornerstone-core init state if it has been manually
  * initialized to force use the cpu rendering (e.g., for tests)
@@ -256,6 +305,14 @@ function _updateRenderingPipelinesForAllViewports(): void {
   );
 }
 
+function getWebWorkerManager() {
+  if (!webWorkerManager) {
+    webWorkerManager = new CentralizedWebWorkerManager();
+  }
+
+  return webWorkerManager;
+}
+
 export {
   init,
   getShouldUseCPURendering,
@@ -268,4 +325,6 @@ export {
   resetUseSharedArrayBuffer,
   getConfiguration,
   setConfiguration,
+  getWebWorkerManager,
+  canRenderFloatTextures,
 };

package/src/loaders/ProgressiveRetrieveImages.ts

@@ -0,0 +1,380 @@
+import {
+  IRetrieveConfiguration,
+  IImagesLoader,
+  IImage,
+  RetrieveStage,
+  EventTypes,
+  ImageLoadListener,
+  RetrieveOptions,
+} from '../types';
+import singleRetrieveStages from './configuration/singleRetrieve';
+import sequentialRetrieveStages from './configuration/sequentialRetrieve';
+import interleavedRetrieveStages from './configuration/interleavedRetrieve';
+import { loadAndCacheImage } from './imageLoader';
+import { triggerEvent, ProgressiveIterator, decimate } from '../utilities';
+import imageLoadPoolManager from '../requestPool/imageLoadPoolManager';
+import { ImageQualityStatus, RequestType, Events } from '../enums';
+import cache from '../cache';
+import eventTarget from '../eventTarget';
+import { fillNearbyFrames } from './fillNearbyFrames';
+
+export {
+  sequentialRetrieveStages,
+  interleavedRetrieveStages,
+  singleRetrieveStages,
+};
+
+type StageStatus = {
+  stageId: string;
+  // startTime is the overall start of loading a given image id
+  startTime?: number;
+  // stageStartTime is the time to start loading this stage item
+  stageStartTime?: number;
+  totalImageCount: number;
+  imageLoadFailedCount: number;
+  imageLoadPendingCount: number;
+};
+
+/**
+ * A nearby request is a request that can be fulfilled by copying another image
+ */
+export type NearbyRequest = {
+  // The item id to fill
+  itemId: string;
+  linearId?: string;
+  // The new status of the filled image (will only fill if the existing status
+  // is less than this one)
+  imageQualityStatus: ImageQualityStatus;
+};
+
+export type ProgressiveRequest = {
+  imageId: string;
+  stage: RetrieveStage;
+  next?: ProgressiveRequest;
+  /**
+   * Nearby requests are a set of requests for filling nearby images which
+   * could be filled by using this image as a copied image to generate the
+   * nearby data as a low-resolution alternative image.
+   */
+  nearbyRequests?: NearbyRequest[];
+};
+
+/**
+ * A progressive loader is given some number of images to load,
+ * and calls a success or failure callback some number of times in some
+ * ordering, possibly calling back multiple times.
+ * This allows the progressive loader to be configured for different setups
+ * and to return render results for various images.
+ *
+ * When used by the a stack viewport, the progressive loader can return multiple
+ * representations to the viewport, replacing earlier/more lossy versions with better ones.
+ *
+ * When used by a streaming loader, the progressive loader can change the ordering
+ * of the rendering to retrieve high priority images first, and the lower priority
+ * images later to provide a complete final rendering.
+ *
+ * Requests are held in a queue, such that subsequent requests for a given
+ * image can be cancelled or ensured to be not initiated until the higher
+ * priority image sets have been completed.
+ *
+ * This loader is also used for the base streamimg image volume, configured with
+ * a minimal interleaved load order, combined with filling nearby volume slices
+ * on load, resulting in much faster initial apparent display.
+ *
+ * The loader will load images from existing cached images, cached volumes, and
+ * from other nearby images or one or more calls to back end services.
+ *
+ * @param imageIds - the set of images to load.  For a volume, these should be
+ *                   ordered from top to bottom.
+ * @param listener - has success and failure callbacks to listen for image deliver events, and may
+ *                   have a getTargetOptions to get information on the retrieve
+ * @param retrieveOptions - is a set of retrieve options to use
+ */
+export class ProgressiveRetrieveImages
+  implements IImagesLoader, IRetrieveConfiguration
+{
+  public static createProgressive = createProgressive;
+
+  public static interleavedRetrieveStages = {
+    stages: interleavedRetrieveStages,
+  };
+
+  public static singleRetrieveStages = {
+    stages: singleRetrieveStages,
+  };
+
+  public static sequentialRetrieveStages = {
+    stages: sequentialRetrieveStages,
+  };
+
+  stages: RetrieveStage[];
+  retrieveOptions: Record<string, RetrieveOptions>;
+
+  constructor(imageRetrieveConfiguration: IRetrieveConfiguration) {
+    this.stages = imageRetrieveConfiguration.stages || singleRetrieveStages;
+    this.retrieveOptions = imageRetrieveConfiguration.retrieveOptions || {};
+  }
+
+  public loadImages(imageIds: string[], listener: ImageLoadListener) {
+    const instance = new ProgressiveRetrieveImagesInstance(
+      this,
+      imageIds,
+      listener
+    );
+    return instance.loadImages();
+  }
+}
+
+class ProgressiveRetrieveImagesInstance {
+  imageIds: string[];
+  listener: ImageLoadListener;
+  stages: RetrieveStage[];
+  retrieveOptions: Record<string, RetrieveOptions>;
+  outstandingRequests = 0;
+
+  stageStatusMap = new Map<string, StageStatus>();
+  imageQualityStatusMap = new Map<string, ImageQualityStatus>();
+  displayedIterator = new ProgressiveIterator<void | IImage>('displayed');
+
+  constructor(configuration: IRetrieveConfiguration, imageIds, listener) {
+    this.stages = configuration.stages;
+    this.retrieveOptions = configuration.retrieveOptions;
+    this.imageIds = imageIds;
+    this.listener = listener;
+  }
+
+  public async loadImages() {
+    // The actual function is to just setup the interleave and add the
+    // requests, with all the actual work being handled by the nested functions
+    const interleaved = this.createStageRequests();
+    this.outstandingRequests = interleaved.length;
+    for (const request of interleaved) {
+      this.addRequest(request);
+    }
+    if (this.outstandingRequests === 0) {
+      return Promise.resolve(null);
+    }
+
+    return this.displayedIterator.getDonePromise();
+  }
+
+  protected sendRequest(request, options) {
+    const { imageId, next } = request;
+    const errorCallback = (reason, done) => {
+      this.listener.errorCallback(imageId, complete || !next, reason);
+      if (done) {
+        this.updateStageStatus(request.stage, reason);
+      }
+    };
+    const loadedPromise = (options.loader || loadAndCacheImage)(
+      imageId,
+      options
+    );
+    const uncompressedIterator = ProgressiveIterator.as(loadedPromise);
+    let complete = false;
+
+    uncompressedIterator
+      .forEach(async (image, done) => {
+        const oldStatus = this.imageQualityStatusMap.get(imageId);
+        if (!image) {
+          console.warn('No image retrieved', imageId);
+          return;
+        }
+        const { imageQualityStatus } = image;
+        complete ||= imageQualityStatus === ImageQualityStatus.FULL_RESOLUTION;
+        if (oldStatus !== undefined && oldStatus > imageQualityStatus) {
+          // We already have a better status, so don't update it
+          this.updateStageStatus(request.stage, null, true);
+          return;
+        }
+
+        this.listener.successCallback(imageId, image);
+        this.imageQualityStatusMap.set(imageId, imageQualityStatus);
+        this.displayedIterator.add(image);
+        if (done) {
+          this.updateStageStatus(request.stage);
+        }
+        fillNearbyFrames(
+          this.listener,
+          this.imageQualityStatusMap,
+          request,
+          image,
+          options
+        );
+      }, errorCallback)
+      .finally(() => {
+        if (!complete && next) {
+          if (cache.getImageLoadObject(imageId)) {
+            cache.removeImageLoadObject(imageId);
+          }
+          this.addRequest(next, options.streamingData);
+        } else {
+          if (!complete) {
+            this.listener.errorCallback(imageId, true, "Couldn't decode");
+          }
+          this.outstandingRequests--;
+          for (let skip = next; skip; skip = skip.next) {
+            this.updateStageStatus(skip.stage, null, true);
+          }
+        }
+        if (this.outstandingRequests <= 0) {
+          this.displayedIterator.resolve();
+        }
+      });
+    const doneLoad = uncompressedIterator.getDonePromise();
+    // Errors already handled above in the callback
+    return doneLoad.catch((e) => null);
+  }
+
+  /** Adds a rquest to the image load pool manager */
+  protected addRequest(request, streamingData = {}) {
+    const { imageId, stage } = request;
+    const baseOptions = this.listener.getLoaderImageOptions(imageId);
+    if (!baseOptions) {
+      // Image no longer of interest
+      return;
+    }
+    const { retrieveType = 'default' } = stage;
+    const { retrieveOptions: keyedRetrieveOptions } = this;
+    const retrieveOptions =
+      keyedRetrieveOptions[retrieveType] || keyedRetrieveOptions.default;
+    const options = {
+      ...baseOptions,
+      retrieveType,
+      retrieveOptions,
+      streamingData,
+    };
+    const priority = stage.priority ?? -5;
+    const requestType = stage.requestType || RequestType.Interaction;
+    const additionalDetails = { imageId };
+
+    imageLoadPoolManager.addRequest(
+      this.sendRequest.bind(this, request, options),
+      requestType,
+      additionalDetails,
+      priority
+    );
+  }
+
+  protected updateStageStatus(stage, failure?, skipped = false) {
+    const { id } = stage;
+    const stageStatus = this.stageStatusMap.get(id);
+    if (!stageStatus) {
+      return;
+    }
+    stageStatus.imageLoadPendingCount--;
+    if (failure) {
+      stageStatus.imageLoadFailedCount++;
+    } else if (!skipped) {
+      stageStatus.totalImageCount++;
+    }
+    if (!skipped && !stageStatus.stageStartTime) {
+      stageStatus.stageStartTime = Date.now();
+    }
+    if (!stageStatus.imageLoadPendingCount) {
+      const {
+        imageLoadFailedCount: numberOfFailures,
+        totalImageCount: numberOfImages,
+        stageStartTime = Date.now(),
+        startTime,
+      } = stageStatus;
+      const detail: EventTypes.ImageLoadStageEventDetail = {
+        stageId: id,
+        numberOfFailures,
+        numberOfImages,
+        stageDurationInMS: stageStartTime ? Date.now() - stageStartTime : null,
+        startDurationInMS: Date.now() - startTime,
+      };
+      triggerEvent(eventTarget, Events.IMAGE_RETRIEVAL_STAGE, detail);
+      this.stageStatusMap.delete(id);
+    }
+  }
+
+  /** Interleaves the values according to the stages definition */
+  protected createStageRequests() {
+    const interleaved = new Array<ProgressiveRequest>();
+    // Maps image id to the LAST progressive request - to allow tail append
+    const imageRequests = new Map<string, ProgressiveRequest>();
+
+    const addStageInstance = (stage, position) => {
+      const index =
+        position < 0
+          ? this.imageIds.length + position
+          : position < 1
+          ? Math.floor((this.imageIds.length - 1) * position)
+          : position;
+      const imageId = this.imageIds[index];
+      if (!imageId) {
+        throw new Error(`No value found to add to requests at ${position}`);
+      }
+      const request: ProgressiveRequest = {
+        imageId,
+        stage,
+        nearbyRequests: this.findNearbyRequests(index, stage),
+      };
+      this.addStageStatus(stage);
+      const existingRequest = imageRequests.get(imageId);
+      if (existingRequest) {
+        existingRequest.next = request;
+      } else {
+        interleaved.push(request);
+      }
+      imageRequests.set(imageId, request);
+    };
+
+    for (const stage of this.stages) {
+      const indices =
+        stage.positions ||
+        decimate(this.imageIds, stage.decimate || 1, stage.offset ?? 0);
+      indices.forEach((index) => addStageInstance(stage, index));
+    }
+    return interleaved;
+  }
+
+  /**
+   * Finds nearby requests to fulfill to show the merge information earlier.
+   * @param index - to use as the base value
+   * @param imageIds - set of image ids to request
+   * @param stage - to find information from
+   * @returns Array of nearby frames to fill when the main stage is done
+   */
+  protected findNearbyRequests(index: number, stage): NearbyRequest[] {
+    const nearby = new Array<NearbyRequest>();
+    if (!stage.nearbyFrames) {
+      return nearby;
+    }
+    for (const nearbyItem of stage.nearbyFrames) {
+      const nearbyIndex = index + nearbyItem.offset;
+      if (nearbyIndex < 0 || nearbyIndex >= this.imageIds.length) {
+        continue;
+      }
+      nearby.push({
+        itemId: this.imageIds[nearbyIndex],
+        imageQualityStatus: nearbyItem.imageQualityStatus,
+      });
+    }
+
+    return nearby;
+  }
+
+  protected addStageStatus(stage) {
+    const { id } = stage;
+    const stageStatus = this.stageStatusMap.get(id) || {
+      stageId: id,
+      startTime: Date.now(),
+      stageStartTime: null,
+      totalImageCount: 0,
+      imageLoadFailedCount: 0,
+      imageLoadPendingCount: 0,
+    };
+    stageStatus.imageLoadPendingCount++;
+    this.stageStatusMap.set(id, stageStatus);
+    return stageStatus;
+  }
+}
+
+export function createProgressive(configuration: IRetrieveConfiguration) {
+  return new ProgressiveRetrieveImages(configuration);
+}
+
+export default ProgressiveRetrieveImages;

package/src/loaders/configuration/interleavedRetrieve.ts

@@ -0,0 +1,96 @@
+import { RetrieveStage, NearbyFrames } from '../../types';
+import { RequestType, ImageQualityStatus } from '../../enums';
+
+// Defines some nearby frames to replicate to
+const nearbyFrames: NearbyFrames[] = [
+  {
+    offset: -1,
+    imageQualityStatus: ImageQualityStatus.ADJACENT_REPLICATE,
+  },
+  {
+    offset: +1,
+    imageQualityStatus: ImageQualityStatus.ADJACENT_REPLICATE,
+  },
+  { offset: +2, imageQualityStatus: ImageQualityStatus.FAR_REPLICATE },
+];
+
+/**
+ * This configuration is designed to interleave the data requests, using
+ * lossy/thumbnail requests when available, but falling back to full retrieve
+ * requests in an interleaved manner.
+ * The basic ordering is:
+ *   1. Retrieve middle image, first, last
+ *   2. Retrieve every 4th image, offset 1, lossy if available
+ *   3. Retrieve every 4th image, offset 3, lossy if available
+ *   4. Retrieve every 4th image, offset 0, full images
+ *   5. Retrieve every 4th image, offset 2, full images
+ *   6. Retrieve every 4th image, offsets 1 and 3, full images if not already done
+ */
+const interleavedRetrieveConfiguration: RetrieveStage[] = [
+  {
+    id: 'initialImages',
+    // Values between -1 and 1 are relative to size, so 0.5 is middle image
+    // and 0 is first image, -1 is last image
+    positions: [0.5, 0, -1],
+    retrieveType: 'default',
+    requestType: RequestType.Thumbnail,
+    priority: 5,
+    nearbyFrames,
+  },
+  {
+    id: 'quarterThumb',
+    decimate: 4,
+    offset: 3,
+    requestType: RequestType.Thumbnail,
+    retrieveType: 'multipleFast',
+    priority: 6,
+    nearbyFrames,
+  },
+  {
+    id: 'halfThumb',
+    decimate: 4,
+    offset: 1,
+    priority: 7,
+    requestType: RequestType.Thumbnail,
+    retrieveType: 'multipleFast',
+    nearbyFrames,
+  },
+  {
+    id: 'quarterFull',
+    decimate: 4,
+    offset: 2,
+    priority: 8,
+    requestType: RequestType.Thumbnail,
+    retrieveType: 'multipleFinal',
+  },
+  {
+    id: 'halfFull',
+    decimate: 4,
+    offset: 0,
+    priority: 9,
+    requestType: RequestType.Thumbnail,
+    retrieveType: 'multipleFinal',
+  },
+  {
+    id: 'threeQuarterFull',
+    decimate: 4,
+    offset: 1,
+    priority: 10,
+    requestType: RequestType.Thumbnail,
+    retrieveType: 'multipleFinal',
+  },
+  {
+    id: 'finalFull',
+    decimate: 4,
+    offset: 3,
+    priority: 11,
+    requestType: RequestType.Thumbnail,
+    retrieveType: 'multipleFinal',
+  },
+  {
+    // This goes back to basic retrieve to recover from retrieving against
+    // servers returning errors for any of the above requests.
+    id: 'errorRetrieve',
+  },
+];
+export default interleavedRetrieveConfiguration;

package/src/loaders/configuration/sequentialRetrieve.ts

@@ -0,0 +1,17 @@
+import type { RetrieveStage } from '../../types';
+
+/**
+ * This simply retrieves the images sequentially as provided.
+ */
+const sequentialRetrieveStages: RetrieveStage[] = [
+  {
+    id: 'lossySequential',
+    retrieveType: 'singleFast',
+  },
+  {
+    id: 'finalSequential',
+    retrieveType: 'singleFinal',
+  },
+];
+
+export default sequentialRetrieveStages;

package/src/loaders/configuration/singleRetrieve.ts

@@ -0,0 +1,18 @@
+import type { RetrieveStage } from '../../types';
+
+/**
+ * This simply retrieves the images sequentially as provided.
+ */
+const singleRetrieveStages: RetrieveStage[] = [
+  {
+    id: 'initialImages',
+    retrieveType: 'single',
+  },
+  // Shouldn't be necessary, but if the server returns an error for the above
+  // configuration, this will ensure the image is still fetched.
+  {
+    id: 'errorRetrieve',
+  },
+];
+
+export default singleRetrieveStages;