@radicalimaging/static-wado-creator 0.6.4

package/lib/operation/StudyData.js

@@ -0,0 +1,416 @@
+const path = require("path");
+const fs = require("fs");
+const hashFactory = require("node-object-hash");
+const { JSONReader } = require("@radicalimaging/static-wado-util");
+const Tags = require("../dictionary/Tags");
+const TagLists = require("../model/TagLists");
+const JSONWriter = require("../writer/JSONWriter");
+
+const {getValue, setValue, getList, setList} = Tags;
+const hasher = hashFactory();
+
+const getSeriesInstanceUid = (seriesInstance) =>
+  getValue(seriesInstance,Tags.SeriesInstanceUID);
+
+/**
+ * StudyData contains information about the grouped study data.  It is used to create
+ * study level information such as study or series level metadata, and to group things
+ * together during processing, such as handling the study level deduplicated group
+ * files.
+ *
+ * It also has methods on performing study level operations such as patient name updates
+ * or instance remove operations, and has methods to handle avoiding needing to store instance
+ * level data multiple times when it already exists.
+ */
+class StudyData {
+  constructor({ studyInstanceUid, studyPath, deduplicatedPath, deduplicatedInstancesPath }, { isGroup, clean }) {
+    this.studyInstanceUid = studyInstanceUid;
+    this.studyPath = studyPath;
+    this.isGroup = isGroup;
+    this.deduplicatedPath = deduplicatedPath;
+    this.deduplicatedInstancesPath = deduplicatedInstancesPath;
+    this.clean = clean;
+    this.clear();
+  }
+
+  clear() {
+    this.deduplicated = [];
+    this.extractData = {};
+    // The list of already existing files read in to create this object
+    this.existingFiles = [];
+    // The read hashes is the hashes of the files that are read, both as groups and internally
+    this.readHashes = {};
+    // The deduplicated hashes are just the hashes for individual items
+    this.deduplicatedHashes = {};
+    this.sopInstances = {};
+
+    // Used to track if new instances have been added.
+    this.newInstancesAdded = 0;
+  }
+
+  /**
+   * Clean the directory, and/or read existing data in the isInstances or isGroup files.
+   * TODO - implement this.
+   */
+  async init({ clean }) {
+    if (clean) {
+      // Wipe out the study directory entirely, as well as the deduplicatedRoot and instancesRoot
+    }
+    if (this.deduplicatedPath) {
+      await this.readDeduplicated(this.deduplicatedPath);
+    }
+    if (this.deduplicatedInstancesPath) {
+      await this.readDeduplicated(this.deduplicatedInstancesPath);
+    }
+  }
+
+  get numberOfInstances() {
+    return this.deduplicated.length;
+  }
+
+  /**
+   * Indicate if this study is 'dirty', that is, has been updated.
+   * That is NOT to say that the metadata files generated from this study data are up to date, that is
+   * a separate type of check.
+   */
+  get dirty() {
+    return this.newInstancesAdded > 0 || this.existingFiles.length > 1;
+  }
+
+  async dirtyMetadata() {
+    if (this.dirty) {
+      // console.log("Study data is dirty - need to write updated file");
+      return true;
+    }
+    try {
+      const deduplicatedTopFile = await JSONReader(this.studyPath, "deduplicated.gz", null);
+      if (!deduplicatedTopFile) {
+        return true;
+      }
+      const info = deduplicatedTopFile[0];
+      if (!info || getValue(info,Tags.DeduppedHash) || getValue(info,Tags.DeduppedType) != "info") {
+        return true;
+      }
+      const hashValue = getValue(info,Tags.DeduppedHash);
+      return this.existingFiles[0].indexOf(hashValue) == -1;
+    } catch (e) {
+      console.log("Assume study metadata is dirty", e);
+      return true;
+    }
+  }
+
+  async reject(seriesInstanceUid, sopInstanceUid, reason) {
+    console.log("Rejecting series instance UID", seriesInstanceUid, "because", reason);
+    // TODO - actually add a reject note...
+    this.newInstancesAdded += 1;
+    for (let i = 0; i < this.deduplicated.length; i++) {
+      const recombined = await this.recombine(i);
+      const reSeriesUid = getValue(recombined,Tags.SeriesInstanceUID);
+      if (reSeriesUid !== seriesInstanceUid) continue;
+      const reSop = getValue(recombined,Tags.SOPInstanceUID);
+      if (!reSop) continue;
+      if (reSop !== sopInstanceUid) continue;
+      setValue(this.deduplicated[i], Tags.DeduppedType,"deleted");
+    }
+  }
+
+  async delete() {
+    await fs.rmSync(this.studyPath, { recursive: true, force: true });
+    await fs.rmSync(this.deduplicatedInstancesPath, { recursive: true, force: true });
+    await fs.rmSync(this.deduplicatedPath, { recursive: true, force: true });
+    this.clear();
+  }
+
+  async getOrLoadExtract(hashKey) {
+    let item = this.extractData[hashKey];
+    if (!item) {
+      item = await JSONReader.readHashData(this.studyPath, hashKey);
+      if (!item) {
+        console.error("Unable to read hashdata", hashKey);
+        return item;
+      }
+      this.extractData[hashKey] = item;
+    }
+    return item;
+  }
+
+  assignUndefined(ret, item) {
+    if (!item) return;
+    Object.keys(item).forEach((key) => {
+      if (ret[key]) return;
+      ret[key] = item[key];
+    });
+  }
+
+  /**
+   * Create a full study instance data for instance at index
+   */
+  async recombine(index) {
+    const deduplicated = this.deduplicated[index];
+    if (index < 0 || index >= this.deduplicated.length) {
+      throw new Error(`Can't read index ${index}, out of bounds [0..${this.deduplicated.length})`);
+    }
+    const refs = getList(deduplicated,Tags.DeduppedRef);
+    if (!refs) {
+      console.log("No refs for", deduplicated);
+      return deduplicated;
+    }
+    const ret = { ...deduplicated };
+    for (const hashKey of refs) {
+      const item = await this.getOrLoadExtract(hashKey);
+      this.assignUndefined(ret, item);
+    }
+    return ret;
+  }
+
+  async addExtracted(callback, hashKey, item) {
+    if (this.extractData[hashKey]) {
+      if( this.verbose ) console.log("Already have extracted", hashKey, getValue(item,Tags.DeduppedType));
+      return;
+    }
+    await callback.bulkdata(this, hashKey, item);
+    this.extractData[hashKey] = item;
+  }
+
+  sopExists(sopUID) {
+    const sopValue = (sopUID && sopUID.Value && sopUID.Value[0]) || sopUID;
+    return sopValue && this.sopInstances[sopValue] !== undefined;
+  }
+
+  addDeduplicated(data, filename) {
+    if (!this.isGroup) return;
+    // TODO - check the hash code on the added data, if it has already been seen then ignore this item.
+    if (this.internalAddDeduplicated(data, filename)) {
+      this.newInstancesAdded += 1;
+    }
+  }
+
+  /** Add the instance if not already present */
+  internalAddDeduplicated(data, filename = "internal") {
+    const hashValue = TagLists.addHash(data, Tags.InstanceType);
+    if (!hashValue || this.deduplicatedHashes[hashValue]) {
+      // console.log('Not adding', hashValue, 'because the hash exists');
+      return false;
+    }
+    const sopValue = getValue(data,Tags.SOPInstanceUID);
+    const sopIndex = this.sopInstances[sopValue];
+    if (!sopValue) {
+      console.warn("No sop value in", filename, "reading", data);
+      return false;
+    }
+    this.deduplicatedHashes[hashValue] = data;
+    this.readHashes[hashValue] = filename;
+    if (sopIndex !== undefined) {
+      console.log("Replacing SOP", sopValue, "at index", sopIndex);
+      this.deduplicated[sopIndex] = data;
+    } else {
+      this.sopInstances[sopValue] = this.deduplicated.length;
+      this.deduplicated.push(data);
+    }
+
+    return !!hashValue;
+  }
+
+  async listJsonFiles(dir) {
+    const names = await fs.promises.readdir(dir);
+    if (!names || names.length == 0) return [];
+    const gzNames = names.filter((name) => name.indexOf(".gz") > 0);
+    const statInfo = gzNames
+      .map((name) => {
+        const stat = fs.statSync(path.join(dir, name));
+        return {
+          ...stat,
+          name,
+          hash: this.removeGz(name),
+        };
+      })
+      .sort((a, b) => b.mtimeMs - a.mtimeMs);
+    return statInfo;
+  }
+
+  /**
+   * Reads the deduplicated directory, finding the newest file in the directory of type JSON,
+   * and then reads it in.  It then reads in all un-referenced files, and adds references to those to the
+   * current path, NOT changing the hash value of the top level element.
+   *
+   * Records the files read in referencedFiles list.
+   * Sets the dirty deduplicated flag if at least two instances were read.
+   *
+   * Does NOT call the readInstances, which is a separate step which can be executed after this one.
+   *
+   * @param {*} deduplicatedDirectory
+   */
+  async readDeduplicated(deduplicatedDirectory) {
+    try {
+      const files = await this.listJsonFiles(deduplicatedDirectory);
+      if (!files || !files.length) {
+        console.log("No deduplicated for", deduplicatedDirectory);
+        return;
+      }
+      console.log("There are", files.length, "files to check");
+      for (let i = 0; i < files.length; i++) {
+        const stat = files[i];
+        const { hash } = stat;
+        if (this.readHashes[hash]) {
+          continue;
+        }
+        await this.readDeduplicatedFile(deduplicatedDirectory, stat);
+      }
+      console.log("Done checking", deduplicatedDirectory);
+    } catch (e) {
+      // No-op console.log(e);
+    }
+  }
+
+  async readDeduplicatedFile(dir, stat) {
+    const { hash, name } = stat;
+    try {
+      if (this.verbose) console.log("Reading deduplicated file", name);
+      const data = await JSONReader(dir, name);
+      this.readHashes[hash] = name;
+      this.existingFiles.push(name);
+      const listData = (Array.isArray(data) && data) || [data];
+      listData.forEach((item) => {
+        const type = getValue(item, Tags.DeduppedType);
+        if (type == Tags.InstanceType) {
+          this.internalAddDeduplicated(item, name);
+        } else if (type == "info") {
+          const refs = getList(item,Tags.DeduppedRef);
+          if (refs) {
+            refs.forEach((hashValue) => {
+              this.readHashes[hashValue] = `${hashValue}.gz`;
+            });
+          }
+        } else {
+          const hashValue = getValue(item,Tags.DeduppedHash);
+          if (hashValue) {
+            this.extractData[hashValue] = item;
+          }
+        }
+      });
+    } catch (e) {
+      console.error("Unable to read", dir, name);
+    }
+  }
+
+  async writeMetadata() {
+    const anInstance = await this.recombine(0);
+    const series = {};
+
+    for (let i = 0; i < this.numberOfInstances; i++) {
+      const seriesInstance = await this.recombine(i);
+      const type = getValue(seriesInstance,Tags.DeduppedType);
+      if (type == "deleted") {
+        console.log("Skipping", type);
+        continue;
+      }
+      const seriesInstanceUid = getSeriesInstanceUid(seriesInstance);
+      if (!seriesInstanceUid) {
+        console.log("Cant get seriesUid from", Tags.SeriesInstanceUID, seriesInstance);
+        continue;
+      }
+      if (!series[seriesInstanceUid]) {
+        const seriesQuery = TagLists.extract(seriesInstance, "series", TagLists.SeriesQuery);
+        const seriesPath = path.join(this.studyPath, "series", seriesInstanceUid);
+        series[seriesInstanceUid] = {
+          seriesPath,
+          seriesQuery,
+          instances: [],
+          instancesQuery: [],
+        };
+      }
+      series[seriesInstanceUid].instances.push(seriesInstance);
+      series[seriesInstanceUid].instancesQuery.push(TagLists.extract(seriesInstance, "instance", TagLists.InstanceQuery));
+    }
+
+    const seriesList = [];
+    const modalitiesInStudy = [];
+    let numberOfInstances = 0;
+    let numberOfSeries = 0;
+    for (const seriesUid of Object.keys(series)) {
+      const singleSeries = series[seriesUid];
+      const { seriesQuery, seriesPath, instances, instancesQuery } = singleSeries;
+      seriesQuery[Tags.NumberOfSeriesRelatedInstances] = {
+        vr: "IS",
+        Value: [instances.length],
+      };
+      numberOfInstances += instances.length;
+      numberOfSeries += 1;
+      seriesList.push(seriesQuery);
+      const modality = getValue(seriesQuery,Tags.Modality);
+      if (modalitiesInStudy.indexOf(modality) == -1) modalitiesInStudy.push(modality);
+      await JSONWriter(seriesPath, "metadata", instances, {
+        gzip: true,
+        index: false,
+      });
+      await JSONWriter(seriesPath, "series", [seriesQuery]);
+      await JSONWriter(seriesPath, "instances", instancesQuery);
+    }
+
+    await JSONWriter(this.studyPath, "series", seriesList);
+    console.log("Wrote series with", seriesList.length);
+
+    const studyQuery = TagLists.extract(anInstance, "study", TagLists.PatientStudyQuery);
+    studyQuery[Tags.ModalitiesInStudy] = { Value: modalitiesInStudy, vr: "CS" };
+    studyQuery[Tags.NumberOfStudyRelatedInstances] = {
+      Value: [numberOfInstances],
+      vr: "IS",
+    };
+    studyQuery[Tags.NumberOfStudyRelatedSeries] = {
+      Value: [numberOfSeries],
+      vr: "IS",
+    };
+    // Write the index.json directly in the studyPath directory.
+    await JSONWriter(this.studyPath, "index.json", [studyQuery], {
+      gzip: true,
+      index: false,
+    });
+
+    const infoItem = this.createInfo();
+    console.log("Writing deduplicated study data with", Object.values(this.extractData).length, "extract items and", this.deduplicated.length, "instance items");
+    await JSONWriter(this.studyPath, "deduplicated", [infoItem, ...Object.values(this.extractData), ...this.deduplicated]);
+
+    return studyQuery;
+  }
+
+  /**
+   * Creates an information item containing the hash value of this item type.
+   */
+  createInfo() {
+    const data = {};
+    const hashValue = hasher.hash(this.deduplicated);
+    setValue(data,Tags.DeduppedHash,hashValue);
+    setValue(data,Tags.DeduppedType,"info");
+    return data;
+  }
+
+  /* eslint-disable-next-line class-methods-use-this */
+  removeGz(name) {
+    const gzIndex = name.indexOf(".gz");
+    return (gzIndex > 0 && name.substring(0, gzIndex)) || name;
+  }
+
+  /** Writes the deduplicated group */
+  async writeDeduplicatedGroup() {
+    const data = this.createInfo();
+    const hashValue = getValue(data,Tags.DeduppedHash);
+    console.log(
+      "Writing deduplicated hash data set data with",
+      Object.values(this.extractData).length,
+      "extract items and",
+      this.deduplicated.length,
+      "instance items"
+    );
+    setList(data,Tags.DeduppedRef,
+      Object.keys(this.readHashes).filter(() => this.deduplicatedHashes[hashValue] == undefined));
+    const deduplicatedList = [data, ...Object.values(this.extractData), ...this.deduplicated];
+    // const naturalList = deduplicatedList.map(Tags.naturalizeDataset);
+    // // console.log("naturalList=", JSON.stringify(naturalList,null,2));
+    // console.log("Going to write study data", naturalList.length);
+    // await JSONWriter(this.deduplicatedPath, hashValue, naturalList, { gzip: true, index: false });
+    await JSONWriter(this.deduplicatedPath, hashValue, deduplicatedList, { gzip: true, index: false });
+    console.log("Wrote naturalized dataset");
+  }
+}
+
+module.exports = StudyData;

package/lib/operation/ThumbnailService.js

@@ -0,0 +1,184 @@
+const path = require("path");
+const glob = require("glob");
+const dicomCodec = require("@cornerstonejs/dicom-codec");
+const staticCS = require("@radicalimaging/static-cs-lite");
+const fs = require("fs");
+const { exec } = require("child_process");
+const decodeImage = require("./adapter/decodeImage");
+const { shouldThumbUseTranscoded } = require("./adapter/transcodeImage");
+const { isVideo } = require("../writer/VideoWriter");
+const Tags = require("../dictionary/Tags");
+
+/**
+ * Return the middle index of given list
+ * @param {*} listThumbs
+ * @returns
+ */
+function getThumbIndex(listThumbs) {
+  return Math.trunc(listThumbs / 2);
+}
+function internalGenerateThumbnail(originalImageFrame, dataset, metadata, transferSyntaxUid, doneCallback) {
+  decodeImage(originalImageFrame, dataset, transferSyntaxUid)
+    .then((decodeResult = {}) => {
+      if (isVideo(transferSyntaxUid)) {
+        console.log("Video data - no thumbnail generator yet");
+      } else {
+        const { imageFrame, imageInfo } = decodeResult;
+        const pixelData = dicomCodec.getPixelData(imageFrame, imageInfo, transferSyntaxUid);
+        staticCS.getRenderedBuffer(transferSyntaxUid, pixelData, metadata, doneCallback);
+      }
+    })
+    .catch((error) => {
+      console.log(`Error while generating thumbnail:: ${error}`);
+    });
+}
+
+/**
+ * ThumbObj wrapper containing both original and transcoded content.
+ *
+ * @typedef {Object} ThumbObjWrapper
+ * @property {Object} id object containing study properties such as seriesUid, instanceUid
+ * @property {*} transcodedId transcoded object containing study properties such as seriesUid, instanceUid (this might be the same as original id depending on options)
+ * @property {*} imageFrame original image frame
+ * @property {*} transcodedImageFrame transcoded image frame (this might be the same as original frame depending on options)
+ * @property {number} frameIndex
+ */
+
+/**
+ * ThumbObj containing imageFrame, id to be used.
+ *
+ * @typedef {Object} ThumbObj
+ * @property {Object} id object containing study properties such as seriesUid, instanceUid
+ * @property {*} imageFrame
+ * @property {number} frameIndex
+ */
+
+/**
+ * Service to handle thumbnail creation.
+ * @class
+ * @constructor
+ * @property {Array<ThumbObj>} framesThumbnailObj Auxiliary list that contains all thumb obj.
+ * @property {ThumbObj} favoriteThumbnailObj Favorite thumb obj to be used on thumbnail creation
+ * @property {String} thumbFileName thumb file (with extension) to be used when creating the file.
+ * @public
+ */
+class ThumbnailService {
+  constructor() {
+    this.framesThumbnailObj = [];
+    this.favoriteThumbnailObj = {};
+    this.thumbFileName = "thumbnail";
+  }
+
+  /**
+   * Add thumbObj to list of possible thumbs.
+   * It might use original content or transcoded depending on options flags
+   * @param {ThumbObjWrapper} thumbObjWrapper
+   * @param {Object} programOpts
+   */
+  queueThumbnail(thumbObjWrapper, programOpts) {
+    const { id, imageFrame, transcodedId, transcodedImageFrame, frameIndex } = thumbObjWrapper;
+    const getThumbContent = (originalContent, trancodedContent) => (shouldThumbUseTranscoded(id, programOpts) ? trancodedContent : originalContent);
+
+    const thumbObj = {
+      imageFrame: getThumbContent(imageFrame, transcodedImageFrame),
+      id: getThumbContent(id, transcodedId),
+      frameIndex,
+    };
+
+    this.framesThumbnailObj.push(thumbObj);
+
+    this.setFavoriteThumbnailObj();
+  }
+
+  setFavoriteThumbnailObj() {
+    const favIndex = getThumbIndex(this.framesThumbnailObj.length);
+
+    this.favoriteThumbnailObj = this.framesThumbnailObj[favIndex];
+  }
+
+  ffmpeg(input, output) {
+    exec(`ffmpeg -i "${input}" -vf  "thumbnail,scale=640:360" -frames:v 1 -f singlejpeg "${output}"`, (error, stdout, stderr) => {
+      if (error) {
+        console.log(`error: ${error.message}`);
+        return;
+      }
+      if (stderr) {
+        console.log(`stderr: ${stderr}`);
+        return;
+      }
+      console.log(`stdout: ${stdout}`);
+    });
+  }
+
+  /**
+   * Generates thumbnails for the levels: instances, series, study
+   *
+   * @param {*} dataSet
+   * @param {*} metadata
+   * @param {*} callback
+   */
+  generateThumbnails(itemId, dataSet, metadata, callback) {
+    const { imageFrame, id } = this.favoriteThumbnailObj;
+
+    // There are various reasons no thumbnails might be generated, so just return
+    if (!id) {
+      const pixelData = metadata[Tags.PixelData];
+      if (pixelData) {
+        const { BulkDataURI } = pixelData;
+        if (BulkDataURI?.indexOf("mp4")) {
+          const mp4Path = path.join(itemId.sopInstanceRootPath, "pixeldata.mp4");
+          const thumbPath = path.join(itemId.sopInstanceRootPath, "thumbnail");
+          console.log("MP4 - converting video format", mp4Path);
+          this.ffmpeg(mp4Path, thumbPath);
+        } else {
+          console.log("pixelData = ", pixelData, Tags.PixelData);
+        }
+      } else {
+        console.log("Series is of other type...", metadata[Tags.Modality]);
+      }
+      return;
+    }
+    internalGenerateThumbnail(imageFrame, dataSet, metadata, id.transferSyntaxUid, async (thumbBuffer) => {
+      if (thumbBuffer) {
+        await callback.thumbWriter(id.sopInstanceRootPath, this.thumbFileName, thumbBuffer);
+
+        this.copySyncThumbnail(id.sopInstanceRootPath, id.seriesRootPath);
+        this.copySyncThumbnail(id.seriesRootPath, id.studyPath);
+      }
+    });
+  }
+
+  /**
+   * Copy thumbnail from sourceFolder to targetFolderPath. It usually is used to copy from child to parent folder.
+   * If there is already some existing thumbnails under targetFolderPath it actually get thumb on the index defined by getThumbIndex.
+   * Copy is a syncrhonous process.
+   *
+   * @param {*} sourceFolderPath
+   * @param {*} targetFolderPath
+   * @returns
+   */
+  async copySyncThumbnail(sourceFolderPath, targetFolderPath) {
+    const parentPathLevel = path.join(sourceFolderPath, "../");
+    const thumbFilesPath = glob.sync(`${parentPathLevel}*/${this.thumbFileName}`);
+
+    const thumbIndex = getThumbIndex(thumbFilesPath.length);
+    const thumbFilePath = thumbFilesPath[thumbIndex];
+
+    if (!fs.existsSync(thumbFilePath)) {
+      console.log("Thumbnail to copy does not exists");
+      return;
+    }
+
+    try {
+      if (!fs.lstatSync(targetFolderPath).isDirectory()) {
+        throw new Error(`Target path: ${targetFolderPath} is not a directory`);
+      }
+
+      fs.copyFileSync(thumbFilePath, `${targetFolderPath}/${this.thumbFileName}`);
+    } catch (e) {
+      console.log("The file could not be copied", e);
+    }
+  }
+}
+
+module.exports = ThumbnailService;