@sjcrh/proteinpaint-server 2.118.3-2 → 2.119.0-0

package/dataset/termdb.test.js

@@ -155,11 +155,11 @@ function termdb_test_default() {
         file: "files/hg38/TermdbTest/TermdbTest_CNV_gene.gz"
       },
       /*
-      on the fly cnv calls from gene body probe signals are no longer used
-      probe2cnv:{
-      	file: 'files/hg19/pnet/PNET.probesignals.gz'
-      }
-      */
+               on the fly cnv calls from gene body probe signals are no longer used
+               probe2cnv:{
+                       file: 'files/hg19/pnet/PNET.probesignals.gz'
+               }
+               */
       singleSampleMutation: {
         src: "native",
         sample_id_key: "sample_id",
@@ -197,6 +197,10 @@ function termdb_test_default() {
       topVariablyExpressedGenes: {
         src: "native"
       },
+      rnaseqGeneCount: {
+        storage_type: "HDF5",
+        file: "files/hg38/TermdbTest/TermdbTest.geneCounts.h5"
+      },
       WSImages: {
         type: "H&E",
         imageBySampleFolder: "files/hg38/TermdbTest/wsimages"

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sjcrh/proteinpaint-server",
-  "version": "2.118.3-2",
+  "version": "2.119.0-0",
   "type": "module",
   "description": "a genomics visualization tool for exploring a cohort's genotype and phenotype data",
   "main": "src/app.js",
@@ -65,11 +65,11 @@
     "typescript": "^5.6.3"
   },
   "dependencies": {
-    "@sjcrh/augen": "2.118.3-2",
+    "@sjcrh/augen": "2.119.0-0",
     "@sjcrh/proteinpaint-python": "2.118.0",
-    "@sjcrh/proteinpaint-rust": "2.117.0",
-    "@sjcrh/proteinpaint-shared": "2.118.3-2",
-    "@sjcrh/proteinpaint-types": "2.118.3-2",
+    "@sjcrh/proteinpaint-rust": "2.119.0-0",
+    "@sjcrh/proteinpaint-shared": "2.119.0-0",
+    "@sjcrh/proteinpaint-types": "2.119.0-0",
     "@types/express": "^5.0.0",
     "@types/express-session": "^1.18.1",
     "better-sqlite3": "^9.4.1",

package/routes/samplewsimages.js

@@ -23,8 +23,16 @@ function init({ genomes }) {
       if (!ds)
         throw "invalid dataset name";
       const sampleId = query.sample_id;
-      const images = await ds.queries.WSImages.getWSImages(sampleId);
-      res.send({ sampleWSImages: images });
+      const wsimages = await ds.queries.WSImages.getWSImages(sampleId);
+      if (ds.queries.WSImages.getWSIAnnotations) {
+        for (const wsimage of wsimages) {
+          const annotations = await ds.queries.WSImages.getWSIAnnotations(sampleId, wsimage.filename);
+          if (annotations) {
+            wsimage.overlays = annotations;
+          }
+        }
+      }
+      res.send({ sampleWSImages: wsimages });
     } catch (e) {
       console.log(e);
       res.status(404).send("Sample images not found");

package/routes/termdb.DE.js

@@ -9,6 +9,7 @@ import { mayLog } from "#src/helpers.ts";
 import serverconfig from "../src/serverconfig.js";
 import imagesize from "image-size";
 import { get_header_txt } from "#src/utils.js";
+import { formatElapsedTime } from "@sjcrh/proteinpaint-shared/time.js";
 const api = {
   endpoint: "DEanalysis",
   methods: {
@@ -186,6 +187,10 @@ async function run_DE(param, ds, term_results, term_results2) {
     throw "sample size of group1 < 1";
   if (sample_size2 < 1)
     throw "sample size of group2 < 1";
+  const commonnames = group1names.filter((element) => group2names.includes(element));
+  if (commonnames.length > 0) {
+    throw "Common elements found between both groups:" + commonnames.map((i) => i).join(",");
+  }
   const cases_string = group1names.map((i) => i).join(",");
   const controls_string = group2names.map((i) => i).join(",");
   const expression_input = {
@@ -221,7 +226,7 @@ async function run_DE(param, ds, term_results, term_results2) {
     const result2 = JSON.parse(
       await run_R(path.join(serverconfig.binpath, "utils", "edge.R"), JSON.stringify(expression_input))
     );
-    mayLog("Time taken to run edgeR:", Date.now() - time12, "ms");
+    mayLog("Time taken to run edgeR:", formatElapsedTime(Date.now() - time12));
     param.method = "edgeR";
     const ql_imagePath = path.join(serverconfig.cachedir, result2.edgeR_ql_image_name[0]);
     mayLog("ql_imagePath:", ql_imagePath);
@@ -244,7 +249,7 @@ async function run_DE(param, ds, term_results, term_results2) {
   }
   const time1 = (/* @__PURE__ */ new Date()).valueOf();
   const result = JSON.parse(await run_rust("DEanalysis", JSON.stringify(expression_input)));
-  mayLog("Time taken to run rust DE pipeline:", Date.now() - time1, "ms");
+  mayLog("Time taken to run rust DE pipeline:", formatElapsedTime(Date.now() - time1));
   param.method = "wilcoxon";
   return { data: result, sample_size1, sample_size2, method: param.method };
 }

package/routes/termdb.cluster.js

@@ -12,6 +12,7 @@ import { getData } from "#src/termdb.matrix.js";
 import { termType2label } from "#shared/terms.js";
 import { mayLog } from "#src/helpers.ts";
 import { formatElapsedTime } from "#shared/time.js";
+import { getResult as getResultGene } from "#src/gene.js";
 const api = {
   endpoint: "termdb/cluster",
   methods: {
@@ -219,7 +220,7 @@ async function validate_query_geneExpression(ds, genome) {
     return;
   }
   if (q.src == "native") {
-    await validateNative(q, ds);
+    await validateNative(q, ds, genome);
     return;
   }
   throw "unknown queries.geneExpression.src";
@@ -255,7 +256,7 @@ async function queryGeneExpression(hdf5_file, geneNames) {
     throw error;
   }
 }
-async function validateNative(q, ds) {
+async function validateNative(q, ds, genome) {
   q.file = path.join(serverconfig.tpmasterdir, q.file);
   q.samples = [];
   await utils.file_is_readable(q.file);
@@ -271,9 +272,102 @@ async function validateNative(q, ds) {
         throw "unknown sample from HDF5: " + sn;
       q.samples.push(si);
     }
-    console.log(`${ds.label}: HDF5 file validated. Format: ${vr.format}, Samples:`, vr.sampleNames.length);
+    console.log(
+      `${ds.label}: geneExpression HDF5 file validated. Format: ${vr.format}, Samples:`,
+      vr.sampleNames.length
+    );
   } catch (error) {
-    throw `${ds.label}: Failed to validate HDF5 file: ${error}`;
+    console.log(`${ds.label}: geneExpression HDF5 validation failed, falling back to tabix handling: ${error}`);
+    try {
+      q.samples = [];
+      await utils.validate_tabixfile(q.file);
+      q.nochr = await utils.tabix_is_nochr(q.file, null, genome);
+      const lines = await utils.get_header_tabix(q.file);
+      if (!lines[0])
+        throw "Header line missing from " + q.file;
+      const l = lines[0].split("	");
+      if (l.slice(0, 4).join("	") != "#chr	start	stop	gene") {
+        throw "Header line has wrong content for columns 1-4";
+      }
+      for (let i = 4; i < l.length; i++) {
+        const id = ds.cohort.termdb.q.sampleName2id(l[i]);
+        if (id == void 0) {
+          throw "queries.geneExpression: unknown sample from header: " + l[i];
+        }
+        q.samples.push(id);
+      }
+    } catch (e) {
+      throw `${ds.label} geneExpression file cannot be validated as tabix file`;
+    }
+    console.log(`${ds.label}: Tabix file successfully initialized. Samples: ${q.samples.length}`);
+    q.get = async (param) => {
+      const limitSamples = await mayLimitSamples(param, q.samples, ds);
+      if (limitSamples?.size == 0) {
+        return { term2sample2value: /* @__PURE__ */ new Map(), byTermId: {}, bySampleId: {} };
+      }
+      const bySampleId = {};
+      const samples = q.samples || [];
+      if (limitSamples) {
+        for (const sid of limitSamples) {
+          bySampleId[sid] = { label: ds.cohort.termdb.q.id2sampleName(sid) };
+        }
+      } else {
+        for (const sid of samples) {
+          bySampleId[sid] = { label: ds.cohort.termdb.q.id2sampleName(sid) };
+        }
+      }
+      const term2sample2value = /* @__PURE__ */ new Map();
+      const byTermId = {};
+      for (const geneTerm of param.terms) {
+        if (!geneTerm.gene)
+          continue;
+        if (!geneTerm.chr || !Number.isInteger(geneTerm.start) || !Number.isInteger(geneTerm.stop)) {
+          const re = getResultGene(genome, { input: geneTerm.gene, deep: 1 });
+          if (!re.gmlst || re.gmlst.length == 0) {
+            console.warn("Unknown gene:" + geneTerm.gene);
+            continue;
+          }
+          const i = re.gmlst.find((i2) => i2.isdefault) || re.gmlst[0];
+          geneTerm.start = i.start;
+          geneTerm.stop = i.stop;
+          geneTerm.chr = i.chr;
+        }
+        if (!geneTerm.chr || !Number.isInteger(geneTerm.start) || !Number.isInteger(geneTerm.stop)) {
+          throw "Missing chr/start/stop";
+        }
+        const s2v = {};
+        await utils.get_lines_bigfile({
+          args: [
+            q.file,
+            (q.nochr ? geneTerm.chr.replace("chr", "") : geneTerm.chr) + ":" + geneTerm.start + "-" + geneTerm.stop
+          ],
+          callback: (line) => {
+            const l = line.split("	");
+            if (l[3].toLowerCase() != geneTerm.gene.toLowerCase())
+              return;
+            for (let i = 4; i < l.length; i++) {
+              const sampleId = samples[i - 4];
+              if (limitSamples && !limitSamples.has(sampleId))
+                continue;
+              if (!l[i])
+                continue;
+              const v = Number(l[i]);
+              if (Number.isNaN(v))
+                throw "Expression value not number";
+              s2v[sampleId] = v;
+            }
+          }
+        });
+        if (Object.keys(s2v).length) {
+          term2sample2value.set(geneTerm.gene, s2v);
+        }
+      }
+      if (term2sample2value.size == 0) {
+        throw "No data available for the input " + param.terms?.map((g) => g.gene).join(", ");
+      }
+      return { term2sample2value, byTermId, bySampleId };
+    };
+    return;
   }
   q.get = async (param) => {
     const limitSamples = await mayLimitSamples(param, q.samples, ds);

package/routes/tileserver.js

@@ -6,7 +6,7 @@ import SessionManager from "#src/wsisessions/SessionManager.ts";
 import serverconfig from "#src/serverconfig.js";
 import path from "path";
 const api = {
-  endpoint: `tileserver/layer/slide/:sessionId/zoomify/:TileGroup/:z-:x-:y@1x.jpg`,
+  endpoint: `tileserver/layer/:type/:sessionId/zoomify/:TileGroup/:z-:x-:y@1x.jpg`,
   methods: {
     get: {
       ...tilePayload,
@@ -21,7 +21,7 @@ const api = {
 function init({ genomes }) {
   return async (req, res) => {
     try {
-      const { sessionId, TileGroup, z, x, y } = req.params;
+      const { type, sessionId, TileGroup, z, x, y } = req.params;
       const query = req.query;
       const wsiImage = query.wsi_image;
       if (!wsiImage)
@@ -49,7 +49,7 @@ function init({ genomes }) {
       if (!tileServer) {
         throw new Error("No tile server");
       }
-      const url = `${tileServer.url}/tileserver/layer/slide/${sessionId}/zoomify/${TileGroup}/${z}-${x}-${y}@1x.jpg`;
+      const url = `${tileServer.url}/tileserver/layer/${type}/${sessionId}/zoomify/${TileGroup}/${z}-${x}-${y}@1x.jpg`;
       const response = await ky.get(url, { timeout: 12e4 });
       await SessionManager.getInstance().updateSession(wsiImagePath);
       const buffer = await response.arrayBuffer();

package/routes/wsimages.js

@@ -45,7 +45,7 @@ function init({ genomes }) {
       if (!mount)
         throw new Error("No mount available for TileServer");
       const wsiImagePath = path.join(`${mount}/${ds.queries.WSImages.imageBySampleFolder}/${sampleId}`, wsimage);
-      const sessionId = await getSessionId(cookieJar, getCookieString, setCookie, wsiImagePath);
+      const sessionId = await getSessionId(ds, sampleId, cookieJar, getCookieString, setCookie, wsiImagePath);
       const getWsiImageResponse = await getWsiImageDimensions(sessionId, getCookieString, wsiImagePath);
       const payload = {
         status: "ok",
@@ -62,7 +62,7 @@ function init({ genomes }) {
     }
   };
 }
-async function getSessionId(cookieJar, getCookieString, setCookie, wsimage) {
+async function getSessionId(ds, sampleId, cookieJar, getCookieString, setCookie, wsimage) {
   const sessionManager = SessionManager.getInstance();
   const invalidateResult = await sessionManager.syncAndInvalidateSessions(wsimage);
   if (!invalidateResult)
@@ -93,6 +93,49 @@ async function getSessionId(cookieJar, getCookieString, setCookie, wsimage) {
     hooks: getHooks(cookieJar, getCookieString, setCookie)
   });
   await sessionManager.setSession(wsimage, sessionId, tileServer);
+  if (ds.queries.WSImages.getWSIAnnotations) {
+    const annotationFiles = await ds.queries.WSImages.getWSIAnnotations(sampleId, wsimage);
+    if (!annotationFiles)
+      throw new Error("No annotations files found");
+    const mount = serverconfig.features?.tileserver?.mount;
+    if (!mount)
+      throw new Error("No mount available for TileServer");
+    if (annotationFiles.length > 0) {
+      for (const annotationFile of annotationFiles) {
+        const annotationsFilePath = path.join(
+          `${mount}/${ds.queries.WSImages.imageBySampleFolder}/${sampleId}`,
+          annotationFile
+        );
+        const annotationsData = qs.stringify({
+          overlay_path: annotationsFilePath
+        });
+        await ky.put(`${tileServer.url}/tileserver/overlay`, {
+          body: annotationsData,
+          timeout: 5e4,
+          headers: {
+            "Content-Type": "application/x-www-form-urlencoded",
+            Cookie: `session_id=${sessionId}`
+          },
+          hooks: getHooks(cookieJar, getCookieString, setCookie)
+        });
+      }
+      const cmapData = qs.stringify({
+        cmap: JSON.stringify({
+          keys: ["annotation"],
+          values: [ds.queries.WSImages.annotationsColor]
+        })
+      });
+      await ky.put(`${tileServer.url}/tileserver/cmap`, {
+        body: cmapData,
+        timeout: 5e4,
+        headers: {
+          "Content-Type": "application/x-www-form-urlencoded",
+          Cookie: `session_id=${sessionId}`
+        },
+        hooks: getHooks(cookieJar, getCookieString, setCookie)
+      });
+    }
+  }
   return sessionId;
 }
 async function getWsiImageDimensions(sessionId, getCookieString, wsimage) {