@sjcrh/proteinpaint-server 2.71.0 → 2.73.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sjcrh/proteinpaint-server",
-  "version": "2.71.0",
+  "version": "2.73.0",
   "type": "module",
   "description": "a genomics visualization tool for exploring a cohort's genotype and phenotype data",
   "main": "src/app.js",
@@ -44,6 +44,7 @@
     "@babel/preset-typescript": "^7.21.4",
     "@babel/register": "^7.14.5",
     "@types/node": "^20.11.24",
+    "@types/tough-cookie": "^4.0.5",
     "@typescript-eslint/eslint-plugin": "^5.60.0",
     "babel-loader": "^8.2.2",
     "esbuild": "^0.19.12",
@@ -61,7 +62,7 @@
   },
   "dependencies": {
     "@sjcrh/augen": "2.46.0",
-    "@sjcrh/proteinpaint-rust": "2.61.1",
+    "@sjcrh/proteinpaint-rust": "2.73.0",
     "better-sqlite3": "^9.4.1",
     "body-parser": "^1.15.2",
     "canvas": "~2.11.2",
@@ -82,6 +83,7 @@
     "node-fetch": "^2.6.1",
     "partjson": "^0.58.2",
     "tiny-async-pool": "^1.2.0",
+    "tough-cookie": "^4.1.4",
     "typedoc-plugin-missing-exports": "^2.0.1"
   },
   "repository": {

package/routes/samplewsimages.js

@@ -0,0 +1,51 @@
+import fs from "fs";
+import path from "path";
+import serverconfig from "#src/serverconfig.js";
+const api = {
+  endpoint: "samplewsimages",
+  methods: {
+    get: {
+      init,
+      request: {
+        typeId: "GetSampleWSImagesRequest"
+      },
+      response: {
+        typeId: "GetSampleWSImagesResponse"
+      }
+    },
+    post: {
+      alternativeFor: "get",
+      init
+    }
+  }
+};
+function init({ genomes }) {
+  return async (req, res) => {
+    try {
+      const query = req.query;
+      const g = genomes[query.genome];
+      if (!g)
+        throw "invalid genome name";
+      const ds = g.datasets[query.dslabel];
+      if (!ds)
+        throw "invalid dataset name";
+      const sampleId = query.sample_id;
+      const sampleWSImagesPath = path.join(
+        `${serverconfig.tpmasterdir}/${ds.queries.WSImages.imageBySampleFolder}`,
+        sampleId
+      );
+      const sampleWSImages = await getWSImages(sampleWSImagesPath);
+      res.send({ sampleWSImages });
+    } catch (e) {
+      console.log(e);
+      res.status(404).send("Sample images not found");
+    }
+  };
+}
+async function getWSImages(sampleImagesPath) {
+  const files = await fs.promises.readdir(sampleImagesPath);
+  return files.filter((file) => [".svs", ".mrxs", ".scn", ".ndpi", ".tiff"].includes(path.extname(file)));
+}
+export {
+  api
+};

package/routes/termdb.categories.js

@@ -92,7 +92,7 @@ async function trigger_getcategories(q, res, tdb, ds, genome) {
   if (data.error)
     throw data.error;
   const lst = [];
-  if (q.tw.term.type == "geneVariant" && !q.tw.q.groupsetting?.inuse) {
+  if (q.tw.term.type == "geneVariant" && q.tw.q.type != "predefined-groupset" && q.tw.q.type != "custom-groupset") {
     const samples = data.samples;
     const dtClassMap = /* @__PURE__ */ new Map();
     if (ds.assayAvailability?.byDt) {

package/routes/termdb.cluster.js

@@ -192,11 +192,10 @@ async function validateNative(q, ds, genome) {
       }
     }
     const term2sample2value = /* @__PURE__ */ new Map();
-    for (const g of param.terms) {
-      const geneTerm = g;
+    for (const geneTerm of param.terms) {
       if (!geneTerm.gene)
         continue;
-      if (!geneTerm.chr) {
+      if (!geneTerm.chr || !Number.isInteger(geneTerm.start) || !Number.isInteger(geneTerm.stop)) {
         const re = getResultGene(genome, { input: geneTerm.gene, deep: 1 });
         if (!re.gmlst || re.gmlst.length == 0) {
           console.warn("unknown gene:" + geneTerm.gene);
@@ -208,12 +207,13 @@ async function validateNative(q, ds, genome) {
         geneTerm.chr = i.chr;
       }
       const s2v = {};
+      if (!geneTerm.chr || !Number.isInteger(geneTerm.start) || !Number.isInteger(geneTerm.stop))
+        throw "missing chr/start/stop";
       await utils.get_lines_bigfile({
         args: [
           q.file,
-          (q.nochr ? geneTerm.chr?.replace("chr", "") : geneTerm.chr) + ":" + geneTerm.start + "-" + geneTerm.stop
+          (q.nochr ? geneTerm.chr.replace("chr", "") : geneTerm.chr) + ":" + geneTerm.start + "-" + geneTerm.stop
         ],
-        // must do g.chr?.replace to avoid tsc error
         callback: (line) => {
           const l = line.split("	");
           if (l[3].toLowerCase() != geneTerm.gene.toLowerCase())

package/routes/termdb.config.js

@@ -177,11 +177,16 @@ function addNonDictionaryQueries(c, ds, genome) {
       q2.NIdata[k] = JSON.parse(JSON.stringify(q.NIdata[k]));
     }
   }
-  if (q.DZImages) {
+  if (q.DZImages && serverconfig.features.showDZImages) {
     q2.DZImages = {
       type: q.DZImages.type
     };
   }
+  if (q.WSImages && serverconfig.features.showWSImages) {
+    q2.WSImages = {
+      type: q.WSImages.type
+    };
+  }
   if (q.singleSampleGbtk) {
     q2.singleSampleGbtk = {};
     for (const k in q.singleSampleGbtk) {

package/routes/termdb.getdescrstats.js

@@ -84,7 +84,11 @@ function init({ genomes }) {
         }
         values.push(Number(value));
       }
-      result = Summarystats(values);
+      if (values.length) {
+        result = Summarystats(values);
+      } else {
+        result = {};
+      }
     } catch (e) {
       if (e instanceof Error && e.stack)
         console.log(e);

package/routes/termdb.termsbyids.js

@@ -45,7 +45,7 @@ async function trigger_gettermsbyid(q, res, tdb) {
   for (const id of q.ids) {
     const term = tdb.q.termjsonByOneid(id);
     if (term) {
-      if (term.type == "categorical" && !term.values && !term.groupsetting?.inuse) {
+      if (term.type == "categorical" && !term.values) {
         term.values = {};
         term.samplecount = {};
       }

package/routes/termdb.topVariablyExpressedGenes.js

@@ -1,9 +1,9 @@
 import path from "path";
 import { run_rust } from "@sjcrh/proteinpaint-rust";
-import got from "got";
 import serverconfig from "#src/serverconfig.js";
 import { get_samples } from "#src/termdb.sql.js";
 import { makeFilter } from "#src/mds3.gdc.js";
+import { cachedFetch } from "#src/utils.js";
 const api = {
   endpoint: "termdb/topVariablyExpressedGenes",
   methods: {
@@ -61,6 +61,7 @@ function nativeValidateQuery(ds) {
     throw "topVariablyExpressedGenes query given but geneExpression missing";
   if (gE.src != "native")
     throw "topVariablyExpressedGenes is native but geneExpression.src is not native";
+  addTopVEarg(ds.queries.topVariablyExpressedGenes);
   ds.queries.topVariablyExpressedGenes.getGenes = async (q) => {
     const samples = [];
     if (q.filter) {
@@ -81,18 +82,81 @@ function nativeValidateQuery(ds) {
         samples.push(n);
       }
     }
-    const genes = await computeGenes4nativeDs(q, ds, gE.file, samples);
+    const genes = await computeGenes4nativeDs(q, gE.file, samples);
     return genes;
   };
 }
-async function computeGenes4nativeDs(q, ds, matrixFile, samples) {
+function addTopVEarg(q) {
+  const arglst = [
+    { id: "maxGenes", label: "Gene Count", type: "number", value: 100 },
+    {
+      id: "filter_extreme_values",
+      label: "Filter Extreme Values",
+      type: "boolean",
+      value: true,
+      options: [
+        {
+          id: "min_count",
+          label: "Min count",
+          type: "number",
+          value: 10
+        },
+        {
+          id: "min_total_count",
+          label: "Min total count",
+          type: "number",
+          value: 15
+        }
+      ]
+    },
+    {
+      id: "rank_type",
+      label: "Rank by:",
+      type: "radio",
+      options: [
+        /** The param option in input JSON is very important.
+         * It instructs what method will be used to calculate variation in the counts for a particular gene.
+         * It supports variance as well as interquartile region.
+         * This is based on the recommendation of this article:
+         * https://www.frontiersin.org/articles/10.3389/fgene.2021.632620/full.
+         * This article recommends using interquartile region over variance.*/
+        {
+          type: "boolean",
+          label: "Variance",
+          value: "var"
+        },
+        {
+          type: "boolean",
+          label: "Interquartile Region",
+          value: "iqr"
+        }
+      ]
+    }
+  ];
+  if (q.arguments) {
+    for (const a of q.arguments) {
+      if (!a.id)
+        throw "missing id of topVE.arguments[]";
+      const item = arglst.find((i) => i.id == a.id);
+      if (!item)
+        throw "unknown id of topVE.arguments[]";
+      Object.assign(item, a);
+    }
+  }
+  q.arguments = arglst;
+}
+async function computeGenes4nativeDs(q, matrixFile, samples) {
   const input_json = {
     input_file: matrixFile,
     samples: samples.join(","),
-    filter_extreme_values: true,
+    filter_extreme_values: q.filter_extreme_values,
     num_genes: q.maxGenes,
-    param: "var"
+    rank_type: q.rank_type?.type
   };
+  if (q.filter_extreme_values == 1) {
+    input_json["min_count"] = q.min_count;
+    input_json["min_total_count"] = q.min_total_count;
+  }
   const rust_output = await run_rust("topGeneByExpressionVariance", JSON.stringify(input_json));
   const rust_output_list = rust_output.split("\n");
   let output_json;
@@ -109,7 +173,7 @@ async function computeGenes4nativeDs(q, ds, matrixFile, samples) {
 function gdcValidateQuery(ds, genome) {
   ds.queries.topVariablyExpressedGenes.getGenes = async (q) => {
     if (serverconfig.features.gdcGenes) {
-      console.log(
+      console.error(
         "!!GDC!! using serverconfig.features.gdcGenes[] but not live api query. only use this on DEV and never on PROD!"
       );
       return serverconfig.features.gdcGenes;
@@ -120,11 +184,12 @@ function gdcValidateQuery(ds, genome) {
     const { host, headers } = ds.getHostHeaders(q);
     const url = path.join(host.geneExp, "/gene_expression/gene_selection");
     try {
-      const response = await got.post(url, {
+      const response = await cachedFetch(url, {
+        method: "POST",
         headers,
-        body: JSON.stringify(getGeneSelectionArg(q))
+        body: getGeneSelectionArg(q)
       });
-      const re = JSON.parse(response.body);
+      const re = response.body;
       const genes = [];
       if (!Array.isArray(re.gene_selection))
         throw "re.gene_selection[] is not array";
@@ -141,7 +206,7 @@ function gdcValidateQuery(ds, genome) {
       }
       return genes;
     } catch (e) {
-      console.log(e.stack || e);
+      console.error(e.stack || e);
       throw e;
     }
   };

package/routes/tileserver.js

@@ -0,0 +1,42 @@
+import ky from "ky";
+import serverconfig from "#src/serverconfig.js";
+const routePath = "tileserver";
+const api = {
+  endpoint: `${routePath}/layer/slide/:sampleId/zoomify/:TileGroup/:z-:x-:y@1x.jpg`,
+  methods: {
+    get: {
+      init,
+      request: {
+        typeId: "any"
+      },
+      response: {
+        typeId: "any"
+      }
+    },
+    post: {
+      alternativeFor: "get",
+      init
+    }
+  }
+};
+function init() {
+  return async (req, res) => {
+    try {
+      const { sampleId, TileGroup, z, x, y } = req.params;
+      const url = `${serverconfig.tileServerURL}/tileserver/layer/slide/${sampleId}/zoomify/${TileGroup}/${z}-${x}-${y}@1x.jpg`;
+      const response = await ky.get(url);
+      const buffer = await response.arrayBuffer();
+      res.status(response.status).send(Buffer.from(buffer));
+    } catch (error) {
+      if (error.response) {
+        const errorBody = await error.response.arrayBuffer();
+        res.status(error.response.status).send(Buffer.from(errorBody));
+      } else {
+        res.status(500).send("Internal Server Error");
+      }
+    }
+  };
+}
+export {
+  api
+};

package/routes/wsimages.js

@@ -0,0 +1,115 @@
+import ky from "ky";
+import qs from "qs";
+import path from "path";
+import serverconfig from "#src/serverconfig.js";
+import { CookieJar } from "tough-cookie";
+import { promisify } from "util";
+const routePath = "wsimages";
+const api = {
+  endpoint: `${routePath}`,
+  methods: {
+    get: {
+      init,
+      request: {
+        typeId: "any"
+      },
+      response: {
+        typeId: "any"
+      }
+    },
+    post: {
+      alternativeFor: "get",
+      init
+    }
+  }
+};
+function init({ genomes }) {
+  return async (req, res) => {
+    try {
+      const g = genomes[req.query.genome];
+      if (!g)
+        throw "invalid genome name";
+      const ds = g.datasets[req.query.dslabel];
+      if (!ds)
+        throw "invalid dataset name";
+      const sampleId = req.query.sampleId;
+      if (!sampleId)
+        throw "invalid sampleId";
+      const wsimage = req.query.wsimage;
+      if (!wsimage)
+        throw "invalid wsimage";
+      const cookieJar = new CookieJar();
+      const setCookie = promisify(cookieJar.setCookie.bind(cookieJar));
+      const getCookieString = promisify(cookieJar.getCookieString.bind(cookieJar));
+      await ky.get(`${serverconfig.tileServerURL}/tileserver/session_id`, {
+        hooks: {
+          beforeRequest: [
+            async (request) => {
+              const cookie = await getCookieString(request.url);
+              request.headers.set("Cookie", cookie);
+            }
+          ],
+          afterResponse: [
+            async (request, options, response) => {
+              const setCookieHeader = response.headers.get("set-cookie");
+              if (setCookieHeader) {
+                await setCookie(setCookieHeader, request.url);
+              }
+            }
+          ]
+        }
+      });
+      const cookieString = await getCookieString(`${serverconfig.tileServerURL}/tileserver/session_id`);
+      const sessionId = cookieString.match(/session_id=([^;]*)/)?.[1];
+      const sampleWsiTileServer = path.join(
+        `${serverconfig.tileServerMount}/${ds.queries.WSImages.imageBySampleFolder}/${sampleId}`,
+        wsimage
+      );
+      const data = qs.stringify({ slide_path: sampleWsiTileServer });
+      await ky.put(`${serverconfig.tileServerURL}/tileserver/slide`, {
+        body: data,
+        headers: {
+          "Content-Type": "application/x-www-form-urlencoded",
+          Cookie: `session_id=${sessionId}`
+          // Include the session_id in the headers
+        },
+        hooks: {
+          beforeRequest: [
+            async (request) => {
+              const cookie = await getCookieString(request.url);
+              request.headers.set("Cookie", cookie);
+            }
+          ],
+          afterResponse: [
+            async (request, options, response) => {
+              const setCookieHeader = response.headers.get("set-cookie");
+              if (setCookieHeader) {
+                await setCookie(setCookieHeader, request.url);
+              }
+            }
+          ]
+        }
+      });
+      const getWsiImageResponse = await ky.get(`${serverconfig.tileServerURL}/tileserver/slide`, {
+        hooks: {
+          beforeRequest: [
+            async (request) => {
+              const cookie = await getCookieString(request.url);
+              request.headers.set("Cookie", cookie);
+            }
+          ]
+        }
+      }).json();
+      res.status(200).json({ sessionId, slide_dimensions: getWsiImageResponse.slide_dimensions });
+    } catch (e) {
+      console.log(e);
+      res.send({
+        status: "error",
+        error: e.error || e
+      });
+    }
+  };
+}
+export {
+  api
+};