@sjcrh/proteinpaint-server 2.59.0 → 2.61.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sjcrh/proteinpaint-server",
-  "version": "2.59.0",
+  "version": "2.61.1",
   "type": "module",
   "description": "a genomics visualization tool for exploring a cohort's genotype and phenotype data",
   "main": "src/app.js",
@@ -61,7 +61,7 @@
   },
   "dependencies": {
     "@sjcrh/augen": "2.46.0",
-    "@sjcrh/proteinpaint-rust": "2.59.0",
+    "@sjcrh/proteinpaint-rust": "2.61.1",
     "better-sqlite3": "^9.4.1",
     "body-parser": "^1.15.2",
     "canvas": "~2.11.2",

package/routes/dzimages.js

@@ -0,0 +1,64 @@
+import path from "path";
+import serverconfig from "#src/serverconfig.js";
+import { illegalpath } from "#src/utils.js";
+const routePath = "dzimages";
+const api = {
+  endpoint: `${routePath}/:sampleId`,
+  methods: {
+    get: {
+      init,
+      request: {
+        typeId: "any"
+      },
+      response: {
+        typeId: "any"
+      }
+    },
+    post: {
+      alternativeFor: "get",
+      init
+    }
+  }
+};
+function init({ genomes }) {
+  return async (req, res) => {
+    let imagePath;
+    try {
+      const g = genomes[req.query.genome];
+      if (!g)
+        throw "invalid genome name";
+      const ds = g.datasets[req.query.dslabel];
+      if (!ds)
+        throw "invalid dataset name";
+      const sampleId = req.params.sampleId;
+      if (!sampleId)
+        throw "invalid sampleId";
+      if (illegalpath(req.query.file))
+        throw `illegalpath filepath`;
+      const filename = path.basename(req.query.file);
+      const allowedExtensions = [".dzi", ".jpeg", ".png"];
+      const extension = path.extname(filename);
+      if (!allowedExtensions.includes(extension)) {
+        throw `Invalid file extension. Allowed extensions are ${allowedExtensions.join(", ")}`;
+      }
+      imagePath = path.join(
+        `${serverconfig.tpmasterdir}/${ds.queries.DZImages.imageBySampleFolder}`,
+        `${sampleId}/${req.query.file}`
+      );
+      res.sendFile(imagePath, (err) => {
+        if (err) {
+          res.status(404).send("Image not found");
+        }
+      });
+    } catch (e) {
+      console.log(e);
+      res.send({
+        status: "error",
+        error: e.error || e
+      });
+    }
+  };
+}
+export {
+  api
+};

package/routes/sampledzimages.js

@@ -0,0 +1,50 @@
+import fs from "fs";
+import path from "path";
+import serverconfig from "#src/serverconfig.js";
+const api = {
+  endpoint: "sampledzimages",
+  methods: {
+    get: {
+      init,
+      request: {
+        typeId: "GetSampleDZImagesRequest"
+      },
+      response: {
+        typeId: "GetSampleDZImagesResponse"
+      }
+    },
+    post: {
+      alternativeFor: "get",
+      init
+    }
+  }
+};
+function init({ genomes }) {
+  return async (req, res) => {
+    try {
+      const g = genomes[req.query.genome];
+      if (!g)
+        throw "invalid genome name";
+      const ds = g.datasets[req.query.dslabel];
+      if (!ds)
+        throw "invalid dataset name";
+      const sampleId = req.query.sample_id;
+      const sampleDZImagesPath = path.join(
+        `${serverconfig.tpmasterdir}/${ds.queries.DZImages.imageBySampleFolder}`,
+        sampleId
+      );
+      const sampleDZImages = getDZImages(sampleDZImagesPath);
+      res.send({ sampleDZImages });
+    } catch (e) {
+      console.log(e);
+      res.status(404).send("Sample images not found");
+    }
+  };
+}
+function getDZImages(sampleImagesPath) {
+  const files = fs.readdirSync(sampleImagesPath);
+  return files.filter((file) => path.extname(file) === ".dzi");
+}
+export {
+  api
+};

package/routes/termdb.DE.js

@@ -0,0 +1,129 @@
+import path from "path";
+import { run_rust } from "@sjcrh/proteinpaint-rust";
+import { get_ds_tdb } from "../src/termdb.js";
+import run_R from "../src/run_R.js";
+import serverconfig from "../src/serverconfig.js";
+const api = {
+  endpoint: "DEanalysis",
+  methods: {
+    all: {
+      init,
+      request: {
+        typeId: "DERequest"
+      },
+      response: {
+        typeId: "DEResponse"
+        // will combine this with type checker
+        //valid: (t) => {}
+      }
+    }
+  }
+};
+function init({ genomes }) {
+  return async (req, res) => {
+    try {
+      const q = req.query;
+      const genome = genomes[q.genome];
+      if (!genome)
+        throw "invalid genome";
+      const [ds] = get_ds_tdb(genome, q);
+      const results = await run_DE(req.query, ds);
+      res.send(results);
+    } catch (e) {
+      res.send({ status: "error", error: e.message || e });
+      if (e instanceof Error && e.stack)
+        console.log(e);
+    }
+  };
+}
+async function run_DE(param, ds) {
+  if (param.samplelst?.groups?.length != 2)
+    throw ".samplelst.groups.length!=2";
+  if (param.samplelst.groups[0].values?.length < 1)
+    throw "samplelst.groups[0].values.length<1";
+  if (param.samplelst.groups[1].values?.length < 1)
+    throw "samplelst.groups[1].values.length<1";
+  const q = ds.queries.rnaseqGeneCount;
+  if (!q)
+    return;
+  if (!q.file)
+    throw "unknown data type for rnaseqGeneCount";
+  const group1names = [];
+  for (const s of param.samplelst.groups[0].values) {
+    if (!Number.isInteger(s.sampleId))
+      continue;
+    const n = ds.cohort.termdb.q.id2sampleName(s.sampleId);
+    if (!n)
+      continue;
+    if (q.allSampleSet.has(n)) {
+      group1names.push(n);
+    } else {
+    }
+  }
+  const group2names = [];
+  for (const s of param.samplelst.groups[1].values) {
+    if (!Number.isInteger(s.sampleId))
+      continue;
+    const n = ds.cohort.termdb.q.id2sampleName(s.sampleId);
+    if (!n)
+      continue;
+    if (q.allSampleSet.has(n)) {
+      group2names.push(n);
+    } else {
+    }
+  }
+  const sample_size1 = group1names.length;
+  const sample_size2 = group2names.length;
+  if (sample_size1 < 1)
+    throw "sample size of group1 < 1";
+  if (sample_size2 < 1)
+    throw "sample size of group2 < 1";
+  const cases_string = group1names.map((i) => i).join(",");
+  const controls_string = group2names.map((i) => i).join(",");
+  const expression_input = {
+    case: cases_string,
+    control: controls_string,
+    input_file: q.file,
+    min_count: param.min_count,
+    min_total_count: param.min_total_count
+  };
+  const sample_size_limit = 8;
+  let result;
+  if (group1names.length <= sample_size_limit && group2names.length <= sample_size_limit || param.method == "edgeR") {
+    const time1 = (/* @__PURE__ */ new Date()).valueOf();
+    result = JSON.parse(
+      await run_R(path.join(serverconfig.binpath, "utils", "edge.R"), JSON.stringify(expression_input))
+    );
+    const time2 = (/* @__PURE__ */ new Date()).valueOf();
+    console.log("Time taken to run edgeR:", time2 - time1, "ms");
+    param.method = "edgeR";
+  } else if (param.method == "wilcoxon") {
+    const time1 = (/* @__PURE__ */ new Date()).valueOf();
+    const rust_output = await run_rust("DEanalysis", JSON.stringify(expression_input));
+    const time2 = (/* @__PURE__ */ new Date()).valueOf();
+    for (const line of rust_output.split("\n")) {
+      if (line.startsWith("adjusted_p_values:")) {
+        result = JSON.parse(line.replace("adjusted_p_values:", ""));
+      } else {
+      }
+    }
+    console.log("Time taken to run rust DE pipeline:", time2 - time1, "ms");
+    param.method = "wilcoxon";
+  } else {
+    const time1 = (/* @__PURE__ */ new Date()).valueOf();
+    const rust_output = await run_rust("DEanalysis", JSON.stringify(expression_input));
+    const time2 = (/* @__PURE__ */ new Date()).valueOf();
+    for (const line of rust_output.split("\n")) {
+      if (line.startsWith("adjusted_p_values:")) {
+        result = JSON.parse(line.replace("adjusted_p_values:", ""));
+      } else {
+      }
+    }
+    console.log("Time taken to run rust DE pipeline:", time2 - time1, "ms");
+    param.method = "wilcoxon";
+  }
+  return { data: result, sample_size1, sample_size2, method: param.method };
+}
+export {
+  api
+};

package/routes/termdb.cluster.js

@@ -4,9 +4,9 @@ import * as utils from "#src/utils.js";
 import serverconfig from "#src/serverconfig.js";
 import { gdc_validate_query_geneExpression } from "#src/mds3.gdc.js";
 import { mayLimitSamples } from "#src/mds3.filter.js";
-import { dtgeneexpression } from "#shared/common.js";
 import { clusterMethodLst, distanceMethodLst } from "#shared/clustering.js";
 import { getResult as getResultGene } from "#src/gene.js";
+import { TermTypes } from "#shared/terms.js";
 const api = {
   endpoint: "termdb/cluster",
   methods: {
@@ -34,9 +34,9 @@ function init({ genomes }) {
         throw "invalid dataset name";
       if (ds.__gdc && !ds.__gdc.doneCaching)
         throw "The server has not finished caching the case IDs: try again in about 2 minutes.";
-      if (q.dataType == dtgeneexpression) {
-        if (!ds.queries?.geneExpression)
-          throw "no geneExpression data on this dataset";
+      if (q.dataType == TermTypes.GENE_EXPRESSION || q.dataType == TermTypes.METABOLITE_INTENSITY) {
+        if (!ds.queries?.[q.dataType])
+          throw `no ${q.dataType} data on this dataset`;
         result = await getResult(q, ds);
       } else {
         throw "unknown q.dataType " + q.dataType;
@@ -53,15 +53,16 @@ function init({ genomes }) {
   };
 }
 async function getResult(q, ds) {
-  const { gene2sample2value, byTermId, bySampleId } = await ds.queries.geneExpression.get(q);
-  if (gene2sample2value.size == 0)
+  const type = q.dataType;
+  const { term2sample2value, byTermId, bySampleId } = await ds.queries[type].get(q);
+  if (term2sample2value.size == 0)
     throw "no data";
-  if (gene2sample2value.size == 1) {
-    const g = Array.from(gene2sample2value.keys())[0];
-    return { gene: g, data: gene2sample2value.get(g) };
+  if (term2sample2value.size == 1) {
+    const g = Array.from(term2sample2value.keys())[0];
+    return { term: { gene: g, type: TermTypes.GENE_EXPRESSION }, data: term2sample2value.get(g) };
   }
   const t = Date.now();
-  const clustering = await doClustering(gene2sample2value, q);
+  const clustering = await doClustering(term2sample2value, q);
   if (serverconfig.debugmode)
     console.log("clustering done:", Date.now() - t, "ms");
   return { clustering, byTermId, bySampleId };
@@ -137,7 +138,7 @@ async function validate_query_geneExpression(ds, genome) {
   const q = ds.queries.geneExpression;
   if (!q)
     return;
-  q.gene2bins = {};
+  q.geneExpression2bins = {};
   if (q.src == "gdcapi") {
     gdc_validate_query_geneExpression(ds, genome);
     return;
@@ -169,12 +170,11 @@ async function validateNative(q, ds, genome) {
         throw "queries.geneExpression: unknown sample from header: " + l[i];
       q.samples.push(id);
     }
-    console.log(q.samples.length, "samples from geneExpression of", ds.label);
   }
   q.get = async (param) => {
     const limitSamples = await mayLimitSamples(param, q.samples, ds);
     if (limitSamples?.size == 0) {
-      return { gene2sample2value: /* @__PURE__ */ new Set(), byTermId: {}, bySampleId: {} };
+      return { term2sample2value: /* @__PURE__ */ new Set(), byTermId: {}, bySampleId: {} };
     }
     const bySampleId = {};
     const samples = q.samples || [];
@@ -187,26 +187,32 @@ async function validateNative(q, ds, genome) {
         bySampleId[sid] = { label: ds.cohort.termdb.q.id2sampleName(sid) };
       }
     }
-    const gene2sample2value = /* @__PURE__ */ new Map();
-    for (const g of param.genes) {
-      if (!g.gene)
+    const term2sample2value = /* @__PURE__ */ new Map();
+    for (const g of param.terms) {
+      const geneTerm = g;
+      if (!geneTerm.gene)
         continue;
-      if (!g.chr) {
-        const re = getResultGene(genome, { input: g.gene, deep: 1 });
-        if (!re.gmlst || re.gmlst.length == 0)
-          throw "unknown gene";
+      if (!geneTerm.chr) {
+        const re = getResultGene(genome, { input: geneTerm.gene, deep: 1 });
+        if (!re.gmlst || re.gmlst.length == 0) {
+          console.warn("unknown gene:" + geneTerm.gene);
+          continue;
+        }
         const i = re.gmlst.find((i2) => i2.isdefault) || re.gmlst[0];
-        g.start = i.start;
-        g.stop = i.stop;
-        g.chr = i.chr;
+        geneTerm.start = i.start;
+        geneTerm.stop = i.stop;
+        geneTerm.chr = i.chr;
       }
       const s2v = {};
       await utils.get_lines_bigfile({
-        args: [q.file, (q.nochr ? g.chr?.replace("chr", "") : g.chr) + ":" + g.start + "-" + g.stop],
+        args: [
+          q.file,
+          (q.nochr ? geneTerm.chr?.replace("chr", "") : geneTerm.chr) + ":" + geneTerm.start + "-" + geneTerm.stop
+        ],
         // must do g.chr?.replace to avoid tsc error
         callback: (line) => {
           const l = line.split("	");
-          if (l[3].toLowerCase() != g.gene.toLowerCase())
+          if (l[3].toLowerCase() != geneTerm.gene.toLowerCase())
             return;
           for (let i = 4; i < l.length; i++) {
             const sampleId = samples[i - 4];
@@ -222,10 +228,12 @@ async function validateNative(q, ds, genome) {
         }
       });
       if (Object.keys(s2v).length)
-        gene2sample2value.set(g.gene, s2v);
+        term2sample2value.set(geneTerm.gene, s2v);
     }
     const byTermId = {};
-    return { gene2sample2value, byTermId, bySampleId };
+    if (term2sample2value.size == 0)
+      throw "no data available for the input " + param.terms?.map((g) => g.gene).join(", ");
+    return { term2sample2value, byTermId, bySampleId };
   };
 }
 export {

package/routes/termdb.config.js

@@ -3,6 +3,7 @@ import { authApi } from "#src/auth.js";
 import { get_ds_tdb } from "#src/termdb.js";
 import { mayCopyFromCookie } from "#src/utils.js";
 import { mayComputeTermtypeByCohort } from "#src/termdb.server.init.js";
+import { TermTypes } from "#shared/terms.js";
 const api = {
   endpoint: "termdb/config",
   methods: {
@@ -168,6 +169,17 @@ function addGenomicQueries(c, ds, genome) {
       delete q2.singleSampleGenomeQuantification[k].folder;
     }
   }
+  if (q.NIdata) {
+    q2.NIdata = {};
+    for (const k in q.NIdata) {
+      q2.NIdata[k] = JSON.parse(JSON.stringify(q.NIdata[k]));
+    }
+  }
+  if (q.DZImages) {
+    q2.DZImages = {
+      type: q.DZImages.type
+    };
+  }
   if (q.singleSampleGbtk) {
     q2.singleSampleGbtk = {};
     for (const k in q.singleSampleGbtk) {
@@ -207,7 +219,9 @@ function getAllowedTermTypes(ds) {
     typeSet.add("geneVariant");
   }
   if (ds?.queries?.geneExpression)
-    typeSet.add("geneExpression");
+    typeSet.add(TermTypes.GENE_EXPRESSION);
+  if (ds?.queries?.metaboliteIntensity)
+    typeSet.add(TermTypes.METABOLITE_INTENSITY);
   return [...typeSet];
 }
 export {

package/routes/termdb.getTopTermsByType.js

@@ -0,0 +1,106 @@
+import { run_rust } from "@sjcrh/proteinpaint-rust";
+import { get_samples } from "#src/termdb.sql.js";
+import { TermTypes } from "#shared/terms.js";
+const api = {
+  endpoint: "termdb/getTopTermsByType",
+  methods: {
+    all: {
+      init,
+      request: {
+        typeId: "TermdbTopTermsByTypeRequest"
+      },
+      response: {
+        typeId: "TermdbTopTermsByTypeResponse"
+      }
+    }
+  }
+};
+function init({ genomes }) {
+  return async (req, res) => {
+    try {
+      const q = req.query;
+      const type = q.type;
+      const genome = genomes[q.genome];
+      if (!genome)
+        throw "invalid genome";
+      const ds = genome.datasets?.[q.dslabel];
+      if (!ds)
+        throw "invalid dslabel";
+      if (!ds.queries[type])
+        throw "not supported on dataset";
+      const t = Date.now();
+      const terms = await ds.queries[type].getTopTerms(q);
+      res.send({ terms });
+    } catch (e) {
+      res.send({ status: "error", error: e.message || e });
+    }
+  };
+}
+function validate_query_getTopTermsByType(ds, genome) {
+  const types = [TermTypes.METABOLITE_INTENSITY];
+  for (const type of types) {
+    if (ds.queries[type]) {
+      const q = ds.queries[type];
+      if (!q)
+        return;
+      if (q.src == "gdcapi")
+        gdcValidateQuery(ds, genome, type);
+      else if (q.src == "native")
+        nativeValidateQuery(ds, type);
+      else
+        throw "unknown topVariablyExpressedGenes.src";
+    }
+  }
+}
+function nativeValidateQuery(ds, type) {
+  ds.queries[type].getTopTerms = async (q) => {
+    const typeQuery = ds.queries[type];
+    const samples = [];
+    if (q.filter) {
+      const sidlst = await get_samples(q.filter, ds);
+      for (const i of sidlst) {
+        if (typeQuery.samples.includes(i.id)) {
+          const n = ds.cohort.termdb.q.id2sampleName(i.id);
+          if (!n)
+            throw "sample id cannot convert to string name";
+          samples.push(n);
+        }
+      }
+    } else {
+      for (const i of typeQuery.samples) {
+        const n = ds.cohort.termdb.q.id2sampleName(i.id);
+        if (!n)
+          throw "sample id cannot convert to string name";
+        samples.push(n);
+      }
+    }
+    const terms = await computeTopTerms(typeQuery.file, samples, type);
+    return terms;
+  };
+}
+async function computeTopTerms(file, samples, type) {
+  const input_json = {
+    input_file: file,
+    samples: samples.join(","),
+    param: "var"
+  };
+  const rust_result = await run_rust("computeTopTerms", JSON.stringify(input_json));
+  const rust_result_list = rust_result.split("\n");
+  let output_json;
+  for (const item of rust_result_list) {
+    if (item.includes("output_json")) {
+      output_json = JSON.parse(item.replace("output_json:", ""));
+    }
+  }
+  const varMetabolite = output_json.map((i) => ({ name: i.metabolite, type }));
+  return varMetabolite;
+}
+function gdcValidateQuery(ds, genome, type) {
+  ds.queries[type].getTopTerms = async (q) => {
+    return [];
+  };
+}
+export {
+  api,
+  validate_query_getTopTermsByType
+};

package/routes/termdb.getdescrstats.js

@@ -81,7 +81,7 @@ async function trigger_getdescrstats(q, res, ds, genome) {
   for (const key in data.samples) {
     const sample = data.samples[key];
     const value = sample[q.tw.$id].value;
-    if (q.tw.values?.[value]?.uncomputable) {
+    if (q.tw.q.hiddenValues?.[value]) {
       continue;
     }
     if (q.settings?.violin?.unit === "log") {