@sjcrh/proteinpaint-server 2.112.1-0 → 2.114.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sjcrh/proteinpaint-server",
-  "version": "2.112.1-0",
+  "version": "2.114.0",
   "type": "module",
   "description": "a genomics visualization tool for exploring a cohort's genotype and phenotype data",
   "main": "src/app.js",
@@ -18,6 +18,9 @@
     "prestart": "tsx emitImports.js dev > server.js",
     "start": "tsx watch . /start.js",
     "test:unit": "tsx emitImports.js unit > serverTests.js && c8 tsx serverTests.js && rm -rf ./cache",
+    "precombined:coverage": "tsx emitImports.js unit > serverTests.js",
+    "combined:coverage": "closeCoverageKey=test c8 --all --src=proteinpaint/server --experimental-monocart -r=v8 -r=html -r=json -r=markdown-summary -r=markdown-details -o=./.coverage tsx ./coverage.js & ",
+    "postcombined:coverage": "rm -rf ./cache",
     "getconf": "../build/getConfigProp.js",
     "doc": "../augen/build.sh routes shared/types/routes shared/checkers ../public/docs/server",
     "mjs": "esbuild \"$DIR/*.ts\" --platform=node --outdir=\"$DIR\" --format=esm",
@@ -49,6 +52,8 @@
     "c8": "^10.1.3",
     "esbuild": "^0.19.12",
     "glob": "^10.4.5",
+    "monocart-coverage-reports": "^2.12.1",
+    "node-notifier": "^9.0.1",
     "node-watch": "^0.7.1",
     "nodemon": "^2.0.19",
     "prettier": "^2.8.8",
@@ -59,10 +64,10 @@
     "typescript": "^5.6.3"
   },
   "dependencies": {
-    "@sjcrh/augen": "2.109.1-0",
-    "@sjcrh/proteinpaint-rust": "2.112.0",
-    "@sjcrh/proteinpaint-shared": "2.112.1-0",
-    "@sjcrh/proteinpaint-types": "2.112.1-0",
+    "@sjcrh/augen": "2.113.0",
+    "@sjcrh/proteinpaint-rust": "2.114.0",
+    "@sjcrh/proteinpaint-shared": "2.114.0",
+    "@sjcrh/proteinpaint-types": "2.114.0",
     "@types/express": "^5.0.0",
     "@types/express-session": "^1.18.1",
     "better-sqlite3": "^9.4.1",

package/routes/termdb.DE.js

@@ -196,7 +196,10 @@ async function run_DE(param, ds, term_results, term_results2) {
     cachedir: serverconfig.cachedir,
     min_count: param.min_count,
     min_total_count: param.min_total_count,
-    storage_type: param.storage_type
+    storage_type: param.storage_type,
+    DE_method: param.method,
+    mds_cutoff: 1e4
+    // If the dimensions of the read counts matrix is below this threshold, only then the mds image will be generated as its very compute intensive. Number of genes * Number of samples < mds_cutoff for mds generation
   };
   if (param.tw) {
     expression_input.conf1 = [...conf1_group2, ...conf1_group1];
@@ -213,7 +216,7 @@ async function run_DE(param, ds, term_results, term_results2) {
     }
   }
   const sample_size_limit = 8;
-  if (group1names.length <= sample_size_limit && group2names.length <= sample_size_limit || param.method == "edgeR") {
+  if (group1names.length <= sample_size_limit && group2names.length <= sample_size_limit || param.method == "edgeR" || param.method == "limma") {
     const time12 = (/* @__PURE__ */ new Date()).valueOf();
     const result2 = JSON.parse(
       await run_R(path.join(serverconfig.binpath, "utils", "edge.R"), JSON.stringify(expression_input))
@@ -222,21 +225,21 @@ async function run_DE(param, ds, term_results, term_results2) {
     param.method = "edgeR";
     const ql_imagePath = path.join(serverconfig.cachedir, result2.edgeR_ql_image_name[0]);
     mayLog("ql_imagePath:", ql_imagePath);
-    const mds_imagePath = path.join(serverconfig.cachedir, result2.edgeR_mds_image_name[0]);
-    mayLog("mds_imagePath:", mds_imagePath);
-    await readFileAndDelete(mds_imagePath, "mds_image", result2);
     await readFileAndDelete(ql_imagePath, "ql_image", result2);
+    if (result2.edgeR_mds_image_name) {
+      const mds_imagePath = path.join(serverconfig.cachedir, result2.edgeR_mds_image_name[0]);
+      mayLog("mds_imagePath:", mds_imagePath);
+      await readFileAndDelete(mds_imagePath, "mds_image", result2);
+    }
+    const images = [result2.ql_image];
+    if (result2.mds_image)
+      images.push(result2.mds_image);
     return {
       data: result2.gene_data,
       sample_size1,
       sample_size2,
       method: param.method,
-      images: [
-        result2.mds_image,
-        // MDS image
-        result2.ql_image
-        // QL fit image
-      ]
+      images
     };
   }
   const time1 = (/* @__PURE__ */ new Date()).valueOf();
@@ -259,7 +262,7 @@ async function readFileAndDelete(file, key, response) {
       throw err;
   });
 }
-async function validate_query_rnaseqGeneCount(ds, genome) {
+async function validate_query_rnaseqGeneCount(ds) {
   const q = ds.queries.rnaseqGeneCount;
   if (!q)
     return;
@@ -278,6 +281,7 @@ async function validate_query_rnaseqGeneCount(ds, genome) {
       const time1 = (/* @__PURE__ */ new Date()).valueOf();
       const result = await run_rust("DEanalysis", JSON.stringify(get_samples_from_hdf5));
       const time2 = (/* @__PURE__ */ new Date()).valueOf();
+      mayLog("Time taken to query gene expression:", time2 - time1, "ms");
       samples = result.split(",");
     } else
       throw "unknown storage type:" + ds.queries.rnaseqGeneCount.storage_type;

package/routes/termdb.cluster.js

@@ -1,5 +1,6 @@
 import path from "path";
 import run_R from "#src/run_R.js";
+import { run_rust } from "@sjcrh/proteinpaint-rust";
 import { termdbClusterPayload } from "#types/checkers";
 import * as utils from "#src/utils.js";
 import serverconfig from "#src/serverconfig.js";
@@ -67,22 +68,32 @@ async function getResult(q, ds, genome) {
     _q = JSON.parse(JSON.stringify(q));
     _q.forClusteringAnalysis = true;
   }
-  let term2sample2value, byTermId, bySampleId;
+  let term2sample2value, byTermId, bySampleId, skippedSexChrGenes;
   if (q.dataType == NUMERIC_DICTIONARY_TERM) {
     ;
     ({ term2sample2value, byTermId, bySampleId } = await getNumericDictTermAnnotation(q, ds, genome));
   } else {
     ;
-    ({ term2sample2value, byTermId, bySampleId } = await ds.queries[q.dataType].get(_q));
+    ({ term2sample2value, byTermId, bySampleId, skippedSexChrGenes } = await ds.queries[q.dataType].get(_q));
   }
-  const removedHierClusterTerms = [];
+  const noValueTerms = [];
   for (const [term, obj] of term2sample2value) {
     if (Object.keys(obj).length === 0) {
-      removedHierClusterTerms.push(term);
+      noValueTerms.push(term);
       term2sample2value.delete(term);
       delete byTermId[term];
     }
   }
+  const removedHierClusterTerms = [];
+  if (noValueTerms.length) {
+    removedHierClusterTerms.push({
+      text: `Skipped ${q.dataType == TermTypes.GENE_EXPRESSION ? "genes" : "items"} with no data`,
+      lst: noValueTerms
+    });
+  }
+  if (skippedSexChrGenes?.length) {
+    removedHierClusterTerms.push({ text: "Skipped sex chromosome genes", lst: skippedSexChrGenes });
+  }
   if (term2sample2value.size == 0)
     throw "no data";
   if (term2sample2value.size == 1) {
@@ -213,92 +224,199 @@ async function validate_query_geneExpression(ds, genome) {
   }
   throw "unknown queries.geneExpression.src";
 }
+async function validateHDF5File(filePath) {
+  try {
+    const jsonInput = JSON.stringify({
+      hdf5_file: filePath
+    });
+    const result = await run_rust("validateHDF5", jsonInput);
+    return JSON.parse(result);
+  } catch (error) {
+    console.error(`Error validating file: ${error}`);
+    return {
+      status: "error",
+      message: `Validation error`
+    };
+  }
+}
+async function queryGeneExpression(hdf5_file, geneName) {
+  const jsonInput = JSON.stringify({
+    hdf5_file,
+    gene: geneName
+  });
+  try {
+    const result = await run_rust("readHDF5", jsonInput);
+    if (!result || Object.keys(result).length === 0) {
+      throw new Error("Failed to retrieve expression data: Empty or missing response");
+    }
+    return result;
+  } catch (error) {
+    console.error(`Error querying gene expression for ${geneName}`);
+    throw error;
+  }
+}
 async function validateNative(q, ds, genome) {
-  if (!q.file.startsWith(serverconfig.tpmasterdir))
+  if (q.hdf5File === true) {
     q.file = path.join(serverconfig.tpmasterdir, q.file);
-  if (!q.samples)
     q.samples = [];
-  await utils.validate_tabixfile(q.file);
-  q.nochr = await utils.tabix_is_nochr(q.file, null, genome);
-  q.samples = [];
-  {
-    const lines = await utils.get_header_tabix(q.file);
-    if (!lines[0])
-      throw "header line missing from " + q.file;
-    const l = lines[0].split("	");
-    if (l.slice(0, 4).join("	") != "#chr	start	stop	gene")
-      throw "header line has wrong content for columns 1-4";
-    for (let i = 4; i < l.length; i++) {
-      const id = ds.cohort.termdb.q.sampleName2id(l[i]);
-      if (id == void 0)
-        throw "queries.geneExpression: unknown sample from header: " + l[i];
-      q.samples.push(id);
-    }
-  }
-  q.get = async (param) => {
-    const limitSamples = await mayLimitSamples(param, q.samples, ds);
-    if (limitSamples?.size == 0) {
-      return { term2sample2value: /* @__PURE__ */ new Map(), byTermId: {}, bySampleId: {} };
-    }
-    const bySampleId = {};
-    const samples = q.samples || [];
-    if (limitSamples) {
-      for (const sid of limitSamples) {
-        bySampleId[sid] = { label: ds.cohort.termdb.q.id2sampleName(sid) };
-      }
-    } else {
-      for (const sid of samples) {
-        bySampleId[sid] = { label: ds.cohort.termdb.q.id2sampleName(sid) };
+    await utils.file_is_readable(q.file);
+    try {
+      const vr = await validateHDF5File(q.file);
+      if (vr.status !== "success")
+        throw vr.message;
+      if (!vr.sampleNames?.length)
+        throw "HDF5 file has no samples";
+      for (const sn of vr.sampleNames) {
+        const si = ds.cohort.termdb.q.sampleName2id(sn);
+        if (si == void 0)
+          throw "unknown sample from HDF5: " + sn;
+        q.samples.push(si);
       }
+      console.log(`${ds.label}: HDF5 file validated. Format: ${vr.format}, Samples:`, vr.sampleNames.length);
+    } catch (error) {
+      throw `${ds.label}: Failed to validate HDF5 file: ${error}`;
     }
-    const term2sample2value = /* @__PURE__ */ new Map();
-    for (const geneTerm of param.terms) {
-      if (!geneTerm.gene)
-        continue;
-      if (!geneTerm.chr || !Number.isInteger(geneTerm.start) || !Number.isInteger(geneTerm.stop)) {
-        const re = getResultGene(genome, { input: geneTerm.gene, deep: 1 });
-        if (!re.gmlst || re.gmlst.length == 0) {
-          console.warn("unknown gene:" + geneTerm.gene);
-          continue;
+    q.get = async (param) => {
+      const limitSamples = await mayLimitSamples(param, q.samples, ds);
+      if (limitSamples?.size == 0) {
+        return { term2sample2value: /* @__PURE__ */ new Map(), byTermId: {}, bySampleId: {} };
+      }
+      const bySampleId = {};
+      const samples = q.samples || [];
+      if (limitSamples) {
+        for (const sid of limitSamples) {
+          bySampleId[sid] = { label: ds.cohort.termdb.q.id2sampleName(sid) };
+        }
+      } else {
+        for (const sid of samples) {
+          bySampleId[sid] = { label: ds.cohort.termdb.q.id2sampleName(sid) };
         }
-        const i = re.gmlst.find((i2) => i2.isdefault) || re.gmlst[0];
-        geneTerm.start = i.start;
-        geneTerm.stop = i.stop;
-        geneTerm.chr = i.chr;
       }
-      const s2v = {};
-      if (!geneTerm.chr || !Number.isInteger(geneTerm.start) || !Number.isInteger(geneTerm.stop))
-        throw "missing chr/start/stop";
-      await utils.get_lines_bigfile({
-        args: [
-          q.file,
-          (q.nochr ? geneTerm.chr.replace("chr", "") : geneTerm.chr) + ":" + geneTerm.start + "-" + geneTerm.stop
-        ],
-        callback: (line) => {
-          const l = line.split("	");
-          if (l[3].toLowerCase() != geneTerm.gene.toLowerCase())
-            return;
-          for (let i = 4; i < l.length; i++) {
-            const sampleId = samples[i - 4];
-            if (limitSamples && !limitSamples.has(sampleId))
+      const term2sample2value = /* @__PURE__ */ new Map();
+      const byTermId = {};
+      for (const geneTerm of param.terms) {
+        if (!geneTerm.gene)
+          continue;
+        try {
+          const geneQuery = await queryGeneExpression(q.file, geneTerm.gene);
+          const geneData = JSON.parse(geneQuery);
+          const samplesData = geneData.samples || {};
+          const s2v = {};
+          for (const [sampleName, value] of Object.entries(samplesData)) {
+            const sampleId = ds.cohort.termdb.q.sampleName2id(sampleName);
+            if (!sampleId)
               continue;
-            if (!l[i])
+            if (limitSamples && !limitSamples.has(sampleId))
               continue;
-            const v = Number(l[i]);
-            if (Number.isNaN(v))
-              throw "exp value not number";
-            s2v[sampleId] = v;
+            s2v[sampleId] = Number(value);
+          }
+          console.log(`Gene ${geneTerm.gene} has ${Object.keys(s2v).length} samples with data`);
+          if (Object.keys(s2v).length) {
+            term2sample2value.set(geneTerm.gene, s2v);
           }
+        } catch (error) {
+          console.warn(`Error processing gene ${geneTerm.gene}:`, error);
+          continue;
         }
-      });
-      if (Object.keys(s2v).length)
-        term2sample2value.set(geneTerm.gene, s2v);
+      }
+      if (term2sample2value.size == 0) {
+        throw "No data available for the input " + param.terms?.map((g) => g.gene).join(", ");
+      }
+      return { term2sample2value, byTermId, bySampleId };
+    };
+  } else {
+    if (!q.file.startsWith(serverconfig.tpmasterdir)) {
+      q.file = path.join(serverconfig.tpmasterdir, q.file);
     }
-    const byTermId = {};
-    if (term2sample2value.size == 0)
-      throw "no data available for the input " + param.terms?.map((g) => g.gene).join(", ");
-    return { term2sample2value, byTermId, bySampleId };
-  };
+    if (!q.samples)
+      q.samples = [];
+    await utils.validate_tabixfile(q.file);
+    q.nochr = await utils.tabix_is_nochr(q.file, null, genome);
+    q.samples = [];
+    {
+      const lines = await utils.get_header_tabix(q.file);
+      if (!lines[0])
+        throw "Header line missing from " + q.file;
+      const l = lines[0].split("	");
+      if (l.slice(0, 4).join("	") != "#chr	start	stop	gene") {
+        throw "Header line has wrong content for columns 1-4";
+      }
+      for (let i = 4; i < l.length; i++) {
+        const id = ds.cohort.termdb.q.sampleName2id(l[i]);
+        if (id == void 0) {
+          throw "queries.geneExpression: unknown sample from header: " + l[i];
+        }
+        q.samples.push(id);
+      }
+    }
+    q.get = async (param) => {
+      const limitSamples = await mayLimitSamples(param, q.samples, ds);
+      if (limitSamples?.size == 0) {
+        return { term2sample2value: /* @__PURE__ */ new Map(), byTermId: {}, bySampleId: {} };
+      }
+      const bySampleId = {};
+      const samples = q.samples || [];
+      if (limitSamples) {
+        for (const sid of limitSamples) {
+          bySampleId[sid] = { label: ds.cohort.termdb.q.id2sampleName(sid) };
+        }
+      } else {
+        for (const sid of samples) {
+          bySampleId[sid] = { label: ds.cohort.termdb.q.id2sampleName(sid) };
+        }
+      }
+      const term2sample2value = /* @__PURE__ */ new Map();
+      for (const geneTerm of param.terms) {
+        if (!geneTerm.gene)
+          continue;
+        if (!geneTerm.chr || !Number.isInteger(geneTerm.start) || !Number.isInteger(geneTerm.stop)) {
+          const re = getResultGene(genome, { input: geneTerm.gene, deep: 1 });
+          if (!re.gmlst || re.gmlst.length == 0) {
+            console.warn("Unknown gene:" + geneTerm.gene);
+            continue;
+          }
+          const i = re.gmlst.find((i2) => i2.isdefault) || re.gmlst[0];
+          geneTerm.start = i.start;
+          geneTerm.stop = i.stop;
+          geneTerm.chr = i.chr;
+        }
+        const s2v = {};
+        if (!geneTerm.chr || !Number.isInteger(geneTerm.start) || !Number.isInteger(geneTerm.stop)) {
+          throw "Missing chr/start/stop";
+        }
+        await utils.get_lines_bigfile({
+          args: [
+            q.file,
+            (q.nochr ? geneTerm.chr.replace("chr", "") : geneTerm.chr) + ":" + geneTerm.start + "-" + geneTerm.stop
+          ],
+          callback: (line) => {
+            const l = line.split("	");
+            if (l[3].toLowerCase() != geneTerm.gene.toLowerCase())
+              return;
+            for (let i = 4; i < l.length; i++) {
+              const sampleId = samples[i - 4];
+              if (limitSamples && !limitSamples.has(sampleId))
+                continue;
+              if (!l[i])
+                continue;
+              const v = Number(l[i]);
+              if (Number.isNaN(v))
+                throw "Expression value not number";
+              s2v[sampleId] = v;
+            }
+          }
+        });
+        if (Object.keys(s2v).length) {
+          term2sample2value.set(geneTerm.gene, s2v);
+        }
+      }
+      const byTermId = {};
+      if (term2sample2value.size == 0) {
+        throw "No data available for the input " + param.terms?.map((g) => g.gene).join(", ");
+      }
+      return { term2sample2value, byTermId, bySampleId };
+    };
+  }
 }
 export {
   api,

package/routes/termdb.config.js

@@ -155,6 +155,8 @@ function addNonDictionaryQueries(c, ds, genome) {
       details: q.snvindel.details,
       populations: q.snvindel.populations
     };
+    if (q.snvindel.byisoform?.processTwsInOneQuery)
+      q2.snvindel.byisoform = { processTwsInOneQuery: true };
   }
   if (q.trackLst) {
     q2.trackLst = q.trackLst;