@sjcrh/proteinpaint-server 2.113.0 → 2.115.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sjcrh/proteinpaint-server",
-  "version": "2.113.0",
+  "version": "2.115.0",
   "type": "module",
   "description": "a genomics visualization tool for exploring a cohort's genotype and phenotype data",
   "main": "src/app.js",
@@ -19,8 +19,9 @@
     "start": "tsx watch . /start.js",
     "test:unit": "tsx emitImports.js unit > serverTests.js && c8 tsx serverTests.js && rm -rf ./cache",
     "precombined:coverage": "tsx emitImports.js unit > serverTests.js",
-    "combined:coverage": "closeCoverageKey=test c8 --all --src=proteinpaint/server --experimental-monocart -r=v8 -r=html -r=json -r=markdown-summary -r=markdown-details -o=./.coverage tsx ./coverage.js & ",
+    "combined:coverage": "coverageKey=test c8 --all --src=proteinpaint/server --experimental-monocart -r=v8 -r=html -r=json -r=markdown-summary -r=markdown-details -o=./.coverage tsx ./coverage.js & ",
     "postcombined:coverage": "rm -rf ./cache",
+    "spec:coverage": "tsx test/relevant.js",
     "getconf": "../build/getConfigProp.js",
     "doc": "../augen/build.sh routes shared/types/routes shared/checkers ../public/docs/server",
     "mjs": "esbuild \"$DIR/*.ts\" --platform=node --outdir=\"$DIR\" --format=esm",
@@ -64,10 +65,10 @@
     "typescript": "^5.6.3"
   },
   "dependencies": {
-    "@sjcrh/augen": "2.113.0",
-    "@sjcrh/proteinpaint-rust": "2.112.0",
-    "@sjcrh/proteinpaint-shared": "2.113.0",
-    "@sjcrh/proteinpaint-types": "2.113.0",
+    "@sjcrh/augen": "2.115.0",
+    "@sjcrh/proteinpaint-rust": "2.114.0",
+    "@sjcrh/proteinpaint-shared": "2.115.0",
+    "@sjcrh/proteinpaint-types": "2.115.0",
     "@types/express": "^5.0.0",
     "@types/express-session": "^1.18.1",
     "better-sqlite3": "^9.4.1",

package/routes/correlationVolcano.js

@@ -5,6 +5,7 @@ import serverconfig from "../src/serverconfig.js";
 import { mayLog } from "#src/helpers.ts";
 import path from "path";
 import { getStdDev } from "#shared/descriptive.stats.js";
+import { formatElapsedTime } from "#shared/time.js";
 const minArrayLength = 3;
 const minSD = 0.05;
 const api = {
@@ -92,7 +93,9 @@ async function compute(q, ds, genome) {
   const output = {
     terms: JSON.parse(await run_R(path.join(serverconfig.binpath, "utils", "corr.R"), JSON.stringify(input)))
   };
-  mayLog("Time taken to run correlation analysis:", Date.now() - time1);
+  const elapsedMs = Date.now() - time1;
+  const formattedTime = formatElapsedTime(elapsedMs);
+  mayLog("Time taken to run correlation analysis:", formattedTime);
   for (const t of output.terms) {
     const t2 = {
       tw$id: t.id,

package/routes/samplewsimages.js

@@ -23,18 +23,7 @@ function init({ genomes }) {
       if (!ds)
         throw "invalid dataset name";
       const sampleId = query.sample_id;
-      if (ds.queries.WSImages.sources) {
-        const images2 = [];
-        if (ds.queries.WSImages.sources) {
-          images2.push({
-            filename: sampleId + "_fsspec.json",
-            metadata: ""
-          });
-        }
-        res.send({ sampleWSImages: images2 });
-        return;
-      }
-      const images = await ds.queries.WSImages.getWSImages({ sampleId });
+      const images = await ds.queries.WSImages.getWSImages(sampleId);
       res.send({ sampleWSImages: images });
     } catch (e) {
       console.log(e);
@@ -42,32 +31,31 @@ function init({ genomes }) {
     }
   };
 }
-function validate_query_getSampleWSImages(ds) {
-  const q = ds.queries?.WSImages;
-  if (!q)
+async function validate_query_getSampleWSImages(ds) {
+  if (!ds.queries?.WSImages)
     return;
-  nativeValidateQuery(ds);
+  validateQuery(ds);
 }
-function nativeValidateQuery(ds) {
-  ds.queries.WSImages.getWSImages = async (q) => {
-    return await getWSImages(ds, q.sampleId);
-  };
-}
-async function getWSImages(ds, sampleName) {
-  const sql = `SELECT wsimages.filename as filename, wsimages.metadata as metadata
-                 FROM wsimages
-                          INNER JOIN sampleidmap
-                                     ON wsimages.sample = sampleidmap.id
-                 WHERE sampleidmap.name = ?`;
-  const rows = ds.cohort.db.connection.prepare(sql).all(sampleName);
-  const images = [];
-  for (const row of rows) {
-    images.push({
-      filename: row.filename,
-      metadata: row.metadata
-    });
+function validateQuery(ds) {
+  if (typeof ds.queries.WSImages.getWSImages == "function") {
+    return;
   }
-  return images;
+  ds.queries.WSImages.getWSImages = async (sampleName) => {
+    const sql = `SELECT wsimages.filename as filename, wsimages.metadata as metadata
+					 FROM wsimages
+							  INNER JOIN sampleidmap
+										 ON wsimages.sample = sampleidmap.id
+					 WHERE sampleidmap.name = ?`;
+    const rows = ds.cohort.db.connection.prepare(sql).all(sampleName);
+    const images = [];
+    for (const row of rows) {
+      images.push({
+        filename: row.filename,
+        metadata: row.metadata
+      });
+    }
+    return images;
+  };
 }
 export {
   api,

package/routes/termdb.cluster.js

@@ -1,5 +1,6 @@
 import path from "path";
 import run_R from "#src/run_R.js";
+import { run_rust } from "@sjcrh/proteinpaint-rust";
 import { termdbClusterPayload } from "#types/checkers";
 import * as utils from "#src/utils.js";
 import serverconfig from "#src/serverconfig.js";
@@ -223,92 +224,199 @@ async function validate_query_geneExpression(ds, genome) {
   }
   throw "unknown queries.geneExpression.src";
 }
+async function validateHDF5File(filePath) {
+  try {
+    const jsonInput = JSON.stringify({
+      hdf5_file: filePath
+    });
+    const result = await run_rust("validateHDF5", jsonInput);
+    return JSON.parse(result);
+  } catch (error) {
+    console.error(`Error validating file: ${error}`);
+    return {
+      status: "error",
+      message: `Validation error`
+    };
+  }
+}
+async function queryGeneExpression(hdf5_file, geneName) {
+  const jsonInput = JSON.stringify({
+    hdf5_file,
+    gene: geneName
+  });
+  try {
+    const result = await run_rust("readHDF5", jsonInput);
+    if (!result || Object.keys(result).length === 0) {
+      throw new Error("Failed to retrieve expression data: Empty or missing response");
+    }
+    return result;
+  } catch (error) {
+    console.error(`Error querying gene expression for ${geneName}`);
+    throw error;
+  }
+}
 async function validateNative(q, ds, genome) {
-  if (!q.file.startsWith(serverconfig.tpmasterdir))
+  if (q.hdf5File === true) {
     q.file = path.join(serverconfig.tpmasterdir, q.file);
-  if (!q.samples)
     q.samples = [];
-  await utils.validate_tabixfile(q.file);
-  q.nochr = await utils.tabix_is_nochr(q.file, null, genome);
-  q.samples = [];
-  {
-    const lines = await utils.get_header_tabix(q.file);
-    if (!lines[0])
-      throw "header line missing from " + q.file;
-    const l = lines[0].split("	");
-    if (l.slice(0, 4).join("	") != "#chr	start	stop	gene")
-      throw "header line has wrong content for columns 1-4";
-    for (let i = 4; i < l.length; i++) {
-      const id = ds.cohort.termdb.q.sampleName2id(l[i]);
-      if (id == void 0)
-        throw "queries.geneExpression: unknown sample from header: " + l[i];
-      q.samples.push(id);
-    }
-  }
-  q.get = async (param) => {
-    const limitSamples = await mayLimitSamples(param, q.samples, ds);
-    if (limitSamples?.size == 0) {
-      return { term2sample2value: /* @__PURE__ */ new Map(), byTermId: {}, bySampleId: {} };
-    }
-    const bySampleId = {};
-    const samples = q.samples || [];
-    if (limitSamples) {
-      for (const sid of limitSamples) {
-        bySampleId[sid] = { label: ds.cohort.termdb.q.id2sampleName(sid) };
-      }
-    } else {
-      for (const sid of samples) {
-        bySampleId[sid] = { label: ds.cohort.termdb.q.id2sampleName(sid) };
+    await utils.file_is_readable(q.file);
+    try {
+      const vr = await validateHDF5File(q.file);
+      if (vr.status !== "success")
+        throw vr.message;
+      if (!vr.sampleNames?.length)
+        throw "HDF5 file has no samples";
+      for (const sn of vr.sampleNames) {
+        const si = ds.cohort.termdb.q.sampleName2id(sn);
+        if (si == void 0)
+          throw "unknown sample from HDF5: " + sn;
+        q.samples.push(si);
       }
+      console.log(`${ds.label}: HDF5 file validated. Format: ${vr.format}, Samples:`, vr.sampleNames.length);
+    } catch (error) {
+      throw `${ds.label}: Failed to validate HDF5 file: ${error}`;
     }
-    const term2sample2value = /* @__PURE__ */ new Map();
-    for (const geneTerm of param.terms) {
-      if (!geneTerm.gene)
-        continue;
-      if (!geneTerm.chr || !Number.isInteger(geneTerm.start) || !Number.isInteger(geneTerm.stop)) {
-        const re = getResultGene(genome, { input: geneTerm.gene, deep: 1 });
-        if (!re.gmlst || re.gmlst.length == 0) {
-          console.warn("unknown gene:" + geneTerm.gene);
-          continue;
+    q.get = async (param) => {
+      const limitSamples = await mayLimitSamples(param, q.samples, ds);
+      if (limitSamples?.size == 0) {
+        return { term2sample2value: /* @__PURE__ */ new Map(), byTermId: {}, bySampleId: {} };
+      }
+      const bySampleId = {};
+      const samples = q.samples || [];
+      if (limitSamples) {
+        for (const sid of limitSamples) {
+          bySampleId[sid] = { label: ds.cohort.termdb.q.id2sampleName(sid) };
+        }
+      } else {
+        for (const sid of samples) {
+          bySampleId[sid] = { label: ds.cohort.termdb.q.id2sampleName(sid) };
         }
-        const i = re.gmlst.find((i2) => i2.isdefault) || re.gmlst[0];
-        geneTerm.start = i.start;
-        geneTerm.stop = i.stop;
-        geneTerm.chr = i.chr;
       }
-      const s2v = {};
-      if (!geneTerm.chr || !Number.isInteger(geneTerm.start) || !Number.isInteger(geneTerm.stop))
-        throw "missing chr/start/stop";
-      await utils.get_lines_bigfile({
-        args: [
-          q.file,
-          (q.nochr ? geneTerm.chr.replace("chr", "") : geneTerm.chr) + ":" + geneTerm.start + "-" + geneTerm.stop
-        ],
-        callback: (line) => {
-          const l = line.split("	");
-          if (l[3].toLowerCase() != geneTerm.gene.toLowerCase())
-            return;
-          for (let i = 4; i < l.length; i++) {
-            const sampleId = samples[i - 4];
-            if (limitSamples && !limitSamples.has(sampleId))
+      const term2sample2value = /* @__PURE__ */ new Map();
+      const byTermId = {};
+      for (const geneTerm of param.terms) {
+        if (!geneTerm.gene)
+          continue;
+        try {
+          const geneQuery = await queryGeneExpression(q.file, geneTerm.gene);
+          const geneData = JSON.parse(geneQuery);
+          const samplesData = geneData.samples || {};
+          const s2v = {};
+          for (const [sampleName, value] of Object.entries(samplesData)) {
+            const sampleId = ds.cohort.termdb.q.sampleName2id(sampleName);
+            if (!sampleId)
               continue;
-            if (!l[i])
+            if (limitSamples && !limitSamples.has(sampleId))
               continue;
-            const v = Number(l[i]);
-            if (Number.isNaN(v))
-              throw "exp value not number";
-            s2v[sampleId] = v;
+            s2v[sampleId] = Number(value);
+          }
+          console.log(`Gene ${geneTerm.gene} has ${Object.keys(s2v).length} samples with data`);
+          if (Object.keys(s2v).length) {
+            term2sample2value.set(geneTerm.gene, s2v);
           }
+        } catch (error) {
+          console.warn(`Error processing gene ${geneTerm.gene}:`, error);
+          continue;
+        }
+      }
+      if (term2sample2value.size == 0) {
+        throw "No data available for the input " + param.terms?.map((g) => g.gene).join(", ");
+      }
+      return { term2sample2value, byTermId, bySampleId };
+    };
+  } else {
+    if (!q.file.startsWith(serverconfig.tpmasterdir)) {
+      q.file = path.join(serverconfig.tpmasterdir, q.file);
+    }
+    if (!q.samples)
+      q.samples = [];
+    await utils.validate_tabixfile(q.file);
+    q.nochr = await utils.tabix_is_nochr(q.file, null, genome);
+    q.samples = [];
+    {
+      const lines = await utils.get_header_tabix(q.file);
+      if (!lines[0])
+        throw "Header line missing from " + q.file;
+      const l = lines[0].split("	");
+      if (l.slice(0, 4).join("	") != "#chr	start	stop	gene") {
+        throw "Header line has wrong content for columns 1-4";
+      }
+      for (let i = 4; i < l.length; i++) {
+        const id = ds.cohort.termdb.q.sampleName2id(l[i]);
+        if (id == void 0) {
+          throw "queries.geneExpression: unknown sample from header: " + l[i];
         }
-      });
-      if (Object.keys(s2v).length)
-        term2sample2value.set(geneTerm.gene, s2v);
+        q.samples.push(id);
+      }
     }
-    const byTermId = {};
-    if (term2sample2value.size == 0)
-      throw "no data available for the input " + param.terms?.map((g) => g.gene).join(", ");
-    return { term2sample2value, byTermId, bySampleId };
-  };
+    q.get = async (param) => {
+      const limitSamples = await mayLimitSamples(param, q.samples, ds);
+      if (limitSamples?.size == 0) {
+        return { term2sample2value: /* @__PURE__ */ new Map(), byTermId: {}, bySampleId: {} };
+      }
+      const bySampleId = {};
+      const samples = q.samples || [];
+      if (limitSamples) {
+        for (const sid of limitSamples) {
+          bySampleId[sid] = { label: ds.cohort.termdb.q.id2sampleName(sid) };
+        }
+      } else {
+        for (const sid of samples) {
+          bySampleId[sid] = { label: ds.cohort.termdb.q.id2sampleName(sid) };
+        }
+      }
+      const term2sample2value = /* @__PURE__ */ new Map();
+      for (const geneTerm of param.terms) {
+        if (!geneTerm.gene)
+          continue;
+        if (!geneTerm.chr || !Number.isInteger(geneTerm.start) || !Number.isInteger(geneTerm.stop)) {
+          const re = getResultGene(genome, { input: geneTerm.gene, deep: 1 });
+          if (!re.gmlst || re.gmlst.length == 0) {
+            console.warn("Unknown gene:" + geneTerm.gene);
+            continue;
+          }
+          const i = re.gmlst.find((i2) => i2.isdefault) || re.gmlst[0];
+          geneTerm.start = i.start;
+          geneTerm.stop = i.stop;
+          geneTerm.chr = i.chr;
+        }
+        const s2v = {};
+        if (!geneTerm.chr || !Number.isInteger(geneTerm.start) || !Number.isInteger(geneTerm.stop)) {
+          throw "Missing chr/start/stop";
+        }
+        await utils.get_lines_bigfile({
+          args: [
+            q.file,
+            (q.nochr ? geneTerm.chr.replace("chr", "") : geneTerm.chr) + ":" + geneTerm.start + "-" + geneTerm.stop
+          ],
+          callback: (line) => {
+            const l = line.split("	");
+            if (l[3].toLowerCase() != geneTerm.gene.toLowerCase())
+              return;
+            for (let i = 4; i < l.length; i++) {
+              const sampleId = samples[i - 4];
+              if (limitSamples && !limitSamples.has(sampleId))
+                continue;
+              if (!l[i])
+                continue;
+              const v = Number(l[i]);
+              if (Number.isNaN(v))
+                throw "Expression value not number";
+              s2v[sampleId] = v;
+            }
+          }
+        });
+        if (Object.keys(s2v).length) {
+          term2sample2value.set(geneTerm.gene, s2v);
+        }
+      }
+      const byTermId = {};
+      if (term2sample2value.size == 0) {
+        throw "No data available for the input " + param.terms?.map((g) => g.gene).join(", ");
+      }
+      return { term2sample2value, byTermId, bySampleId };
+    };
+  }
 }
 export {
   api,

package/routes/termdb.config.js

@@ -155,6 +155,8 @@ function addNonDictionaryQueries(c, ds, genome) {
       details: q.snvindel.details,
       populations: q.snvindel.populations
     };
+    if (q.snvindel.byisoform?.processTwsInOneQuery)
+      q2.snvindel.byisoform = { processTwsInOneQuery: true };
   }
   if (q.trackLst) {
     q2.trackLst = q.trackLst;

package/routes/termdb.topMutatedGenes.js

@@ -0,0 +1,51 @@
+import { topMutatedGenePayload } from "#types/checkers";
+const api = {
+  endpoint: "termdb/topMutatedGenes",
+  methods: {
+    get: {
+      init,
+      ...topMutatedGenePayload
+    },
+    post: {
+      init,
+      ...topMutatedGenePayload
+    }
+  }
+};
+function init({ genomes }) {
+  return async (req, res) => {
+    try {
+      const q = req.query;
+      const g = genomes[q.genome];
+      if (!g)
+        throw "genome missing";
+      const ds = g.datasets?.[q.dslabel];
+      if (!ds)
+        throw "ds missing";
+      if (!ds.queries?.topMutatedGenes)
+        throw "not supported by ds";
+      const genes = await ds.queries.topMutatedGenes.get(q);
+      const payload = { genes };
+      res.send(payload);
+    } catch (e) {
+      res.send({ status: "error", error: e.message || e });
+      if (e.stack)
+        console.log(e.stack);
+      else
+        console.trace(e);
+    }
+  };
+}
+function validate_query_getTopMutatedGenes(ds, genome) {
+  const q = ds.queries?.topMutatedGenes;
+  if (!q)
+    return;
+  if (typeof q.get == "function")
+    return;
+  q.get = async (param) => {
+  };
+}
+export {
+  api,
+  validate_query_getTopMutatedGenes
+};

package/routes/wsisamples.js

@@ -0,0 +1,74 @@
+import { wsiSamplesPayload } from "#types/checkers";
+const routePath = "wsisamples";
+const api = {
+  endpoint: `${routePath}`,
+  methods: {
+    get: {
+      ...wsiSamplesPayload,
+      init
+    },
+    post: {
+      ...wsiSamplesPayload,
+      init
+    }
+  }
+};
+function init({ genomes }) {
+  return async (req, res) => {
+    try {
+      const query = req.query;
+      const g = genomes[query.genome];
+      if (!g)
+        throw new Error("Invalid genome name");
+      const ds = g.datasets[query.dslabel];
+      if (!ds)
+        throw "Invalid dslabel";
+      const images = await ds.queries.WSImages.getSamples();
+      const payload = {
+        samples: images
+      };
+      res.status(200).json(payload);
+    } catch (e) {
+      console.warn(e);
+      res.status(500).send({
+        status: "error",
+        error: e.message || e
+      });
+    }
+  };
+}
+async function validate_query_getWSISamples(ds) {
+  const q = ds.queries?.WSImages;
+  if (!q)
+    return;
+  validateQuery(ds);
+}
+function validateQuery(ds) {
+  if (typeof ds.queries.WSImages.getSamples == "function") {
+    return;
+  }
+  ds.queries.WSImages.getSamples = async () => {
+    const sql = `SELECT wsimages.sample as sample, wsimages.filename as filename, wsimages.metadata as metadata, sampleidmap.name as sampleName
+					 FROM wsimages
+					 INNER JOIN sampleidmap
+					 ON wsimages.sample = sampleidmap.id`;
+    try {
+      const rows = ds.cohort.db.connection.prepare(sql).all();
+      const sampleMap = {};
+      for (const row of rows) {
+        if (!sampleMap[row.sample]) {
+          sampleMap[row.sample] = { sampleId: row.sampleName, wsimages: [] };
+        }
+        sampleMap[row.sample].wsimages.push(row.filename);
+      }
+      return Object.values(sampleMap);
+    } catch (error) {
+      console.error("Error fetching samples:", error);
+      throw error;
+    }
+  };
+}
+export {
+  api,
+  validate_query_getWSISamples
+};