@sjcrh/proteinpaint-server 2.140.1 → 2.142.0

package/dataset/termdb.test.js

@@ -265,7 +265,8 @@ function termdb_test_default() {
       },
       geneExpression: {
         src: "native",
-        file: "files/hg38/TermdbTest/rnaseq/TermdbTest.fpkm.matrix.h5",
+        file: "files/hg38/TermdbTest/rnaseq/TermdbTest.fpkm.matrix.new.h5",
+        newformat: true,
         unit: "FPKM"
       },
       ssGSEA: {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sjcrh/proteinpaint-server",
-  "version": "2.140.1",
+  "version": "2.142.0",
   "type": "module",
   "description": "a genomics visualization tool for exploring a cohort's genotype and phenotype data",
   "main": "src/app.js",
@@ -62,9 +62,9 @@
     "@sjcrh/augen": "2.136.0",
     "@sjcrh/proteinpaint-python": "2.139.1",
     "@sjcrh/proteinpaint-r": "2.137.2-0",
-    "@sjcrh/proteinpaint-rust": "2.140.1",
-    "@sjcrh/proteinpaint-shared": "2.140.1-1",
-    "@sjcrh/proteinpaint-types": "2.140.1",
+    "@sjcrh/proteinpaint-rust": "2.142.0",
+    "@sjcrh/proteinpaint-shared": "2.142.0",
+    "@sjcrh/proteinpaint-types": "2.142.0",
     "@types/express": "^5.0.0",
     "@types/express-session": "^1.18.1",
     "better-sqlite3": "^9.4.1",

package/routes/correlationVolcano.js

@@ -3,9 +3,7 @@ import { getData } from "../src/termdb.matrix.js";
 import { run_R } from "@sjcrh/proteinpaint-r";
 import { mayLog } from "#src/helpers.ts";
 import { getStdDev } from "#shared/descriptive.stats.js";
-import { formatElapsedTime } from "#shared/time.js";
 const minArrayLength = 3;
-const minSD = 0.05;
 const api = {
   endpoint: "termdb/correlationVolcano",
   methods: {
@@ -39,12 +37,15 @@ function init({ genomes }) {
   };
 }
 async function compute(q, ds) {
-  const terms = [q.featureTw, ...q.variableTwLst];
+  if (!q.featureTw.$id)
+    throw "featureTw.$id missing";
+  if (!ds.cohort.correlationVolcano.feature.termTypes.includes(q.featureTw?.term.type))
+    throw "unsupported featureTw.term.type";
   const data = await getData(
     {
       filter: q.filter,
       filter0: q.filter0,
-      terms,
+      terms: [q.featureTw, ...q.variableTwLst],
       __protected__: q.__protected__
     },
     ds
@@ -56,30 +57,27 @@ async function compute(q, ds) {
     vtid2array.set(tw.$id, { id: tw.$id, v1: [], v2: [] });
   }
   for (const sid in data.samples) {
-    const featureValue = q.featureTw.$id === void 0 ? void 0 : data.samples[sid][q.featureTw.$id]?.value;
+    const featureValue = data.samples[sid][q.featureTw.$id]?.value;
     if (!Number.isFinite(featureValue))
       continue;
     for (const tw of q.variableTwLst) {
-      const variableValue = tw.$id === void 0 ? void 0 : data.samples[sid][tw.$id]?.value;
+      if (!tw.$id)
+        throw "variableTwLst[].$id missing";
+      const variableValue = data.samples[sid][tw.$id]?.value;
       if (!Number.isFinite(variableValue))
         continue;
       vtid2array.get(tw.$id).v1.push(featureValue);
       vtid2array.get(tw.$id).v2.push(variableValue);
     }
   }
-  const [acceptedVariables, skippedVariables] = Array.from(vtid2array.values()).reduce(
-    ([accepted, skipped], t) => {
-      const grterThanOne = t.v1.length > minArrayLength && t.v2.length > minArrayLength;
-      const significantSD = getStdDev(t.v1) > minSD && getStdDev(t.v2) > minSD;
-      const v = grterThanOne && significantSD ? accepted : skipped;
-      if (v === accepted)
-        accepted.push(t);
-      if (v === skipped)
-        skipped.push({ tw$id: t.id });
-      return [accepted, skipped];
-    },
-    [[], []]
-  );
+  const acceptedVariables = [], skippedVariables = [];
+  for (const [tid, v] of vtid2array) {
+    if (v.v1.length < minArrayLength || v.v2.length < minArrayLength || getStdDev(v.v1) == 0 || getStdDev(v.v2) == 0) {
+      skippedVariables.push({ tw$id: tid });
+      continue;
+    }
+    acceptedVariables.push(v);
+  }
   const result = { skippedVariables, variableItems: [] };
   if (!acceptedVariables.length)
     return result;
@@ -91,9 +89,7 @@ async function compute(q, ds) {
   const output = {
     terms: JSON.parse(await run_R("corr.R", JSON.stringify(input)))
   };
-  const elapsedMs = Date.now() - time1;
-  const formattedTime = formatElapsedTime(elapsedMs);
-  mayLog("Time taken to run correlation analysis:", formattedTime);
+  mayLog("Time taken to run correlation analysis:", Date.now() - time1);
   for (const t of output.terms) {
     const t2 = {
       tw$id: t.id,
@@ -113,11 +109,18 @@ function validate_correlationVolcano(ds) {
     return;
   if (typeof cv.feature != "object")
     throw "cv.feature not obj";
-  if (cv.feature.termType == "geneExpression") {
-    if (!ds.queries?.geneExpression)
-      throw "cv.feature.termType=geneExpression not supported by ds";
-  } else {
-    throw "unknown cv.feature.termType";
+  if (!Array.isArray(cv.feature.termTypes))
+    throw "cv.feature.termTypes[] not array";
+  for (const t of cv.feature.termTypes) {
+    if (t == "geneExpression") {
+      if (!ds.queries?.geneExpression)
+        throw "geneExpression cv.feature is not supported";
+    } else if (t == "ssGSEA") {
+      if (!ds.queries?.ssGSEA)
+        throw "ssGSEA cv.feature is not supported";
+    } else {
+      throw "unknown cv.feature.termType";
+    }
   }
   if (typeof cv.variables != "object")
     throw "cv.variables not obj";

package/routes/termdb.cluster.js

@@ -240,27 +240,60 @@ async function queryGeneExpression(hdf5_file, geneNames) {
     throw error;
   }
 }
+async function queryHDF5(hdf5_file, query) {
+  const jsonInput = JSON.stringify({
+    hdf5_file,
+    query
+  });
+  try {
+    const result = await run_rust("readH5", jsonInput);
+    if (!result || result.length === 0) {
+      throw new Error("Failed to retrieve expression data: Empty or missing response");
+    }
+    return result;
+  } catch (error) {
+    console.error(`Error querying HDF5 for ${query}`);
+    throw error;
+  }
+}
 async function validateNative(q, ds) {
   q.file = path.join(serverconfig.tpmasterdir, q.file);
   q.samples = [];
   try {
     await utils.file_is_readable(q.file);
-    const tmp = await run_rust("validateHDF5", JSON.stringify({ hdf5_file: q.file }));
+    let tmp;
+    if (q.newformat) {
+      tmp = await run_rust("readH5", JSON.stringify({ hdf5_file: q.file, validate: true }));
+    } else {
+      tmp = await run_rust("validateHDF5", JSON.stringify({ hdf5_file: q.file }));
+    }
     const vr = JSON.parse(tmp);
     if (vr.status !== "success")
       throw vr.message;
-    if (!vr.sampleNames?.length)
-      throw "HDF5 file has no samples, please check file.";
-    for (const sn of vr.sampleNames) {
-      const si = ds.cohort.termdb.q.sampleName2id(sn);
-      if (si == void 0)
-        throw `unknown sample ${sn} from HDF5 ${q.file}`;
-      q.samples.push(si);
+    if (q.newformat) {
+      if (!vr.samples?.length)
+        throw "HDF5 file has no samples, please check file.";
+      for (const sn of vr.samples) {
+        const si = ds.cohort.termdb.q.sampleName2id(sn);
+        if (si == void 0)
+          throw `unknown sample ${sn} from HDF5 ${q.file}`;
+        q.samples.push(si);
+      }
+      console.log(`${ds.label}: geneExpression HDF5 file validated. Format: ${vr.format}, Samples:`, vr.samples.length);
+    } else {
+      if (!vr.sampleNames?.length)
+        throw "HDF5 file has no samples, please check file.";
+      for (const sn of vr.sampleNames) {
+        const si = ds.cohort.termdb.q.sampleName2id(sn);
+        if (si == void 0)
+          throw `unknown sample ${sn} from HDF5 ${q.file}`;
+        q.samples.push(si);
+      }
+      console.log(
+        `${ds.label}: geneExpression HDF5 file validated. Format: ${vr.format}, Samples:`,
+        vr.sampleNames.length
+      );
     }
-    console.log(
-      `${ds.label}: geneExpression HDF5 file validated. Format: ${vr.format}, Samples:`,
-      vr.sampleNames.length
-    );
   } catch (error) {
     throw `${ds.label}: Failed to validate geneExpression HDF5 file: ${error}`;
   }
@@ -293,34 +326,70 @@ async function validateNative(q, ds) {
       return { term2sample2value, byTermId };
     }
     const time1 = Date.now();
-    const geneData = JSON.parse(await queryGeneExpression(q.file, geneNames));
+    let geneData;
+    if (q.newformat) {
+      geneData = JSON.parse(await queryHDF5(q.file, geneNames));
+    } else {
+      geneData = JSON.parse(await queryGeneExpression(q.file, geneNames));
+    }
     mayLog("Time taken to run gene query:", formatElapsedTime(Date.now() - time1));
-    const genesData = geneData.genes || { [geneNames[0]]: geneData };
-    for (const tw of param.terms) {
-      if (!tw.term.gene)
-        continue;
-      const geneResult = genesData[tw.term.gene];
-      if (!geneResult) {
-        console.warn(`No data found for gene ${tw.term.gene} in the response`);
-        continue;
+    if (q.newformat) {
+      const genesData = geneData.query_output || {};
+      if (!genesData)
+        throw "No expression data returned from HDF5 query";
+      for (const tw of param.terms) {
+        if (!tw.term.gene)
+          continue;
+        const geneResult = genesData[tw.term.gene];
+        if (!geneResult) {
+          console.warn(`No data found for gene ${tw.term.gene} in the response`);
+          continue;
+        }
+        const samplesData = geneResult.samples || {};
+        const s2v = {};
+        for (const sampleName in samplesData) {
+          const sampleId = ds.cohort.termdb.q.sampleName2id(sampleName);
+          if (!sampleId)
+            continue;
+          if (limitSamples && !limitSamples.has(sampleId))
+            continue;
+          s2v[sampleId] = samplesData[sampleName];
+        }
+        if (Object.keys(s2v).length) {
+          term2sample2value.set(tw.$id, s2v);
+        }
+      }
+      if (term2sample2value.size == 0) {
+        throw "No data available for the input " + param.terms?.map((tw) => tw.term.gene).join(", ");
       }
-      const samplesData = geneResult.samples || {};
-      const s2v = {};
-      for (const [sampleName, value] of Object.entries(samplesData)) {
-        const sampleId = ds.cohort.termdb.q.sampleName2id(sampleName);
-        if (!sampleId)
+    } else {
+      const genesData = geneData.genes || { [geneNames[0]]: geneData };
+      for (const tw of param.terms) {
+        if (!tw.term.gene)
           continue;
-        if (limitSamples && !limitSamples.has(sampleId))
+        const geneResult = genesData[tw.term.gene];
+        if (!geneResult) {
+          console.warn(`No data found for gene ${tw.term.gene} in the response`);
           continue;
-        s2v[sampleId] = value;
+        }
+        const samplesData = geneResult.samples || {};
+        const s2v = {};
+        for (const [sampleName, value] of Object.entries(samplesData)) {
+          const sampleId = ds.cohort.termdb.q.sampleName2id(sampleName);
+          if (!sampleId)
+            continue;
+          if (limitSamples && !limitSamples.has(sampleId))
+            continue;
+          s2v[sampleId] = value;
+        }
+        if (Object.keys(s2v).length) {
+          term2sample2value.set(tw.$id, s2v);
+        }
       }
-      if (Object.keys(s2v).length) {
-        term2sample2value.set(tw.$id, s2v);
+      if (term2sample2value.size == 0) {
+        throw "No data available for the input " + param.terms?.map((tw) => tw.term.gene).join(", ");
       }
     }
-    if (term2sample2value.size == 0) {
-      throw "No data available for the input " + param.terms?.map((tw) => tw.term.gene).join(", ");
-    }
     return { term2sample2value, byTermId, bySampleId };
   };
 }

package/routes/termdb.topVariablyExpressedGenes.js

@@ -83,8 +83,7 @@ function nativeValidateQuery(ds) {
         samples.push(n);
       }
     }
-    const genes = await computeGenes4nativeDs(q, gE.file, samples);
-    return genes;
+    return await computeGenes4nativeDs(q, gE, samples);
   };
 }
 function addTopVEarg(q) {
@@ -146,9 +145,9 @@ function addTopVEarg(q) {
   }
   q.arguments = arglst;
 }
-async function computeGenes4nativeDs(q, matrixFile, samples) {
+async function computeGenes4nativeDs(q, gE, samples) {
   const input_json = {
-    input_file: matrixFile,
+    input_file: gE.file,
     samples: samples.join(","),
     filter_extreme_values: q.filter_extreme_values,
     num_genes: q.maxGenes,
@@ -158,6 +157,9 @@ async function computeGenes4nativeDs(q, matrixFile, samples) {
     input_json["min_count"] = q.min_count;
     input_json["min_total_count"] = q.min_total_count;
   }
+  if (gE.newformat) {
+    input_json["newformat"] = true;
+  }
   const rust_output = await run_rust("topGeneByExpressionVariance", JSON.stringify(input_json));
   const rust_output_list = rust_output.split("\n");
   let output_json;