@sjcrh/proteinpaint-server 2.116.1-2 → 2.118.0

package/dataset/termdb.test.js

@@ -194,7 +194,8 @@ var termdb_test_default = {
     },
     geneExpression: {
       src: "native",
-      file: "files/hg38/TermdbTest/TermdbTest.fpkm.matrix.gz"
+      hdf5File: true,
+      file: "files/hg38/TermdbTest/TermdbTest.fpkm.matrix.h5"
     },
     topVariablyExpressedGenes: {
       src: "native"

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sjcrh/proteinpaint-server",
-  "version": "2.116.1-2",
+  "version": "2.118.0",
   "type": "module",
   "description": "a genomics visualization tool for exploring a cohort's genotype and phenotype data",
   "main": "src/app.js",
@@ -66,9 +66,10 @@
   },
   "dependencies": {
     "@sjcrh/augen": "2.116.0",
-    "@sjcrh/proteinpaint-rust": "2.116.0",
-    "@sjcrh/proteinpaint-shared": "2.116.1-1",
-    "@sjcrh/proteinpaint-types": "2.116.1-2",
+    "@sjcrh/proteinpaint-python": "2.118.0",
+    "@sjcrh/proteinpaint-rust": "2.117.0",
+    "@sjcrh/proteinpaint-shared": "2.118.0",
+    "@sjcrh/proteinpaint-types": "2.117.0",
     "@types/express": "^5.0.0",
     "@types/express-session": "^1.18.1",
     "better-sqlite3": "^9.4.1",

package/routes/brainImaging.js

@@ -1,10 +1,10 @@
 import path from "path";
 import serverconfig from "#src/serverconfig.js";
 import { brainImagingPayload } from "#types/checkers";
-import { spawn } from "child_process";
 import { getData } from "../src/termdb.matrix.js";
 import { isNumericTerm } from "@sjcrh/proteinpaint-shared/terms.js";
 import { getColors } from "#shared/common.js";
+import { run_python } from "@sjcrh/proteinpaint-python";
 const api = {
   endpoint: "brainImaging",
   methods: {
@@ -124,7 +124,20 @@ async function getBrainImage(query, genomes, plane, index) {
         if (!legend[category])
           legend[category] = { color: filesByCat[category].color, maxLength };
       }
-      const url = await generateBrainImage(refFile, plane, index, maxLength, JSON.stringify(filesByCat));
+      const arg = {
+        refFile,
+        plane,
+        index,
+        maxLength,
+        filesByCat: JSON.stringify(filesByCat)
+      };
+      let url;
+      try {
+        url = await run_python("plotBrainImaging.py", JSON.stringify(arg));
+      } catch (error) {
+        const errmsg = "Error running Python script:" + error;
+        throw new Error(errmsg);
+      }
       brainImageDict[dcategory] = { url, catNum };
     }
     if (query.legendFilter) {
@@ -141,30 +154,6 @@ async function getBrainImage(query, genomes, plane, index) {
     throw "no reference or sample files";
   }
 }
-async function generateBrainImage(refFile, plane, index, maxLength, filesJson) {
-  return new Promise((resolve, reject) => {
-    const cmd = [`${serverconfig.binpath}/utils/plotBrainImaging.py`, refFile, plane, index, maxLength, filesJson];
-    const ps = spawn(serverconfig.python, cmd);
-    const imgData = [];
-    ps.stdout.on("data", (data) => {
-      imgData.push(data);
-    });
-    ps.stderr.on("data", (data) => {
-      console.error(`stderr: ${data}`);
-      reject(new Error(`Python script filed: ${data}`));
-    });
-    ps.on("close", (code) => {
-      if (code === 0) {
-        const imageBuffer = Buffer.concat(imgData);
-        const base64Data = imageBuffer.toString("base64");
-        const imgUrl = `data:image/png;base64,${base64Data}`;
-        resolve(imgUrl);
-      } else {
-        reject(new Error(`Python script exited with code ${code}`));
-      }
-    });
-  });
-}
 export {
   api
 };

package/routes/brainImagingSamples.js

@@ -1,7 +1,6 @@
 import fs from "fs";
 import path from "path";
 import serverconfig from "#src/serverconfig.js";
-import { spawn } from "child_process";
 const api = {
   endpoint: "brainImagingSamples",
   methods: {
@@ -62,59 +61,6 @@ async function getBrainImageSamples(query, genomes) {
     throw "no reference or sample files";
   }
 }
-async function validate_query_NIdata(ds) {
-  const q = ds.queries.NIdata;
-  if (!q || !serverconfig.features?.showBrainImaging)
-    return;
-  for (const key in q) {
-    if (q[key].referenceFile && q[key].samples) {
-      q[key].get = async (sampleName, l, f, t) => {
-        const refFile = path.join(serverconfig.tpmasterdir, q[key].referenceFile);
-        const sampleFile = path.join(serverconfig.tpmasterdir, q[key].samples, sampleName);
-        try {
-          await fs.promises.stat(sampleFile);
-        } catch (e) {
-          if (e.code == "EACCES")
-            throw "cannot read file, permission denied";
-          if (e.code == "ENOENT")
-            throw "no data for this sample";
-          throw "failed to load data";
-        }
-        return new Promise((resolve, reject) => {
-          const ps = spawn(serverconfig.python, [
-            `${serverconfig.binpath}/utils/plotBrainImaging.py`,
-            refFile,
-            sampleFile,
-            l,
-            f,
-            t
-          ]);
-          const imgData = [];
-          ps.stdout.on("data", (data) => {
-            imgData.push(data);
-          });
-          ps.stderr.on("data", (data) => {
-            console.error(`stderr: ${data}`);
-            reject(new Error(`Python script filed: ${data}`));
-          });
-          ps.on("close", (code) => {
-            if (code === 0) {
-              const imageBuffer = Buffer.concat(imgData);
-              const base64Data = imageBuffer.toString("base64");
-              const imgUrl = `data:image/png;base64,${base64Data}`;
-              resolve(imgUrl);
-            } else {
-              reject(new Error(`Python script exited with code ${code}`));
-            }
-          });
-        });
-      };
-    } else {
-      throw "no reference or sample files";
-    }
-  }
-}
 export {
-  api,
-  validate_query_NIdata
+  api
 };

package/routes/genesetEnrichment.js

@@ -1,9 +1,8 @@
 import { genesetEnrichmentPayload } from "#types/checkers";
 import fs from "fs";
 import path from "path";
-import { spawn } from "child_process";
-import { Readable } from "stream";
 import serverconfig from "#src/serverconfig.js";
+import { run_python } from "@sjcrh/proteinpaint-python";
 const api = {
   endpoint: "genesetEnrichment",
   methods: {
@@ -60,11 +59,7 @@ async function run_genesetEnrichment_analysis(q, genomes) {
     genedb: path.join(serverconfig.tpmasterdir, genomes[q.genome].genedb.dbfile),
     filter_non_coding_genes: q.filter_non_coding_genes
   };
-  const gsea_output = await run_gsea(
-    `${serverconfig.binpath}/utils/gsea.py`,
-    "/" + JSON.stringify(genesetenrichment_input)
-    // "/" is needed for python to accept the bracket "{" as a bracket
-  );
+  const gsea_output = await run_python("gsea.py", "/" + JSON.stringify(genesetenrichment_input));
   let result;
   let data_found = false;
   let image_found = false;
@@ -88,55 +83,6 @@ async function run_genesetEnrichment_analysis(q, genomes) {
     throw ``;
   }
 }
-async function run_gsea(path2, data) {
-  try {
-    await fs.promises.stat(path2);
-  } catch (_) {
-    throw `${path2} does not exist`;
-  }
-  return new Promise((resolve, reject) => {
-    const _stdout = [];
-    const _stderr = [];
-    const sp = spawn(serverconfig.python, [path2]);
-    if (data) {
-      try {
-        const input = data.endsWith("\n") ? data : data + "\n";
-        Readable.from(input).pipe(sp.stdin);
-      } catch (e) {
-        sp.kill();
-        let errmsg = e;
-        const stderr = _stderr.join("").trim();
-        if (stderr)
-          errmsg += `
-python stderr: ${stderr}`;
-        reject(errmsg);
-      }
-    }
-    sp.stdout.on("data", (data2) => _stdout.push(data2));
-    sp.stderr.on("data", (data2) => _stderr.push(data2));
-    sp.on("error", (err) => reject(err));
-    sp.on("close", (code) => {
-      const stdout = _stdout.join("").trim();
-      const stderr = _stderr.join("").trim();
-      if (code !== 0) {
-        let errmsg = `python process exited with non-zero status code=${code}`;
-        if (stdout)
-          errmsg += `
-python stdout: ${stdout}`;
-        if (stderr)
-          errmsg += `
-python stderr: ${stderr}`;
-        reject(errmsg);
-      }
-      if (stderr) {
-        const errmsg = `python process emitted standard error
-python stderr: ${stderr}`;
-        reject(errmsg);
-      }
-      resolve(stdout);
-    });
-  });
-}
 export {
   api
 };

package/routes/termdb.cluster.js

@@ -12,6 +12,7 @@ import { TermTypes, NUMERIC_DICTIONARY_TERM } from "#shared/terms.js";
 import { getData } from "#src/termdb.matrix.js";
 import { termType2label } from "#shared/terms.js";
 import { mayLog } from "#src/helpers.ts";
+import { formatElapsedTime } from "#shared/time.js";
 const api = {
   endpoint: "termdb/cluster",
   methods: {
@@ -102,7 +103,7 @@ async function getResult(q, ds, genome) {
   }
   const t = Date.now();
   const clustering = await doClustering(term2sample2value, q, Object.keys(bySampleId).length);
-  mayLog("clustering done:", Date.now() - t, "ms");
+  mayLog("clustering done:", formatElapsedTime(Date.now() - t));
   const result = { clustering, byTermId, bySampleId };
   if (removedHierClusterTerms.length)
     result.removedHierClusterTerms = removedHierClusterTerms;
@@ -239,19 +240,19 @@ async function validateHDF5File(filePath) {
     };
   }
 }
-async function queryGeneExpression(hdf5_file, geneName) {
+async function queryGeneExpression(hdf5_file, geneNames) {
   const jsonInput = JSON.stringify({
     hdf5_file,
-    gene: geneName
+    gene: geneNames
   });
   try {
     const result = await run_rust("readHDF5", jsonInput);
-    if (!result || Object.keys(result).length === 0) {
+    if (!result || result.length === 0) {
       throw new Error("Failed to retrieve expression data: Empty or missing response");
     }
     return result;
   } catch (error) {
-    console.error(`Error querying gene expression for ${geneName}`);
+    console.error(`Error querying gene expression for ${geneNames}`);
     throw error;
   }
 }
@@ -294,13 +295,30 @@ async function validateNative(q, ds, genome) {
       }
       const term2sample2value = /* @__PURE__ */ new Map();
       const byTermId = {};
+      const geneNames = [];
       for (const geneTerm of param.terms) {
-        if (!geneTerm.gene)
-          continue;
-        try {
-          const geneQuery = await queryGeneExpression(q.file, geneTerm.gene);
-          const geneData = JSON.parse(geneQuery);
-          const samplesData = geneData.samples || {};
+        if (geneTerm.gene) {
+          geneNames.push(geneTerm.gene);
+        }
+      }
+      if (geneNames.length === 0) {
+        console.log("No genes to query");
+        return { term2sample2value, byTermId };
+      }
+      const time1 = Date.now();
+      try {
+        const geneData = JSON.parse(await queryGeneExpression(q.file, geneNames));
+        mayLog("Time taken to run gene query:", formatElapsedTime(Date.now() - time1));
+        const genesData = geneData.genes || { [geneNames[0]]: geneData };
+        for (const geneTerm of param.terms) {
+          if (!geneTerm.gene)
+            continue;
+          const geneResult = genesData[geneTerm.gene];
+          if (!geneResult) {
+            console.warn(`No data found for gene ${geneTerm.gene} in the response`);
+            continue;
+          }
+          const samplesData = geneResult.samples || {};
           const s2v = {};
           for (const [sampleName, value] of Object.entries(samplesData)) {
             const sampleId = ds.cohort.termdb.q.sampleName2id(sampleName);
@@ -308,16 +326,14 @@ async function validateNative(q, ds, genome) {
               continue;
             if (limitSamples && !limitSamples.has(sampleId))
               continue;
-            s2v[sampleId] = Number(value);
+            s2v[sampleId] = value;
           }
-          console.log(`Gene ${geneTerm.gene} has ${Object.keys(s2v).length} samples with data`);
           if (Object.keys(s2v).length) {
             term2sample2value.set(geneTerm.gene, s2v);
           }
-        } catch (error) {
-          console.warn(`Error processing gene ${geneTerm.gene}:`, error);
-          continue;
         }
+      } catch (error) {
+        console.error(`Error processing batch gene query:`, error);
       }
       if (term2sample2value.size == 0) {
         throw "No data available for the input " + param.terms?.map((g) => g.gene).join(", ");

package/routes/termdb.config.js

@@ -41,6 +41,7 @@ function make(q, req, res, ds, genome) {
   const c = {
     selectCohort: getSelectCohort(ds, req),
     supportedChartTypes: tdb.q?.getSupportedChartTypes(req),
+    hiddenTermIds: tdb.hiddenTermIds,
     renamedChartTypes: ds.cohort.renamedChartTypes,
     allowedTermTypes: getAllowedTermTypes(ds),
     termMatch2geneSet: tdb.termMatch2geneSet,

package/routes/termdb.topVariablyExpressedGenes.js

@@ -5,6 +5,8 @@ import { get_samples } from "#src/termdb.sql.js";
 import { makeFilter } from "#src/mds3.gdc.js";
 import { cachedFetch } from "#src/utils.js";
 import { joinUrl } from "#shared/joinUrl.js";
+import { formatElapsedTime } from "#shared/time.js";
+import { mayLog } from "#src/helpers.ts";
 const api = {
   endpoint: "termdb/topVariablyExpressedGenes",
   methods: {
@@ -35,8 +37,7 @@ function init({ genomes }) {
       result = {
         genes: await ds.queries.topVariablyExpressedGenes.getGenes(q)
       };
-      if (serverconfig.debugmode)
-        console.log("topVariablyExpressedGenes", Date.now() - t, "ms");
+      mayLog("topVariablyExpressedGenes", formatElapsedTime(Date.now() - t));
     } catch (e) {
       result = { status: e.status || 400, error: e.message || e };
     }