@sjcrh/proteinpaint-server 2.64.0 → 2.66.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sjcrh/proteinpaint-server",
-  "version": "2.64.0",
+  "version": "2.66.0",
   "type": "module",
   "description": "a genomics visualization tool for exploring a cohort's genotype and phenotype data",
   "main": "src/app.js",
@@ -27,7 +27,7 @@
     "postpack": "./dedupjs.sh",
     "dedup": "./dedupjs.sh",
     "//todo": "refactor or deprecate the scripts below",
-    "pretest": "tsc --esModuleInterop genome/*.ts dataset/*.ts && ./test/pretest.js",
+    "pretest": "tsc && ./test/pretest.js",
     "prepare": "ts-patch install",
     "pretest:type": "npm run checkers",
     "pretest:integration": "tsc --esModuleInterop genome/*.ts dataset/*.ts",

package/routes/hicdata.js

@@ -12,24 +12,25 @@ const api = {
       },
       response: {
         typeId: "HicdataResponse"
-      }
-      /*
+      },
       examples: [
-      	{
-      		request: {
-      			body: {
-      				genome: 'hg38-test',
-      				dslabel: 'TermdbTest',
-      				embedder: 'localhost',
-      				gettermbyid: 'subcohort'
-      			}
-      		},
-      		response: {
-      			header: { status: 200 }
-      		}
-      	}
+        {
+          request: {
+            body: {
+              embedder: "localhost",
+              url: "https://proteinpaint.stjude.org/ppdemo/hg19/hic/hic_demo.hic",
+              matrixType: "observed",
+              nmeth: "NONE",
+              pos1: "3",
+              pos2: "2",
+              resolution: 1e6
+            }
+          },
+          response: {
+            header: { status: 200 }
+          }
+        }
       ]
-      */
     },
     post: {
       alternativeFor: "get",

package/routes/hicgenome.js

@@ -8,11 +8,29 @@ const api = {
     get: {
       init,
       request: {
-        typeId: "HicdataRequest"
+        typeId: "HicGenomeRequest"
       },
       response: {
-        typeId: "HicdataResponse"
-      }
+        typeId: "HicGenomeResponse"
+      },
+      examples: [
+        {
+          request: {
+            body: {
+              chrlst: ["chr1", "chr2"],
+              embedder: "localhost",
+              url: "https://proteinpaint.stjude.org/ppdemo/hg19/hic/hic_demo.hic",
+              matrixType: "observed",
+              nmeth: "NONE",
+              nochr: true,
+              resolution: 25e5
+            }
+          },
+          response: {
+            header: { status: 200 }
+          }
+        }
+      ]
     },
     post: {
       alternativeFor: "get",
@@ -34,15 +52,7 @@ function init() {
           return new Promise((resolve, reject) => {
             const pos1 = req.query.nochr ? lead.replace("chr", "") : lead;
             const pos2 = req.query.nochr ? follow.replace("chr", "") : follow;
-            const par = [
-              matrixType,
-              req.query.nmeth || "NONE",
-              file,
-              pos1,
-              pos2,
-              req.query.isfrag ? "FRAG" : "BP",
-              req.query.resolution
-            ];
+            const par = [matrixType, req.query.nmeth || "NONE", file, pos1, pos2, "BP", req.query.resolution];
             const ps = spawn(serverconfig.hicstraw, par);
             const rl = readline.createInterface({ input: ps.stdout });
             const items = [];
@@ -61,9 +71,6 @@ function init() {
                 fieldnotnumerical++;
                 return;
               }
-              if (req.query.mincutoff != void 0 && v <= req.query.mincutoff) {
-                return;
-              }
               items.push([n1, n2, v]);
             });
             data.push({ items, lead, follow });

package/routes/termdb.categories.js

@@ -1,5 +1,6 @@
 import { getOrderedLabels } from "#src/termdb.barchart.js";
 import { getData } from "#src/termdb.matrix.js";
+import { TermTypes } from "#shared/terms.js";
 const api = {
   endpoint: "termdb/categories",
   methods: {
@@ -195,6 +196,8 @@ function getDefaultQ(term, q) {
   }
   if (term.type == "geneVariant")
     return {};
+  if (term.type == TermTypes.SINGLECELL_CELLTYPE)
+    return {};
   throw "unknown term type";
 }
 export {

package/routes/termdb.cluster.js

@@ -53,8 +53,12 @@ function init({ genomes }) {
   };
 }
 async function getResult(q, ds) {
-  const type = q.dataType;
-  const { term2sample2value, byTermId, bySampleId } = await ds.queries[type].get(q);
+  let _q = q;
+  if (q.dataType == TermTypes.GENE_EXPRESSION) {
+    _q = JSON.parse(JSON.stringify(q));
+    _q.forClusteringAnalysis = true;
+  }
+  const { term2sample2value, byTermId, bySampleId } = await ds.queries[q.dataType].get(_q);
   if (term2sample2value.size == 0)
     throw "no data";
   if (term2sample2value.size == 1) {

package/routes/termdb.config.js

@@ -191,6 +191,10 @@ function addNonDictionaryQueries(c, ds, genome) {
     q2.rnaseqGeneCount = true;
   }
   if (q.singleCell) {
+    const plots = "plots" in q.singleCell.data ? q.singleCell.data.plots.map((p) => ({
+      name: p.name,
+      colorColumn: p.colorColumn.name
+    })) : [];
     q2.singleCell = {
       samples: {
         firstColumnName: q.singleCell.samples.firstColumnName,
@@ -199,7 +203,8 @@ function addNonDictionaryQueries(c, ds, genome) {
       },
       data: {
         sameLegend: q.singleCell.data.sameLegend,
-        refName: q.singleCell.data.refName
+        refName: q.singleCell.data.refName,
+        plots
       }
     };
     if (q.singleCell.geneExpression) {

package/routes/termdb.singlecellSamples.js

@@ -1,7 +1,7 @@
 import fs from "fs";
 import path from "path";
-import { spawn } from "child_process";
-import { read_file, get_header_txt } from "#src/utils.js";
+import { read_file } from "#src/utils.js";
+import run_R from "#src/run_R.js";
 import serverconfig from "#src/serverconfig.js";
 import { gdc_validate_query_singleCell_samples, gdc_validate_query_singleCell_data } from "#src/mds3.gdc.js";
 const api = {
@@ -61,7 +61,7 @@ async function validate_query_singleCell(ds, genome) {
   if (q.data.src == "gdcapi") {
     gdc_validate_query_singleCell_data(ds, genome);
   } else if (q.data.src == "native") {
-    validateDataNative(q.data);
+    validateDataNative(q.data, ds);
   } else {
     throw "unknown singleCell.data.src";
   }
@@ -90,7 +90,7 @@ async function validateSamplesNative(S, ds) {
     return { samples: Object.values(samples) };
   };
 }
-function validateDataNative(D) {
+function validateDataNative(D, ds) {
   const nameSet = /* @__PURE__ */ new Set();
   for (const plot of D.plots) {
     if (nameSet.has(plot.name))
@@ -100,29 +100,45 @@ function validateDataNative(D) {
   D.get = async (q) => {
     try {
       const plots = [];
+      let geneExpMap;
+      if (ds.queries.singleCell.geneExpression && q.gene) {
+        geneExpMap = await ds.queries.singleCell.geneExpression.get({ sample: q.sample, gene: q.gene });
+      }
+      const file2Lines = {};
       for (const plot of D.plots) {
+        if (!q.plots.includes(plot.name))
+          continue;
         const tsvfile = path.join(serverconfig.tpmasterdir, plot.folder, q.sample + plot.fileSuffix);
-        try {
-          await fs.promises.stat(tsvfile);
-        } catch (e) {
-          if (e.code == "ENOENT") {
-            continue;
+        if (!file2Lines[tsvfile]) {
+          try {
+            await fs.promises.stat(tsvfile);
+          } catch (e) {
+            if (e.code == "ENOENT") {
+              continue;
+            }
+            if (e.code == "EACCES")
+              throw "cannot read file, permission denied";
+            throw "failed to load sc data file";
           }
-          if (e.code == "EACCES")
-            throw "cannot read file, permission denied";
-          throw "failed to load sc data file";
+          file2Lines[tsvfile] = (await read_file(tsvfile)).trim().split("\n");
         }
-        const lines = (await read_file(tsvfile)).trim().split("\n");
+        const lines = file2Lines[tsvfile];
         const cells = [];
         for (let i = 1; i < lines.length; i++) {
           const l = lines[i].split("	");
-          const cellId = l[0], x = Number(l[plot.coordsColumns.x]), y = Number(l[plot.coordsColumns.y]);
+          const cellId = lines.length > 3 ? l[0] : void 0, x = Number(l[plot.coordsColumns.x]), y = Number(l[plot.coordsColumns.y]);
           const category = l[plot.colorColumn?.index] || "";
           if (!cellId)
             throw "cell id missing";
           if (!Number.isFinite(x) || !Number.isFinite(y))
             throw "x/y not number";
-          cells.push({ cellId, x, y, category });
+          const cell = { cellId, x, y, category };
+          if (geneExpMap) {
+            if (geneExpMap[cellId] !== void 0) {
+              cell.geneExp = geneExpMap[cellId];
+            }
+          }
+          cells.push(cell);
         }
         plots.push({ name: plot.name, cells, colorBy: plot.colorColumn?.name, colorMap: plot.colorColumn?.colorMap });
       }
@@ -138,41 +154,25 @@ function validateDataNative(D) {
   };
 }
 function validateGeneExpressionNative(G) {
+  G.sample2gene2expressionBins = {};
   G.get = async (q) => {
-    const tsvfile = path.join(serverconfig.tpmasterdir, G.folder, q.sample);
+    const rdsfile = path.join(serverconfig.tpmasterdir, G.folder, q.sample + ".rds");
     try {
-      await fs.promises.stat(tsvfile);
+      await fs.promises.stat(rdsfile);
     } catch (e) {
-      throw "geneExp matrix file not found or readable for this sample";
+      return {};
     }
-    const header = await get_header_txt(tsvfile);
-    return await grepMatrix4geneExpression(tsvfile, q.gene, header);
+    let out;
+    try {
+      out = JSON.parse(
+        await run_R(path.join(serverconfig.binpath, "utils", "getGeneFromMatrix.R"), null, [rdsfile, q.gene])
+      );
+    } catch (e) {
+      return {};
+    }
+    return out;
   };
 }
-function grepMatrix4geneExpression(tsvfile, gene, header) {
-  return new Promise((resolve, reject) => {
-    const cp = spawn("grep", ["-m", "1", gene + "	", tsvfile]);
-    const out = [], err = [];
-    cp.stdout.on("data", (d) => out.push(d));
-    cp.stderr.on("data", (d) => err.push(d));
-    cp.on("close", () => {
-      const e = err.join("");
-      if (e)
-        reject(e);
-      const l = out.join("").split("	");
-      if (l.length != header.length)
-        reject(`number of fields differ between data line and header: ${l.length} ${header.length}`);
-      const cell2value = {};
-      for (let i = 1; i < l.length; i++) {
-        const v = Number(l[i]);
-        if (Number.isNaN(v))
-          continue;
-        cell2value[header[i]] = v;
-      }
-      resolve(cell2value);
-    });
-  });
-}
 export {
   api,
   validate_query_singleCell

package/routes/termdb.topVariablyExpressedGenes.js

@@ -1,9 +1,9 @@
-import { gdcGetCasesWithExpressionDataFromCohort } from "../src/mds3.gdc.js";
 import path from "path";
 import { run_rust } from "@sjcrh/proteinpaint-rust";
 import got from "got";
 import serverconfig from "#src/serverconfig.js";
 import { get_samples } from "#src/termdb.sql.js";
+import { makeFilter } from "#src/mds3.gdc.js";
 const api = {
   endpoint: "termdb/topVariablyExpressedGenes",
   methods: {
@@ -106,21 +106,20 @@ async function computeGenes4nativeDs(q, ds, matrixFile, samples) {
 function gdcValidateQuery(ds, genome) {
   ds.queries.topVariablyExpressedGenes.getGenes = async (q) => {
     if (serverconfig.features.gdcGenes) {
-      console.log("!!GDC!! using serverconfig.features.gdcGenes[]");
+      console.log(
+        "!!GDC!! using serverconfig.features.gdcGenes[] but not live api query. only use this on DEV and never on PROD!"
+      );
       return serverconfig.features.gdcGenes;
     }
-    if (!ds.__gdc.doneCaching)
+    if (!ds.__gdc.doneCaching) {
       throw "The server has not finished caching the case IDs: try again in about 2 minutes.";
-    const caseLst = await gdcGetCasesWithExpressionDataFromCohort(q, ds);
-    if (caseLst.length == 0) {
-      return [];
     }
     const { host, headers } = ds.getHostHeaders(q);
     const url = path.join(host.geneExp, "/gene_expression/gene_selection");
     try {
       const response = await got.post(url, {
         headers,
-        body: JSON.stringify(getGeneSelectionArg(q, caseLst))
+        body: JSON.stringify(getGeneSelectionArg(q))
       });
       const re = JSON.parse(response.body);
       const genes = [];
@@ -143,13 +142,14 @@ function gdcValidateQuery(ds, genome) {
       throw e;
     }
   };
-  function getGeneSelectionArg(q, caseLst) {
-    return {
-      case_ids: caseLst,
-      //.slice(0, 20),
-      gene_ids: tempGetCGCgenes(genome),
+  function getGeneSelectionArg(q) {
+    const arg = {
+      // add any to avoid tsc err
+      case_filters: makeFilter(q),
       selection_size: Number(q.maxGenes)
     };
+    arg.gene_ids = tempGetCGCgenes(genome);
+    return arg;
   }
 }
 function tempGetCGCgenes(genome) {