@sjcrh/proteinpaint-server 2.65.0 → 2.66.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sjcrh/proteinpaint-server",
-  "version": "2.65.0",
+  "version": "2.66.0",
   "type": "module",
   "description": "a genomics visualization tool for exploring a cohort's genotype and phenotype data",
   "main": "src/app.js",
@@ -27,7 +27,7 @@
     "postpack": "./dedupjs.sh",
     "dedup": "./dedupjs.sh",
     "//todo": "refactor or deprecate the scripts below",
-    "pretest": "tsc --esModuleInterop genome/*.ts dataset/*.ts && ./test/pretest.js",
+    "pretest": "tsc && ./test/pretest.js",
     "prepare": "ts-patch install",
     "pretest:type": "npm run checkers",
     "pretest:integration": "tsc --esModuleInterop genome/*.ts dataset/*.ts",

package/routes/termdb.categories.js

@@ -1,5 +1,6 @@
 import { getOrderedLabels } from "#src/termdb.barchart.js";
 import { getData } from "#src/termdb.matrix.js";
+import { TermTypes } from "#shared/terms.js";
 const api = {
   endpoint: "termdb/categories",
   methods: {
@@ -195,6 +196,8 @@ function getDefaultQ(term, q) {
   }
   if (term.type == "geneVariant")
     return {};
+  if (term.type == TermTypes.SINGLECELL_CELLTYPE)
+    return {};
   throw "unknown term type";
 }
 export {

package/routes/termdb.cluster.js

@@ -53,8 +53,12 @@ function init({ genomes }) {
   };
 }
 async function getResult(q, ds) {
-  const type = q.dataType;
-  const { term2sample2value, byTermId, bySampleId } = await ds.queries[type].get(q);
+  let _q = q;
+  if (q.dataType == TermTypes.GENE_EXPRESSION) {
+    _q = JSON.parse(JSON.stringify(q));
+    _q.forClusteringAnalysis = true;
+  }
+  const { term2sample2value, byTermId, bySampleId } = await ds.queries[q.dataType].get(_q);
   if (term2sample2value.size == 0)
     throw "no data";
   if (term2sample2value.size == 1) {

package/routes/termdb.config.js

@@ -191,6 +191,10 @@ function addNonDictionaryQueries(c, ds, genome) {
     q2.rnaseqGeneCount = true;
   }
   if (q.singleCell) {
+    const plots = "plots" in q.singleCell.data ? q.singleCell.data.plots.map((p) => ({
+      name: p.name,
+      colorColumn: p.colorColumn.name
+    })) : [];
     q2.singleCell = {
       samples: {
         firstColumnName: q.singleCell.samples.firstColumnName,
@@ -199,7 +203,8 @@ function addNonDictionaryQueries(c, ds, genome) {
       },
       data: {
         sameLegend: q.singleCell.data.sameLegend,
-        refName: q.singleCell.data.refName
+        refName: q.singleCell.data.refName,
+        plots
       }
     };
     if (q.singleCell.geneExpression) {

package/routes/termdb.singlecellSamples.js

@@ -1,7 +1,7 @@
 import fs from "fs";
 import path from "path";
-import { spawn } from "child_process";
-import { read_file, get_header_txt } from "#src/utils.js";
+import { read_file } from "#src/utils.js";
+import run_R from "#src/run_R.js";
 import serverconfig from "#src/serverconfig.js";
 import { gdc_validate_query_singleCell_samples, gdc_validate_query_singleCell_data } from "#src/mds3.gdc.js";
 const api = {
@@ -104,19 +104,25 @@ function validateDataNative(D, ds) {
       if (ds.queries.singleCell.geneExpression && q.gene) {
         geneExpMap = await ds.queries.singleCell.geneExpression.get({ sample: q.sample, gene: q.gene });
       }
+      const file2Lines = {};
       for (const plot of D.plots) {
+        if (!q.plots.includes(plot.name))
+          continue;
         const tsvfile = path.join(serverconfig.tpmasterdir, plot.folder, q.sample + plot.fileSuffix);
-        try {
-          await fs.promises.stat(tsvfile);
-        } catch (e) {
-          if (e.code == "ENOENT") {
-            continue;
+        if (!file2Lines[tsvfile]) {
+          try {
+            await fs.promises.stat(tsvfile);
+          } catch (e) {
+            if (e.code == "ENOENT") {
+              continue;
+            }
+            if (e.code == "EACCES")
+              throw "cannot read file, permission denied";
+            throw "failed to load sc data file";
           }
-          if (e.code == "EACCES")
-            throw "cannot read file, permission denied";
-          throw "failed to load sc data file";
+          file2Lines[tsvfile] = (await read_file(tsvfile)).trim().split("\n");
         }
-        const lines = (await read_file(tsvfile)).trim().split("\n");
+        const lines = file2Lines[tsvfile];
         const cells = [];
         for (let i = 1; i < lines.length; i++) {
           const l = lines[i].split("	");
@@ -148,47 +154,25 @@ function validateDataNative(D, ds) {
   };
 }
 function validateGeneExpressionNative(G) {
+  G.sample2gene2expressionBins = {};
   G.get = async (q) => {
-    const tsvfile = path.join(serverconfig.tpmasterdir, G.folder, q.sample);
+    const rdsfile = path.join(serverconfig.tpmasterdir, G.folder, q.sample + ".rds");
+    try {
+      await fs.promises.stat(rdsfile);
+    } catch (e) {
+      return {};
+    }
+    let out;
     try {
-      await fs.promises.stat(tsvfile);
+      out = JSON.parse(
+        await run_R(path.join(serverconfig.binpath, "utils", "getGeneFromMatrix.R"), null, [rdsfile, q.gene])
+      );
     } catch (e) {
-      throw "geneExp matrix file not found or readable for this sample";
+      return {};
     }
-    const header = (await get_header_txt(tsvfile)).split("	");
-    if (header.length == 0)
-      throw "blank header line";
-    return await grepMatrix4geneExpression(tsvfile, q.gene, header);
+    return out;
   };
 }
-function grepMatrix4geneExpression(tsvfile, gene, header) {
-  return new Promise((resolve, reject) => {
-    const cp = spawn("grep", ["-m", "1", gene + "	", tsvfile]);
-    const out = [], err = [];
-    cp.stdout.on("data", (d) => out.push(d));
-    cp.stderr.on("data", (d) => err.push(d));
-    cp.on("close", () => {
-      const e = err.join("");
-      if (e)
-        reject(e);
-      const cell2value = {};
-      const line = out.join("").trim();
-      if (!line) {
-        resolve(cell2value);
-      }
-      const l = line.split("	");
-      if (l.length != header.length)
-        reject(`number of fields differ between data line and header: ${l.length} ${header.length}`);
-      for (let i = 1; i < l.length; i++) {
-        const v = Number(l[i]);
-        if (Number.isNaN(v))
-          continue;
-        cell2value[header[i]] = v;
-      }
-      resolve(cell2value);
-    });
-  });
-}
 export {
   api,
   validate_query_singleCell

package/routes/termdb.topVariablyExpressedGenes.js

@@ -1,9 +1,9 @@
-import { gdcGetCasesWithExpressionDataFromCohort } from "../src/mds3.gdc.js";
 import path from "path";
 import { run_rust } from "@sjcrh/proteinpaint-rust";
 import got from "got";
 import serverconfig from "#src/serverconfig.js";
 import { get_samples } from "#src/termdb.sql.js";
+import { makeFilter } from "#src/mds3.gdc.js";
 const api = {
   endpoint: "termdb/topVariablyExpressedGenes",
   methods: {
@@ -106,21 +106,20 @@ async function computeGenes4nativeDs(q, ds, matrixFile, samples) {
 function gdcValidateQuery(ds, genome) {
   ds.queries.topVariablyExpressedGenes.getGenes = async (q) => {
     if (serverconfig.features.gdcGenes) {
-      console.log("!!GDC!! using serverconfig.features.gdcGenes[]");
+      console.log(
+        "!!GDC!! using serverconfig.features.gdcGenes[] but not live api query. only use this on DEV and never on PROD!"
+      );
       return serverconfig.features.gdcGenes;
     }
-    if (!ds.__gdc.doneCaching)
+    if (!ds.__gdc.doneCaching) {
       throw "The server has not finished caching the case IDs: try again in about 2 minutes.";
-    const caseLst = await gdcGetCasesWithExpressionDataFromCohort(q, ds);
-    if (caseLst.length == 0) {
-      return [];
     }
     const { host, headers } = ds.getHostHeaders(q);
     const url = path.join(host.geneExp, "/gene_expression/gene_selection");
     try {
       const response = await got.post(url, {
         headers,
-        body: JSON.stringify(getGeneSelectionArg(q, caseLst))
+        body: JSON.stringify(getGeneSelectionArg(q))
       });
       const re = JSON.parse(response.body);
       const genes = [];
@@ -143,13 +142,14 @@ function gdcValidateQuery(ds, genome) {
       throw e;
     }
   };
-  function getGeneSelectionArg(q, caseLst) {
-    return {
-      case_ids: caseLst,
-      //.slice(0, 20),
-      gene_ids: tempGetCGCgenes(genome),
+  function getGeneSelectionArg(q) {
+    const arg = {
+      // add any to avoid tsc err
+      case_filters: makeFilter(q),
       selection_size: Number(q.maxGenes)
     };
+    arg.gene_ids = tempGetCGCgenes(genome);
+    return arg;
   }
 }
 function tempGetCGCgenes(genome) {