@sjcrh/proteinpaint-server 2.65.0 → 2.67.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sjcrh/proteinpaint-server",
-  "version": "2.65.0",
+  "version": "2.67.0",
   "type": "module",
   "description": "a genomics visualization tool for exploring a cohort's genotype and phenotype data",
   "main": "src/app.js",
@@ -27,10 +27,10 @@
     "postpack": "./dedupjs.sh",
     "dedup": "./dedupjs.sh",
     "//todo": "refactor or deprecate the scripts below",
-    "pretest": "tsc --esModuleInterop genome/*.ts dataset/*.ts && ./test/pretest.js",
+    "pretest": "tsc && ./test/pretest.js",
     "prepare": "ts-patch install",
     "pretest:type": "npm run checkers",
-    "pretest:integration": "tsc --esModuleInterop genome/*.ts dataset/*.ts",
+    "pretest:integration": "tsc",
     "test:integration": "echo 'TODO: server integration tests'",
     "test:tsc": "tsc --esModuleInterop --noEmit --allowImportingTsExtensions ./shared/types/test/*.type.spec.ts"
   },

package/routes/termdb.categories.js

@@ -1,5 +1,6 @@
 import { getOrderedLabels } from "#src/termdb.barchart.js";
 import { getData } from "#src/termdb.matrix.js";
+import { TermTypes } from "#shared/terms.js";
 const api = {
   endpoint: "termdb/categories",
   methods: {
@@ -78,11 +79,10 @@ function init({ genomes }) {
   };
 }
 async function trigger_getcategories(q, res, tdb, ds, genome) {
-  const $id = Math.random().toString();
-  const tw = { $id, term: q.term, q: q.term1_q || getDefaultQ(q.term, q) };
+  const $id = q.tw.$id;
   const arg = {
     filter: q.filter,
-    terms: [tw],
+    terms: [q.tw],
     currentGeneNames: q.currentGeneNames,
     // optional, from mds3 mayAddGetCategoryArgs()
     rglst: q.rglst
@@ -92,7 +92,7 @@ async function trigger_getcategories(q, res, tdb, ds, genome) {
   if (data.error)
     throw data.error;
   const lst = [];
-  if (q.term.type == "geneVariant") {
+  if (q.tw.term.type == "geneVariant" && !q.tw.q.groupsetting.inuse) {
     const samples = data.samples;
     const dtClassMap = /* @__PURE__ */ new Map();
     if (ds.assayAvailability?.byDt) {
@@ -153,15 +153,15 @@ async function trigger_getcategories(q, res, tdb, ds, genome) {
       lst.push({
         samplecount: count,
         key,
-        label: data.refs?.byTermId?.[$id]?.events?.find((e) => e.event === key).label || q.term?.values?.[key]?.label || key
+        label: data.refs?.byTermId?.[$id]?.events?.find((e) => e.event === key).label || q.tw.term?.values?.[key]?.label || key
       });
     }
   }
   const orderedLabels = getOrderedLabels(
-    q.term,
+    q.tw.term,
     data.refs?.byTermId?.[$id]?.bins || [],
     data.refs?.byTermId?.[$id]?.events,
-    q.term1_q
+    q.tw.q
   );
   if (orderedLabels.length) {
     lst.sort((a, b) => orderedLabels.indexOf(a.label) - orderedLabels.indexOf(b.label));
@@ -195,6 +195,8 @@ function getDefaultQ(term, q) {
   }
   if (term.type == "geneVariant")
     return {};
+  if (term.type == TermTypes.SINGLECELL_CELLTYPE)
+    return {};
   throw "unknown term type";
 }
 export {

package/routes/termdb.cluster.js

@@ -53,8 +53,12 @@ function init({ genomes }) {
   };
 }
 async function getResult(q, ds) {
-  const type = q.dataType;
-  const { term2sample2value, byTermId, bySampleId } = await ds.queries[type].get(q);
+  let _q = q;
+  if (q.dataType == TermTypes.GENE_EXPRESSION) {
+    _q = JSON.parse(JSON.stringify(q));
+    _q.forClusteringAnalysis = true;
+  }
+  const { term2sample2value, byTermId, bySampleId } = await ds.queries[q.dataType].get(_q);
   if (term2sample2value.size == 0)
     throw "no data";
   if (term2sample2value.size == 1) {

package/routes/termdb.config.js

@@ -199,12 +199,18 @@ function addNonDictionaryQueries(c, ds, genome) {
       },
       data: {
         sameLegend: q.singleCell.data.sameLegend,
-        refName: q.singleCell.data.refName
+        refName: q.singleCell.data.refName,
+        plots: q.singleCell.data.plots.map((p) => {
+          return { name: p.name, colorColumn: p.colorColumn.name };
+        })
       }
     };
     if (q.singleCell.geneExpression) {
       q2.singleCell.geneExpression = {};
     }
+    if (q.singleCell.DEgenes) {
+      q2.singleCell.DEgenes = { columnName: q.singleCell.DEgenes.columnName };
+    }
   }
   if (q.images) {
     q2.images = {};

package/routes/termdb.singlecellDEgenes.js

@@ -0,0 +1,51 @@
+import { gdc_validate_query_singleCell_DEgenes } from "#src/mds3.gdc.js";
+const api = {
+  endpoint: "termdb/singlecellDEgenes",
+  methods: {
+    all: {
+      init,
+      request: {
+        typeId: "TermdbSinglecellDEgenesRequest"
+      },
+      response: {
+        typeId: "TermdbSinglecellDEgenesResponse"
+      }
+    }
+  }
+};
+function init({ genomes }) {
+  return async (req, res) => {
+    const q = req.query;
+    let result;
+    try {
+      const g = genomes[q.genome];
+      if (!g)
+        throw "invalid genome name";
+      const ds = g.datasets[q.dslabel];
+      if (!ds)
+        throw "invalid dataset name";
+      if (!ds.queries?.singleCell?.DEgenes)
+        throw "not supported on this dataset";
+      result = await ds.queries.singleCell.DEgenes.get(q);
+    } catch (e) {
+      if (e.stack)
+        console.log(e.stack);
+      result = {
+        status: e.status || 400,
+        error: e.message || e
+      };
+    }
+    res.send(result);
+  };
+}
+async function validate_query_singleCell_DEgenes(ds) {
+  if (ds.queries.singleCell.DEgenes.src == "gdcapi") {
+    gdc_validate_query_singleCell_DEgenes(ds);
+  } else {
+    throw "unknown singleCell.DEgenes.src";
+  }
+}
+export {
+  api,
+  validate_query_singleCell_DEgenes
+};

package/routes/termdb.singlecellSamples.js

@@ -1,13 +1,14 @@
 import fs from "fs";
 import path from "path";
-import { spawn } from "child_process";
-import { read_file, get_header_txt } from "#src/utils.js";
+import { read_file } from "#src/utils.js";
+import run_R from "#src/run_R.js";
 import serverconfig from "#src/serverconfig.js";
+import { validate_query_singleCell_DEgenes } from "./termdb.singlecellDEgenes.ts";
 import { gdc_validate_query_singleCell_samples, gdc_validate_query_singleCell_data } from "#src/mds3.gdc.js";
 const api = {
   endpoint: "termdb/singlecellSamples",
   methods: {
-    get: {
+    all: {
       init,
       request: {
         typeId: "TermdbSinglecellsamplesRequest"
@@ -15,10 +16,6 @@ const api = {
       response: {
         typeId: "TermdbSinglecellsamplesResponse"
       }
-    },
-    post: {
-      alternativeFor: "get",
-      init
     }
   }
 };
@@ -73,6 +70,9 @@ async function validate_query_singleCell(ds, genome) {
       throw "unknown singleCell.geneExpression.src";
     }
   }
+  if (q.DEgenes) {
+    validate_query_singleCell_DEgenes(ds);
+  }
 }
 async function validateSamplesNative(S, ds) {
   const samples = {};
@@ -104,19 +104,25 @@ function validateDataNative(D, ds) {
       if (ds.queries.singleCell.geneExpression && q.gene) {
         geneExpMap = await ds.queries.singleCell.geneExpression.get({ sample: q.sample, gene: q.gene });
       }
+      const file2Lines = {};
       for (const plot of D.plots) {
+        if (!q.plots.includes(plot.name))
+          continue;
         const tsvfile = path.join(serverconfig.tpmasterdir, plot.folder, q.sample + plot.fileSuffix);
-        try {
-          await fs.promises.stat(tsvfile);
-        } catch (e) {
-          if (e.code == "ENOENT") {
-            continue;
+        if (!file2Lines[tsvfile]) {
+          try {
+            await fs.promises.stat(tsvfile);
+          } catch (e) {
+            if (e.code == "ENOENT") {
+              continue;
+            }
+            if (e.code == "EACCES")
+              throw "cannot read file, permission denied";
+            throw "failed to load sc data file";
           }
-          if (e.code == "EACCES")
-            throw "cannot read file, permission denied";
-          throw "failed to load sc data file";
+          file2Lines[tsvfile] = (await read_file(tsvfile)).trim().split("\n");
         }
-        const lines = (await read_file(tsvfile)).trim().split("\n");
+        const lines = file2Lines[tsvfile];
         const cells = [];
         for (let i = 1; i < lines.length; i++) {
           const l = lines[i].split("	");
@@ -148,47 +154,25 @@ function validateDataNative(D, ds) {
   };
 }
 function validateGeneExpressionNative(G) {
+  G.sample2gene2expressionBins = {};
   G.get = async (q) => {
-    const tsvfile = path.join(serverconfig.tpmasterdir, G.folder, q.sample);
+    const rdsfile = path.join(serverconfig.tpmasterdir, G.folder, q.sample + ".rds");
+    try {
+      await fs.promises.stat(rdsfile);
+    } catch (e) {
+      return {};
+    }
+    let out;
     try {
-      await fs.promises.stat(tsvfile);
+      out = JSON.parse(
+        await run_R(path.join(serverconfig.binpath, "utils", "getGeneFromMatrix.R"), null, [rdsfile, q.gene])
+      );
     } catch (e) {
-      throw "geneExp matrix file not found or readable for this sample";
+      return {};
     }
-    const header = (await get_header_txt(tsvfile)).split("	");
-    if (header.length == 0)
-      throw "blank header line";
-    return await grepMatrix4geneExpression(tsvfile, q.gene, header);
+    return out;
   };
 }
-function grepMatrix4geneExpression(tsvfile, gene, header) {
-  return new Promise((resolve, reject) => {
-    const cp = spawn("grep", ["-m", "1", gene + "	", tsvfile]);
-    const out = [], err = [];
-    cp.stdout.on("data", (d) => out.push(d));
-    cp.stderr.on("data", (d) => err.push(d));
-    cp.on("close", () => {
-      const e = err.join("");
-      if (e)
-        reject(e);
-      const cell2value = {};
-      const line = out.join("").trim();
-      if (!line) {
-        resolve(cell2value);
-      }
-      const l = line.split("	");
-      if (l.length != header.length)
-        reject(`number of fields differ between data line and header: ${l.length} ${header.length}`);
-      for (let i = 1; i < l.length; i++) {
-        const v = Number(l[i]);
-        if (Number.isNaN(v))
-          continue;
-        cell2value[header[i]] = v;
-      }
-      resolve(cell2value);
-    });
-  });
-}
 export {
   api,
   validate_query_singleCell

package/routes/termdb.termsbyids.js

@@ -39,7 +39,9 @@ function init({ genomes }) {
   };
 }
 async function trigger_gettermsbyid(q, res, tdb) {
-  const terms = {};
+  const terms = {
+    terms: {}
+  };
   for (const id of q.ids) {
     const term = tdb.q.termjsonByOneid(id);
     if (term) {

package/routes/termdb.topVariablyExpressedGenes.js

@@ -1,9 +1,9 @@
-import { gdcGetCasesWithExpressionDataFromCohort } from "../src/mds3.gdc.js";
 import path from "path";
 import { run_rust } from "@sjcrh/proteinpaint-rust";
 import got from "got";
 import serverconfig from "#src/serverconfig.js";
 import { get_samples } from "#src/termdb.sql.js";
+import { makeFilter } from "#src/mds3.gdc.js";
 const api = {
   endpoint: "termdb/topVariablyExpressedGenes",
   methods: {
@@ -106,21 +106,20 @@ async function computeGenes4nativeDs(q, ds, matrixFile, samples) {
 function gdcValidateQuery(ds, genome) {
   ds.queries.topVariablyExpressedGenes.getGenes = async (q) => {
     if (serverconfig.features.gdcGenes) {
-      console.log("!!GDC!! using serverconfig.features.gdcGenes[]");
+      console.log(
+        "!!GDC!! using serverconfig.features.gdcGenes[] but not live api query. only use this on DEV and never on PROD!"
+      );
       return serverconfig.features.gdcGenes;
     }
-    if (!ds.__gdc.doneCaching)
+    if (!ds.__gdc.doneCaching) {
       throw "The server has not finished caching the case IDs: try again in about 2 minutes.";
-    const caseLst = await gdcGetCasesWithExpressionDataFromCohort(q, ds);
-    if (caseLst.length == 0) {
-      return [];
     }
     const { host, headers } = ds.getHostHeaders(q);
     const url = path.join(host.geneExp, "/gene_expression/gene_selection");
     try {
       const response = await got.post(url, {
         headers,
-        body: JSON.stringify(getGeneSelectionArg(q, caseLst))
+        body: JSON.stringify(getGeneSelectionArg(q))
       });
       const re = JSON.parse(response.body);
       const genes = [];
@@ -143,13 +142,14 @@ function gdcValidateQuery(ds, genome) {
       throw e;
     }
   };
-  function getGeneSelectionArg(q, caseLst) {
-    return {
-      case_ids: caseLst,
-      //.slice(0, 20),
-      gene_ids: tempGetCGCgenes(genome),
+  function getGeneSelectionArg(q) {
+    const arg = {
+      // add any to avoid tsc err
+      case_filters: makeFilter(q),
       selection_size: Number(q.maxGenes)
     };
+    arg.gene_ids = tempGetCGCgenes(genome);
+    return arg;
   }
 }
 function tempGetCGCgenes(genome) {