@sjcrh/proteinpaint-server 2.106.0 → 2.108.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sjcrh/proteinpaint-server",
-  "version": "2.106.0",
+  "version": "2.108.0",
   "type": "module",
   "description": "a genomics visualization tool for exploring a cohort's genotype and phenotype data",
   "main": "src/app.js",
@@ -60,8 +60,8 @@
   "dependencies": {
     "@sjcrh/augen": "2.87.0",
     "@sjcrh/proteinpaint-rust": "2.99.0",
-    "@sjcrh/proteinpaint-shared": "2.106.0",
-    "@sjcrh/proteinpaint-types": "2.106.0",
+    "@sjcrh/proteinpaint-shared": "2.108.0",
+    "@sjcrh/proteinpaint-types": "2.108.0",
     "@types/express": "^5.0.0",
     "@types/express-session": "^1.18.1",
     "better-sqlite3": "^9.4.1",

package/routes/burden.js

@@ -15,6 +15,7 @@ const api = {
     }
   }
 };
+const MAXBOOTNUM = 20;
 function init({ genomes }) {
   return async function handler(req, res) {
     try {
@@ -27,72 +28,137 @@ function init({ genomes }) {
         throw `invalid q.genome=${req.query.dslabel}`;
       if (!ds.cohort.cumburden?.files)
         throw `missing ds.cohort.cumburden.files`;
-      const estimates = await getBurdenEstimates(req, ds);
-      const { keys, rows } = formatPayload(estimates);
-      res.send({ status: "ok", keys, rows });
+      if (!ds.cohort?.cumburden?.db)
+        throw `missing ds.cohort.cumburden.db`;
+      if (!ds.cohort?.cumburden?.bootsubdir)
+        throw `missing ds.cohort.cumburden.bootsubdir`;
+      const result = await getBurdenResult(q, ds.cohort.cumburden);
+      if (!q.showCI) {
+        res.send({
+          status: "ok",
+          /*estimate: result.estimate,*/
+          ...formatPayload(result.estimate)
+        });
+      } else {
+        if (!result.ci95)
+          await compute95ci(result, ds.cohort.cumburden);
+        res.send({
+          status: "ok",
+          /*ci95: result.ci95,*/
+          ...formatPayload(result.ci95)
+        });
+      }
     } catch (e) {
       res.send({ status: "error", error: e.message || e });
     }
   };
 }
-async function getBurdenEstimates(q, ds) {
-  for (const k in q.query) {
-    q.query[k] = Number(q.query[k]);
+async function getBurdenResult(q, cumburden) {
+  const { id, jsonInput } = normalizeInput(q, cumburden);
+  let result = cumburden.db.connection.prepare("SELECT * FROM estimates WHERE id=?").get(id);
+  if (!result) {
+    result = { id, status: null, input: jsonInput };
+    const estJson = await run_R(path.join(serverconfig.binpath, "utils", "burden-main.R"), jsonInput, []);
+    const estimate = JSON.parse(estJson);
+    const ages = Object.keys(estimate[0]).filter((k) => k.startsWith("["));
+    const overall = { chc: 0 };
+    for (const age of ages) {
+      overall[age] = [0];
+      for (const est of estimate)
+        overall[age][0] += est[age];
+    }
+    estimate.push(overall);
+    const burden = {};
+    for (const est of estimate) {
+      burden[est.chc] = est;
+    }
+    cumburden.db.connection.prepare("INSERT INTO estimates (id, input, status, estimate) VALUES (?, ?, ?, ?)").run([result.id, jsonInput, 0, JSON.stringify(burden)]);
+    result.status = 0;
+    result.estimate = burden;
   }
-  const data = Object.assign({}, defaults, q.query);
-  const { fit, surv, sample } = ds.cohort.cumburden.files;
-  if (!fit || !surv || !sample)
-    throw `missing one or more of ds.cohort.burden.files.{fit, surv, sample}`;
-  const args = [
-    `${serverconfig.tpmasterdir}/${fit}`,
-    `${serverconfig.tpmasterdir}/${surv}`,
-    `${serverconfig.tpmasterdir}/${sample}`
-  ];
-  const estimates = JSON.parse(
-    await run_R(path.join(serverconfig.binpath, "utils", "burden.R"), JSON.stringify(data), args)
-  );
-  return estimates;
+  for (const [k, v] of Object.entries(result)) {
+    if (k !== "id" && typeof v == "string")
+      result[k] = JSON.parse(v);
+  }
+  return result;
 }
-function formatPayload(estimates) {
-  const rawKeys = Object.keys(estimates[0]);
-  const outKeys = [];
-  const keys = [];
-  for (const k of rawKeys) {
-    if (k == "chc") {
-      keys.push(k);
-      outKeys.push(k);
-    } else {
-      const age = Number(k.slice(1).split(",")[0]);
-      if (age <= 60 && age % 2 == 0) {
-        keys.push(k);
-        outKeys.push(`burden${age}`);
+function normalizeInput(q, cumburden) {
+  const keys = Object.keys(q).filter((k) => k in defaultInputValues).sort();
+  const id = keys.map((k) => q[k]).join("-");
+  const normalized = {};
+  for (const k of keys)
+    normalized[k] = q[k];
+  normalized.datafiles = {
+    dir: path.join(serverconfig.tpmasterdir, cumburden.dir),
+    files: cumburden.files,
+    boosubdir: cumburden.bootsubdir
+  };
+  normalized.binpath = serverconfig.binpath;
+  const jsonInput = JSON.stringify(normalized);
+  return { id, jsonInput };
+}
+async function compute95ci(result, cumburden) {
+  try {
+    if (!result.input)
+      throw "result{} does not have .input";
+    const input = structuredClone(result.input);
+    input.burden = Object.values(result.estimate).filter((est) => est.chc !== 0);
+    const lowup = await run_R(path.join(serverconfig.binpath, "utils", "burden-ci95.R"), JSON.stringify(input), []);
+    const { low, up, overall } = JSON.parse(lowup);
+    const ci95 = { 0: {} };
+    for (const est of Object.values(result.estimate)) {
+      if (!ci95[est.chc])
+        ci95[est.chc] = {};
+      const lower = low.find((l) => l.chc === est.chc);
+      const upper = up.find((u) => u.chc === est.chc);
+      for (const [age, val] of Object.entries(est)) {
+        if (!age.startsWith("["))
+          continue;
+        const burden = est.chc === 0 ? overall[0][age] : val;
+        ci95[est.chc][age] = [burden, lower[age], upper[age]];
       }
     }
+    result.ci95 = ci95;
+  } catch (e) {
+    console.log(e);
   }
+  await cumburden.db.connection.prepare(`UPDATE estimates SET ci95=? WHERE id=?`).run(JSON.stringify(result.ci95), result.id);
+}
+function sortNumericValue(a, b) {
+  return a < b ? -1 : 1;
+}
+function formatPayload(estimates) {
+  const rawKeys = Object.keys(estimates["1"]).filter((k) => k.startsWith("["));
+  const renamedKeys = rawKeys.map((k) => `burden${k.split(",")[0].slice(1)}`);
+  const outKeys = ["chc", ...renamedKeys];
   const rows = [];
-  for (const v of estimates) {
-    rows.push(keys.map((k) => v[k]));
+  for (const [chc, burdenByAge] of Object.entries(estimates)) {
+    const arr = [chc];
+    for (const age of rawKeys)
+      arr.push(Array.isArray(burdenByAge[age]) ? burdenByAge[age] : [burdenByAge[age]]);
+    rows.push(arr);
   }
   return { keys: outKeys, rows };
 }
-const defaults = Object.freeze({
+const defaultInputValues = Object.freeze({
+  // showCI: false, do not track so it's not computed as part of unique ID
   diaggrp: 5,
-  sex: 0,
+  sex: 1,
   white: 1,
-  agedx: 1,
+  agedx: 6,
   // chemotherapy
   steriod: 0,
   bleo: 0,
-  vcr: 0,
-  //12, // Vincristine
-  etop: 0,
-  //2500, // Etoposide
+  vcr: 12,
+  // Vincristine
+  etop: 2500,
+  // Etoposide
   itmt: 0,
   // Intrathecal methothrexate_grp: 0,
-  ced: 0,
-  //1.6, // Cyclophosphamide, 0.7692 mean 7692.
-  cisp: 0,
-  //300, // Cisplatin
+  ced: 1.6,
+  // Cyclophosphamide, 0.7692 mean 7692.
+  cisp: 300,
+  // Cisplatin
   dox: 0,
   // Anthracycline, 3 mean 300 ml/m2
   carbo: 0,
@@ -100,14 +166,11 @@ const defaults = Object.freeze({
   hdmtx: 0,
   // High-Dose Methotrexate
   // radiation
-  brain: 0,
-  //5.4,
-  chest: 0,
-  //2.4,
+  brain: 5.4,
+  chest: 2.4,
   heart: 0,
   pelvis: 0,
-  abd: 0
-  //2.4
+  abd: 2.4
 });
 export {
   api

package/routes/correlationVolcano.js

@@ -3,7 +3,10 @@ import { getData } from "../src/termdb.matrix.js";
 import run_R from "../src/run_R.js";
 import serverconfig from "../src/serverconfig.js";
 import { mayLog } from "#src/helpers.ts";
+import { stdDev } from "#shared/violin.bins.js";
 import path from "path";
+const minArrayLength = 3;
+const minSD = 0.05;
 const api = {
   endpoint: "termdb/correlationVolcano",
   methods: {
@@ -65,16 +68,31 @@ async function compute(q, ds, genome) {
       vtid2array.get(tw.$id).v2.push(variableValue);
     }
   }
+  const [acceptedVariables, skippedVariables] = Array.from(vtid2array.values()).reduce(
+    ([accepted, skipped], t) => {
+      const grterThanOne = t.v1.length > minArrayLength && t.v2.length > minArrayLength;
+      const significantSD = stdDev(t.v1) > minSD && stdDev(t.v2) > minSD;
+      const v = grterThanOne && significantSD ? accepted : skipped;
+      if (v === accepted)
+        accepted.push(t);
+      if (v === skipped)
+        skipped.push({ tw$id: t.id });
+      return [accepted, skipped];
+    },
+    [[], []]
+  );
+  const result = { skippedVariables, variableItems: [] };
+  if (!acceptedVariables.length)
+    return result;
   const input = {
     method: q.correlationMethod || "pearson",
-    terms: [...vtid2array.values()]
+    terms: acceptedVariables
   };
   const time1 = Date.now();
   const output = {
     terms: JSON.parse(await run_R(path.join(serverconfig.binpath, "utils", "corr.R"), JSON.stringify(input)))
   };
   mayLog("Time taken to run correlation analysis:", Date.now() - time1);
-  const result = { variableItems: [] };
   for (const t of output.terms) {
     const t2 = {
       tw$id: t.id,

package/routes/termdb.DE.js

@@ -58,12 +58,14 @@ async function run_DE(param, ds, term_results) {
     throw "samplelst.groups[0].values.length<1";
   if (param.samplelst.groups[1].values?.length < 1)
     throw "samplelst.groups[1].values.length<1";
-  param.storage_type = ds.queries.rnaseqGeneCount.storage_type;
   const q = ds.queries.rnaseqGeneCount;
   if (!q)
     return;
   if (!q.file)
     throw "unknown data type for rnaseqGeneCount";
+  if (!q.storage_type)
+    throw "storage_type is not defined";
+  param.storage_type = q.storage_type;
   const group1names = [];
   const conf1_group1 = [];
   for (const s of param.samplelst.groups[0].values) {
@@ -73,9 +75,17 @@ async function run_DE(param, ds, term_results) {
     if (!n)
       continue;
     if (q.allSampleSet.has(n)) {
-      group1names.push(n);
       if (param.tw) {
-        conf1_group1.push(term_results.samples[s.sampleId][param.tw.$id]["value"]);
+        if (term_results.samples[s.sampleId]) {
+          if (param.tw.q.mode == "continuous") {
+            conf1_group1.push(term_results.samples[s.sampleId][param.tw.$id]["value"]);
+          } else {
+            conf1_group1.push(term_results.samples[s.sampleId][param.tw.$id]["key"]);
+          }
+          group1names.push(n);
+        }
+      } else {
+        group1names.push(n);
       }
     } else {
     }
@@ -89,9 +99,17 @@ async function run_DE(param, ds, term_results) {
     if (!n)
       continue;
     if (q.allSampleSet.has(n)) {
-      group2names.push(n);
       if (param.tw) {
-        conf1_group2.push(term_results.samples[s.sampleId][param.tw.$id]["value"]);
+        if (term_results.samples[s.sampleId]) {
+          if (param.tw.q.mode == "continuous") {
+            conf1_group2.push(term_results.samples[s.sampleId][param.tw.$id]["value"]);
+          } else {
+            conf1_group2.push(term_results.samples[s.sampleId][param.tw.$id]["key"]);
+          }
+          group2names.push(n);
+        }
+      } else {
+        group2names.push(n);
       }
     } else {
     }
@@ -115,7 +133,10 @@ async function run_DE(param, ds, term_results) {
   };
   if (param.tw) {
     expression_input.conf1 = [...conf1_group2, ...conf1_group1];
-    expression_input.conf1_type = param.tw.term.type;
+    expression_input.conf1_mode = param.tw.q.mode;
+    if (new Set(expression_input.conf1).size === 1) {
+      throw "Confounding variable has only one value";
+    }
   }
   const sample_size_limit = 8;
   let result;

package/routes/termdb.cluster.js

@@ -9,6 +9,7 @@ import { clusterMethodLst, distanceMethodLst } from "#shared/clustering.js";
 import { getResult as getResultGene } from "#src/gene.js";
 import { TermTypes, NUMERIC_DICTIONARY_TERM } from "#shared/terms.js";
 import { getData } from "#src/termdb.matrix.js";
+import { termType2label } from "#shared/terms.js";
 const api = {
   endpoint: "termdb/cluster",
   methods: {
@@ -106,14 +107,24 @@ async function getNumericDictTermAnnotation(q, ds, genome) {
 }
 async function doClustering(data, q, numCases = 1e3) {
   const sampleSet = /* @__PURE__ */ new Set();
+  let firstTerm = true;
   for (const o of data.values()) {
-    for (const s in o)
-      sampleSet.add(s);
-    if (sampleSet.size >= numCases)
-      break;
+    const currentSampleIds = new Set(Object.keys(o));
+    if (firstTerm) {
+      currentSampleIds.forEach((id) => sampleSet.add(id));
+      firstTerm = false;
+    } else {
+      for (const id of sampleSet) {
+        if (!currentSampleIds.has(id)) {
+          sampleSet.delete(id);
+        }
+      }
+    }
   }
   if (sampleSet.size == 0)
-    throw "termdb.cluster: no samples";
+    throw `termdb.cluster: There are no overlapping tested samples shared across the selected ${termType2label(
+      q.dataType
+    )}`;
   if (!clusterMethodLst.find((i) => i.value == q.clusterMethod))
     throw "Invalid cluster method";
   if (!distanceMethodLst.find((i) => i.value == q.distanceMethod))
@@ -122,7 +133,7 @@ async function doClustering(data, q, numCases = 1e3) {
     matrix: [],
     row_names: [],
     // genes
-    col_names: [...sampleSet],
+    col_names: [...sampleSet].slice(0, numCases),
     // samples
     cluster_method: q.clusterMethod,
     distance_method: q.distanceMethod,
@@ -133,7 +144,7 @@ async function doClustering(data, q, numCases = 1e3) {
     inputData.row_names.push(gene);
     const row = [];
     for (const s of inputData.col_names) {
-      row.push(o[s] || 0);
+      row.push(o[s]);
     }
     inputData.matrix.push(q.zScoreTransformation ? getZscore(row) : row);
   }