@sjcrh/proteinpaint-server 2.188.1 → 2.190.0

package/routes/termdb.DE.js

@@ -1,24 +1,15 @@
-import path from "path";
-import { diffExpPayload } from "#types/checkers";
 import { mayLog } from "#src/helpers.ts";
 import serverconfig from "../src/serverconfig.js";
-import { get_header_txt } from "#src/utils.js";
+import { run_R } from "@sjcrh/proteinpaint-r";
 import { run_rust } from "@sjcrh/proteinpaint-rust";
+import { formatElapsedTime } from "#shared";
 import { renderVolcano } from "../src/renderVolcano.ts";
-import { readCacheFileOrRecompute, resolveDaContext, resolveSampleGroups } from "../src/diffAnalysis.ts";
-const api = {
-  endpoint: "termdb/DE",
-  methods: {
-    get: {
-      ...diffExpPayload,
-      init
-    },
-    post: {
-      ...diffExpPayload,
-      init
-    }
-  }
-};
+import { cacheOrRecompute } from "#src/utils/cacheOrRecompute.ts";
+import {
+  buildGroupValues,
+  canonicalizeSamplelst,
+  resolveDaContext
+} from "#src/utils/sampleGroups.ts";
 function init({ genomes }) {
   return async (req, res) => {
     try {
@@ -37,57 +28,149 @@ function init({ genomes }) {
         });
         return;
       }
-      const result = await readCacheFileOrRecompute({ daRequest: q, genomes });
-      if (result.kind !== "DE") throw new Error("expected DE result from readCacheFileOrRecompute");
-      const rendered = await renderVolcano(result.geneData, q.volcanoRender);
-      rendered.cacheId = result.cacheId;
+      const { result, cacheId, images } = await loadDeForResponse(q, genomes);
+      const rendered = await renderVolcano(result.geneRows, q.volcanoRender);
+      rendered.cacheId = cacheId;
       const output = {
         data: rendered,
         sample_size1: result.sample_size1,
         sample_size2: result.sample_size2,
-        method: result.method,
-        images: result.images
+        method: result.method
       };
+      if (images.length) output.images = images;
       if (result.bcv != null) output.bcv = result.bcv;
       res.send(output);
     } catch (e) {
-      res.send({ status: "error", error: e.message || e });
+      res.status(e.status || 500).send({ status: "error", error: e.message || e, code: e.code });
       if (e instanceof Error && e.stack) console.log(e);
     }
   };
 }
-async function validate_query_rnaseqGeneCount(ds) {
-  const q = ds.queries.rnaseqGeneCount;
-  if (!q) return;
-  if (!q.file) throw new Error("unknown data type for rnaseqGeneCount");
-  q.file = path.join(serverconfig.tpmasterdir, q.file);
-  {
-    let samples = [];
-    if (ds.queries.rnaseqGeneCount.storage_type == "text") {
-      samples = (await get_header_txt(q.file, null)).split("	").slice(4);
-    } else if (ds.queries.rnaseqGeneCount.storage_type == "HDF5") {
-      const get_samples_from_hdf5 = {
-        hdf5_file: q.file,
-        validate: true
-      };
-      const time1 = (/* @__PURE__ */ new Date()).valueOf();
-      const result = await run_rust("readH5", JSON.stringify(get_samples_from_hdf5));
-      const time2 = (/* @__PURE__ */ new Date()).valueOf();
-      mayLog("Time taken to query gene expression:", time2 - time1, "ms");
-      const vr = JSON.parse(result);
-      if (vr.status !== "success") throw new Error(vr.message);
-      if (!Array.isArray(vr.samples)) throw new Error("HDF5 file has no samples, please check file.");
-      samples = vr.samples;
-    } else throw new Error("unknown storage type:" + ds.queries.rnaseqGeneCount.storage_type);
-    q.allSampleSet = new Set(samples);
-    const unknownSamples = [];
-    for (const n of q.allSampleSet) {
-      if (!ds.cohort.termdb.q.sampleName2id(n)) unknownSamples.push(n);
+function deKeyInputs(req) {
+  return {
+    genome: req.genome,
+    dslabel: req.dslabel,
+    samplelst: canonicalizeSamplelst(req.samplelst),
+    min_count: req.min_count,
+    min_total_count: req.min_total_count,
+    cpm_cutoff: req.cpm_cutoff,
+    method: req.method ?? null,
+    tw: req.tw ?? null,
+    tw2: req.tw2 ?? null,
+    filter: req.filter ?? null,
+    filter0: req.filter0 ?? null
+  };
+}
+async function loadDeForResponse(req, genomes) {
+  const { result, cacheId } = await getDeCacheResult(req, genomes);
+  const images = [];
+  if (result.qlImage) images.push(result.qlImage);
+  if (result.mdsImage) images.push(result.mdsImage);
+  return { result, cacheId, images };
+}
+async function getDeCacheResult(req, genomes) {
+  const { result, cacheId } = await cacheOrRecompute({
+    computeArgument: deKeyInputs(req),
+    cacheSubdir: "de",
+    computeFresh: async () => {
+      const { ds, term_results, term_results2 } = await resolveDaContext(req, genomes);
+      return runDeFresh(req, ds, term_results, term_results2);
     }
-    console.log(q.allSampleSet.size, `rnaseqGeneCount samples from ${ds.label}`);
+  });
+  return { result, cacheId };
+}
+async function runDeFresh(param, ds, term_results, term_results2) {
+  const groups = resolveSampleGroups(param, ds, term_results, term_results2);
+  if (groups.alerts.length) throw new Error(groups.alerts.join(" | "));
+  const q = ds.queries.rnaseqGeneCount;
+  const expression_input = {
+    case: groups.group2names.join(","),
+    control: groups.group1names.join(","),
+    data_type: "do_DE",
+    input_file: q.file,
+    cachedir: serverconfig.cachedir,
+    storage_type: q.storage_type,
+    DE_method: param.method,
+    mds_cutoff: 1e4,
+    min_count: param.min_count,
+    min_total_count: param.min_total_count,
+    cpm_cutoff: param.cpm_cutoff
+  };
+  if (param.tw) {
+    expression_input.conf1 = [...groups.conf1_group2, ...groups.conf1_group1];
+    expression_input.conf1_mode = param.tw.q.mode;
+    if (new Set(expression_input.conf1).size === 1) throw new Error("Confounding variable 1 has only one value");
+  }
+  if (param.tw2) {
+    expression_input.conf2 = [...groups.conf2_group2, ...groups.conf2_group1];
+    expression_input.conf2_mode = param.tw2.q.mode;
+    if (new Set(expression_input.conf2).size === 1) throw new Error("Confounding variable 2 has only one value");
   }
+  const small = groups.group1names.length <= 8 && groups.group2names.length <= 8;
+  const engine = small || param.method === "edgeR" || param.method === "limma" ? "edgeR" : "wilcoxon";
+  if (engine === "edgeR") {
+    const time12 = (/* @__PURE__ */ new Date()).valueOf();
+    const result2 = JSON.parse(await run_R("edge_newh5.R", JSON.stringify(expression_input)));
+    mayLog("Time taken to run edgeR:", formatElapsedTime(Date.now() - time12));
+    param.method = "edgeR";
+    const qlImage = deImageFromB64(result2.ql_image_b64, "ql_image");
+    const mdsImage = deImageFromB64(result2.mds_image_b64, "mds_image");
+    const cacheResult2 = {
+      geneRows: result2.gene_data,
+      sample_size1: result2.num_controls[0],
+      sample_size2: result2.num_cases[0],
+      method: param.method,
+      bcv: result2.bcv && result2.bcv[0] != null ? result2.bcv[0] : void 0,
+      ...qlImage ? { qlImage } : {},
+      ...mdsImage ? { mdsImage } : {}
+    };
+    return cacheResult2;
+  }
+  const time1 = (/* @__PURE__ */ new Date()).valueOf();
+  const result = JSON.parse(await run_rust("DEanalysis", JSON.stringify(expression_input)));
+  mayLog("Time taken to run rust DE pipeline:", formatElapsedTime(Date.now() - time1));
+  param.method = "wilcoxon";
+  const cacheResult = {
+    geneRows: result,
+    sample_size1: groups.group1names.length,
+    sample_size2: groups.group2names.length,
+    method: param.method
+  };
+  return cacheResult;
+}
+function resolveSampleGroups(param, ds, term_results, term_results2) {
+  if (param.samplelst?.groups?.length != 2) throw new Error(".samplelst.groups.length!=2");
+  if (param.samplelst.groups[0].values?.length < 1) throw new Error("samplelst.groups[0].values.length<1");
+  if (param.samplelst.groups[1].values?.length < 1) throw new Error("samplelst.groups[1].values.length<1");
+  const q = ds.queries.rnaseqGeneCount;
+  if (!q) throw new Error("rnaseqGeneCount query missing on ds");
+  if (!q.file) throw new Error("unknown data type for rnaseqGeneCount");
+  if (!q.storage_type) throw new Error("storage_type is not defined");
+  const g1 = buildGroupValues(param.samplelst.groups[0].values, q, ds, param.tw, param.tw2, term_results, term_results2);
+  const g2 = buildGroupValues(param.samplelst.groups[1].values, q, ds, param.tw, param.tw2, term_results, term_results2);
+  const alerts = [];
+  if (g1.names.length < 1) alerts.push("sample size of group1 < 1");
+  if (g2.names.length < 1) alerts.push("sample size of group2 < 1");
+  const commonnames = g1.names.filter((x) => g2.names.includes(x));
+  if (commonnames.length) alerts.push(`Common elements found between both groups: ${commonnames.join(", ")}`);
+  return {
+    group1names: g1.names,
+    group2names: g2.names,
+    conf1_group1: g1.conf1,
+    conf1_group2: g2.conf1,
+    conf2_group1: g1.conf2,
+    conf2_group2: g2.conf2,
+    alerts
+  };
+}
+function deImageFromB64(b64, key) {
+  if (!b64) return null;
+  const padding = b64.endsWith("==") ? 2 : b64.endsWith("=") ? 1 : 0;
+  const size = Math.floor(b64.length * 3 / 4) - padding;
+  return { src: `data:image/png;base64,${b64}`, size, key };
 }
 export {
-  api,
-  validate_query_rnaseqGeneCount
+  getDeCacheResult,
+  init,
+  resolveSampleGroups
 };

package/routes/termdb.categories.js

@@ -1,19 +1,5 @@
-import { termdbCategoriesPayload } from "#types/checkers";
 import { getOrderedLabels } from "#src/termdb.barchart.js";
 import { getData } from "#src/termdb.matrix.js";
-const api = {
-  endpoint: "termdb/categories",
-  methods: {
-    get: {
-      ...termdbCategoriesPayload,
-      init
-    },
-    post: {
-      ...termdbCategoriesPayload,
-      init
-    }
-  }
-};
 function init({ genomes }) {
   return async (req, res) => {
     const q = req.query;
@@ -132,6 +118,6 @@ function getCategories(data, q, ds, $id) {
   return [lst, orderedLabels];
 }
 export {
-  api,
-  getCategories
+  getCategories,
+  init
 };