@sjcrh/proteinpaint-server 2.185.0 → 2.187.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sjcrh/proteinpaint-server",
-  "version": "2.185.0",
+  "version": "2.187.0",
   "type": "module",
   "description": "a genomics visualization tool for exploring a cohort's genotype and phenotype data",
   "main": "src/app.js",
@@ -62,11 +62,11 @@
   },
   "dependencies": {
     "@sjcrh/augen": "2.181.1",
-    "@sjcrh/proteinpaint-python": "2.185.0",
+    "@sjcrh/proteinpaint-python": "2.187.0",
     "@sjcrh/proteinpaint-r": "2.181.0",
-    "@sjcrh/proteinpaint-rust": "2.185.0",
-    "@sjcrh/proteinpaint-shared": "2.185.0",
-    "@sjcrh/proteinpaint-types": "2.185.0",
+    "@sjcrh/proteinpaint-rust": "2.186.0",
+    "@sjcrh/proteinpaint-shared": "2.187.0",
+    "@sjcrh/proteinpaint-types": "2.187.0",
     "@types/express": "^5.0.0",
     "@types/express-session": "^1.18.1",
     "better-sqlite3": "^12.4.1",

package/routes/genesetEnrichment.js

@@ -31,6 +31,10 @@ function init({ genomes }) {
         res.send(results);
         return;
       }
+      if (typeof results === "object" && results.error) {
+        res.send({ status: "error", error: results.error });
+        return;
+      }
       if (typeof results != "string") throw new Error("gsea result is not string");
       res.sendFile(results, (err) => {
         fs.unlink(results, () => {
@@ -77,12 +81,24 @@ async function resolveGseaGenesAndFoldChange({
 }) {
   if (q.cacheId) {
     if (!q.daRequest) throw new Error("daCacheMissing");
-    const result = await readCacheFileOrRecompute({ daRequest: q.daRequest, genomes });
+    const result = await readCacheFileOrRecompute({
+      daRequest: q.daRequest,
+      genomes
+    });
     if (result.cacheId !== q.cacheId) throw new Error("cacheId does not match daRequest");
-    return {
-      genes: result.geneData.map((g) => g.gene_name),
-      fold_change: result.geneData.map((g) => g.fold_change)
-    };
+    if (result.kind === "DE") {
+      return {
+        genes: result.geneData.map((g) => g.gene_name),
+        fold_change: result.geneData.map((g) => g.fold_change)
+      };
+    }
+    if (result.kind === "DM") {
+      return {
+        genes: result.promoterData.map((p) => p.gene_name),
+        fold_change: result.promoterData.map((p) => p.fold_change)
+      };
+    }
+    throw new Error(`unexpected result kind: ${result.kind}`);
   }
   if (!q.genes || !q.fold_change) throw new Error("requires genes and fold_change when cacheId is absent");
   return { genes: q.genes, fold_change: q.fold_change };

package/routes/profile.forms2.js

@@ -0,0 +1,107 @@
+import { ProfileForms2ScoresPayload } from "#types/checkers";
+import { getData } from "../src/termdb.matrix.js";
+const api = {
+  endpoint: "termdb/profileForms2Scores",
+  methods: {
+    get: {
+      ...ProfileForms2ScoresPayload,
+      init
+    },
+    post: {
+      ...ProfileForms2ScoresPayload,
+      init
+    }
+  }
+};
+function init({ genomes }) {
+  return async (req, res) => {
+    try {
+      const g = genomes[req.query.genome];
+      if (!g) throw "invalid genome name";
+      const ds = g.datasets?.[req.query.dslabel];
+      const result = await getScores(req.query, ds);
+      res.send(result);
+    } catch (e) {
+      console.log(e);
+      res.send({ status: "error", error: e.message || e });
+    }
+  };
+}
+async function getScores(query, ds) {
+  const { activeCohort, clientAuthResult } = query.__protected__;
+  const facilityTermId = ds.cohort.termdb.plotConfigByCohort?.[activeCohort]?.profileForms2?.facilityTW?.id;
+  if (!facilityTermId) throw `profileForms2.facilityTW.id missing for cohort '${activeCohort}'`;
+  const facilityTW = { term: { id: facilityTermId }, q: {} };
+  const terms = [facilityTW, ...query.scoreTerms];
+  if (query.scScoreTerms) terms.push(...query.scScoreTerms);
+  if (!query.filterByUserSites) query.__protected__.ignoredTermIds.push(facilityTermId);
+  const cohortAuth = clientAuthResult[activeCohort];
+  const isPublic = !cohortAuth?.role || cohortAuth.role === "public";
+  const userSites = cohortAuth?.sites;
+  const raw = await getData(
+    {
+      terms,
+      filter: query.filter,
+      __protected__: query.__protected__
+    },
+    ds
+  );
+  if (raw.error) throw raw.error;
+  const samples = Object.values(raw.samples);
+  let sites = samples.map((s) => {
+    const val = s[facilityTW.$id].value;
+    let label = facilityTW.term.values?.[val]?.label || val;
+    if (label.length > 50) label = label.slice(0, 47) + "...";
+    return { value: val, label };
+  });
+  if (userSites && query.filterByUserSites) sites = sites.filter((s) => userSites.includes(s.value));
+  sites.sort((a, b) => a.label.localeCompare(b.label));
+  const eligibleSamples = userSites && query.filterByUserSites ? samples.filter((s) => userSites.includes(s[facilityTW.$id].value)) : samples;
+  const term2Score = {};
+  for (const d of query.scoreTerms) {
+    term2Score[d.term.id] = getPercentsDict((sample) => getDict(d.$id, sample), eligibleSamples);
+  }
+  if (query.scScoreTerms) {
+    for (const d of query.scScoreTerms) {
+      term2Score[d.term.id] = getSCPercentsDict(d, eligibleSamples);
+    }
+  }
+  return {
+    term2Score,
+    sites: isPublic ? [] : sites,
+    // public never sees site IDs
+    n: eligibleSamples.length
+  };
+}
+function getDict(key, sample) {
+  if (!sample[key]) return null;
+  const termData = sample[key].value;
+  return JSON.parse(termData);
+}
+function getPercentsDict(getDictFunc, samples) {
+  const percentageDict = {};
+  for (const sample of samples) {
+    const percents = getDictFunc(sample);
+    if (!percents) continue;
+    for (const key in percents) {
+      const value = percents[key];
+      if (!percentageDict[key]) percentageDict[key] = 0;
+      percentageDict[key] += value;
+    }
+  }
+  return percentageDict;
+}
+function getSCPercentsDict(tw, samples) {
+  if (!tw) throw "tw not defined";
+  const percentageDict = {};
+  for (const sample of samples) {
+    const key = sample[tw.$id]?.value;
+    if (key == null) continue;
+    if (!percentageDict[key]) percentageDict[key] = 0;
+    percentageDict[key] += 1;
+  }
+  return percentageDict;
+}
+export {
+  api
+};

package/routes/termdb.DE.js

@@ -5,7 +5,7 @@ import serverconfig from "../src/serverconfig.js";
 import { get_header_txt } from "#src/utils.js";
 import { run_rust } from "@sjcrh/proteinpaint-rust";
 import { renderVolcano } from "../src/renderVolcano.ts";
-import { readCacheFileOrRecompute, resolveDeContext, resolveSampleGroups } from "../src/diffAnalysis.ts";
+import { readCacheFileOrRecompute, resolveDaContext, resolveSampleGroups } from "../src/diffAnalysis.ts";
 const api = {
   endpoint: "termdb/DE",
   methods: {
@@ -24,7 +24,7 @@ function init({ genomes }) {
     try {
       const q = req.query;
       if (q.preAnalysis) {
-        const { ds, term_results, term_results2 } = await resolveDeContext(q, genomes);
+        const { ds, term_results, term_results2 } = await resolveDaContext(q, genomes);
         const groups = resolveSampleGroups(q, ds, term_results, term_results2);
         const group1Name = q.samplelst.groups[0].name;
         const group2Name = q.samplelst.groups[1].name;
@@ -37,20 +37,18 @@ function init({ genomes }) {
         });
         return;
       }
-      const { cacheId, geneData, sample_size1, sample_size2, method, images, bcv } = await readCacheFileOrRecompute({
-        daRequest: q,
-        genomes
-      });
-      const rendered = await renderVolcano(geneData, q.volcanoRender);
-      rendered.cacheId = cacheId;
+      const result = await readCacheFileOrRecompute({ daRequest: q, genomes });
+      if (result.kind !== "DE") throw new Error("expected DE result from readCacheFileOrRecompute");
+      const rendered = await renderVolcano(result.geneData, q.volcanoRender);
+      rendered.cacheId = result.cacheId;
       const output = {
         data: rendered,
-        sample_size1,
-        sample_size2,
-        method,
-        images
+        sample_size1: result.sample_size1,
+        sample_size2: result.sample_size2,
+        method: result.method,
+        images: result.images
       };
-      if (bcv != null) output.bcv = bcv;
+      if (result.bcv != null) output.bcv = result.bcv;
       res.send(output);
     } catch (e) {
       res.send({ status: "error", error: e.message || e });

package/routes/termdb.cluster.js

@@ -261,18 +261,24 @@ async function validateNative(q, ds) {
     const vr = JSON.parse(tmp);
     if (vr.status !== "success") throw vr.message;
     if (!vr.samples?.length) throw "HDF5 file has no samples, please check file.";
+    const unknownSamples = /* @__PURE__ */ new Set();
     for (const sn of vr.samples) {
       const si = ds.cohort.termdb.q.sampleName2id(sn);
-      if (si == void 0) {
+      if (si === void 0) {
         if (ds.cohort.db) {
           throw `unknown sample ${sn} from HDF5 ${q.file}`;
         } else {
+          unknownSamples.add(sn);
           continue;
         }
       }
       q.samples.push(si);
     }
     console.log(`${ds.label}: geneExpression HDF5 file validated. Format: ${vr.format}, Samples:`, q.samples.length);
+    if (unknownSamples.size) {
+      const arr = [...unknownSamples];
+      console.log(`unknown samples from geneExpression HDF5 file (${arr.length}): ${arr.join(", ")}`);
+    }
   } catch (error) {
     throw `${ds.label}: Failed to validate geneExpression HDF5 file: ${error}`;
   }

package/routes/termdb.config.js

@@ -195,23 +195,37 @@ function addNonDictionaryQueries(c, ds, genome) {
   }
   if (q.proteome) {
     q2.proteome = {};
-    if (q.proteome.overlayTerm) {
-      q2.proteome.overlayTerm = JSON.parse(JSON.stringify(q.proteome.overlayTerm));
-    }
-    if (q.proteome.assays) {
-      q2.proteome.assays = {};
-      for (const assay in q.proteome.assays) {
-        q2.proteome.assays[assay] = {};
-        if (q.proteome.assays[assay].cohorts) {
-          q2.proteome.assays[assay].cohorts = {};
-          for (const cohort in q.proteome.assays[assay].cohorts) {
-            q2.proteome.assays[assay].cohorts[cohort] = {};
-            const src = q.proteome.assays[assay].cohorts[cohort];
-            if ("controlFilter" in src) {
-              q2.proteome.assays[assay].cohorts[cohort].controlFilter = JSON.parse(JSON.stringify(src.controlFilter));
-            }
-            if ("caseFilter" in src) {
-              q2.proteome.assays[assay].cohorts[cohort].caseFilter = JSON.parse(JSON.stringify(src.caseFilter));
+    if (q.proteome.organisms) {
+      q2.proteome.organisms = {};
+      for (const organism in q.proteome.organisms) {
+        q2.proteome.organisms[organism] = {};
+        const orgSrc = q.proteome.organisms[organism];
+        if (orgSrc.overlayTerm) {
+          q2.proteome.organisms[organism].overlayTerm = JSON.parse(JSON.stringify(orgSrc.overlayTerm));
+        }
+        if (orgSrc.genomeName) {
+          q2.proteome.organisms[organism].genomeName = orgSrc.genomeName;
+        }
+        if (orgSrc.assays) {
+          q2.proteome.organisms[organism].assays = {};
+          for (const assay in orgSrc.assays) {
+            q2.proteome.organisms[organism].assays[assay] = {};
+            if (orgSrc.assays[assay].cohorts) {
+              q2.proteome.organisms[organism].assays[assay].cohorts = {};
+              for (const cohort in orgSrc.assays[assay].cohorts) {
+                q2.proteome.organisms[organism].assays[assay].cohorts[cohort] = {};
+                const src = orgSrc.assays[assay].cohorts[cohort];
+                if ("controlFilter" in src) {
+                  q2.proteome.organisms[organism].assays[assay].cohorts[cohort].controlFilter = JSON.parse(
+                    JSON.stringify(src.controlFilter)
+                  );
+                }
+                if ("caseFilter" in src) {
+                  q2.proteome.organisms[organism].assays[assay].cohorts[cohort].caseFilter = JSON.parse(
+                    JSON.stringify(src.caseFilter)
+                  );
+                }
+              }
             }
           }
         }
@@ -305,7 +319,7 @@ function getDsAllowedTermTypes(ds) {
   if (ds.queries?.geneExpression) typeSet.add(GENE_EXPRESSION);
   if (ds.queries?.isoformExpression) typeSet.add(ISOFORM_EXPRESSION);
   if (ds.queries?.metaboliteIntensity) typeSet.add(METABOLITE_INTENSITY);
-  if (ds.queries?.proteome?.assays) typeSet.add(PROTEOME_ABUNDANCE);
+  if (ds.queries?.proteome) typeSet.add(PROTEOME_ABUNDANCE);
   if (ds.queries?.ssGSEA) typeSet.add(SSGSEA);
   if (ds.queries?.dnaMethylation) typeSet.add(DNA_METHYLATION);
   if (ds.queries?.singleCell) {

package/routes/termdb.diffMeth.js

@@ -1,9 +1,6 @@
 import { diffMethPayload } from "#types/checkers";
-import { getData } from "../src/termdb.matrix.js";
-import { run_R } from "@sjcrh/proteinpaint-r";
-import { mayLog } from "#src/helpers.ts";
-import { formatElapsedTime } from "#shared";
 import { renderVolcano } from "../src/renderVolcano.ts";
+import { readCacheFileOrRecompute, resolveDaContext, resolveDmSampleGroups } from "../src/diffAnalysis.ts";
 const api = {
   endpoint: "termdb/diffMeth",
   methods: {
@@ -21,173 +18,38 @@ function init({ genomes }) {
   return async (req, res) => {
     try {
       const q = req.query;
-      const genome = genomes[q.genome];
-      if (!genome) throw "unknown genome";
-      const ds = genome.datasets?.[q.dslabel];
-      if (!ds) throw "unknown ds";
-      let term_results = [];
-      if (q.tw) {
-        term_results = await getData({ filter: q.filter, filter0: q.filter0, terms: [q.tw] }, ds);
-        if (term_results.error) throw new Error(term_results.error);
-      }
-      let term_results2 = [];
-      if (q.tw2) {
-        term_results2 = await getData({ filter: q.filter, filter0: q.filter0, terms: [q.tw2] }, ds);
-        if (term_results2.error) throw new Error(term_results2.error);
-      }
-      const results = await run_diffMeth(req.query, ds, term_results, term_results2);
-      if (!results || !results.data)
-        throw new Error(
-          "Differential methylation analysis returned no data. Please verify sample selections and try again."
-        );
-      if ("totalRows" in results.data && results.data.totalRows === 0)
+      if (q.preAnalysis) {
+        const { ds, term_results, term_results2 } = await resolveDaContext(q, genomes);
+        const groups = resolveDmSampleGroups(q, ds, term_results, term_results2);
+        const group1Name = q.samplelst.groups[0].name;
+        const group2Name = q.samplelst.groups[1].name;
+        res.send({
+          data: {
+            [group1Name]: groups.group1names.length,
+            [group2Name]: groups.group2names.length,
+            ...groups.alerts.length ? { alert: groups.alerts.join(" | ") } : {}
+          }
+        });
+        return;
+      }
+      const result = await readCacheFileOrRecompute({ daRequest: q, genomes });
+      if (result.kind !== "DM") throw new Error("expected DM result from readCacheFileOrRecompute");
+      const rendered = await renderVolcano(result.promoterData, q.volcanoRender);
+      rendered.cacheId = result.cacheId;
+      if (rendered.totalRows === 0)
         throw new Error("No promoters passed filtering. Try relaxing group criteria or selecting more samples.");
-      res.send(results);
+      const output = {
+        data: rendered,
+        sample_size1: result.sample_size1,
+        sample_size2: result.sample_size2
+      };
+      res.send(output);
     } catch (e) {
       res.send({ status: "error", error: e.message || e });
       if (e instanceof Error && e.stack) console.log(e);
     }
   };
 }
-async function run_diffMeth(param, ds, term_results, term_results2) {
-  if (param.samplelst?.groups?.length != 2)
-    throw new Error("Exactly 2 sample groups are required for differential methylation analysis.");
-  if (param.samplelst.groups[0].values?.length < 1)
-    throw new Error("Group 1 has no samples. Please select at least one sample.");
-  if (param.samplelst.groups[1].values?.length < 1)
-    throw new Error("Group 2 has no samples. Please select at least one sample.");
-  const q = ds.queries.dnaMethylation?.promoter;
-  if (!q) throw new Error("This dataset does not have promoter-level methylation data configured.");
-  if (!q.file) throw new Error("Promoter methylation data file is not configured for this dataset.");
-  const group1names = [];
-  const conf1_group1 = [];
-  const conf2_group1 = [];
-  for (const s of param.samplelst.groups[0].values) {
-    if (!Number.isInteger(s.sampleId)) continue;
-    const n = ds.cohort.termdb.q.id2sampleName(s.sampleId);
-    if (!n) continue;
-    if (!q.allSampleSet.has(n)) continue;
-    if (param.tw && param.tw2) {
-      if (term_results.samples[s.sampleId] && term_results2.samples[s.sampleId]) {
-        conf1_group1.push(
-          param.tw.q.mode == "continuous" ? term_results.samples[s.sampleId][param.tw.$id]["value"] : term_results.samples[s.sampleId][param.tw.$id]["key"]
-        );
-        conf2_group1.push(
-          param.tw2.q.mode == "continuous" ? term_results2.samples[s.sampleId][param.tw2.$id]["value"] : term_results2.samples[s.sampleId][param.tw2.$id]["key"]
-        );
-        group1names.push(n);
-      }
-    } else if (param.tw && !param.tw2) {
-      if (term_results.samples[s.sampleId]) {
-        conf1_group1.push(
-          param.tw.q.mode == "continuous" ? term_results.samples[s.sampleId][param.tw.$id]["value"] : term_results.samples[s.sampleId][param.tw.$id]["key"]
-        );
-        group1names.push(n);
-      }
-    } else if (!param.tw && param.tw2) {
-      if (term_results2.samples[s.sampleId]) {
-        conf2_group1.push(
-          param.tw2.q.mode == "continuous" ? term_results2.samples[s.sampleId][param.tw2.$id]["value"] : term_results2.samples[s.sampleId][param.tw2.$id]["key"]
-        );
-        group1names.push(n);
-      }
-    } else {
-      group1names.push(n);
-    }
-  }
-  const group2names = [];
-  const conf1_group2 = [];
-  const conf2_group2 = [];
-  for (const s of param.samplelst.groups[1].values) {
-    if (!Number.isInteger(s.sampleId)) continue;
-    const n = ds.cohort.termdb.q.id2sampleName(s.sampleId);
-    if (!n) continue;
-    if (!q.allSampleSet.has(n)) continue;
-    if (param.tw && param.tw2) {
-      if (term_results.samples[s.sampleId] && term_results2.samples[s.sampleId]) {
-        conf1_group2.push(
-          param.tw.q.mode == "continuous" ? term_results.samples[s.sampleId][param.tw.$id]["value"] : term_results.samples[s.sampleId][param.tw.$id]["key"]
-        );
-        conf2_group2.push(
-          param.tw2.q.mode == "continuous" ? term_results2.samples[s.sampleId][param.tw2.$id]["value"] : term_results2.samples[s.sampleId][param.tw2.$id]["key"]
-        );
-        group2names.push(n);
-      }
-    } else if (param.tw && !param.tw2) {
-      if (term_results.samples[s.sampleId]) {
-        conf1_group2.push(
-          param.tw.q.mode == "continuous" ? term_results.samples[s.sampleId][param.tw.$id]["value"] : term_results.samples[s.sampleId][param.tw.$id]["key"]
-        );
-        group2names.push(n);
-      }
-    } else if (!param.tw && param.tw2) {
-      if (term_results2.samples[s.sampleId]) {
-        conf2_group2.push(
-          param.tw2.q.mode == "continuous" ? term_results2.samples[s.sampleId][param.tw2.$id]["value"] : term_results2.samples[s.sampleId][param.tw2.$id]["key"]
-        );
-        group2names.push(n);
-      }
-    } else {
-      group2names.push(n);
-    }
-  }
-  const sample_size1 = group1names.length;
-  const sample_size2 = group2names.length;
-  const alerts = validateGroups(sample_size1, sample_size2, group1names, group2names);
-  if (param.preAnalysis) {
-    const group1Name = param.samplelst.groups[0].name;
-    const group2Name = param.samplelst.groups[1].name;
-    return {
-      data: {
-        [group1Name]: sample_size1,
-        [group2Name]: sample_size2,
-        ...alerts.length ? { alert: alerts.join(" | ") } : {}
-      }
-    };
-  }
-  if (alerts.length) throw new Error(alerts.join(" | "));
-  const diffMethInput = {
-    case: group2names.join(","),
-    control: group1names.join(","),
-    input_file: q.file,
-    min_samples_per_group: param.min_samples_per_group
-  };
-  if (param.tw) {
-    diffMethInput.conf1 = [...conf1_group2, ...conf1_group1];
-    diffMethInput.conf1_mode = param.tw.q.mode;
-    if (new Set(diffMethInput.conf1).size === 1) {
-      throw new Error("Confounding variable 1 has only one value");
-    }
-  }
-  if (param.tw2) {
-    diffMethInput.conf2 = [...conf2_group2, ...conf2_group1];
-    diffMethInput.conf2_mode = param.tw2.q.mode;
-    if (new Set(diffMethInput.conf2).size === 1) {
-      throw new Error("Confounding variable 2 has only one value");
-    }
-  }
-  const time1 = Date.now();
-  const result = JSON.parse(await run_R("diffMeth.R", JSON.stringify(diffMethInput)));
-  mayLog("Time taken to run diffMeth:", formatElapsedTime(Date.now() - time1));
-  const rendered = await renderVolcano(result.promoter_data, param.volcanoRender);
-  const output = {
-    data: rendered,
-    sample_size1,
-    sample_size2
-  };
-  return output;
-}
-function validateGroups(sample_size1, sample_size2, group1names, group2names) {
-  const alerts = [];
-  if (sample_size1 < 1) alerts.push("No samples in group 1 have methylation data available.");
-  if (sample_size2 < 1) alerts.push("No samples in group 2 have methylation data available.");
-  const commonnames = group1names.filter((x) => group2names.includes(x));
-  if (commonnames.length)
-    alerts.push(
-      `${commonnames.length} sample(s) appear in both groups: ${commonnames.join(", ")}. Please remove duplicates.`
-    );
-  return alerts;
-}
 export {
   api
 };