@sjcrh/proteinpaint-server 2.166.0 → 2.167.0

package/dataset/termdb.test.js

@@ -369,20 +369,7 @@ function termdb_test_default() {
         jsonFile: "files/hg38/TermdbTest/trackLst/facet.json",
         activeTracks: ["bw 1", "bed 1"]
       },
-      chat: {},
-      alphaGenome: {
-        default: {
-          gene: "FLT3",
-          chromosome: "chr13",
-          position: 28034105,
-          reference: "A",
-          alternate: "AACTCCCATTTGAGATCATACC",
-          ontologyTerm: "EFO:0000572",
-          // lymphoblast
-          outputType: 4
-          //RNA SEQ
-        }
-      }
+      chat: {}
     }
   };
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sjcrh/proteinpaint-server",
-  "version": "2.166.0",
+  "version": "2.167.0",
   "type": "module",
   "description": "a genomics visualization tool for exploring a cohort's genotype and phenotype data",
   "main": "src/app.js",
@@ -64,9 +64,9 @@
     "@sjcrh/augen": "2.143.0",
     "@sjcrh/proteinpaint-python": "2.164.1",
     "@sjcrh/proteinpaint-r": "2.152.1-0",
-    "@sjcrh/proteinpaint-rust": "2.166.0",
-    "@sjcrh/proteinpaint-shared": "2.166.0",
-    "@sjcrh/proteinpaint-types": "2.166.0",
+    "@sjcrh/proteinpaint-rust": "2.167.0",
+    "@sjcrh/proteinpaint-shared": "2.167.0",
+    "@sjcrh/proteinpaint-types": "2.167.0",
     "@types/express": "^5.0.0",
     "@types/express-session": "^1.18.1",
     "better-sqlite3": "^12.4.1",

package/routes/termdb.boxplot.js

@@ -3,6 +3,7 @@ import { getData } from "../src/termdb.matrix.js";
 import { boxplot_getvalue } from "../src/utils.js";
 import { sortPlot2Values } from "./termdb.violin.ts";
 import { getDescrStats } from "./termdb.descrstats.ts";
+import { run_rust } from "@sjcrh/proteinpaint-rust";
 const api = {
   endpoint: "termdb/boxplot",
   methods: {
@@ -21,78 +22,18 @@ function init({ genomes }) {
     const q = req.query;
     try {
       const genome = genomes[q.genome];
-      if (!genome) throw "invalid genome name";
+      if (!genome) throw new Error("invalid genome name");
       const ds = genome.datasets?.[q.dslabel];
-      if (!ds) throw "invalid ds";
+      if (!ds) throw new Error("invalid ds");
       const terms = [q.tw];
       if (q.overlayTw) terms.push(q.overlayTw);
       if (q.divideTw) terms.push(q.divideTw);
       const data = await getData({ filter: q.filter, filter0: q.filter0, terms, __protected__: q.__protected__ }, ds);
-      if (data.error) throw data.error;
-      const samples = Object.values(data.samples);
-      const values = samples.map((s) => s?.[q.tw.$id]?.value).filter((v) => typeof v === "number" && !q.tw.term.values?.[v]?.uncomputable);
-      const descrStats = getDescrStats(values, q.removeOutliers);
-      const sampleType = `All ${data.sampleType?.plural_name || "samples"}`;
-      const overlayTerm = q.overlayTw;
-      const divideTerm = q.divideTw;
-      const { absMin, absMax, chart2plot2values, uncomputableValues } = parseValues(
-        q,
+      if (data.error) throw new Error(data.error);
+      const { absMin, absMax, charts, uncomputableValues, descrStats, outlierMin, outlierMax } = await processData(
         data,
-        sampleType,
-        q.isLogScale,
-        overlayTerm,
-        divideTerm
+        q
       );
-      if (!absMin && absMin !== 0) throw "absMin is undefined [termdb.boxplot init()]";
-      if (!absMax && absMax !== 0) throw "absMax is undefined [termdb.boxplot init()]";
-      const charts = {};
-      let outlierMin = Number.POSITIVE_INFINITY, outlierMax = Number.NEGATIVE_INFINITY;
-      for (const [chart, plot2values] of chart2plot2values) {
-        const plots = [];
-        for (const [key, values2] of sortPlot2Values(data, plot2values, overlayTerm)) {
-          const sortedValues = values2.sort((a, b) => a - b);
-          const vs = sortedValues.map((v) => {
-            const value = { value: v };
-            return value;
-          });
-          if (q.removeOutliers) {
-            outlierMin = Math.min(outlierMin, descrStats.outlierMin.value);
-            outlierMax = Math.max(outlierMax, descrStats.outlierMax.value);
-          }
-          const boxplot = boxplot_getvalue(vs, q.removeOutliers);
-          if (!boxplot) throw "boxplot_getvalue failed [termdb.boxplot init()]";
-          const _plot = {
-            boxplot,
-            descrStats
-          };
-          if (overlayTerm) {
-            const _key = overlayTerm?.term?.values?.[key]?.label || key;
-            const plotLabel = `${_key}, n=${values2.length}`;
-            const overlayBins = numericBins(overlayTerm, data);
-            const plot = Object.assign(_plot, {
-              color: overlayTerm?.term?.values?.[key]?.color || null,
-              key: _key,
-              overlayBins: overlayBins.has(key) ? overlayBins.get(key) : null,
-              seriesId: key
-            });
-            plot.boxplot.label = plotLabel;
-            plots.push(plot);
-          } else {
-            const plotLabel = `${sampleType}, n=${values2.length}`;
-            const plot = Object.assign(_plot, {
-              key: sampleType
-            });
-            plot.boxplot.label = plotLabel;
-            plots.push(plot);
-          }
-        }
-        if (q.tw.term?.values) setHiddenPlots(q.tw, plots);
-        if (overlayTerm && overlayTerm.term?.values) setHiddenPlots(overlayTerm, plots);
-        if (q.orderByMedian == true) {
-          plots.sort((a, b) => a.boxplot.p50 - b.boxplot.p50);
-        }
-        charts[chart] = { chartId: chart, plots };
-      }
       const returnData = {
         absMin: q.removeOutliers ? outlierMin : absMin,
         absMax: q.removeOutliers ? outlierMax : absMax,
@@ -107,6 +48,94 @@ function init({ genomes }) {
     }
   };
 }
+async function processData(data, q) {
+  const samples = Object.values(data.samples);
+  const values = samples.map((s) => s?.[q.tw.$id]?.value).filter((v) => typeof v === "number" && !q.tw.term.values?.[v]?.uncomputable);
+  const descrStats = getDescrStats(values, q.removeOutliers);
+  const sampleType = `All ${data.sampleType?.plural_name || "samples"}`;
+  const overlayTerm = q.overlayTw;
+  const divideTerm = q.divideTw;
+  const { absMin, absMax, chart2plot2values, uncomputableValues } = parseValues(
+    q,
+    data,
+    sampleType,
+    q.isLogScale,
+    overlayTerm,
+    divideTerm
+  );
+  if (!absMin && absMin !== 0) throw new Error("absMin is undefined");
+  if (!absMax && absMax !== 0) throw new Error("absMax is undefined");
+  const charts = {};
+  let outlierMin = Number.POSITIVE_INFINITY, outlierMax = Number.NEGATIVE_INFINITY;
+  for (const [chart, plot2values] of chart2plot2values) {
+    const plots = [];
+    for (const [key, values2] of sortPlot2Values(data, plot2values, overlayTerm)) {
+      ;
+      [outlierMax, outlierMin] = setPlotData(
+        plots,
+        values2,
+        key,
+        sampleType,
+        descrStats,
+        q,
+        data,
+        outlierMin,
+        outlierMax,
+        overlayTerm
+      );
+    }
+    if (q.tw.term?.values) setHiddenPlots(q.tw, plots);
+    if (overlayTerm && overlayTerm.term?.values) setHiddenPlots(overlayTerm, plots);
+    if (q.orderByMedian == true) {
+      plots.sort((a, b) => a.boxplot.p50 - b.boxplot.p50);
+    }
+    charts[chart] = { chartId: chart, plots };
+  }
+  if (q.showAssocTests == true && overlayTerm) await getWilcoxonData(charts);
+  Object.keys(charts).forEach((c) => charts[c].plots.forEach((p) => delete p.tempValues));
+  return { absMin, absMax, charts, uncomputableValues, descrStats, outlierMin, outlierMax };
+}
+function setPlotData(plots, values, key, sampleType, descrStats, q, data, outlierMin, outlierMax, overlayTerm) {
+  const sortedValues = values.sort((a, b) => a - b);
+  const vs = sortedValues.map((v) => {
+    const value = { value: v };
+    return value;
+  });
+  if (q.removeOutliers) {
+    outlierMin = Math.min(outlierMin, descrStats.outlierMin.value);
+    outlierMax = Math.max(outlierMax, descrStats.outlierMax.value);
+  }
+  const boxplot = boxplot_getvalue(vs, q.removeOutliers);
+  if (!boxplot) throw new Error("boxplot_getvalue failed [termdb.boxplot init()]");
+  const _plot = {
+    boxplot,
+    descrStats,
+    //quick fix
+    //to delete later
+    tempValues: sortedValues
+  };
+  if (overlayTerm) {
+    const _key = overlayTerm?.term?.values?.[key]?.label || key;
+    const plotLabel = `${_key}, n=${values.length}`;
+    const overlayBins = numericBins(overlayTerm, data);
+    const plot = Object.assign(_plot, {
+      color: overlayTerm?.term?.values?.[key]?.color || null,
+      key: _key,
+      overlayBins: overlayBins.has(key) ? overlayBins.get(key) : null,
+      seriesId: key
+    });
+    plot.boxplot.label = plotLabel;
+    plots.push(plot);
+  } else {
+    const plotLabel = `${sampleType}, n=${values.length}`;
+    const plot = Object.assign(_plot, {
+      key: sampleType
+    });
+    plot.boxplot.label = plotLabel;
+    plots.push(plot);
+  }
+  return [outlierMax, outlierMin];
+}
 function setHiddenPlots(term, plots) {
   for (const v of Object.values(term.term?.values)) {
     const plot = plots.find((p) => p.key === v.label);
@@ -125,6 +154,35 @@ function setUncomputableValues(values) {
     return Object.entries(values).map(([label, v]) => ({ label, value: v }));
   } else return null;
 }
+async function getWilcoxonData(charts) {
+  for (const chart of Object.values(charts)) {
+    const numPlots = chart.plots?.length;
+    if (numPlots < 2) continue;
+    const wilcoxonInput = [];
+    for (let i = 0; i < numPlots; i++) {
+      const group1_id = chart.plots[i].boxplot.label.replace(/, n=\d+$/, "");
+      const group1_values = chart.plots[i].tempValues;
+      for (let j = i + 1; j < numPlots; j++) {
+        const group2_id = chart.plots[j].boxplot.label.replace(/, n=\d+$/, "");
+        const group2_values = chart.plots[j].tempValues;
+        wilcoxonInput.push({ group1_id, group1_values, group2_id, group2_values });
+      }
+    }
+    const wilcoxonOutput = JSON.parse(await run_rust("wilcoxon", JSON.stringify(wilcoxonInput)));
+    chart.wilcoxon = [];
+    for (const test of wilcoxonOutput) {
+      if (test.pvalue == null || test.pvalue == "null") {
+        chart.wilcoxon.push([{ value: test.group1_id }, { value: test.group2_id }, { html: "NA" }]);
+      } else {
+        chart.wilcoxon.push([
+          { value: test.group1_id },
+          { value: test.group2_id },
+          { html: test.pvalue.toPrecision(4) }
+        ]);
+      }
+    }
+  }
+}
 function parseValues(q, data, sampleType, isLog, overlayTerm, divideTerm) {
   const chart2plot2values = /* @__PURE__ */ new Map();
   const uncomputableValues = {};

package/routes/termdb.chat.js

@@ -41,7 +41,6 @@ function init({ genomes }) {
         return;
       }
       const serverconfig_ds_entries = serverconfig.genomes.find((genome) => genome.name == q.genome).datasets.find((dslabel) => dslabel.name == ds.label);
-      console.log("serverconfig_ds_entries:", serverconfig_ds_entries);
       if (!serverconfig_ds_entries.aifiles) {
         throw "aifiles are missing for chatbot to work";
       }
@@ -75,14 +74,41 @@ function init({ genomes }) {
       const ai_output_data = await run_rust("aichatbot", JSON.stringify(chatbot_input));
       const time2 = (/* @__PURE__ */ new Date()).valueOf();
       mayLog("Time taken to run rust AI chatbot:", time2 - time1, "ms");
-      let ai_output_json = "";
+      let ai_output_json;
       for (const line of ai_output_data.split("\n")) {
         if (line.startsWith("final_output:") == true) {
-          ai_output_json = JSON.parse(line.replace("final_output:", ""));
+          ai_output_json = JSON.parse(JSON.parse(line.replace("final_output:", "")));
         } else {
           mayLog(line);
         }
       }
+      if (ai_output_json.type == "plot") {
+        if (typeof ai_output_json.plot != "object") throw ".plot{} missing when .type=plot";
+        if (ai_output_json.plot.simpleFilter) {
+          if (!Array.isArray(ai_output_json.plot.simpleFilter)) throw "ai_output_json.plot.simpleFilter is not array";
+          const localfilter = { type: "tvslst", in: true, join: "", lst: [] };
+          for (const f of ai_output_json.plot.simpleFilter) {
+            const term = ds.cohort.termdb.q.termjsonByOneid(f.term);
+            if (!term) throw "invalid term id from simpleFilter[].term";
+            if (term.type != "categorical") throw "term not categorical";
+            let cat;
+            for (const ck in term.values) {
+              if (ck == f.category) cat = ck;
+              else if (term.values[ck].label == f.category) cat = ck;
+            }
+            if (!cat) throw "invalid category from " + JSON.stringify(f);
+            localfilter.lst.push({
+              type: "tvs",
+              tvs: {
+                term,
+                values: [{ key: cat }]
+              }
+            });
+          }
+          delete ai_output_json.plot.simpleFilter;
+          ai_output_json.plot.filter = localfilter;
+        }
+      }
       res.send(ai_output_json);
     } catch (e) {
       if (e.stack) console.log(e.stack);

package/routes/termdb.cluster.js

@@ -105,7 +105,8 @@ async function getResult(q, ds) {
 }
 async function getNumericDictTermAnnotation(q, ds) {
   const getDataArgs = {
-    terms: q.terms.map((term) => ({ term, q: { mode: "continuous" } })),
+    // TODO: figure out when term is not a termwrapper
+    terms: q.terms.map((tw) => tw.term ? tw : { term: tw, q: { mode: "continuous" } }),
     filter: q.filter,
     filter0: q.filter0,
     __protected__: q.__protected__

package/routes/termdb.config.js

@@ -77,6 +77,7 @@ function make(q, req, res, ds, genome) {
   if (tdb.numericTermCollections) c.numericTermCollections = tdb.numericTermCollections;
   if (ds.assayAvailability) c.assayAvailability = ds.assayAvailability;
   if (ds.cohort.correlationVolcano) c.correlationVolcano = ds.cohort.correlationVolcano;
+  if (ds.cohort.boxplots) c.boxplots = ds.cohort.boxplots;
   addRestrictAncestries(c, tdb);
   addScatterplots(c, ds);
   addMatrixplots(c, ds);

package/routes/termdb.violin.js

@@ -27,9 +27,9 @@ function init({ genomes }) {
     let data;
     try {
       const g = genomes[q.genome];
-      if (!g) throw "invalid genome name";
+      if (!g) throw new Error("invalid genome name");
       const ds = g.datasets?.[q.dslabel];
-      if (!ds) throw "invalid ds";
+      if (!ds) throw new Error("invalid ds");
       data = await getViolin(q, ds);
     } catch (e) {
       data = { error: e?.message || e };
@@ -39,10 +39,10 @@ function init({ genomes }) {
   };
 }
 async function getViolin(q, ds) {
-  if (typeof q.tw?.term != "object" || typeof q.tw?.q != "object") throw "q.tw not of {term,q}";
+  if (typeof q.tw?.term != "object" || typeof q.tw?.q != "object") throw new Error("q.tw not of {term,q}");
   const term = q.tw.term;
   if (!q.tw.q.mode) q.tw.q.mode = "continuous";
-  if (!isNumericTerm(term) && term.type !== "survival") throw "term type is not numeric or survival";
+  if (!isNumericTerm(term) && term.type !== "survival") throw new Error("term type is not numeric or survival");
   const terms = [q.tw];
   if (q.overlayTw) terms.push(q.overlayTw);
   if (q.divideTw) terms.push(q.divideTw);
@@ -56,14 +56,14 @@ async function getViolin(q, ds) {
     },
     ds
   );
-  if (!data) throw "getData() returns nothing";
-  if (data.error) throw data.error;
+  if (!data) throw new Error("getData() returns nothing");
+  if (data.error) throw new Error(data.error);
   const samples = Object.values(data.samples);
   let values = samples.map((s) => s?.[q.tw.$id]?.value).filter((v) => typeof v === "number" && !q.tw.term.values?.[v]?.uncomputable);
   if (q.unit == "log") values = values.filter((v) => v > 0);
   const descrStats = getDescrStats(values);
   const sampleType = `All ${data.sampleType?.plural_name || "samples"}`;
-  if (data.error) throw data.error;
+  if (data.error) throw new Error(data.error);
   if (q.overlayTw && data.refs.byTermId[q.overlayTw.$id]) {
     data.refs.byTermId[q.overlayTw.$id].orderedLabels = getOrderedLabels(
       q.overlayTw,
@@ -174,7 +174,7 @@ function setResponse(valuesObject, data, q) {
 }
 async function createCanvasImg(q, result, ds) {
   if (!q.radius) q.radius = 5;
-  if (q.radius <= 0) throw "q.radius is not a number";
+  if (q.radius <= 0) throw new Error("q.radius is not a number");
   else q.radius = +q.radius;
   const isH = q.orientation == "horizontal";
   for (const k of Object.keys(result.charts)) {