@sjcrh/proteinpaint-server 2.182.0 → 2.182.2

package/routes/termdb.violinBox.js

@@ -1,14 +1,15 @@
 import { violinBoxPayload } from "#types/checkers";
 import { scaleLinear, scaleLog } from "d3";
 import { run_rust } from "@sjcrh/proteinpaint-rust";
+import { run_R } from "@sjcrh/proteinpaint-r";
 import { getData } from "../src/termdb.matrix.js";
 import { createCanvas } from "canvas";
 import { getOrderedLabels } from "../src/termdb.barchart.js";
 import { getDescrStats, getStdDev, getMean } from "./termdb.descrstats.ts";
 import { isNumericTerm } from "#shared/terms.js";
 import { boxplot_getvalue } from "../src/utils.js";
-import { run_R } from "@sjcrh/proteinpaint-r";
 import { roundValueAuto } from "#shared/roundValue.js";
+import { mayLog } from "#src/helpers.ts";
 const api = {
   endpoint: "termdb/violinBox",
   methods: {
@@ -33,7 +34,8 @@ function init({ genomes }) {
       if (!ds) throw new Error("invalid ds");
       if (typeof q.tw?.term != "object" || typeof q.tw?.q != "object") throw new Error("q.tw not of {term,q}");
       const term = q.tw.term;
-      if (!isNumericTerm(term) && term.type !== "survival") throw new Error("term type is not numeric or survival");
+      if (!isNumericTerm(term) && term.type !== "survival" && term.type !== "termCollection")
+        throw new Error("term type is not numeric, survival, or termCollection");
       const arg = {
         terms: [q.tw],
         filter: q.filter,
@@ -47,6 +49,7 @@ function init({ genomes }) {
       const data = await getData(arg, ds);
       if (!data) throw new Error("getData() returns nothing");
       if (data.error) throw new Error(data.error);
+      if (q.tw.term.type === "termCollection") expandNumericTermCollection(q, data);
       if (q.plotType === "violin") {
         result = await getViolin(q, data, ds);
       } else if (q.plotType === "box") {
@@ -61,17 +64,61 @@ function init({ genomes }) {
     res.send(result);
   };
 }
+function expandNumericTermCollection(q, data) {
+  const term = q.tw.term;
+  if (term.memberType !== "numeric") throw new Error("only numeric termCollection is supported for violinBox");
+  if (q.overlayTw)
+    throw new Error("overlayTw is not supported with numeric termCollection; member terms are used as the overlay");
+  if (q.divideTw) throw new Error("divideTw is not supported with numeric termCollection");
+  const termlst = term.termlst || [];
+  mayLog("termlst", termlst);
+  mayLog(
+    `Expanding numeric termCollection with ${termlst.length} member terms and ${Object.keys(data.samples).length} samples`
+  );
+  const propsByTermId = term.propsByTermId || {};
+  const tcId = q.tw.$id;
+  const memberNameById = {};
+  const overlayValues = {};
+  for (const mt of termlst) {
+    const name = mt.name || mt.id;
+    memberNameById[mt.id] = name;
+    overlayValues[name] = { label: name, color: propsByTermId[mt.id]?.color };
+  }
+  const newSamples = {};
+  for (const [sampleId, sampleData] of Object.entries(data.samples)) {
+    const tcEntry = sampleData[tcId];
+    const memberValues = tcEntry?.value;
+    if (!memberValues || typeof memberValues !== "object") continue;
+    for (const [memberId, memberVal] of Object.entries(memberValues)) {
+      if (typeof memberVal !== "number" || !Number.isFinite(memberVal)) continue;
+      const memberName = memberNameById[memberId] || memberId;
+      newSamples[`${sampleId}__${memberId}`] = {
+        ...sampleData,
+        [tcId]: { key: memberVal, value: memberVal },
+        ["__tcOverlay"]: { key: memberName, value: memberName }
+      };
+    }
+  }
+  data.samples = newSamples;
+  q.overlayTw = {
+    $id: "__tcOverlay",
+    term: { type: "categorical", values: overlayValues, name: term.name },
+    q: {}
+  };
+  data.refs.byTermId["__tcOverlay"] = {
+    keyOrder: termlst.map((t) => t.name || t.id)
+  };
+}
 async function getViolin(q, data, ds) {
   const samples = Object.values(data.samples);
-  let values = samples.map((s) => s?.[q.tw.$id]?.value).filter((v) => typeof v === "number" && !q.tw.term.values?.[v]?.uncomputable);
-  if (q.isLogScale) values = values.filter((v) => v > 0);
+  const values = extractNumericValues(samples, q.tw, q.isLogScale);
   const descrStats = getDescrStats(values);
-  const sampleType = `All ${data.sampleType?.plural_name || "samples"}`;
-  if (data.error) throw new Error(data.error);
+  const sampleType = computeSampleType(data);
   if (q.overlayTw && data.refs.byTermId[q.overlayTw.$id]) {
+    ;
     data.refs.byTermId[q.overlayTw.$id].orderedLabels = getOrderedLabels(
-      q.overlayTw,
-      data.refs.byTermId[q.overlayTw.$id]?.bins,
+      q.overlayTw.term,
+      data.refs.byTermId[q.overlayTw.$id]?.bins || [],
       void 0,
       q.overlayTw.q
     );
@@ -102,8 +149,7 @@ function divideValues(q, data, sampleType) {
     q,
     data,
     sampleType,
-    useLog == true ? true : false,
-    // avoid tsc err
+    !!useLog,
     overlayTerm,
     divideTerm
   );
@@ -134,23 +180,17 @@ function setViolinResponse(valuesObject, data, q) {
     const plots = [];
     for (const [plot, values] of sortPlot2Values(data, plot2values, overlayTerm)) {
       plots.push({
-        label: overlayTerm?.term?.values?.[plot]?.label || plot,
-        // avoid strange tsc err
+        label: String(overlayTerm?.term?.values?.[plot]?.label || plot),
         values,
-        seriesId: plot,
-        chartId: chart,
-        //quick fix to get list samples working
+        seriesId: String(plot),
+        chartId: String(chart),
         plotValueCount: values?.length,
         color: overlayTerm?.term?.values?.[plot]?.color || ""
       });
     }
-    charts[chart] = { chartId: chart, plots };
+    charts[String(chart)] = { chartId: String(chart), plots };
   }
-  const bins = {
-    term1: numericBins(q.tw, data)
-  };
-  if (overlayTerm) bins.term2 = numericBins(overlayTerm, data);
-  if (divideTw) bins.term0 = numericBins(divideTw, data);
+  const bins = buildBins(q.tw, data, overlayTerm, divideTw);
   const result = {
     min: valuesObject.min,
     max: valuesObject.max,
@@ -218,30 +258,11 @@ async function createCanvasImg(q, result, ds) {
   }
 }
 async function getViolinWilcoxonData(result) {
-  for (const k of Object.keys(result.charts)) {
-    const chart = result.charts[k];
-    const numPlots = chart.plots.length;
-    if (numPlots < 2) continue;
-    const wilcoxInput = [];
-    for (let i = 0; i < numPlots; i++) {
-      const group1_id = chart.plots[i].label;
-      const group1_values = chart.plots[i].values;
-      for (let j = i + 1; j < numPlots; j++) {
-        const group2_id = chart.plots[j].label;
-        const group2_values = chart.plots[j].values;
-        wilcoxInput.push({ group1_id, group1_values, group2_id, group2_values });
-      }
-    }
-    const wilcoxOutput = JSON.parse(await run_rust("wilcoxon", JSON.stringify(wilcoxInput)));
-    chart.pvalues = [];
-    for (const test of wilcoxOutput) {
-      if (test.pvalue == null || test.pvalue == "null") {
-        chart.pvalues.push([{ value: test.group1_id }, { value: test.group2_id }, { html: "NA" }]);
-      } else {
-        chart.pvalues.push([{ value: test.group1_id }, { value: test.group2_id }, { html: test.pvalue.toPrecision(4) }]);
-      }
-    }
-  }
+  await runWilcoxonTests(
+    result.charts,
+    { getGroupId: (plot) => plot.label, getGroupValues: (plot) => plot.values },
+    "pvalues"
+  );
 }
 async function getDensity(values) {
   const result = await getDensities({ plot: values });
@@ -297,17 +318,16 @@ async function getBoxPlot(q, data) {
 }
 async function processBoxPlotData(data, q) {
   const samples = Object.values(data.samples);
-  const values = samples.map((s) => s?.[q.tw.$id]?.value).filter((v) => typeof v === "number" && !q.tw.term.values?.[v]?.uncomputable);
+  const values = extractNumericValues(samples, q.tw);
   const descrStats = getDescrStats(values, q.removeOutliers);
-  const sampleType = `All ${data.sampleType?.plural_name || "samples"}`;
+  const sampleType = computeSampleType(data);
   const overlayTw = q.overlayTw;
   const divideTw = q.divideTw;
   const { absMin, absMax, chart2plot2values, uncomputableValues } = parseValues(
     q,
     data,
     sampleType,
-    q.isLogScale == true ? true : false,
-    // avoid tsc err
+    !!q.isLogScale,
     overlayTw,
     divideTw
   );
@@ -338,16 +358,12 @@ async function processBoxPlotData(data, q) {
       plots.sort((a, b) => a.boxplot.p50 - b.boxplot.p50);
     }
     const sampleCount = plots.reduce((total, p) => {
-      if (p.hidden) return total;
+      if (p.isHidden) return total;
       return total + p.descrStats.total.value;
     }, 0);
-    charts[chart] = { chartId: chart, plots, sampleCount };
+    charts[String(chart)] = { chartId: String(chart), plots, sampleCount };
   }
-  const bins = {
-    term1: numericBins(q.tw, data)
-  };
-  if (overlayTw) bins.term2 = numericBins(overlayTw, data);
-  if (divideTw) bins.term0 = numericBins(divideTw, data);
+  const bins = buildBins(q.tw, data, overlayTw, divideTw);
   if (q.showAssocTests && overlayTw) await getBoxPlotWilcoxonData(charts);
   Object.keys(charts).forEach((c) => charts[c].plots.forEach((p) => delete p.tempValues));
   return { absMin, absMax, bins, charts, uncomputableValues, descrStats, outlierMin, outlierMax };
@@ -367,14 +383,13 @@ function setPlotData(plots, values, key, sampleType, descrStats, q, outlierMin,
   const plot = {
     boxplot,
     descrStats: setIndividualBoxPlotStats(boxplot, sortedValues),
-    //quick fix
-    //to delete later
+    // See comment in processBoxPlotData about tempValues
     tempValues: sortedValues
   };
   if (overlayTw) {
     const _key = overlayTw?.term?.values?.[key]?.label || key;
-    plot.color = overlayTw?.term?.values?.[key]?.color || null;
-    plot.key = _key;
+    plot.color = overlayTw?.term?.values?.[key]?.color || void 0;
+    plot.key = String(_key);
     plot.seriesId = key;
     plot.boxplot.label = `${_key}, n=${values.length}`;
   } else {
@@ -419,26 +434,36 @@ function setUncomputableValues(values) {
   } else return null;
 }
 async function getBoxPlotWilcoxonData(charts) {
+  await runWilcoxonTests(
+    charts,
+    {
+      getGroupId: (plot) => plot.boxplot.label.replace(/, n=\d+$/, ""),
+      getGroupValues: (plot) => plot.tempValues
+    },
+    "wilcoxon"
+  );
+}
+async function runWilcoxonTests(charts, accessors, resultKey) {
   for (const chart of Object.values(charts)) {
     const numPlots = chart.plots?.length;
-    if (numPlots < 2) continue;
-    const wilcoxonInput = [];
+    if (!numPlots || numPlots < 2) continue;
+    const wilcoxInput = [];
     for (let i = 0; i < numPlots; i++) {
-      const group1_id = chart.plots[i].boxplot.label.replace(/, n=\d+$/, "");
-      const group1_values = chart.plots[i].tempValues;
+      const group1_id = accessors.getGroupId(chart.plots[i]);
+      const group1_values = accessors.getGroupValues(chart.plots[i]);
       for (let j = i + 1; j < numPlots; j++) {
-        const group2_id = chart.plots[j].boxplot.label.replace(/, n=\d+$/, "");
-        const group2_values = chart.plots[j].tempValues;
-        wilcoxonInput.push({ group1_id, group1_values, group2_id, group2_values });
+        const group2_id = accessors.getGroupId(chart.plots[j]);
+        const group2_values = accessors.getGroupValues(chart.plots[j]);
+        wilcoxInput.push({ group1_id, group1_values, group2_id, group2_values });
       }
     }
-    const wilcoxonOutput = JSON.parse(await run_rust("wilcoxon", JSON.stringify(wilcoxonInput)));
-    chart.wilcoxon = [];
-    for (const test of wilcoxonOutput) {
+    const wilcoxOutput = JSON.parse(await run_rust("wilcoxon", JSON.stringify(wilcoxInput)));
+    chart[resultKey] = [];
+    for (const test of wilcoxOutput) {
       if (test.pvalue == null || test.pvalue == "null") {
-        chart.wilcoxon.push([{ value: test.group1_id }, { value: test.group2_id }, { html: "NA" }]);
+        chart[resultKey].push([{ value: test.group1_id }, { value: test.group2_id }, { html: "NA" }]);
       } else {
-        chart.wilcoxon.push([
+        chart[resultKey].push([
           { value: test.group1_id },
           { value: test.group2_id },
           { html: test.pvalue.toPrecision(4) }
@@ -447,37 +472,49 @@ async function getBoxPlotWilcoxonData(charts) {
     }
   }
 }
+function computeSampleType(data) {
+  return `All ${data.sampleType?.plural_name || "samples"}`;
+}
+function buildBins(tw, data, overlayTw, divideTw) {
+  const bins = {
+    term1: numericBins(tw, data)
+  };
+  if (overlayTw) bins.term2 = numericBins(overlayTw, data);
+  if (divideTw) bins.term0 = numericBins(divideTw, data);
+  return bins;
+}
+function extractNumericValues(samples, tw, isLogScale) {
+  let values = samples.map((s) => s?.[tw.$id]?.value).filter((v) => typeof v === "number" && !tw.term.values?.[v]?.uncomputable);
+  if (isLogScale) values = values.filter((v) => v > 0);
+  return values;
+}
 function parseValues(q, data, sampleType, isLog, overlayTw, divideTw) {
   const chart2plot2values = /* @__PURE__ */ new Map();
   const uncomputableValues = {};
+  function trackUncomputable(tw, key) {
+    if (!tw?.term?.values?.[key]?.uncomputable) return false;
+    const label = tw.term.values[key]?.label;
+    if (label) uncomputableValues[label] = (uncomputableValues[label] || 0) + 1;
+    return true;
+  }
   let absMin = Infinity, absMax = -Infinity;
   for (const val of Object.values(data.samples)) {
     const value = val[q.tw.$id];
     if (!Number.isFinite(value?.value)) continue;
-    if (q.tw.term.values?.[value.value]?.uncomputable) {
-      const label = q.tw.term.values[value.value].label;
-      uncomputableValues[label] = (uncomputableValues[label] || 0) + 1;
-      continue;
-    }
+    if (trackUncomputable(q.tw, value.value)) continue;
     if (isLog && value.value <= 0) continue;
     let chart = "";
     let plot = sampleType;
     if (divideTw) {
-      if (!val[divideTw?.$id]) continue;
+      if (!val[divideTw.$id]) continue;
       const value0 = val[divideTw.$id];
-      if (divideTw.term?.values?.[value0.key]?.uncomputable) {
-        const label = divideTw.term.values[value0?.key]?.label;
-        uncomputableValues[label] = (uncomputableValues[label] || 0) + 1;
-      }
+      trackUncomputable(divideTw, value0.key);
       chart = value0.key;
     }
     if (overlayTw) {
-      if (!val[overlayTw?.$id]) continue;
+      if (!val[overlayTw.$id]) continue;
       const value2 = val[overlayTw.$id];
-      if (overlayTw.term?.values?.[value2.key]?.uncomputable) {
-        const label = overlayTw.term.values[value2?.key]?.label;
-        uncomputableValues[label] = (uncomputableValues[label] || 0) + 1;
-      }
+      trackUncomputable(overlayTw, value2.key);
       plot = value2.key;
     }
     if (!chart2plot2values.has(chart)) chart2plot2values.set(chart, /* @__PURE__ */ new Map());
@@ -493,17 +530,29 @@ function parseValues(q, data, sampleType, isLog, overlayTw, divideTw) {
 function numericBins(tw, data) {
   const bins = {};
   if (!isNumericTerm(tw?.term)) return bins;
-  for (const bin of data.refs.byTermId[tw?.$id]?.bins || []) {
+  for (const bin of data.refs.byTermId[tw.$id]?.bins || []) {
     bins[bin.label] = bin;
   }
   return bins;
 }
 export {
   api,
+  buildBins,
+  computeSampleType,
+  divideValues,
+  expandNumericTermCollection,
+  extractNumericValues,
   getDensities,
   getDensity,
   getViolinWilcoxonData,
   numericBins,
   parseValues,
+  setHiddenPlots,
+  setIndividualBoxPlotStats,
+  setPlotData,
+  setScaleData,
+  setUncomputableValues,
+  setViolinResponse,
+  sortObj,
   sortPlot2Values
 };