@sjcrh/proteinpaint-server 2.196.0 → 2.197.0

package/routes/termdb.violinBox.js

@@ -1,543 +0,0 @@
-import { scaleLinear, scaleLog } from "d3";
-import { run_rust } from "@sjcrh/proteinpaint-rust";
-import { run_R } from "@sjcrh/proteinpaint-r";
-import { getData } from "../src/termdb.matrix.js";
-import { createCanvas } from "canvas";
-import { getOrderedLabels } from "../src/termdb.barchart.js";
-import { getDescrStats, getStdDev, getMean } from "#routes/termdb.descrstats.ts";
-import { isNumericTerm } from "#shared/terms.js";
-import { boxplot_getvalue } from "../src/utils.js";
-import { roundValueAuto } from "#shared/roundValue.js";
-import { mayLog } from "#src/helpers.ts";
-function init({ genomes }) {
-  return async (req, res) => {
-    const q = req.query;
-    let result;
-    try {
-      const g = genomes[q.genome];
-      if (!g) throw new Error("invalid genome name");
-      const ds = g.datasets?.[q.dslabel];
-      if (!ds) throw new Error("invalid ds");
-      if (typeof q.tw?.term != "object" || typeof q.tw?.q != "object") throw new Error("q.tw not of {term,q}");
-      const term = q.tw.term;
-      if (!isNumericTerm(term) && term.type !== "survival" && term.type !== "termCollection")
-        throw new Error("term type is not numeric, survival, or termCollection");
-      const arg = {
-        terms: [q.tw],
-        filter: q.filter,
-        filter0: q.filter0,
-        currentGeneNames: q.currentGeneNames,
-        __protected__: q.__protected__,
-        __abortSignal: q.__abortSignal
-      };
-      if (q.overlayTw) arg.terms.push(q.overlayTw);
-      if (q.divideTw) arg.terms.push(q.divideTw);
-      const data = await getData(arg, ds);
-      if (!data) throw new Error("getData() returns nothing");
-      if (data.error) throw new Error(data.error);
-      if (q.tw.term.type === "termCollection") expandNumericTermCollection(q, data);
-      if (q.plotType === "violin") {
-        result = await getViolin(q, data, ds);
-      } else if (q.plotType === "box") {
-        result = await getBoxPlot(q, data);
-      } else {
-        throw new Error("invalid plotType");
-      }
-    } catch (e) {
-      result = { error: e?.message || e };
-      if (e instanceof Error && e.stack) console.log(e);
-    }
-    res.send(result);
-  };
-}
-function expandNumericTermCollection(q, data) {
-  const term = q.tw.term;
-  if (term.memberType !== "numeric") throw new Error("only numeric termCollection is supported for violinBox");
-  if (q.overlayTw)
-    throw new Error("overlayTw is not supported with numeric termCollection; member terms are used as the overlay");
-  if (q.divideTw) throw new Error("divideTw is not supported with numeric termCollection");
-  const termlst = term.termlst || [];
-  mayLog(
-    `Expanding numeric termCollection with ${termlst.length} member terms and ${Object.keys(data.samples).length} samples`
-  );
-  const propsByTermId = term.propsByTermId || {};
-  const tcId = q.tw.$id;
-  const memberNameById = {};
-  const overlayValues = {};
-  for (const mt of termlst) {
-    const name = mt.name || mt.id;
-    memberNameById[mt.id] = name;
-    overlayValues[name] = { label: name, color: propsByTermId[mt.id]?.color };
-  }
-  const newSamples = {};
-  for (const [sampleId, sampleData] of Object.entries(data.samples)) {
-    const tcEntry = sampleData[tcId];
-    const memberValues = tcEntry?.value;
-    if (!memberValues || typeof memberValues !== "object") continue;
-    for (const [memberId, memberVal] of Object.entries(memberValues)) {
-      if (typeof memberVal !== "number" || !Number.isFinite(memberVal)) continue;
-      const memberName = memberNameById[memberId] || memberId;
-      newSamples[`${sampleId}__${memberId}`] = {
-        ...sampleData,
-        [tcId]: { key: memberVal, value: memberVal },
-        ["__tcOverlay"]: { key: memberName, value: memberName }
-      };
-    }
-  }
-  data.samples = newSamples;
-  q.overlayTw = {
-    $id: "__tcOverlay",
-    term: { type: "categorical", values: overlayValues, name: term.name },
-    q: {}
-  };
-  data.refs.byTermId["__tcOverlay"] = {
-    keyOrder: termlst.map((t) => t.name || t.id)
-  };
-}
-async function getViolin(q, data, ds) {
-  const samples = Object.values(data.samples);
-  const values = extractNumericValues(samples, q.tw, q.isLogScale);
-  const descrStats = getDescrStats(values);
-  const sampleType = computeSampleType(data);
-  if (q.overlayTw && data.refs.byTermId[q.overlayTw.$id]) {
-    ;
-    data.refs.byTermId[q.overlayTw.$id].orderedLabels = getOrderedLabels(
-      q.overlayTw.term,
-      data.refs.byTermId[q.overlayTw.$id]?.bins || [],
-      void 0,
-      q.overlayTw.q
-    );
-  }
-  if (q.scale) setScaleData(q, data, q.tw);
-  const valuesObject = divideValues(q, data, sampleType);
-  const result = setViolinResponse(valuesObject, data, q);
-  if (q.overlayTw) await getViolinWilcoxonData(result);
-  await createCanvasImg(q, result, ds);
-  result["descrStats"] = descrStats;
-  return result;
-}
-function setScaleData(q, data, tw) {
-  if (!q.scale) return;
-  const scale = Number(q.scale);
-  for (const val of Object.values(data.samples)) {
-    if (!tw.$id || !val[tw.$id]) continue;
-    if (tw.term.values?.[val[tw.$id]?.value]?.uncomputable) continue;
-    val[tw.$id].key = val[tw.$id].key / scale;
-    val[tw.$id].value = val[tw.$id].value / scale;
-  }
-}
-function divideValues(q, data, sampleType) {
-  const overlayTerm = q.overlayTw;
-  const divideTerm = q.divideTw;
-  const useLog = q.isLogScale;
-  const { absMax, absMin, chart2plot2values, uncomputableValues } = parseValues(
-    q,
-    data,
-    sampleType,
-    !!useLog,
-    overlayTerm,
-    divideTerm
-  );
-  return {
-    chart2plot2values,
-    min: absMin,
-    max: absMax,
-    uncomputableValues: sortObj(uncomputableValues)
-  };
-}
-function sortObj(object) {
-  return Object.fromEntries(Object.entries(object).sort(([, a], [, b]) => a - b));
-}
-function sortPlot2Values(data, plot2values, overlayTerm) {
-  const orderedLabels = overlayTerm?.$id ? data.refs.byTermId[overlayTerm.$id]?.keyOrder : void 0;
-  plot2values = new Map(
-    [...plot2values].sort(
-      orderedLabels ? (a, b) => orderedLabels.indexOf(a[0]) - orderedLabels.indexOf(b[0]) : overlayTerm?.term?.type === "categorical" ? (a, b) => b[1].length - a[1].length : overlayTerm?.term?.type === "condition" ? (a, b) => Number(a[0]) - Number(b[0]) : (a, b) => a.toString().replace(/[^a-zA-Z0-9<]/g, "").localeCompare(b.toString().replace(/[^a-zA-Z0-9<]/g, ""), void 0, { numeric: true })
-    )
-  );
-  return plot2values;
-}
-function setViolinResponse(valuesObject, data, q) {
-  const charts = {};
-  const overlayTerm = q.overlayTw;
-  const divideTw = q.divideTw;
-  for (const [chart, plot2values] of valuesObject.chart2plot2values) {
-    const plots = [];
-    for (const [plot, values] of sortPlot2Values(data, plot2values, overlayTerm)) {
-      plots.push({
-        label: String(overlayTerm?.term?.values?.[plot]?.label || plot),
-        values,
-        seriesId: String(plot),
-        chartId: String(chart),
-        plotValueCount: values?.length,
-        color: overlayTerm?.term?.values?.[plot]?.color || ""
-      });
-    }
-    charts[String(chart)] = { chartId: String(chart), plots };
-  }
-  const bins = buildBins(q.tw, data, overlayTerm, divideTw);
-  const result = {
-    min: valuesObject.min,
-    max: valuesObject.max,
-    bins,
-    charts,
-    uncomputableValues: Object.keys(valuesObject.uncomputableValues).length > 0 ? valuesObject.uncomputableValues : null
-  };
-  return result;
-}
-async function createCanvasImg(q, result, ds) {
-  if (!q.radius) q.radius = 5;
-  if (q.radius <= 0) throw new Error("q.radius is not a number");
-  else q.radius = +q.radius;
-  const isH = q.orientation == "horizontal";
-  for (const k of Object.keys(result.charts)) {
-    const chart = result.charts[k];
-    const plot2Values = {};
-    for (const plot of chart.plots) plot2Values[plot.label] = plot.values;
-    const useLog = q.isLogScale;
-    const logBase = ds.cohort.termdb.logscaleBase2 ? 2 : 10;
-    const densities = await getDensities(plot2Values, useLog, logBase);
-    let axisScale;
-    if (useLog) {
-      axisScale = scaleLog().base(ds.cohort.termdb.logscaleBase2 ? 2 : 10).domain([result.min, result.max]).range(isH ? [0, q.svgw] : [q.svgw, 0]);
-    } else {
-      axisScale = scaleLinear().domain([result.min, result.max]).range(isH ? [0, q.svgw] : [q.svgw, 0]);
-    }
-    const [width, height] = isH ? [q.svgw * q.devicePixelRatio, q.radius * q.devicePixelRatio] : [q.radius * q.devicePixelRatio, q.svgw * q.devicePixelRatio];
-    for (const plot of chart.plots) {
-      plot.density = densities[plot.label];
-      const canvas = createCanvas(width, height);
-      const ctx = canvas.getContext("2d");
-      if (q.devicePixelRatio != 1) ctx.scale(q.devicePixelRatio, q.devicePixelRatio);
-      ctx.strokeStyle = "black";
-      ctx.lineWidth = 1;
-      if (q.datasymbol === "rug") {
-        ctx.globalAlpha = 0.8;
-        plot.values.forEach((i) => {
-          const s = axisScale(i);
-          ctx.beginPath();
-          if (isH) {
-            ctx.moveTo(s, 0);
-            ctx.lineTo(s, q.radius);
-          } else {
-            ctx.moveTo(0, s);
-            ctx.lineTo(q.radius, s);
-          }
-          ctx.stroke();
-        });
-      } else if (q.datasymbol === "bean") {
-        ctx.globalAlpha = 0.6;
-        ctx.fillStyle = "#ffe6e6";
-        plot.values.forEach((i) => {
-          const s = axisScale(i);
-          ctx.beginPath();
-          if (isH) ctx.arc(s, q.radius / 2, q.radius / 2, 0, 2 * Math.PI);
-          else ctx.arc(q.radius / 2, s, q.radius / 2, 0, 2 * Math.PI);
-          ctx.fill();
-          ctx.stroke();
-        });
-      }
-      plot.src = canvas.toDataURL();
-      plot.summaryStats = getDescrStats(plot.values);
-    }
-  }
-}
-async function getViolinWilcoxonData(result) {
-  await runWilcoxonTests(
-    result.charts,
-    { getGroupId: (plot) => plot.label, getGroupValues: (plot) => plot.values },
-    "pvalues"
-  );
-}
-async function getDensity(values) {
-  const result = await getDensities({ plot: values });
-  return result.plot;
-}
-async function getDensities(plot2Values, useLog = false, logBase = 10) {
-  let transformedPlot2Values = {};
-  if (useLog) {
-    for (const plot in plot2Values) {
-      transformedPlot2Values[plot] = plot2Values[plot].filter((v) => v > 0).map((v) => Math.log(v) / Math.log(logBase));
-    }
-  } else {
-    transformedPlot2Values = plot2Values;
-  }
-  const plot2Density = JSON.parse(
-    await run_R("density.R", JSON.stringify({ plot2Values: transformedPlot2Values }))
-  );
-  const densities = {};
-  for (const plot in plot2Density) {
-    const result = plot2Density[plot];
-    const bins = [];
-    let densityMin = Infinity;
-    let densityMax = -Infinity;
-    let xMin = Infinity;
-    let xMax = -Infinity;
-    for (const [i, x] of Object.entries(result.x)) {
-      const density2 = result.y[i];
-      const x0 = useLog ? Math.pow(logBase, x) : x;
-      xMin = Math.min(xMin, x0);
-      xMax = Math.max(xMax, x0);
-      densityMin = Math.min(densityMin, density2);
-      densityMax = Math.max(densityMax, density2);
-      bins.push({ x0, density: density2 });
-    }
-    bins.unshift({ x0: xMin, density: densityMin });
-    bins.push({ x0: xMax, density: densityMin });
-    const density = { bins, densityMin, densityMax, minvalue: xMin, maxvalue: xMax };
-    densities[plot] = density;
-  }
-  return densities;
-}
-async function getBoxPlot(q, data) {
-  const { absMin, absMax, bins, charts, uncomputableValues, descrStats, outlierMin, outlierMax } = await processBoxPlotData(data, q);
-  const returnData = {
-    absMin: q.removeOutliers ? outlierMin : absMin,
-    absMax: q.removeOutliers ? outlierMax : absMax,
-    bins,
-    charts,
-    uncomputableValues: setUncomputableValues(uncomputableValues),
-    descrStats
-  };
-  return returnData;
-}
-async function processBoxPlotData(data, q) {
-  const samples = Object.values(data.samples);
-  const values = extractNumericValues(samples, q.tw);
-  const descrStats = getDescrStats(values, q.removeOutliers);
-  const sampleType = computeSampleType(data);
-  const overlayTw = q.overlayTw;
-  const divideTw = q.divideTw;
-  const { absMin, absMax, chart2plot2values, uncomputableValues } = parseValues(
-    q,
-    data,
-    sampleType,
-    !!q.isLogScale,
-    overlayTw,
-    divideTw
-  );
-  if (!absMin && absMin !== 0) throw new Error("absMin is undefined");
-  if (!absMax && absMax !== 0) throw new Error("absMax is undefined");
-  const charts = {};
-  let outlierMin = Number.POSITIVE_INFINITY, outlierMax = Number.NEGATIVE_INFINITY;
-  for (const [chart, plot2values] of chart2plot2values) {
-    const plots = [];
-    for (const [key, values2] of sortPlot2Values(data, plot2values, overlayTw)) {
-      ;
-      [outlierMax, outlierMin] = setPlotData(
-        plots,
-        values2,
-        key,
-        sampleType,
-        descrStats,
-        q,
-        outlierMin,
-        outlierMax,
-        overlayTw
-      );
-    }
-    if (q.tw.term?.values) setHiddenPlots(q.tw, plots);
-    if (overlayTw && overlayTw.term?.values) setHiddenPlots(overlayTw, plots);
-    if (divideTw && divideTw.term?.values) setHiddenPlots(divideTw, plots);
-    if (q.orderByMedian == true) {
-      plots.sort((a, b) => a.boxplot.p50 - b.boxplot.p50);
-    }
-    const sampleCount = plots.reduce((total, p) => {
-      if (p.isHidden) return total;
-      return total + p.descrStats.total.value;
-    }, 0);
-    charts[String(chart)] = { chartId: String(chart), plots, sampleCount };
-  }
-  const bins = buildBins(q.tw, data, overlayTw, divideTw);
-  if (q.showAssocTests && overlayTw) await getBoxPlotWilcoxonData(charts);
-  Object.keys(charts).forEach((c) => charts[c].plots.forEach((p) => delete p.tempValues));
-  return { absMin, absMax, bins, charts, uncomputableValues, descrStats, outlierMin, outlierMax };
-}
-function setPlotData(plots, values, key, sampleType, descrStats, q, outlierMin, outlierMax, overlayTw) {
-  const sortedValues = values.sort((a, b) => a - b);
-  const vs = sortedValues.map((v) => {
-    const value = { value: v };
-    return value;
-  });
-  if (q.removeOutliers) {
-    outlierMin = Math.min(outlierMin, descrStats.outlierMin.value);
-    outlierMax = Math.max(outlierMax, descrStats.outlierMax.value);
-  }
-  const boxplot = boxplot_getvalue(vs, q.removeOutliers);
-  if (!boxplot) throw new Error("boxplot_getvalue failed [termdb.violinBox init()]");
-  const plot = {
-    boxplot,
-    descrStats: setIndividualBoxPlotStats(boxplot, sortedValues),
-    // See comment in processBoxPlotData about tempValues
-    tempValues: sortedValues
-  };
-  if (overlayTw) {
-    const _key = overlayTw?.term?.values?.[key]?.label || key;
-    plot.color = overlayTw?.term?.values?.[key]?.color || void 0;
-    plot.key = String(_key);
-    plot.seriesId = key;
-    plot.boxplot.label = `${_key}, n=${values.length}`;
-  } else {
-    plot.key = sampleType;
-    plot.boxplot.label = `${sampleType}, n=${values.length}`;
-  }
-  plots.push(plot);
-  return [outlierMax, outlierMin];
-}
-function setIndividualBoxPlotStats(boxplot, values) {
-  const stats = {
-    total: { key: "total", label: "Total", value: values.length },
-    min: { key: "min", label: "Minimum", value: values[0] },
-    p25: { key: "p25", label: "1st quartile", value: boxplot.p25 },
-    median: { key: "median", label: "Median", value: boxplot.p50 },
-    p75: { key: "p75", label: "3rd quartile", value: boxplot.p75 },
-    mean: { key: "mean", label: "Mean", value: getMean(values) },
-    max: { key: "max", label: "Maximum", value: values[values.length - 1] },
-    stdDev: { key: "stdDev", label: "Standard deviation", value: getStdDev(values) }
-  };
-  for (const key of Object.keys(stats)) {
-    stats[key].value = roundValueAuto(stats[key].value);
-  }
-  return stats;
-}
-function setHiddenPlots(term, plots) {
-  for (const v of Object.values(term.term?.values)) {
-    const plot = plots.find((p) => p.key === v.label);
-    if (plot) plot.isHidden = v?.uncomputable;
-  }
-  if (term.q?.hiddenValues) {
-    for (const key of Object.keys(term.q.hiddenValues)) {
-      const plot = plots.find((p) => p.key === key);
-      if (plot) plot.isHidden = true;
-    }
-  }
-  return plots;
-}
-function setUncomputableValues(values) {
-  if (Object.entries(values)?.length) {
-    return Object.entries(values).map(([label, v]) => ({ label, value: v }));
-  } else return null;
-}
-async function getBoxPlotWilcoxonData(charts) {
-  await runWilcoxonTests(
-    charts,
-    {
-      getGroupId: (plot) => plot.boxplot.label.replace(/, n=\d+$/, ""),
-      getGroupValues: (plot) => plot.tempValues
-    },
-    "wilcoxon"
-  );
-}
-async function runWilcoxonTests(charts, accessors, resultKey) {
-  for (const chart of Object.values(charts)) {
-    const numPlots = chart.plots?.length;
-    if (!numPlots || numPlots < 2) continue;
-    const wilcoxInput = [];
-    for (let i = 0; i < numPlots; i++) {
-      const group1_id = accessors.getGroupId(chart.plots[i]);
-      const group1_values = accessors.getGroupValues(chart.plots[i]);
-      for (let j = i + 1; j < numPlots; j++) {
-        const group2_id = accessors.getGroupId(chart.plots[j]);
-        const group2_values = accessors.getGroupValues(chart.plots[j]);
-        wilcoxInput.push({ group1_id, group1_values, group2_id, group2_values });
-      }
-    }
-    const wilcoxOutput = JSON.parse(await run_rust("wilcoxon", JSON.stringify(wilcoxInput)));
-    chart[resultKey] = [];
-    for (const test of wilcoxOutput) {
-      if (test.pvalue == null || test.pvalue == "null") {
-        chart[resultKey].push([{ value: test.group1_id }, { value: test.group2_id }, { html: "NA" }]);
-      } else {
-        chart[resultKey].push([
-          { value: test.group1_id },
-          { value: test.group2_id },
-          { html: test.pvalue.toPrecision(4) }
-        ]);
-      }
-    }
-  }
-}
-function computeSampleType(data) {
-  return `All ${data.sampleType?.plural_name || "samples"}`;
-}
-function buildBins(tw, data, overlayTw, divideTw) {
-  const bins = {
-    term1: numericBins(tw, data)
-  };
-  if (overlayTw) bins.term2 = numericBins(overlayTw, data);
-  if (divideTw) bins.term0 = numericBins(divideTw, data);
-  return bins;
-}
-function extractNumericValues(samples, tw, isLogScale) {
-  let values = samples.map((s) => s?.[tw.$id]?.value).filter((v) => typeof v === "number" && !tw.term.values?.[v]?.uncomputable);
-  if (isLogScale) values = values.filter((v) => v > 0);
-  return values;
-}
-function parseValues(q, data, sampleType, isLog, overlayTw, divideTw) {
-  const chart2plot2values = /* @__PURE__ */ new Map();
-  const uncomputableValues = {};
-  function trackUncomputable(tw, key) {
-    if (!tw?.term?.values?.[key]?.uncomputable) return false;
-    const label = tw.term.values[key]?.label;
-    if (label) uncomputableValues[label] = (uncomputableValues[label] || 0) + 1;
-    return true;
-  }
-  let absMin = Infinity, absMax = -Infinity;
-  for (const val of Object.values(data.samples)) {
-    const value = val[q.tw.$id];
-    if (!Number.isFinite(value?.value)) continue;
-    if (trackUncomputable(q.tw, value.value)) continue;
-    if (isLog && value.value <= 0) continue;
-    let chart = "";
-    let plot = sampleType;
-    if (divideTw) {
-      if (!val[divideTw.$id]) continue;
-      const value0 = val[divideTw.$id];
-      trackUncomputable(divideTw, value0.key);
-      chart = value0.key;
-    }
-    if (overlayTw) {
-      if (!val[overlayTw.$id]) continue;
-      const value2 = val[overlayTw.$id];
-      trackUncomputable(overlayTw, value2.key);
-      plot = value2.key;
-    }
-    if (!chart2plot2values.has(chart)) chart2plot2values.set(chart, /* @__PURE__ */ new Map());
-    const plot2values = chart2plot2values.get(chart);
-    if (!plot2values.has(plot)) plot2values.set(plot, []);
-    const values = plot2values.get(plot);
-    values.push(value.value);
-    if (value.value < absMin) absMin = value.value;
-    if (value.value > absMax) absMax = value.value;
-  }
-  return { absMax, absMin, chart2plot2values, uncomputableValues };
-}
-function numericBins(tw, data) {
-  const bins = {};
-  if (!isNumericTerm(tw?.term)) return bins;
-  for (const bin of data.refs.byTermId[tw.$id]?.bins || []) {
-    bins[bin.label] = bin;
-  }
-  return bins;
-}
-export {
-  buildBins,
-  computeSampleType,
-  divideValues,
-  expandNumericTermCollection,
-  extractNumericValues,
-  getDensities,
-  getDensity,
-  getViolinWilcoxonData,
-  init,
-  numericBins,
-  parseValues,
-  setHiddenPlots,
-  setIndividualBoxPlotStats,
-  setPlotData,
-  setScaleData,
-  setUncomputableValues,
-  setViolinResponse,
-  sortObj,
-  sortPlot2Values
-};