@sjcrh/proteinpaint-server 2.191.5 → 2.192.0

package/src/serverconfig.js

@@ -53,7 +53,6 @@ if (!serverconfig.tabix) serverconfig.tabix = 'tabix'
 if (!serverconfig.samtools) serverconfig.samtools = 'samtools'
 if (!serverconfig.bcftools) serverconfig.bcftools = 'bcftools'
 if (!serverconfig.hicstraw) serverconfig.hicstraw = 'straw'
-if (!serverconfig.bigwigsummary) serverconfig.bigwigsummary = 'bigWigSummary'
 if (!serverconfig.bigBedToBed) serverconfig.bigBedToBed = 'bigBedToBed'
 if (!serverconfig.bigBedInfo) serverconfig.bigBedInfo = 'bigBedInfo'
 if (!serverconfig.bigBedNamedItems) serverconfig.bigBedNamedItems = 'bigBedNamedItems'
@@ -228,7 +227,6 @@ if (process.env.PP_MODE?.startsWith('container')) {
 		tpmasterdir: '/home/root/pp/tp',
 		cachedir: '/home/root/pp/cache',
 		hicstraw: '/home/root/pp/tools/straw',
-		bigwigsummary: '/home/root/pp/tools/bigWigSummary',
 		bigBedToBed: '/home/root/pp/tools/bigBedToBed',
 		bigBedNamedItems: '/home/root/pp/tools/bigBedNamedItems',
 		bigBedInfo: '/home/root/pp/tools/bigBedInfo'

package/routes/termdb.descrstats.js

@@ -1,115 +0,0 @@
-import { getData } from "#src/termdb.matrix.js";
-import computePercentile from "#shared/compute.percentile.js";
-import { roundValueAuto } from "#shared/roundValue.js";
-function init({ genomes }) {
-  return async (req, res) => {
-    const q = req.query;
-    try {
-      const genome = genomes[req.query.genome];
-      if (!genome) throw "invalid genome name";
-      const ds = genome.datasets[req.query.dslabel];
-      if (!ds) throw "invalid dataset name";
-      const tdb = ds.cohort.termdb;
-      if (!tdb) throw "invalid termdb object";
-      if (!q.tw.$id) q.tw.$id = "_";
-      const result = await trigger_getDescrStats(q, ds);
-      res.send(result);
-    } catch (e) {
-      if (e instanceof Error && e.stack) console.log(e);
-      const result = { error: e?.message || e };
-      res.send(result);
-    }
-  };
-}
-async function trigger_getDescrStats(q, ds) {
-  const data = await getData(
-    {
-      filter: q.filter,
-      filter0: q.filter0,
-      terms: [q.tw],
-      __protected__: q.__protected__,
-      __abortSignal: q.__abortSignal
-    },
-    ds
-  );
-  if (data.error) throw data.error;
-  const values = [];
-  for (const key in data.samples) {
-    const sample = data.samples[key];
-    const v = sample[q.tw.$id || ""];
-    if (!v && v !== 0) {
-      continue;
-    }
-    const value = v.value;
-    if (q.tw.term.values?.[value]?.uncomputable) {
-      continue;
-    }
-    if (q.logScale) {
-      if (value === 0) {
-        continue;
-      }
-    }
-    values.push(Number(value));
-  }
-  const stats = getDescrStats(values);
-  return stats;
-}
-function getDescrStats(values, showOutlierRange = false) {
-  if (!values.length) {
-    return {};
-  }
-  if (values.some((v) => !Number.isFinite(v))) throw new Error("non-numeric values found");
-  const sorted_arr = values.sort((a, b) => a - b);
-  const n = sorted_arr.length;
-  const median = computePercentile(sorted_arr, 50, true);
-  const mean = getMean(sorted_arr);
-  const variance = getVariance(sorted_arr);
-  const stdDev = Math.sqrt(variance);
-  const p25 = computePercentile(sorted_arr, 25, true);
-  const p75 = computePercentile(sorted_arr, 75, true);
-  const IQR = p75 - p25;
-  const min = sorted_arr[0];
-  const max = sorted_arr[sorted_arr.length - 1];
-  const outlierMin = p25 - 1.5 * IQR;
-  const outlierMax = p75 + 1.5 * IQR;
-  const stats = {
-    total: { key: "total", label: "Total", value: n },
-    min: { key: "min", label: "Minimum", value: min },
-    p25: { key: "p25", label: "1st quartile", value: p25 },
-    median: { key: "median", label: "Median", value: median },
-    p75: { key: "p75", label: "3rd quartile", value: p75 },
-    max: { key: "max", label: "Maximum", value: max },
-    mean: { key: "mean", label: "Mean", value: mean },
-    stdDev: { key: "stdDev", label: "Standard deviation", value: stdDev }
-    //variance: { label: 'Variance', value: variance }, // not necessary to report, as it is just stdDev^2
-    //iqr: { label: 'Inter-quartile range', value: IQR } // not necessary to report, as it is just p75-p25
-  };
-  if (showOutlierRange) {
-    stats.outlierMin = { key: "outlierMin", label: "Outlier minimum", value: outlierMin };
-    stats.outlierMax = { key: "outlierMax", label: "Outlier maximum", value: outlierMax };
-  }
-  for (const v of Object.values(stats)) {
-    const rounded = roundValueAuto(v.value);
-    v.value = rounded;
-  }
-  return stats;
-}
-function getMean(data) {
-  return data.reduce((sum, value) => sum + value, 0) / data.length;
-}
-function getVariance(data) {
-  const meanValue = getMean(data);
-  const squaredDifferences = data.map((value) => Math.pow(value - meanValue, 2));
-  return squaredDifferences.reduce((sum, value) => sum + value, 0) / (data.length - 1);
-}
-function getStdDev(data) {
-  const variance = getVariance(data);
-  return Math.sqrt(variance);
-}
-export {
-  getDescrStats,
-  getMean,
-  getStdDev,
-  getVariance,
-  init
-};

package/routes/termdb.sampleScatter.js

@@ -1,434 +0,0 @@
-import { getData } from "#src/termdb.matrix.js";
-import path from "path";
-import serverconfig from "#src/serverconfig.js";
-import { schemeCategory20, getColors, mclass, dt2label, morigin, isNumericTerm } from "#shared";
-import { authApi } from "#src/auth.js";
-import { run_R } from "@sjcrh/proteinpaint-r";
-import { read_file } from "#src/utils.js";
-import { getDescrStats } from "./termdb.descrstats.ts";
-const refColor = "#F5F5DC";
-function init({ genomes }) {
-  return async function(req, res) {
-    try {
-      const q = req.query;
-      if (!q.genome || !q.dslabel) {
-        throw new Error("Genome and dataset label are required for termdb/sampleScatter request.");
-      }
-      const g = genomes[q.genome];
-      const ds = g.datasets[q.dslabel];
-      let refSamples = [], cohortSamples;
-      const terms = [];
-      if (q.colorTW) terms.push(q.colorTW);
-      if (q.shapeTW) terms.push(q.shapeTW);
-      if (q.divideByTW) terms.push(q.divideByTW);
-      if (q.scaleDotTW) terms.push(q.scaleDotTW);
-      if (q.coordTWs) for (const tw of q.coordTWs) terms.push(tw);
-      const data = await getData(
-        { filter: q.filter, filter0: q.filter0, terms, __protected__: q.__protected__, __abortSignal: q.__abortSignal },
-        ds,
-        true
-        // FIXME 3rd arg hardcoded to true
-      );
-      if (data.error) throw new Error(data.error);
-      let result;
-      if (q.coordTWs && q.coordTWs.length > 0) {
-        const tmp = await getSampleCoordinatesByTerms(req, q, ds, data);
-        cohortSamples = tmp[0];
-      } else {
-        if (!q.plotName) throw new Error("Neither plot name or coordinates where provided");
-        if (typeof ds.cohort?.scatterplots?.get == "function") {
-          const allowed = ds.cohort.scatterplots.get(q.__protected__?.clientAuthResult);
-          if (!allowed?.find((i) => i.name == q.plotName)) throw new Error("No permission to display plot");
-        }
-        if (!Array.isArray(ds.cohort?.scatterplots?.plots)) throw new Error("not supported");
-        const plot = ds.cohort.scatterplots.plots.find((p) => p.name == q.plotName);
-        if (!plot) throw new Error(`plot not found with plotName ${q.plotName}`);
-        const tmp = await getSamples(ds, plot);
-        refSamples = tmp[0];
-        cohortSamples = tmp[1];
-        if (q.colorColumn) {
-          let categories = new Set(refSamples.map((s) => s.category));
-          categories = Array.from(categories);
-          const colorMap = {};
-          const k2c = getColors(categories.length);
-          for (const category of categories) {
-            const color = q.colorColumn.colorMap?.[category] || k2c(category);
-            colorMap[category] = {
-              sampleCount: refSamples.filter((s) => s.category == category).length,
-              color,
-              key: category
-            };
-          }
-          const shapeMap = { Ref: { shape: 0, sampleCount: refSamples.length, key: "Ref" } };
-          result = {
-            Default: {
-              samples: refSamples,
-              colorLegend: Object.entries(colorMap),
-              shapeLegend: Object.entries(shapeMap)
-            }
-          };
-        }
-      }
-      const samples = [...cohortSamples, ...refSamples];
-      let range;
-      if (samples.length > 0) {
-        if (q.excludeOutliers) {
-          const ystats = getDescrStats(
-            samples.map((s) => s.y),
-            q.excludeOutliers
-          );
-          cohortSamples = cohortSamples.filter(
-            (sample) => sample.y > ystats.outlierMin.value && sample.y < ystats.outlierMax.value
-          );
-          refSamples = refSamples.filter(
-            (sample) => sample.y > ystats.outlierMin.value && sample.y < ystats.outlierMax.value
-          );
-        }
-        const s0 = samples[0];
-        const [xMin, xMax, yMin, yMax] = samples.reduce(
-          (s, d) => [
-            d.x < s[0] ? d.x : s[0],
-            d.x > s[1] ? d.x : s[1],
-            d.y < s[2] ? d.y : s[2],
-            d.y > s[3] ? d.y : s[3]
-          ],
-          [s0.x, s0.x, s0.y, s0.y]
-        );
-        range = { xMin, xMax, yMin, yMax };
-      }
-      if (!result) result = await colorAndShapeSamples(refSamples, cohortSamples, data, q);
-      res.send({ result, range });
-    } catch (e) {
-      if (e.stack) console.log(e.stack);
-      res.send({ error: e.message || e });
-    }
-  };
-}
-async function getSamples(ds, plot) {
-  if (!plot.filterableSamples) await loadFile(plot, ds);
-  return [readSamples(plot.referenceSamples), readSamples(plot.filterableSamples)];
-  function readSamples(samples) {
-    const result = [];
-    for (const i of JSON.parse(JSON.stringify(samples))) {
-      result.push(i);
-    }
-    return result;
-  }
-}
-async function colorAndShapeSamples(refSamples, cohortSamples, data, q) {
-  const results = {};
-  let fCount = 0;
-  const hasTerms = Object.keys(data.samples).length > 0;
-  for (const sample of cohortSamples) {
-    const dbSample = data.samples[sample.sampleId.toString()];
-    if (!dbSample && hasTerms) {
-      fCount++;
-      continue;
-    }
-    if (q.colorTW && !hasValue(dbSample, q.colorTW) || q.shapeTW && !hasValue(dbSample, q.shapeTW)) continue;
-    let divideBy = "Default";
-    if (q.divideByTW) {
-      sample.z = 0;
-      if (q.divideByTW.term.type == "geneVariant" && q.divideByTW.q.type == "values") {
-        divideBy = getMutation(true, dbSample, q.divideByTW);
-        if (divideBy == null) {
-          divideBy = getMutation(false, dbSample, q.divideByTW);
-        }
-      } else {
-        const field = q.divideByTW.$id;
-        const key = dbSample[field]?.key;
-        if (key == null) continue;
-        if (q.divideByTW.q.mode != "continuous") divideBy = q.divideByTW.term.values?.[key]?.label || key;
-        else sample.z = key;
-      }
-    }
-    if (divideBy == null || divideBy == void 0) {
-      console.log("divideBy is null/undefined for sample " + JSON.stringify(sample));
-      continue;
-    }
-    if (!results[divideBy]) {
-      const samples = refSamples.map((sample2) => ({ ...sample2, category: "Ref", shape: "Ref", z: 0 }));
-      results[divideBy] = { samples, colorMap: {}, shapeMap: {} };
-    }
-    if (!q.divideByTW) sample.z = 0;
-    if (!q.scaleDotTW) sample.scale = 1;
-    else {
-      const value = dbSample?.[q.scaleDotTW.$id]?.key;
-      if (!value || !isComputable(q.scaleDotTW.term, value)) continue;
-      sample.scale = value;
-    }
-    sample.cat_info = {};
-    sample.hidden = {};
-    if (!q.colorTW) {
-      sample.category = "Default";
-    } else {
-      if (q.colorTW?.q?.mode === "continuous") {
-        if (dbSample) sample.category = dbSample[q.colorTW.$id].value;
-      } else processSample(dbSample, sample, q.colorTW, results[divideBy].colorMap, "category");
-    }
-    if (q.shapeTW) processSample(dbSample, sample, q.shapeTW, results[divideBy].shapeMap, "shape");
-    else sample.shape = "Ref";
-    results[divideBy].samples.push(sample);
-  }
-  if (fCount) console.log(fCount + " samples not in the database or filtered");
-  let max = 0;
-  for (const [_, result] of Object.entries(results)) max = Math.max(max, Object.keys(result.colorMap).length);
-  const k2c = getColors(max);
-  const scheme = schemeCategory20;
-  for (const [_, result] of Object.entries(results)) {
-    if (q.colorTW && q.colorTW.q.mode !== "continuous") {
-      let i2 = 20;
-      for (const [category, value] of Object.entries(result.colorMap)) {
-        delete value["sampleIds"];
-        let tvalue;
-        if (q.colorTW.term.values?.[value.key]) {
-          tvalue = q.colorTW.term.values?.[value.key];
-        }
-        if (tvalue && "color" in tvalue) {
-          value.color = tvalue.color;
-        } else if (isNumericTerm(q.colorTW.term)) {
-          const term = data.refs.byTermId[q.colorTW.$id] || data.refs.byTermId[q.colorTW.term.id];
-          const bins = term.bins;
-          const bin = bins.find((bin2) => bin2.label == category);
-          if (bin?.color) value.color = bin.color;
-          else {
-            value.color = scheme[i2 - 1];
-            i2--;
-          }
-        } else if (!(q.colorTW.term.type == "geneVariant" && q.colorTW.q.type == "values")) {
-          value.color = k2c(category);
-        }
-      }
-    }
-    let i = 0;
-    const shapes = Object.entries(result.shapeMap).sort((a, b) => a[0].localeCompare(b[0]));
-    for (const [_2, value] of shapes) {
-      delete value["sampleIds"];
-      if ("shape" in value) continue;
-      if (q.shapeTW.term.values?.[value.key]?.shape != void 0)
-        value.shape = q.shapeTW.term.values?.[value.key].shape;
-      else value.shape = i;
-      i++;
-    }
-    result.colorLegend = q.colorTW ? order(result.colorMap, q.colorTW, data.refs) : [["Default", { sampleCount: cohortSamples.length, color: "blue", key: "Default" }]];
-    result.colorLegend.push([
-      "Ref",
-      {
-        sampleCount: refSamples.length,
-        color: q.colorTW?.term.values?.["Ref"] ? q.colorTW.term.values?.["Ref"].color : refColor,
-        key: "Ref"
-      }
-    ]);
-    result.shapeLegend = shapes;
-    result.shapeLegend.push(["Ref", { sampleCount: refSamples.length, shape: 0, key: "Ref" }]);
-  }
-  return results;
-}
-function hasValue(dbSample, tw) {
-  const key = tw && tw.$id !== void 0 ? dbSample?.[tw.$id]?.key : void 0;
-  const hasKey = key !== void 0;
-  return hasKey;
-}
-function processSample(dbSample, sample, tw, categoryMap, category) {
-  let value = null;
-  if (tw.term.type == "geneVariant" && tw.q["type"] == "values")
-    assignGeneVariantValue(dbSample, sample, tw, categoryMap, category);
-  else {
-    value = dbSample?.[tw.$id]?.key;
-    if (value == null) return;
-    if (tw.term.values?.[value]?.label) {
-      value = tw.term.values?.[value]?.label;
-      sample.hidden[category] = tw.q.hiddenValues ? value in tw.q.hiddenValues : false;
-    } else sample.hidden[category] = tw.q.hiddenValues ? dbSample?.[tw.$id]?.key in tw.q.hiddenValues : false;
-    if (value) {
-      sample[category] = value.toString();
-      if (categoryMap[value] == void 0) categoryMap[value] = { sampleCount: 1, key: dbSample?.[tw.$id]?.key };
-      else categoryMap[value].sampleCount++;
-    }
-  }
-}
-function assignGeneVariantValue(dbSample, sample, tw, categoryMap, category) {
-  if (tw.term.type == "geneVariant") {
-    const mutations = dbSample?.[tw.$id]?.values;
-    sample.cat_info[category] = [];
-    for (const mutation of mutations) {
-      const class_info = mclass[mutation.class];
-      const value = getCategory(mutation);
-      sample.cat_info[category].push(mutation);
-      let mapValue;
-      if (categoryMap[value] == void 0) {
-        const sampleIds = /* @__PURE__ */ new Set();
-        sampleIds.add(dbSample.sample);
-        mapValue = { color: class_info.color, sampleCount: 1, mutation, key: value, sampleIds };
-        categoryMap[value] = mapValue;
-      } else {
-        mapValue = categoryMap[value];
-        mapValue.sampleIds.add(dbSample.sample);
-        mapValue.sampleCount = mapValue.sampleIds.size;
-      }
-    }
-    sample[category] = getMutation(true, dbSample, tw) || getMutation(false, dbSample, tw);
-    if (!sample[category]) sample[category] = getCategory(mutations[0]);
-    sample.hidden[category] = tw.q.hiddenValues ? sample[category] in tw.q.hiddenValues : false;
-  }
-}
-function getMutation(strict, dbSample, tw) {
-  const mutations = dbSample?.[tw.$id]?.values;
-  for (const [dt, _] of Object.entries(dt2label)) {
-    const mutation = mutations.find((mutation2) => {
-      const value2 = getCategory(mutation2);
-      const visible = !(tw.q.hiddenValues && value2 in tw.q.hiddenValues);
-      return mutation2.dt == dt && visible;
-    });
-    if (!mutation) continue;
-    const notImportant = mutation.class == "WT" || mutation.class == "Blank";
-    if (strict && notImportant) continue;
-    const value = getCategory(mutation);
-    return value;
-  }
-}
-function getCategory(mutation) {
-  const dt = mutation.dt;
-  const class_info = mclass[mutation.class];
-  const origin = morigin[mutation.origin]?.label;
-  const dtlabel = origin ? `${origin[0]} ${dt2label[dt]}` : dt2label[dt];
-  return `${class_info.label}, ${dtlabel}`;
-}
-function order(map, tw, refs) {
-  const hasOrder = tw?.term?.values ? Object.keys(tw.term.values).some((key) => tw.term.values[key].order != void 0) : false;
-  let entries = [];
-  if (!tw || map.size == 0) return entries;
-  entries = Object.entries(map);
-  const term = refs?.byTermId[tw.$id] || refs?.byTermId[tw.term.id];
-  if (hasOrder) {
-    entries.sort((a, b) => {
-      let v1, v2;
-      for (const key in tw.term.values) {
-        const value = tw.term.values[key];
-        if (value.label && a[0] == value.label) v1 = value;
-        else if (key == a[0]) v1 = value;
-        if (value.label && b[0] == value.label) v2 = value;
-        else if (key == b[0]) v2 = value;
-      }
-      if (v1?.order < v2?.order) return -1;
-      else if (v1?.order > v2?.order) return 1;
-      else if (v1 > v2) return 1;
-      else if (v1 < v2) return -1;
-      return 0;
-    });
-  } else if (term?.bins) {
-    const bins = term.bins;
-    entries.sort((a, b) => {
-      const binA = bins.findIndex((bin) => bin.label == a[0]);
-      const binB = bins.findIndex((bin) => bin.label == b[0]);
-      if (binA == -1) return 1;
-      if (binB == -1) return -1;
-      return binA - binB;
-    });
-  } else {
-    entries.sort((a, b) => a[0].localeCompare(b[0]));
-  }
-  return entries;
-}
-async function getSampleCoordinatesByTerms(req, q, ds, data) {
-  if (!q.coordTWs || q.coordTWs.length == 0) return [[], data];
-  const canDisplay = authApi.canDisplaySampleIds(req, ds);
-  const samples = [];
-  for (const sampleId in data.samples) {
-    const values = data.samples[sampleId];
-    const x = values[q.coordTWs[0].$id]?.value;
-    let y = values[q.coordTWs[1]?.$id]?.value;
-    if (y === void 0 && (q.chartType == "runchart" || q.chartType == "frequencyChart")) y = 0;
-    const z = q.divideByTW ? values[q.divideByTW?.$id]?.value : 0;
-    if (x == void 0 || y == void 0 || z == void 0) continue;
-    if (!isComputable(q.coordTWs[0].term, x) || !isComputable(q.coordTWs[1]?.term, y) || !isComputable(q.divideByTW?.term, z)) {
-      continue;
-    }
-    const sample = { sampleId, x: Number(x), y: Number(y), z: Number(z) };
-    if (canDisplay) {
-      sample.sample = data.refs.bySampleId[sampleId]?.label || sampleId;
-    }
-    samples.push(sample);
-  }
-  return [samples, data];
-}
-function isComputable(term, value) {
-  if (!term) return true;
-  return !term.values?.[value]?.uncomputable;
-}
-async function loadFile(p, ds) {
-  const lines = (await read_file(path.join(serverconfig.tpmasterdir, p.file))).trim().split("\n");
-  const xColumn = p.coordsColumns?.x || 1;
-  const yColumn = p.coordsColumns?.y || 2;
-  const headerFields = lines[0].split("	");
-  p.filterableSamples = [];
-  p.referenceSamples = [];
-  let invalidXY = 0;
-  for (let i = 1; i < lines.length; i++) {
-    const l = lines[i].trim().split("	");
-    const x = Number(l[xColumn]), y = Number(l[yColumn]);
-    if (Number.isNaN(x) || Number.isNaN(y)) {
-      invalidXY++;
-      continue;
-    }
-    const sample = { sample: l[0], x, y };
-    if (p.colorColumn) {
-      sample["sampleId"] = l[0];
-      sample.category = l[p.colorColumn.index];
-      sample.shape = "Ref";
-      sample.z = 0;
-    }
-    const id = ds.cohort.termdb.q.sampleName2id(l[0]);
-    if (id == void 0) {
-      if (headerFields[3]) {
-        sample.info = {};
-        for (let j = 3; j < headerFields.length; j++) {
-          sample.info[headerFields[j]] = l[j];
-        }
-      }
-      p.referenceSamples.push(sample);
-    } else {
-      sample["sampleId"] = id;
-      p.filterableSamples.push(sample);
-    }
-  }
-  console.log(
-    p.name + " (prebuilt scatter):",
-    p.filterableSamples.length,
-    "lines,",
-    p.referenceSamples.length,
-    "reference cases,",
-    invalidXY,
-    "lines with invalid X/Y values"
-  );
-}
-async function mayInitiateScatterplots(ds) {
-  if (ds.scatterplots) {
-    ds.cohort.scatterplots = ds.scatterplots;
-    delete ds.scatterplots;
-  }
-  if (!ds.cohort.scatterplots) return;
-  if (typeof ds.cohort.scatterplots.get == "function") {
-  }
-  if (!Array.isArray(ds.cohort.scatterplots.plots)) throw new Error("cohort.scatterplots.plots is not array");
-  for (const p of ds.cohort.scatterplots.plots) {
-    if (!p.name) throw new Error(".name missing from one of scatterplots.plots[]");
-    if (p.file) {
-    } else {
-      throw new Error("unknown data source of one of scatterplots.plots[]");
-    }
-  }
-}
-async function trigger_getLowessCurve(q, res) {
-  const data = q.coords;
-  const result = JSON.parse(await run_R("lowess.R", JSON.stringify(data)));
-  const lowessCurve = [];
-  for (const [i, x] of Object.entries(result.x)) lowessCurve.push([x, result.y[i]]);
-  return res.send(lowessCurve);
-}
-export {
-  init,
-  mayInitiateScatterplots,
-  refColor,
-  trigger_getLowessCurve
-};