@sjcrh/proteinpaint-server 2.147.2-0 → 2.148.1

package/routes/termdb.sampleScatter.js

@@ -0,0 +1,436 @@
+import { termdbSampleScatterPayload } from "#types/checkers";
+import { getData } from "../src/termdb.matrix.js";
+import path from "path";
+import serverconfig from "../src/serverconfig.js";
+import { schemeCategory20, getColors } from "#shared/common.js";
+import { mclass, dt2label, morigin } from "#shared/common.js";
+import { authApi } from "../src/auth.js";
+import { run_R } from "@sjcrh/proteinpaint-r";
+import { read_file } from "../src/utils.js";
+import { isNumericTerm } from "@sjcrh/proteinpaint-shared/terms.js";
+import { getDescrStats } from "#routes/termdb.descrstats.ts";
+const api = {
+  endpoint: "termdb/sampleScatter",
+  methods: {
+    get: {
+      ...termdbSampleScatterPayload,
+      init
+    },
+    post: {
+      ...termdbSampleScatterPayload,
+      init
+    }
+  }
+};
+const refColor = "#F5F5DC";
+function init({ genomes }) {
+  return async function(req, res) {
+    const q = req.query;
+    if (!q.genome || !q.dslabel) {
+      throw new Error("Genome and dataset label are required for termdb/sampleScatter request.");
+    }
+    const g = genomes[q.genome];
+    const ds = g.datasets[q.dslabel];
+    try {
+      let refSamples = [], cohortSamples;
+      const terms = [];
+      if (q.colorTW) terms.push(q.colorTW);
+      if (q.shapeTW) terms.push(q.shapeTW);
+      if (q.divideByTW) terms.push(q.divideByTW);
+      if (q.scaleDotTW) terms.push(q.scaleDotTW);
+      if (q.coordTWs) for (const tw of q.coordTWs) terms.push(tw);
+      const data = await getData(
+        { filter: q.filter, filter0: q.filter0, terms, __protected__: q.__protected__ },
+        ds,
+        true
+        // FIXME 3rd arg hardcoded to true
+      );
+      if (data.error) throw data.error;
+      let result;
+      if (q.coordTWs.length > 0) {
+        const tmp = await getSampleCoordinatesByTerms(req, q, ds, data);
+        cohortSamples = tmp[0];
+      } else {
+        if (!q.plotName) throw `Neither plot name or coordinates where provided`;
+        if (!Array.isArray(ds.cohort?.scatterplots?.plots)) throw "not supported";
+        const plot = ds.cohort.scatterplots.plots.find((p) => p.name == q.plotName);
+        if (!plot) throw `plot not found with plotName ${q.plotName}`;
+        const tmp = await getSamples(ds, plot);
+        refSamples = tmp[0];
+        cohortSamples = tmp[1];
+        if (q.colorColumn) {
+          let categories = new Set(refSamples.map((s) => s.category));
+          categories = Array.from(categories);
+          const colorMap = {};
+          const k2c = getColors(categories.length);
+          for (const category of categories) {
+            const color = q.colorColumn.colorMap?.[category] || k2c(category);
+            colorMap[category] = {
+              sampleCount: refSamples.filter((s) => s.category == category).length,
+              color,
+              key: category
+            };
+          }
+          const shapeMap = { Ref: { shape: 0, sampleCount: refSamples.length, key: "Ref" } };
+          result = {
+            Default: {
+              samples: refSamples,
+              colorLegend: Object.entries(colorMap),
+              shapeLegend: Object.entries(shapeMap)
+            }
+          };
+        }
+      }
+      const samples = [...cohortSamples, ...refSamples];
+      let range;
+      if (samples.length > 0) {
+        if (q.excludeOutliers) {
+          const ystats = getDescrStats(
+            samples.map((s) => s.y),
+            q.excludeOutliers
+          );
+          cohortSamples = cohortSamples.filter(
+            (sample) => sample.y > ystats.outlierMin.value && sample.y < ystats.outlierMax.value
+          );
+          refSamples = refSamples.filter(
+            (sample) => sample.y > ystats.outlierMin.value && sample.y < ystats.outlierMax.value
+          );
+        }
+        const s0 = samples[0];
+        const [xMin, xMax, yMin, yMax] = samples.reduce(
+          (s, d) => [
+            d.x < s[0] ? d.x : s[0],
+            d.x > s[1] ? d.x : s[1],
+            d.y < s[2] ? d.y : s[2],
+            d.y > s[3] ? d.y : s[3]
+          ],
+          [s0.x, s0.x, s0.y, s0.y]
+        );
+        range = { xMin, xMax, yMin, yMax };
+      }
+      if (!result) result = await colorAndShapeSamples(refSamples, cohortSamples, data, q);
+      res.send({ result, range });
+    } catch (e) {
+      if (e.stack) console.log(e.stack);
+      res.send({ error: e.message || e });
+    }
+  };
+}
+async function getSamples(ds, plot) {
+  if (!plot.filterableSamples) await loadFile(plot, ds);
+  return [readSamples(plot.referenceSamples), readSamples(plot.filterableSamples)];
+  function readSamples(samples) {
+    const result = [];
+    for (const i of JSON.parse(JSON.stringify(samples))) {
+      result.push(i);
+    }
+    return result;
+  }
+}
+async function colorAndShapeSamples(refSamples, cohortSamples, data, q) {
+  const results = {};
+  let fCount = 0;
+  const hasTerms = Object.keys(data.samples).length > 0;
+  for (const sample of cohortSamples) {
+    const dbSample = data.samples[sample.sampleId.toString()];
+    if (!dbSample && hasTerms) {
+      fCount++;
+      continue;
+    }
+    if (q.colorTW && !hasValue(dbSample, q.colorTW) || q.shapeTW && !hasValue(dbSample, q.shapeTW)) continue;
+    let divideBy = "Default";
+    if (q.divideByTW) {
+      sample.z = 0;
+      if (q.divideByTW.term.type == "geneVariant" && q.divideByTW.q.type == "values") {
+        divideBy = getMutation(true, dbSample, q.divideByTW);
+        if (divideBy == null) {
+          divideBy = getMutation(false, dbSample, q.divideByTW);
+        }
+      } else {
+        const field = q.divideByTW.$id;
+        const key = dbSample[field]?.key;
+        if (key == null) continue;
+        if (q.divideByTW.q.mode != "continuous") divideBy = q.divideByTW.term.values?.[key]?.label || key;
+        else sample.z = key;
+      }
+    }
+    if (divideBy == null || divideBy == void 0) {
+      console.log("divideBy is null/undefined for sample " + JSON.stringify(sample));
+      continue;
+    }
+    if (!results[divideBy]) {
+      const samples = refSamples.map((sample2) => ({ ...sample2, category: "Ref", shape: "Ref", z: 0 }));
+      results[divideBy] = { samples, colorMap: {}, shapeMap: {} };
+    }
+    if (!q.divideByTW) sample.z = 0;
+    if (!q.scaleDotTW) sample.scale = 1;
+    else {
+      const value = dbSample?.[q.scaleDotTW.$id]?.key;
+      if (!value || !isComputable(q.scaleDotTW.term, value)) continue;
+      sample.scale = value;
+    }
+    sample.cat_info = {};
+    sample.hidden = {};
+    if (!q.colorTW) {
+      sample.category = "Default";
+    } else {
+      if (q.colorTW?.q?.mode === "continuous") {
+        if (dbSample) sample.category = dbSample[q.colorTW.$id].value;
+      } else processSample(dbSample, sample, q.colorTW, results[divideBy].colorMap, "category");
+    }
+    if (q.shapeTW) processSample(dbSample, sample, q.shapeTW, results[divideBy].shapeMap, "shape");
+    else sample.shape = "Ref";
+    results[divideBy].samples.push(sample);
+  }
+  if (fCount) console.log(fCount + " samples not in the database or filtered");
+  let max = 0;
+  for (const [_, result] of Object.entries(results)) max = Math.max(max, Object.keys(result.colorMap).length);
+  const k2c = getColors(max);
+  const scheme = schemeCategory20;
+  for (const [_, result] of Object.entries(results)) {
+    if (q.colorTW && q.colorTW.q.mode !== "continuous") {
+      let i2 = 20;
+      for (const [category, value] of Object.entries(result.colorMap)) {
+        delete value["sampleIds"];
+        let tvalue;
+        if (q.colorTW.term.values?.[value.key]) {
+          tvalue = q.colorTW.term.values?.[value.key];
+        }
+        if (tvalue && "color" in tvalue) {
+          value.color = tvalue.color;
+        } else if (isNumericTerm(q.colorTW.term)) {
+          const bins = data.refs.byTermId[q.colorTW.$id].bins;
+          const bin = bins.find((bin2) => bin2.label == category);
+          if (bin?.color) value.color = bin.color;
+          else {
+            value.color = scheme[i2 - 1];
+            i2--;
+          }
+        } else if (!(q.colorTW.term.type == "geneVariant" && q.colorTW.q.type == "values")) {
+          value.color = k2c(category);
+        }
+      }
+    }
+    let i = 0;
+    const shapes = Object.entries(result.shapeMap).sort((a, b) => a[0].localeCompare(b[0]));
+    for (const [_2, value] of shapes) {
+      delete value["sampleIds"];
+      if ("shape" in value) continue;
+      if (q.shapeTW.term.values?.[value.key]?.shape != void 0)
+        value.shape = q.shapeTW.term.values?.[value.key].shape;
+      else value.shape = i;
+      i++;
+    }
+    result.colorLegend = q.colorTW ? order(result.colorMap, q.colorTW, data.refs) : [["Default", { sampleCount: cohortSamples.length, color: "blue", key: "Default" }]];
+    result.colorLegend.push([
+      "Ref",
+      {
+        sampleCount: refSamples.length,
+        color: q.colorTW?.term.values?.["Ref"] ? q.colorTW.term.values?.["Ref"].color : refColor,
+        key: "Ref"
+      }
+    ]);
+    result.shapeLegend = shapes;
+    result.shapeLegend.push(["Ref", { sampleCount: refSamples.length, shape: 0, key: "Ref" }]);
+  }
+  return results;
+}
+function hasValue(dbSample, tw) {
+  const key = tw && tw.$id !== void 0 ? dbSample?.[tw.$id]?.key : void 0;
+  const hasKey = key !== void 0;
+  return hasKey;
+}
+function processSample(dbSample, sample, tw, categoryMap, category) {
+  let value = null;
+  if (tw.term.type == "geneVariant" && tw.q["type"] == "values")
+    assignGeneVariantValue(dbSample, sample, tw, categoryMap, category);
+  else {
+    value = dbSample?.[tw.$id]?.key;
+    if (value == null) return;
+    if (tw.term.values?.[value]?.label) {
+      value = tw.term.values?.[value]?.label;
+      sample.hidden[category] = tw.q.hiddenValues ? value in tw.q.hiddenValues : false;
+    } else sample.hidden[category] = tw.q.hiddenValues ? dbSample?.[tw.$id]?.key in tw.q.hiddenValues : false;
+    if (value) {
+      sample[category] = value.toString();
+      if (categoryMap[value] == void 0) categoryMap[value] = { sampleCount: 1, key: dbSample?.[tw.$id]?.key };
+      else categoryMap[value].sampleCount++;
+    }
+  }
+}
+function assignGeneVariantValue(dbSample, sample, tw, categoryMap, category) {
+  if (tw.term.type == "geneVariant") {
+    const mutations = dbSample?.[tw.$id]?.values;
+    sample.cat_info[category] = [];
+    for (const mutation of mutations) {
+      const class_info = mclass[mutation.class];
+      const value = getCategory(mutation);
+      sample.cat_info[category].push(mutation);
+      let mapValue;
+      if (categoryMap[value] == void 0) {
+        const sampleIds = /* @__PURE__ */ new Set();
+        sampleIds.add(dbSample.sample);
+        mapValue = { color: class_info.color, sampleCount: 1, mutation, key: value, sampleIds };
+        categoryMap[value] = mapValue;
+      } else {
+        mapValue = categoryMap[value];
+        mapValue.sampleIds.add(dbSample.sample);
+        mapValue.sampleCount = mapValue.sampleIds.size;
+      }
+    }
+    sample[category] = getMutation(true, dbSample, tw) || getMutation(false, dbSample, tw);
+    if (!sample[category]) sample[category] = getCategory(mutations[0]);
+    sample.hidden[category] = tw.q.hiddenValues ? sample[category] in tw.q.hiddenValues : false;
+  }
+}
+function getMutation(strict, dbSample, tw) {
+  const mutations = dbSample?.[tw.$id]?.values;
+  for (const [dt, _] of Object.entries(dt2label)) {
+    const mutation = mutations.find((mutation2) => {
+      const value2 = getCategory(mutation2);
+      const visible = !(tw.q.hiddenValues && value2 in tw.q.hiddenValues);
+      return mutation2.dt == dt && visible;
+    });
+    if (!mutation) continue;
+    const notImportant = mutation.class == "WT" || mutation.class == "Blank";
+    if (strict && notImportant) continue;
+    const value = getCategory(mutation);
+    return value;
+  }
+}
+function getCategory(mutation) {
+  const dt = mutation.dt;
+  const class_info = mclass[mutation.class];
+  const origin = morigin[mutation.origin]?.label;
+  const dtlabel = origin ? `${origin[0]} ${dt2label[dt]}` : dt2label[dt];
+  return `${class_info.label}, ${dtlabel}`;
+}
+function order(map, tw, refs) {
+  const hasOrder = tw?.term?.values ? Object.keys(tw.term.values).some((key) => tw.term.values[key].order != void 0) : false;
+  let entries = [];
+  if (!tw || map.size == 0) return entries;
+  entries = Object.entries(map);
+  if (hasOrder) {
+    entries.sort((a, b) => {
+      let v1, v2;
+      for (const key in tw.term.values) {
+        const value = tw.term.values[key];
+        if (value.label && a[0] == value.label) v1 = value;
+        else if (key == a[0]) v1 = value;
+        if (value.label && b[0] == value.label) v2 = value;
+        else if (key == b[0]) v2 = value;
+      }
+      if (v1?.order < v2?.order) return -1;
+      else if (v1?.order > v2?.order) return 1;
+      else if (v1 > v2) return 1;
+      else if (v1 < v2) return -1;
+      return 0;
+    });
+  } else if (refs?.byTermId[tw.$id]?.bins) {
+    const bins = refs.byTermId[tw.$id].bins;
+    entries.sort((a, b) => {
+      const binA = bins.findIndex((bin) => bin.label == a[0]);
+      const binB = bins.findIndex((bin) => bin.label == b[0]);
+      if (binA == -1) return 1;
+      if (binB == -1) return -1;
+      return binA - binB;
+    });
+  } else {
+    entries.sort((a, b) => a[0].localeCompare(b[0]));
+  }
+  return entries;
+}
+async function getSampleCoordinatesByTerms(req, q, ds, data) {
+  if (!q.coordTWs || q.coordTWs.length == 0) return [[], data];
+  const canDisplay = authApi.canDisplaySampleIds(req, ds);
+  const samples = [];
+  for (const sampleId in data.samples) {
+    const values = data.samples[sampleId];
+    const x = values[q.coordTWs[0].$id]?.value;
+    const y = values[q.coordTWs[1]?.$id]?.value || 0;
+    const z = q.divideByTW ? values[q.divideByTW?.$id]?.value : 0;
+    if (x == void 0 || y == void 0 || z == void 0) continue;
+    if (!isComputable(q.coordTWs[0].term, x) || !isComputable(q.coordTWs[1]?.term, y) || !isComputable(q.divideByTW?.term, z)) {
+      continue;
+    }
+    const sample = { sampleId, x: Number(x), y: Number(y), z: Number(z) };
+    if (canDisplay) {
+      sample.sample = data.refs.bySampleId[sampleId]?.label || sampleId;
+    }
+    samples.push(sample);
+  }
+  return [samples, data];
+}
+function isComputable(term, value) {
+  if (!term) return true;
+  return !term.values?.[value]?.uncomputable;
+}
+async function loadFile(p, ds) {
+  const lines = (await read_file(path.join(serverconfig.tpmasterdir, p.file))).trim().split("\n");
+  const xColumn = p.coordsColumns?.x || 1;
+  const yColumn = p.coordsColumns?.y || 2;
+  const headerFields = lines[0].split("	");
+  p.filterableSamples = [];
+  p.referenceSamples = [];
+  let invalidXY = 0;
+  for (let i = 1; i < lines.length; i++) {
+    const l = lines[i].trim().split("	");
+    const x = Number(l[xColumn]), y = Number(l[yColumn]);
+    if (Number.isNaN(x) || Number.isNaN(y)) {
+      invalidXY++;
+      continue;
+    }
+    const sample = { sample: l[0], x, y };
+    if (p.colorColumn) {
+      sample["sampleId"] = l[0];
+      sample.category = l[p.colorColumn.index];
+      sample.shape = "Ref";
+      sample.z = 0;
+    }
+    const id = ds.cohort.termdb.q.sampleName2id(l[0]);
+    if (id == void 0) {
+      if (headerFields[3]) {
+        sample.info = {};
+        for (let j = 3; j < headerFields.length; j++) {
+          sample.info[headerFields[j]] = l[j];
+        }
+      }
+      p.referenceSamples.push(sample);
+    } else {
+      sample["sampleId"] = id;
+      p.filterableSamples.push(sample);
+    }
+  }
+  console.log(
+    p.name + " (prebuilt scatter):",
+    p.filterableSamples.length,
+    "lines,",
+    p.referenceSamples.length,
+    "reference cases,",
+    invalidXY,
+    "lines with invalid X/Y values"
+  );
+}
+async function mayInitiateScatterplots(ds) {
+  if (!ds.cohort.scatterplots) return;
+  if (!Array.isArray(ds.cohort.scatterplots.plots)) throw "cohort.scatterplots.plots is not array";
+  for (const p of ds.cohort.scatterplots.plots) {
+    if (!p.name) throw ".name missing from one of scatterplots.plots[]";
+    if (p.file) {
+    } else {
+      throw "unknown data source of one of scatterplots.plots[]";
+    }
+  }
+}
+async function trigger_getLowessCurve(q, res) {
+  const data = q.coords;
+  const result = JSON.parse(await run_R("lowess.R", JSON.stringify(data)));
+  const lowessCurve = [];
+  for (const [i, x] of Object.entries(result.x)) lowessCurve.push([x, result.y[i]]);
+  return res.send(lowessCurve);
+}
+export {
+  api,
+  mayInitiateScatterplots,
+  trigger_getLowessCurve
+};

package/routes/termdb.singlecellSamples.js

@@ -184,27 +184,17 @@ function validateGeneExpressionNative(G) {
   G.get = async (q) => {
     const h5file = path.join(serverconfig.tpmasterdir, G.folder, (q.sample.eID || q.sample.sID) + ".h5");
     await file_is_readable(h5file);
-    const read_hdf5_input_type = { gene: q.gene, hdf5_file: h5file };
-    let out;
-    try {
-      const time1 = Date.now();
-      const rust_output = await run_rust("readHDF5", JSON.stringify(read_hdf5_input_type));
-      mayLog("Time taken to query HDF5 file:", Date.now() - time1, "ms");
-      for (const line of rust_output.split("\n")) {
-        if (line.startsWith("output_string:")) {
-          out = JSON.parse(line.replace("output_string:", ""));
-        } else {
-          console.log(line);
-        }
-      }
-    } catch (e) {
-      if (typeof e == "string") {
-        const geneNotFound = `Gene '${q.gene}' not found in the HDF5 file`;
-        if (e.includes(geneNotFound)) throw "No expression data for this gene";
-      }
-      console.log(e);
-      throw "error reading h5 file: " + e;
+    const query_gene = q.gene;
+    if (!query_gene) {
+      throw new Error("Gene parameter is undefined");
     }
+    const read_hdf5_input_type = { query: [query_gene], hdf5_file: h5file };
+    const time1 = Date.now();
+    const rust_output = await run_rust("readH5", JSON.stringify(read_hdf5_input_type));
+    mayLog("Time taken to query HDF5 file:", Date.now() - time1, "ms");
+    const result = JSON.parse(rust_output);
+    const out = result.query_output[query_gene]?.samples;
+    if (!out) throw `No expression data for ${query_gene}`;
     return out;
   };
 }