@sjcrh/proteinpaint-server 2.44.0 → 2.46.0

package/routes/termdb.categories.js

@@ -0,0 +1,209 @@
+import { getOrderedLabels } from "#src/termdb.barchart.js";
+import { getData } from "#src/termdb.matrix.js";
+const api = {
+  endpoint: "termdb/categories",
+  methods: {
+    get: {
+      init,
+      request: {
+        typeId: "getcategoriesRequest"
+      },
+      response: {
+        typeId: "getcategoriesResponse"
+      },
+      examples: [
+        {
+          request: {
+            body: {
+              genome: "hg38-test",
+              dslabel: "TermdbTest",
+              embedder: "localhost",
+              getcategories: 1,
+              tid: "diaggrp",
+              term1_q: {
+                isAtomic: true,
+                hiddenValues: {},
+                type: "values",
+                groupsetting: { disabled: true },
+                mode: "discrete"
+              },
+              filter: {
+                type: "tvslst",
+                in: true,
+                join: "",
+                lst: [
+                  {
+                    tag: "cohortFilter",
+                    type: "tvs",
+                    tvs: {
+                      term: {
+                        name: "Cohort",
+                        type: "categorical",
+                        values: { ABC: { label: "ABC" }, XYZ: { label: "XYZ" } },
+                        id: "subcohort",
+                        isleaf: false,
+                        groupsetting: { disabled: true }
+                      },
+                      values: [{ key: "ABC", label: "ABC" }]
+                    }
+                  }
+                ]
+              }
+            }
+          },
+          response: {
+            header: { status: 200 }
+          }
+        }
+      ]
+    },
+    post: {
+      alternativeFor: "get",
+      init
+    }
+  }
+};
+function init({ genomes }) {
+  return async (req, res) => {
+    const q = req.query;
+    try {
+      const g = genomes[req.query.genome];
+      if (!g)
+        throw "invalid genome name";
+      const ds = g.datasets[req.query.dslabel];
+      if (!ds)
+        throw "invalid dataset name";
+      const tdb = ds.cohort.termdb;
+      if (!tdb)
+        throw "invalid termdb object";
+      await trigger_getcategories(q, res, tdb, ds, g);
+    } catch (e) {
+      res.send({ error: e?.message || e });
+      if (e instanceof Error && e.stack)
+        console.log(e);
+    }
+  };
+}
+async function trigger_getcategories(q, res, tdb, ds, genome) {
+  if (!q.tid)
+    throw ".tid missing";
+  const term = q.type == "geneVariant" ? { name: q.tid, type: "geneVariant", isleaf: true } : tdb.q.termjsonByOneid(q.tid);
+  const arg = {
+    filter: q.filter,
+    terms: q.type == "geneVariant" ? [{ term, q: { isAtomic: true } }] : [{ id: q.tid, term, q: q.term1_q || getDefaultQ(term, q) }],
+    currentGeneNames: q.currentGeneNames,
+    // optional, from mds3 mayAddGetCategoryArgs()
+    rglst: q.rglst
+    // optional, from mds3 mayAddGetCategoryArgs()
+  };
+  const data = await getData(arg, ds, genome);
+  if (data.error)
+    throw data.error;
+  const lst = [];
+  if (q.type == "geneVariant") {
+    const samples = data.samples;
+    const dtClassMap = /* @__PURE__ */ new Map();
+    if (ds.assayAvailability?.byDt) {
+      for (const [dtType, dtValue] of Object.entries(ds.assayAvailability.byDt)) {
+        if (dtValue.byOrigin) {
+          dtClassMap.set(parseInt(dtType), { byOrigin: { germline: {}, somatic: {} } });
+        }
+      }
+    }
+    const sampleCountedFor = /* @__PURE__ */ new Set();
+    for (const [sampleId, sampleData] of Object.entries(samples)) {
+      const values = sampleData[q.tid].values;
+      sampleCountedFor.clear();
+      for (const value of values) {
+        if (!dtClassMap.has(value.dt)) {
+          dtClassMap.set(value.dt, {});
+        }
+        const dtClasses = dtClassMap.get(value.dt);
+        if (dtClasses.byOrigin) {
+          if (!dtClasses.byOrigin[value.origin][value.class]) {
+            dtClasses.byOrigin[value.origin][value.class] = 1;
+            sampleCountedFor.add(`${value.dt} ${value.origin} ${value.class}`);
+          }
+          if (!sampleCountedFor.has(`${value.dt} ${value.origin} ${value.class}`)) {
+            sampleCountedFor.add(`${value.dt} ${value.origin} ${value.class}`);
+            dtClasses.byOrigin[value.origin][value.class] += 1;
+          }
+        } else {
+          if (!dtClasses[value.class]) {
+            sampleCountedFor.add(`${value.dt} ${value.class}`);
+            dtClasses[value.class] = 1;
+          }
+          if (!sampleCountedFor.has(`${value.dt} ${value.class}`)) {
+            sampleCountedFor.add(`${value.dt} ${value.class}`);
+            dtClasses[value.class] += 1;
+          }
+        }
+      }
+    }
+    for (const [dt, classes] of dtClassMap) {
+      lst.push({
+        dt,
+        classes
+      });
+    }
+  } else {
+    const key2count = /* @__PURE__ */ new Map();
+    for (const sid in data.samples) {
+      const v = data.samples[sid][q.tid];
+      if (!v)
+        continue;
+      if (!("key" in v))
+        continue;
+      key2count.set(v.key, 1 + (key2count.get(v.key) || 0));
+    }
+    for (const [key, count] of key2count) {
+      lst.push({
+        samplecount: count,
+        key,
+        label: data.refs?.byTermId?.[q.tid]?.events?.find((e) => e.event === key).label || term?.values?.[key]?.label || key
+      });
+    }
+  }
+  const orderedLabels = getOrderedLabels(
+    term,
+    data.refs?.byTermId?.[q.tid]?.bins || [],
+    data.refs?.byTermId?.[q.tid]?.events,
+    q.term1_q
+  );
+  if (orderedLabels.length) {
+    lst.sort((a, b) => orderedLabels.indexOf(a.label) - orderedLabels.indexOf(b.label));
+  }
+  res.send({
+    lst,
+    orderedLabels
+  });
+}
+function getDefaultQ(term, q) {
+  if (term.type == "categorical")
+    return {};
+  if (term.type == "survival")
+    return {};
+  if (term.type == "integer" || term.type == "float")
+    return term.bins.default;
+  if (term.type == "condition") {
+    return {
+      mode: q.mode,
+      breaks: q.breaks,
+      bar_by_grade: q.bar_by_grade,
+      /*Leave this here until bug with term1_q not passing to getCategories is figured out.
+      Commented out b/c tvs condition tests fail.*/
+      //bar_by_children: term.subconditions || q.bar_by_children,
+      bar_by_children: q.bar_by_children,
+      value_by_max_grade: q.value_by_max_grade,
+      value_by_most_recent: q.value_by_most_recent,
+      //value_by_computable_grade: term.subconditions || q.value_by_computable_grade
+      value_by_computable_grade: q.value_by_computable_grade
+    };
+  }
+  if (term.type == "geneVariant")
+    return {};
+  throw "unknown term type";
+}
+export {
+  api
+};

package/routes/termdb.cluster.js

@@ -0,0 +1,228 @@
+import path from "path";
+import fs from "fs";
+import lines2R from "#src/lines2R.js";
+import * as utils from "#src/utils.js";
+import serverconfig from "#src/serverconfig.js";
+import { gdc_validate_query_geneExpression } from "#src/mds3.gdc.js";
+import { mayLimitSamples } from "#src/mds3.filter.js";
+import { dtgeneexpression } from "#shared/common.js";
+const api = {
+  endpoint: "termdb/cluster",
+  methods: {
+    all: {
+      init,
+      request: {
+        typeId: "TermdbClusterRequest"
+      },
+      response: {
+        typeId: "TermdbClusterResponse"
+      }
+    }
+  }
+};
+function init({ genomes }) {
+  return async (req, res) => {
+    const q = req.query;
+    let result;
+    try {
+      const g = genomes[q.genome];
+      if (!g)
+        throw "invalid genome name";
+      const ds = g.datasets[q.dslabel];
+      if (!ds)
+        throw "invalid dataset name";
+      if (ds.__gdc && !ds.__gdc.doneCaching)
+        throw "The server has not finished caching the case IDs: try again in ~2 minutes";
+      if (q.dataType == dtgeneexpression) {
+        if (!ds.queries?.geneExpression)
+          throw "no geneExpression data on this dataset";
+        result = await getResult(q, ds);
+      } else {
+        throw "unknown q.dataType " + q.dataType;
+      }
+    } catch (e) {
+      if (e.stack)
+        console.log(e.stack);
+      result = {
+        status: e.status || 400,
+        error: e.message || e
+      };
+    }
+    res.send(result);
+  };
+}
+async function getResult(q, ds) {
+  const { gene2sample2value, byTermId, bySampleId } = await ds.queries.geneExpression.get(q);
+  if (gene2sample2value.size == 0)
+    throw "no data";
+  if (gene2sample2value.size == 1) {
+    const g = Array.from(gene2sample2value.keys())[0];
+    return { gene: g, data: gene2sample2value.get(g) };
+  }
+  const t = Date.now();
+  const clustering = await doClustering(gene2sample2value, q);
+  if (serverconfig.debugmode)
+    console.log("clustering done:", Date.now() - t, "ms");
+  return { clustering, byTermId, bySampleId };
+}
+async function doClustering(data, q) {
+  const sampleSet = /* @__PURE__ */ new Set();
+  for (const o of data.values()) {
+    for (const s in o)
+      sampleSet.add(s);
+    break;
+  }
+  const inputData = {
+    matrix: [],
+    row_names: [],
+    // genes
+    col_names: [...sampleSet],
+    // samples
+    cluster_method: q.clusterMethod,
+    plot_image: false
+    // When true causes cluster.rs to plot the image into a png file (EXPERIMENTAL)
+  };
+  for (const [gene, o] of data) {
+    inputData.row_names.push(gene);
+    const row = [];
+    for (const s of inputData.col_names) {
+      row.push(o[s] || 0);
+    }
+    inputData.matrix.push(getZscore(row));
+  }
+  const Rinputfile = path.join(serverconfig.cachedir, Math.random().toString() + ".json");
+  await utils.write_file(Rinputfile, JSON.stringify(inputData));
+  const Routput = JSON.parse(await lines2R(path.join(serverconfig.binpath, "utils/hclust.R"), [], [Rinputfile]));
+  await fs.promises.unlink(Rinputfile);
+  const row_names_index = Routput.RowOrder.map((row) => inputData.row_names.indexOf(row.name));
+  const col_names_index = Routput.ColOrder.map((col) => inputData.col_names.indexOf(col.name));
+  const output_matrix = [];
+  for (const rowI of row_names_index) {
+    const newRow = [];
+    for (const colI of col_names_index) {
+      newRow.push(inputData.matrix[rowI][colI]);
+    }
+    output_matrix.push(newRow);
+  }
+  return {
+    row: {
+      merge: Routput.RowMerge,
+      height: Routput.RowHeight,
+      order: Routput.RowOrder,
+      inputOrder: inputData.row_names
+    },
+    col: {
+      merge: Routput.ColumnMerge,
+      height: Routput.ColumnHeight,
+      order: Routput.ColOrder,
+      inputOrder: inputData.col_names
+    },
+    matrix: output_matrix
+  };
+}
+function getZscore(l) {
+  const mean = l.reduce((sum, v) => sum + v, 0) / l.length;
+  const sd = Math.sqrt(l.reduce((sum, v) => sum + Math.pow(v - mean, 2), 0) / l.length);
+  if (sd == 0) {
+    return l;
+  }
+  return l.map((v) => (v - mean) / sd);
+}
+async function validate_query_geneExpression(ds, genome) {
+  const q = ds.queries.geneExpression;
+  if (!q)
+    return;
+  if (q.src == "gdcapi") {
+    gdc_validate_query_geneExpression(ds, genome);
+    return;
+  }
+  if (q.src == "native") {
+    await validateNative(q, ds, genome);
+    return;
+  }
+  throw "unknown queries.geneExpression.src";
+}
+async function validateNative(q, ds, genome) {
+  if (!q.file.startsWith(serverconfig.tpmasterdir))
+    q.file = path.join(serverconfig.tpmasterdir, q.file);
+  if (!q.samples)
+    q.samples = [];
+  await utils.validate_tabixfile(q.file);
+  q.nochr = await utils.tabix_is_nochr(q.file, null, genome);
+  q.samples = [];
+  {
+    const lines = await utils.get_header_tabix(q.file);
+    if (!lines[0])
+      throw "header line missing from " + q.file;
+    const l = lines[0].split("	");
+    if (l.slice(0, 4).join("	") != "#chr	start	stop	gene")
+      throw "header line has wrong content for columns 1-4";
+    for (let i = 4; i < l.length; i++) {
+      const id = ds.cohort.termdb.q.sampleName2id(l[i]);
+      if (id == void 0)
+        throw "queries.geneExpression: unknown sample from header: " + l[i];
+      q.samples.push(id);
+    }
+    console.log(q.samples.length, "samples from geneExpression of", ds.label);
+  }
+  q.get = async (param) => {
+    const limitSamples = await mayLimitSamples(param, q.samples, ds);
+    if (limitSamples?.size == 0) {
+      return { gene2sample2value: /* @__PURE__ */ new Set(), byTermId: {}, bySampleId: {} };
+    }
+    const bySampleId = {};
+    const samples = q.samples || [];
+    if (limitSamples) {
+      for (const sid of limitSamples) {
+        bySampleId[sid] = { label: ds.cohort.termdb.q.id2sampleName(sid) };
+      }
+    } else {
+      for (const sid of samples) {
+        bySampleId[sid] = { label: ds.cohort.termdb.q.id2sampleName(sid) };
+      }
+    }
+    const gene2sample2value = /* @__PURE__ */ new Map();
+    for (const g of param.genes) {
+      if (!g.gene)
+        continue;
+      if (!g.chr) {
+        const lst = genome.genedb.getjsonbyname.all(g.gene);
+        if (lst.length == 0)
+          continue;
+        const j = JSON.parse(lst.find((i) => i.isdefault).genemodel || lst[0].genemodel);
+        g.start = j.start;
+        g.stop = j.stop;
+        g.chr = j.chr;
+      }
+      const s2v = {};
+      await utils.get_lines_bigfile({
+        args: [q.file, (q.nochr ? g.chr?.replace("chr", "") : g.chr) + ":" + g.start + "-" + g.stop],
+        // must do g.chr?.replace to avoid tsc error
+        callback: (line) => {
+          const l = line.split("	");
+          if (l[3].toLowerCase() != g.gene.toLowerCase())
+            return;
+          for (let i = 4; i < l.length; i++) {
+            const sampleId = samples[i - 4];
+            if (limitSamples && !limitSamples.has(sampleId))
+              continue;
+            if (!l[i])
+              continue;
+            const v = Number(l[i]);
+            if (Number.isNaN(v))
+              throw "exp value not number";
+            s2v[sampleId] = v;
+          }
+        }
+      });
+      if (Object.keys(s2v).length)
+        gene2sample2value.set(g.gene, s2v);
+    }
+    const byTermId = {};
+    return { gene2sample2value, byTermId, bySampleId };
+  };
+}
+export {
+  api,
+  validate_query_geneExpression
+};

package/routes/termdb.cohort.summary.js

@@ -0,0 +1,38 @@
+import { get_ds_tdb } from "#src/termdb.js";
+import { mayCopyFromCookie } from "#src/utils.js";
+const api = {
+  endpoint: "termdb/cohort/summary",
+  methods: {
+    get: {
+      init,
+      request: {
+        typeId: "any"
+      },
+      response: {
+        typeId: "any"
+      }
+    }
+  }
+};
+function init({ genomes }) {
+  return async (req, res) => {
+    const q = req.query;
+    mayCopyFromCookie(q, req.cookies);
+    try {
+      const genome = genomes[q.genome];
+      if (!genome)
+        throw "invalid genome";
+      const [ds] = get_ds_tdb(genome, q);
+      res.send({ count: ds.cohort.termdb.q.getcohortsamplecount(q.cohort) });
+    } catch (e) {
+      res.send({ error: e.message || e });
+      if (e.stack)
+        console.log(e.stack);
+      else
+        console.log(e);
+    }
+  };
+}
+export {
+  api
+};

package/routes/termdb.cohorts.js

@@ -0,0 +1,49 @@
+import { get_ds_tdb } from "#src/termdb.js";
+import { mayCopyFromCookie } from "#src/utils.js";
+const api = {
+  endpoint: "termdb/cohorts",
+  methods: {
+    get: {
+      init,
+      request: {
+        typeId: "any"
+      },
+      response: {
+        typeId: "any"
+      }
+    }
+  }
+};
+function init({ genomes }) {
+  return async (req, res) => {
+    const q = req.query;
+    mayCopyFromCookie(q, req.cookies);
+    try {
+      const genome = genomes[q.genome];
+      if (!genome)
+        throw "invalid genome";
+      const [ds] = get_ds_tdb(genome, q);
+      const result = getCohortsData(ds);
+      res.send(result);
+    } catch (e) {
+      res.send({ error: e.message || e });
+      if (e.stack)
+        console.log(e.stack);
+      else
+        console.log(e);
+    }
+  };
+}
+function getCohortsData(ds) {
+  const features = ds.cohort.db.connection.prepare("select * from features").all();
+  const cohorts = ds.cohort.db.connection.prepare(
+    `select * from cohorts where cohort in (select distinct(cohort) from cohort_features)
+		 order by sample_count desc`
+  ).all();
+  const cfeatures = ds.cohort.db.connection.prepare("select * from cohort_features").all();
+  return { cohorts, features, cfeatures };
+}
+export {
+  api,
+  getCohortsData
+};