@sjcrh/proteinpaint-server 2.190.2 → 2.191.1

package/routes/correlationVolcano.js

@@ -1,112 +0,0 @@
-import { getData } from "../src/termdb.matrix.js";
-import { run_R } from "@sjcrh/proteinpaint-r";
-import { mayLog } from "#src/helpers.ts";
-import { getStdDev } from "./termdb.descrstats.ts";
-const minArrayLength = 3;
-function init({ genomes }) {
-  return async (req, res) => {
-    const q = req.query;
-    try {
-      const genome = genomes[q.genome];
-      if (!genome) throw "invalid genome name";
-      const ds = genome.datasets?.[q.dslabel];
-      if (!ds) throw "invalid ds";
-      const result = await compute(q, ds);
-      res.send(result);
-    } catch (e) {
-      res.send({ error: e?.message || e });
-      if (e instanceof Error && e.stack) console.error(e);
-    }
-  };
-}
-async function compute(q, ds) {
-  if (!q.featureTw.$id) throw "featureTw.$id missing";
-  if (!ds.cohort.correlationVolcano.feature.termTypes.includes(q.featureTw?.term.type))
-    throw "unsupported featureTw.term.type";
-  const data = await getData(
-    {
-      filter: q.filter,
-      filter0: q.filter0,
-      terms: [q.featureTw, ...q.variableTwLst],
-      __protected__: q.__protected__
-    },
-    ds
-  );
-  if (data.error) throw data.error;
-  const vtid2array = /* @__PURE__ */ new Map();
-  for (const tw of q.variableTwLst) {
-    vtid2array.set(tw.$id, { id: tw.$id, v1: [], v2: [] });
-  }
-  for (const sid in data.samples) {
-    const featureValue = data.samples[sid][q.featureTw.$id]?.value;
-    if (!Number.isFinite(featureValue)) continue;
-    for (const tw of q.variableTwLst) {
-      if (!tw.$id) throw "variableTwLst[].$id missing";
-      const variableValue = data.samples[sid][tw.$id]?.value;
-      if (!Number.isFinite(variableValue)) continue;
-      vtid2array.get(tw.$id).v1.push(featureValue);
-      vtid2array.get(tw.$id).v2.push(variableValue);
-    }
-  }
-  const acceptedVariables = [], skippedVariables = [];
-  for (const [tid, v] of vtid2array) {
-    if (v.v1.length < minArrayLength || v.v2.length < minArrayLength || getStdDev(v.v1) == 0 || getStdDev(v.v2) == 0) {
-      skippedVariables.push({ tw$id: tid });
-      continue;
-    }
-    acceptedVariables.push(v);
-  }
-  const result = { skippedVariables, variableItems: [] };
-  if (!acceptedVariables.length) return result;
-  const input = {
-    method: q.correlationMethod || "pearson",
-    terms: acceptedVariables
-  };
-  const time1 = Date.now();
-  const output = {
-    terms: JSON.parse(await run_R("corr.R", JSON.stringify(input)))
-  };
-  mayLog("Time taken to run correlation analysis:", Date.now() - time1);
-  for (const t of output.terms) {
-    const t2 = {
-      tw$id: t.id,
-      sampleSize: t.sample_size,
-      //sampleSize: input.terms.get(t.id).v1.length, // This was not working so passed the length of each array from R
-      correlation: t.correlation,
-      original_pvalue: t.original_p_value,
-      adjusted_pvalue: t.adjusted_p_value
-    };
-    result.variableItems.push(t2);
-  }
-  return result;
-}
-function validate_correlationVolcano(ds) {
-  const cv = ds.cohort.correlationVolcano;
-  if (!cv) return;
-  if (typeof cv.feature != "object") throw "cv.feature not obj";
-  if (!Array.isArray(cv.feature.termTypes)) throw "cv.feature.termTypes[] not array";
-  for (const t of cv.feature.termTypes) {
-    if (t == "geneExpression") {
-      if (!ds.queries?.geneExpression) throw "geneExpression cv.feature is not supported";
-    } else if (t == "ssGSEA") {
-      if (!ds.queries?.ssGSEA) throw "ssGSEA cv.feature is not supported";
-    } else {
-      throw "unknown cv.feature.termType";
-    }
-  }
-  if (typeof cv.variables != "object") throw "cv.variables not obj";
-  if (cv.variables.type == "dictionaryTerm") {
-    if (!Array.isArray(cv.variables.termIds)) throw "cv.variables.termIds not array when type=dictionaryTerm";
-    for (const id of cv.variables.termIds) {
-      const t = ds.cohort.termdb.q.termjsonByOneid(id);
-      if (!t) throw "cv.variables.termIds: unknown id: " + id;
-      if (t.type != "integer" && t.type != "float") throw "cv.variables.termIds: not integer/float: " + id;
-    }
-  } else {
-    throw "unknown cv.variables.type";
-  }
-}
-export {
-  init,
-  validate_correlationVolcano
-};

package/routes/dataset.js

@@ -1,233 +0,0 @@
-import * as mds3_init from "#src/mds3.init.js";
-import * as common from "#shared/common.js";
-function init({ genomes }) {
-  return function(req, res) {
-    try {
-      const q = req.query;
-      const genome = genomes[q.genome];
-      if (!genome) throw "unknown genome";
-      if (!genome.datasets) throw "genomeobj.datasets{} missing";
-      let ds;
-      for (const k in genome.datasets) {
-        if (k.toLowerCase() == q.dsname.toLowerCase()) {
-          ds = genome.datasets[k];
-          break;
-        }
-      }
-      if (!ds) throw "invalid dsname";
-      const copy = ds.isMds3 ? mds3_init.client_copy(ds) : ds.isMds ? mds_clientcopy(ds) : copy_legacyDataset(ds);
-      return res.send({ ds: copy });
-    } catch (e) {
-      res.send({ error: e.message || e });
-    }
-  };
-}
-function mds_clientcopy(ds) {
-  const ds2 = {
-    isMds: true,
-    noHandleOnClient: ds.noHandleOnClient,
-    label: ds.label,
-    version: ds.version,
-    annotationsampleset2matrix: ds.annotationsampleset2matrix,
-    mutationAttribute: ds.mutationAttribute,
-    locusAttribute: ds.locusAttribute,
-    alleleAttribute: ds.alleleAttribute,
-    // these are quick fixes and should be deleted later
-    hide_genotypedownload: ds.hide_genotypedownload,
-    hide_phewas: ds.hide_phewas,
-    sample2bam: ds.sample2bam
-  };
-  if (ds.queries) {
-    ds2.queries = {};
-  }
-  if (ds.singlesamplemutationjson) {
-    ds2.singlesamplemutationjson = 1;
-  }
-  if (ds.gene2mutcount) {
-    ds2.gene2mutcount = true;
-    ds2.mutCountType = ds.gene2mutcount.mutationTypes;
-  }
-  if (ds.assayAvailability) {
-    ds2.assayAvailability = 1;
-  }
-  if (ds.cohort && ds.cohort.sampleAttribute) {
-    const toclient = {};
-    for (const k in ds.cohort.sampleAttribute.attributes) {
-      const a = ds.cohort.sampleAttribute.attributes[k];
-      if (!a.clientnoshow) toclient[k] = a;
-    }
-    ds2.sampleAttribute = { attributes: toclient };
-  }
-  if (ds.cohort) {
-    if (ds.cohort.termdb) {
-      ds2.termdb = {
-        selectCohort: ds.cohort.termdb.selectCohort
-      };
-    }
-    if (ds.cohort.attributes && ds.cohort.attributes.defaulthidden) {
-      ds2.cohortHiddenAttr = ds.cohort.attributes.defaulthidden;
-    }
-    if (ds.cohort.survivalplot) {
-      ds2.survivalplot = {
-        samplegroupattrlst: ds.cohort.survivalplot.samplegroupattrlst,
-        plots: []
-      };
-      for (const k in ds.cohort.survivalplot.plots) {
-        const p = ds.cohort.survivalplot.plots[k];
-        ds2.survivalplot.plots.push({
-          key: k,
-          name: p.name,
-          timelabel: p.timelabel
-        });
-      }
-    }
-    if (ds.cohort.mutation_signature) {
-      const sets = {};
-      for (const k in ds.cohort.mutation_signature.sets) {
-        const s = ds.cohort.mutation_signature.sets[k];
-        sets[k] = {
-          name: s.name,
-          signatures: s.signatures
-        };
-      }
-      ds2.mutation_signature = { sets };
-    }
-  }
-  for (const k in ds.queries) {
-    const q = ds.queries[k];
-    const clientquery = {
-      // revealed to client
-      name: q.name,
-      hideforthemoment: q.hideforthemoment
-      // hide track not ready to show on client
-    };
-    if (q.istrack) {
-      clientquery.istrack = true;
-      clientquery.type = q.type;
-      clientquery.isfull = q.isfull;
-      if (q.nochr != void 0) {
-        clientquery.nochr = q.nochr;
-      }
-      if (q.infoFilter) {
-        clientquery.infoFilter = q.infoFilter;
-      }
-      if (q.readcountCutoff) {
-        clientquery.readcountCutoff = q.readcountCutoff;
-      }
-      if (q.valueLabel) {
-        clientquery.valueLabel = q.valueLabel;
-      }
-      if (q.valueCutoff) {
-        clientquery.valueCutoff = q.valueCutoff;
-      }
-      if (q.bplengthUpperLimit) {
-        clientquery.bplengthUpperLimit = q.bplengthUpperLimit;
-      }
-      if (q.segmeanValueCutoff) {
-        clientquery.segmeanValueCutoff = q.segmeanValueCutoff;
-      }
-      if (q.lohLengthUpperLimit) {
-        clientquery.lohLengthUpperLimit = q.lohLengthUpperLimit;
-      }
-      if (q.type == common.tkt.mdssvcnv) {
-        if (q.groupsamplebyattr) {
-          clientquery.groupsamplebyattr = q.groupsamplebyattr;
-        }
-        clientquery.multihidelabel_fusion = q.multihidelabel_fusion;
-        clientquery.multihidelabel_sv = q.multihidelabel_sv;
-        clientquery.multihidelabel_vcf = q.multihidelabel_vcf;
-        clientquery.showfullmode = q.showfullmode;
-        clientquery.legend_vorigin = q.legend_vorigin;
-        clientquery.no_loh = q.no_loh;
-        if (q.expressionrank_querykey) {
-          const e = ds.queries[q.expressionrank_querykey];
-          clientquery.checkexpressionrank = {
-            querykey: q.expressionrank_querykey,
-            datatype: e.datatype
-          };
-          if (e.boxplotbysamplegroup && e.boxplotbysamplegroup.additionals) {
-            const lst = [];
-            if (e.boxplotbysamplegroup.attributes)
-              lst.push(e.boxplotbysamplegroup.attributes.map((i) => i.label).join(", "));
-            for (const i of e.boxplotbysamplegroup.additionals) lst.push(i.label);
-            clientquery.checkexpressionrank.boxplotgroupers = lst;
-          }
-        }
-        if (q.vcf_querykey) {
-          clientquery.checkvcf = {
-            querykey: q.vcf_querykey,
-            info: ds.queries[q.vcf_querykey].info,
-            format: {}
-          };
-          for (const tk of ds.queries[q.vcf_querykey].tracks) {
-            if (tk.format) {
-              for (const k2 in tk.format) {
-                clientquery.checkvcf.format[k2] = tk.format[k2];
-              }
-            }
-          }
-        }
-      }
-    } else if (q.isgenenumeric) {
-      clientquery.isgenenumeric = true;
-      clientquery.datatype = q.datatype;
-      clientquery.no_ase = q.no_ase;
-    } else {
-      continue;
-    }
-    ds2.queries[k] = clientquery;
-  }
-  return ds2;
-}
-function copy_legacyDataset(ds) {
-  const ds2 = {
-    noHandleOnClient: ds.noHandleOnClient,
-    sampleselectable: ds.sampleselectable,
-    label: ds.label,
-    dsinfo: ds.dsinfo,
-    stratify: ds.stratify,
-    cohort: ds.cohort,
-    vcfinfofilter: ds.vcfinfofilter,
-    info2table: ds.info2table,
-    info2singletable: ds.info2singletable,
-    url4variant: ds.url4variant,
-    itemlabelname: ds.itemlabelname
-  };
-  if (ds.snvindel_attributes) {
-    ds2.snvindel_attributes = [];
-    for (const at of ds.snvindel_attributes) {
-      const rep = {};
-      for (const k in at) {
-        if (k == "lst") {
-          rep.lst = [];
-          for (const e of at.lst) {
-            const rep2 = {};
-            for (const k2 in e) rep2[k2] = e[k2];
-            rep.lst.push(rep2);
-          }
-        } else {
-          rep[k] = at[k];
-        }
-      }
-      ds2.snvindel_attributes.push(rep);
-    }
-  }
-  if (ds.snvindel_legend) {
-    ds2.snvindel_legend = ds.snvindel_legend;
-  }
-  const vcfinfo = {};
-  let hasvcf = false;
-  for (const q of ds.queries) {
-    if (q.vcf) {
-      hasvcf = true;
-      vcfinfo[q.vcf.vcfid] = q.vcf;
-    }
-  }
-  if (hasvcf) {
-    ds2.id2vcf = vcfinfo;
-  }
-  return ds2;
-}
-export {
-  init
-};

package/routes/dsdata.js

@@ -1,99 +0,0 @@
-import path from "path";
-import { spawn } from "child_process";
-import serverconfig from "#src/serverconfig.js";
-import * as common from "#shared/common.js";
-function init({ genomes }) {
-  return async function handle_dsdata(req, res) {
-    try {
-      const q = req.query;
-      if (!genomes[q.genome]) throw "invalid genome";
-      if (!q.dsname) throw ".dsname missing";
-      const ds = genomes[q.genome].datasets[q.dsname];
-      if (!ds) throw "invalid dsname";
-      const data = [];
-      for (const query of ds.queries) {
-        if (q.expressiononly && !query.isgeneexpression) {
-          continue;
-        }
-        if (q.noexpression && query.isgeneexpression) {
-          continue;
-        }
-        if (query.dsblocktracklst) {
-          continue;
-        }
-        if (query.vcffile) {
-          const d = await handle_dsdata_vcf(query, req);
-          data.push(d);
-          continue;
-        }
-        if (query.makequery) {
-          const d = handle_dsdata_makequery(ds, query, req, genomes);
-          data.push(d);
-          continue;
-        }
-        throw "unknow type from one of ds.queries[]";
-      }
-      res.send({ data });
-    } catch (e) {
-      if (e.stack) console.log(e.stack);
-      res.send({ error: e.message || e });
-    }
-  };
-}
-function handle_dsdata_makequery(ds, query, req, genomes) {
-  if (req.query.isoform) {
-    if (genomes[req.query.genome].genomicNameRegexp.test(req.query.isoform)) return;
-  }
-  const [sqlstr, values] = query.makequery(req.query);
-  if (!sqlstr) {
-    return;
-  }
-  const rows = ds.newconn.prepare(sqlstr).all(values);
-  let lst;
-  if (query.tidy) {
-    lst = rows.map((i) => query.tidy(i));
-  } else {
-    lst = rows;
-  }
-  const result = {};
-  if (query.isgeneexpression) {
-    result.lst = lst;
-    result.isgeneexpression = true;
-    result.config = query.config;
-    for (const q2 of ds.queries) {
-      if (!q2.dsblocktracklst) continue;
-      for (const tk of q2.dsblocktracklst) {
-        if (tk.type == common.tkt.junction) {
-          result.config.dsjunctiontk = tk;
-        }
-      }
-    }
-  } else {
-    result.lst = lst;
-  }
-  return result;
-}
-function handle_dsdata_vcf(query, req) {
-  const par = [
-    path.join(serverconfig.tpmasterdir, query.vcffile),
-    (query.vcf.nochr ? req.query.range.chr.replace("chr", "") : req.query.range.chr) + ":" + req.query.range.start + "-" + req.query.range.stop
-  ];
-  return new Promise((resolve, reject) => {
-    const ps = spawn(serverconfig.tabix, par);
-    const out = [], out2 = [];
-    ps.stdout.on("data", (i) => out.push(i));
-    ps.stderr.on("data", (i) => out2.push(i));
-    ps.on("close", () => {
-      const e = out2.join("").trim();
-      if (e != "") reject("error querying vcf file");
-      const tmp = out.join("").trim();
-      resolve({
-        lines: tmp == "" ? [] : tmp.split("\n"),
-        vcfid: query.vcf.vcfid
-      });
-    });
-  });
-}
-export {
-  init
-};

package/routes/gdc.maf.js

@@ -1,85 +0,0 @@
-import ky from "ky";
-import { joinUrl } from "#shared/joinUrl.js";
-import serverconfig from "#src/serverconfig.js";
-import { getGdcSampletypes } from "#src/mds3.gdc.js";
-const maxFileNumber = 1e3;
-const allowedWorkflowType = "Aliquot Ensemble Somatic Variant Merging and Masking";
-const maxTotalSizeCompressed = serverconfig.features.gdcMafMaxFileSize || 4e8;
-function init({ genomes }) {
-  return async (req, res) => {
-    try {
-      const g = genomes.hg38;
-      if (!g) throw "hg38 missing";
-      const ds = g.datasets.GDC;
-      if (!ds) throw "hg38 GDC missing";
-      const q = req.query;
-      const payload = await listMafFiles(q, ds);
-      res.send(payload);
-    } catch (e) {
-      res.send({ status: "error", error: e.message || e });
-    }
-  };
-}
-async function listMafFiles(q, ds) {
-  const filters = {
-    op: "and",
-    content: [
-      { op: "=", content: { field: "data_format", value: "MAF" } },
-      { op: "=", content: { field: "experimental_strategy", value: q.experimentalStrategy } },
-      { op: "=", content: { field: "analysis.workflow_type", value: allowedWorkflowType } },
-      { op: "=", content: { field: "access", value: "open" } }
-      // delete if later to support controlled files
-    ]
-  };
-  const case_filters = { op: "and", content: [] };
-  if (q.filter0) {
-    case_filters.content.push(q.filter0);
-  }
-  const { host, headers } = ds.getHostHeaders(q);
-  const body = {
-    filters,
-    size: maxFileNumber,
-    fields: [
-      "id",
-      "file_size",
-      "cases.project.project_id",
-      // for display only
-      "cases.case_id",
-      // case uuid for making case url link to portal
-      "cases.submitter_id",
-      // used when listing all cases & files
-      "cases.samples.tissue_type",
-      "cases.samples.tumor_descriptor"
-    ].join(",")
-  };
-  if (case_filters.content.length) body.case_filters = case_filters;
-  const response = await ky.post(joinUrl(host.rest, "files"), { headers, timeout: false, json: body });
-  if (!response.ok) throw `HTTP Error: ${response.status} ${response.statusText}`;
-  const re = await response.json();
-  if (!Number.isInteger(re.data?.pagination?.total)) throw "re.data.pagination.total is not int";
-  if (!Array.isArray(re.data?.hits)) throw "re.data.hits[] not array";
-  const files = [];
-  for (const h of re.data.hits) {
-    const c = h.cases?.[0];
-    if (!c) throw "h.cases[0] missing";
-    files.push({
-      id: h.id,
-      project_id: c.project.project_id,
-      file_size: h.file_size,
-      case_submitter_id: c.submitter_id,
-      case_uuid: c.case_id,
-      sample_types: getGdcSampletypes(c)
-    });
-  }
-  files.sort((a, b) => b.file_size - a.file_size);
-  const result = {
-    files,
-    filesTotal: re.data.pagination.total,
-    maxTotalSizeCompressed
-  };
-  return result;
-}
-export {
-  init,
-  maxTotalSizeCompressed
-};

package/routes/gdc.mafBuild.js

@@ -1,115 +0,0 @@
-import ky from "ky";
-import { joinUrl } from "#shared/joinUrl.js";
-import { stream_rust } from "@sjcrh/proteinpaint-rust";
-import { maxTotalSizeCompressed } from "./gdc.maf.ts";
-import { mayLog } from "#src/helpers.ts";
-function init({ genomes }) {
-  return async (req, res) => {
-    try {
-      const g = genomes.hg38;
-      if (!g) throw "hg38 missing";
-      const ds = g.datasets.GDC;
-      if (!ds) throw "hg38 GDC missing";
-      const q = req.query;
-      await buildMaf(q, res, ds);
-    } catch (e) {
-      if (e.stack) console.log(e.stack);
-      res.send({ status: "error", error: e.message || e });
-    }
-  };
-}
-async function buildMaf(q, res, ds) {
-  const t0 = Date.now();
-  const { host, headers } = ds.getHostHeaders(q);
-  const fileLst2 = await getFileLstUnderSizeLimit(q.fileIdLst, host, headers);
-  mayLog(`${fileLst2.length} out of ${q.fileIdLst.length} input MAF files accepted by size limit`, Date.now() - t0);
-  const arg = {
-    fileIdLst: fileLst2,
-    columns: q.columns,
-    host: joinUrl(host.rest, "data"),
-    // must use the /data/ endpoint from current host
-    headers
-  };
-  const boundary = "------------------------GDC-MAF-BUILD";
-  res.setHeader("Content-Type", `multipart/form-data; boundary=${boundary}`);
-  res.write(`--${boundary}`);
-  res.write('\r\nContent-Disposition: form-data; name="gzfile"; filename="cohort.maf.gz"');
-  res.write("\r\nContent-Type: application/gzip\r\n\r\n");
-  res.flush();
-  try {
-    const streams = stream_rust("gdcmaf", JSON.stringify(arg), emitJson);
-    if (streams) {
-      const { rustStream, endStream } = streams;
-      res.on("close", () => {
-        if (res.writableEnded) return;
-        try {
-          console.log("\n-- forced res.end() ---\n");
-          res.end();
-        } catch (e) {
-          console.log("error with forced res.end()", e);
-        }
-        try {
-          endStream();
-        } catch (e) {
-          console.log("error in calling endStream()", e);
-        }
-      });
-      rustStream.pipe(res, { end: false }).on("error", (e) => {
-        console.log("rustStream.pipe().on(error)", e);
-      }).on("end", () => {
-        if (res.writableEnded) return;
-        console.log("rustStream.on(end), trigger res.end()");
-        res.end();
-      });
-    } else {
-      emitJson({ error: "server error: undefined rustStream" });
-    }
-  } catch (e) {
-    console.log("error calling stream_rust(gdcmaf)", e);
-  }
-  function emitJson(data, end = true) {
-    if (res.writableEnded) return;
-    if (data) {
-      res.write(`\r
---${boundary}`);
-      res.write('\r\nContent-Disposition: form-data; name="errors"');
-      res.write("\r\nContent-Type: application/x-jsonlines");
-      const json = typeof data === "string" ? data : JSON.stringify(data || { ok: true, status: "ok" });
-      res.write("\r\n\r\n" + json);
-    }
-    res.write(`\r
---${boundary}--\r
-`);
-    mayLog("rust gdcmaf", Date.now() - t0);
-    if (end) res.end();
-  }
-}
-async function getFileLstUnderSizeLimit(lst, host, headers) {
-  if (lst.length == 0) throw "fileIdLst[] not array or blank";
-  const body = {
-    filters: {
-      op: "in",
-      content: { field: "file_id", value: lst }
-    },
-    size: 1e4,
-    fields: "file_size"
-  };
-  const response = await ky.post(joinUrl(host.rest, "files"), { headers, timeout: false, json: body });
-  if (!response.ok) throw `HTTP Error: ${response.status} ${response.statusText}`;
-  const re = await response.json();
-  if (!Array.isArray(re.data?.hits)) throw "re.data.hits[] not array";
-  const out = [];
-  let cumsize = 0;
-  for (const h of re.data.hits) {
-    if (cumsize >= maxTotalSizeCompressed) break;
-    if (!h.id) throw ".id missing";
-    if (!Number.isInteger(h.file_size)) throw ".file_size not integer";
-    cumsize += h.file_size;
-    out.push(h.id);
-  }
-  if (out.length == 0) throw "no file available";
-  return out;
-}
-export {
-  init
-};