@sjcrh/proteinpaint-server 2.44.0 → 2.46.1

package/package.json

@@ -1,34 +1,38 @@
 {
   "name": "@sjcrh/proteinpaint-server",
-  "version": "2.44.0",
+  "version": "2.46.1",
+  "type": "module",
   "description": "a genomics visualization tool for exploring a cohort's genotype and phenotype data",
-  "main": "server.js",
+  "main": "src/app.js",
+  "module": "src/app.js",
   "bin": "start.js",
   "imports": {
-    "#shared/*": "./shared/*",
+    "#types": "./shared/types/index.ts",
     "#shared/types/*": "./shared/types/*",
+    "#shared/*": "./shared/*",
     "#src/*": "./src/*",
     "#routes/*": "./routes/*"
   },
   "scripts": {
-    "prepare": "ts-patch install",
-    "dev": "webpack --env NODE_ENV=development --watch --color",
-    "start": "tsc --esModuleInterop genome/*.ts dataset/*.ts && nodemon --enable-source-maps server.js --watch ./server*.js* --watch dataset/*.ts --watch genome/*.ts",
+    "dev": "npm run start",
+    "prestart": "tsx emitImports.js dev > server.js",
+    "start": "tsx watch src/app.ts",
+    "test:unit": "tsx emitImports.js unit > serverTests.js && tsx serverTests.js && rm -rf ./cache",
+    "getconf": "../build/getConfigProp.js",
+    "doc": "../augen/build.sh routes shared/types/routes shared/checkers ../public/docs/server",
+    "mjs": "esbuild \"$DIR/*.ts\" --platform=node --outdir=\"$DIR\" --format=esm",
+    "cjs": "esbuild \"$DIR/*.ts\" --platform=node --outdir=\"$DIR\" --format=cjs",
+    "build": "./build.sh",
+    "prepack": "npm run build",
+    "postpack": "./dedupjs.sh",
+    "dedup": "./dedupjs.sh",
+    "//todo": "refactor or deprecate the scripts below",
     "pretest": "tsc --esModuleInterop genome/*.ts dataset/*.ts && ./test/pretest.js",
-    "test": "webpack --env NODE_ENV=test exportsFilename=all-test-context.js && node --enable-source-maps test/serverTests.js",
-    "precheckers": "tsc --esModuleInterop genome/*.ts dataset/*.ts",
-    "checkers": "webpack --config=./test/webpack.config.js && node test/emitPrepFiles.bundle.js && typia generate --input shared/checkers-raw --output shared/checkers",
+    "prepare": "ts-patch install",
     "pretest:type": "npm run checkers",
-    "test:type": "webpack --env NODE_ENV=test exportsFilename=type-test-context.js && node --enable-source-maps test/serverTests.js",
-    "pretest:unit": "npm run precheckers",
-    "test:unit": "webpack --env NODE_ENV=test exportsFilename=unit-test-context.js && node --enable-source-maps test/serverTests.js",
     "pretest:integration": "tsc --esModuleInterop genome/*.ts dataset/*.ts",
     "test:integration": "echo 'TODO: server integration tests'",
-    "prepack": "tsc --esModuleInterop genome/*.ts dataset/*.ts && webpack --env NODE_ENV=production",
-    "test:tsc": "tsc --esModuleInterop --noEmit --allowImportingTsExtensions ./shared/types/test/*.type.spec.ts",
-    "response": "nodemon modules/test/test.server.js --watch src",
-    "getconf": "../build/getConfigProp.js",
-    "doc": "../augen/build.sh routes shared/types/routes shared/checkers ../public/docs/server"
+    "test:tsc": "tsc --esModuleInterop --noEmit --allowImportingTsExtensions ./shared/types/test/*.type.spec.ts"
   },
   "author": "",
   "license": "SEE LICENSE IN ./LICENSE",
@@ -39,8 +43,11 @@
     "@babel/preset-env": "^7.9.6",
     "@babel/preset-typescript": "^7.21.4",
     "@babel/register": "^7.14.5",
+    "@types/node": "^20.11.24",
     "@typescript-eslint/eslint-plugin": "^5.60.0",
     "babel-loader": "^8.2.2",
+    "esbuild": "^0.19.12",
+    "glob": "^7.2.3",
     "node-watch": "^0.7.1",
     "nodemon": "^2.0.19",
     "prettier": "^2.8.8",
@@ -50,14 +57,10 @@
     "tsx": "^4.7.1",
     "typedoc": "^0.24.8",
     "typescript": "^5.0.3",
-    "typia": "^4.1.14",
-    "webpack": "^5.76.0",
-    "webpack-cli": "^4.9.2",
-    "webpack-node-externals": "^3.0.0",
-    "webpack-notifier": "^1.15.0"
+    "typia": "^4.1.14"
   },
   "dependencies": {
-    "@sjcrh/augen": "2.43.3-0",
+    "@sjcrh/augen": "2.46.0",
     "@sjcrh/proteinpaint-rust": "2.44.0",
     "better-sqlite3": "^9.4.1",
     "body-parser": "^1.15.2",
@@ -68,7 +71,7 @@
     "deep-object-diff": "^1.1.0",
     "express": "^4.17.1",
     "express-basic-auth": "^1.1.5",
-    "got": "^11.5.1",
+    "got": "^14.2.0",
     "image-size": "^0.5.5",
     "jsonwebtoken": "^9.0.0",
     "jstat": "^1.9.3",
@@ -88,24 +91,17 @@
   },
   "files": [
     "bin.js",
-    "dataset/clinvar*.js",
-    "dataset/gdc*.js",
+    "dataset/*.js",
     "genome/*.js",
+    "routes/*.js",
+    "start.js",
+    "src/app.js",
     "src/serverconfig.js",
     "src/lines2R.js",
     "src/mds3.gdc.filter.js",
     "src/checkReadingFrame.js",
     "src/bedj.parseBed.js",
-    "routes/*",
-    "cards/*",
-    "server.js*",
-    "start.js",
-    "shared/common.js",
-    "shared/termdb.initbinconfig.js",
-    "shared/vcf.js",
     "utils/*.R",
-    "utils/rust/*.*",
-    "utils/rust/src",
     "LICENSE/*"
   ],
   "bugs": {

package/routes/_template_.js

@@ -0,0 +1,30 @@
+const api = {
+  // route endpoint
+  // - no need for trailing slash
+  // - should be a noun (method is based on HTTP GET, POST, etc)
+  // - don't add 'Data' as response is assumed to be data
+  endpoint: "...",
+  methods: {
+    get: {
+      init,
+      request: {
+        typeId: "any"
+      },
+      response: {
+        typeId: "any"
+      }
+    },
+    post: {
+      alternativeFor: "get",
+      init
+    }
+  }
+};
+function init({ genomes }) {
+  return async function(req, res) {
+    console.log(genomes, req, res);
+  };
+}
+export {
+  api
+};

package/routes/burden.js

@@ -0,0 +1,149 @@
+import lines2R from "#src/lines2R.js";
+import path from "path";
+import serverconfig from "#src/serverconfig.js";
+import { write_file } from "#src/utils.js";
+const api = {
+  endpoint: "burden",
+  methods: {
+    get: {
+      init({ genomes }) {
+        return async (req, res) => {
+          try {
+            const genome = genomes[req.query.genome];
+            if (!genome)
+              throw `invalid q.genome=${req.query.genome}`;
+            const q = req.query;
+            const ds = genome.datasets[q.dslabel];
+            if (!ds)
+              throw `invalid q.genome=${req.query.dslabel}`;
+            if (!ds.cohort.cumburden?.files)
+              throw `missing ds.cohort.cumburden.files`;
+            const estimates = await getBurdenEstimates(req, ds);
+            const { keys, rows } = formatPayload(estimates);
+            res.send({ status: "ok", keys, rows });
+          } catch (e) {
+            res.send({ status: "error", error: e.message || e });
+          }
+        };
+      },
+      request: {
+        typeId: "BurdenRequest"
+      },
+      response: {
+        typeId: "BurdenResponse"
+      },
+      examples: [
+        {
+          request: {
+            body: {
+              genome: "hg38",
+              // TODO: !!! use hg38-test and TermdbTest !!!
+              dslabel: "SJLife",
+              diaggrp: 5,
+              sex: 1,
+              white: 1,
+              agedx: 1,
+              bleo: 0,
+              etop: 0,
+              cisp: 0,
+              carbo: 0,
+              steriod: 0,
+              vcr: 0,
+              hdmtx: 0,
+              itmt: 0,
+              ced: 0,
+              dox: 0,
+              heart: 0,
+              brain: 0,
+              abd: 0,
+              pelvis: 0,
+              chest: 0
+            }
+          },
+          response: {
+            header: { status: 200 }
+          }
+        }
+      ]
+    }
+  }
+};
+async function getBurdenEstimates(q, ds) {
+  const infile = path.join(serverconfig.cachedir, Math.random().toString() + ".json");
+  for (const k in q.query) {
+    q.query[k] = Number(q.query[k]);
+  }
+  const data = Object.assign({}, defaults, q.query);
+  await write_file(infile, JSON.stringify(data));
+  const { fit, surv, sample } = ds.cohort.cumburden.files;
+  if (!fit || !surv || !sample)
+    throw `missing one or more of ds.cohort.burden.files.{fit, surv, sample}`;
+  const args = [
+    infile,
+    `${serverconfig.tpmasterdir}/${fit}`,
+    `${serverconfig.tpmasterdir}/${surv}`,
+    `${serverconfig.tpmasterdir}/${sample}`
+  ];
+  const Routput = await lines2R(path.join(serverconfig.binpath, "utils/burden.R"), [], args);
+  const estimates = JSON.parse(Routput[0]);
+  return estimates;
+}
+function formatPayload(estimates) {
+  const rawKeys = Object.keys(estimates[0]);
+  const outKeys = [];
+  const keys = [];
+  for (const k of rawKeys) {
+    if (k == "chc") {
+      keys.push(k);
+      outKeys.push(k);
+    } else {
+      const age = Number(k.slice(1).split(",")[0]);
+      if (age <= 60 && age % 2 == 0) {
+        keys.push(k);
+        outKeys.push(`burden${age}`);
+      }
+    }
+  }
+  const rows = [];
+  for (const v of estimates) {
+    rows.push(keys.map((k) => v[k]));
+  }
+  return { keys: outKeys, rows };
+}
+const defaults = Object.freeze({
+  diaggrp: 5,
+  sex: 0,
+  white: 1,
+  agedx: 1,
+  // chemotherapy
+  steriod: 0,
+  bleo: 0,
+  vcr: 0,
+  //12, // Vincristine
+  etop: 0,
+  //2500, // Etoposide
+  itmt: 0,
+  // Intrathecal methothrexate_grp: 0,
+  ced: 0,
+  //1.6, // Cyclophosphamide, 0.7692 mean 7692.
+  cisp: 0,
+  //300, // Cisplatin
+  dox: 0,
+  // Anthracycline, 3 mean 300 ml/m2
+  carbo: 0,
+  //  Carboplatin
+  hdmtx: 0,
+  // High-Dose Methotrexate
+  // radiation
+  brain: 0,
+  //5.4,
+  chest: 0,
+  //2.4,
+  heart: 0,
+  pelvis: 0,
+  abd: 0
+  //2.4
+});
+export {
+  api
+};

package/routes/dataset.js

@@ -0,0 +1,266 @@
+import * as mds2_init from "#src/mds2.init.js";
+import * as mds3_init from "#src/mds3.init.js";
+import * as common from "#shared/common.js";
+const api = {
+  endpoint: "getDataset",
+  // should rename to simply 'dataset', method is based on HTTP method
+  methods: {
+    get: {
+      init,
+      request: {
+        typeId: "any"
+      },
+      response: {
+        typeId: "any"
+      }
+    },
+    post: {
+      alternativeFor: "get",
+      init
+    }
+  }
+};
+function init({ genomes }) {
+  return function(req, res) {
+    try {
+      const genome = genomes[req.query.genome];
+      if (!genome)
+        throw "unknown genome";
+      if (!genome.datasets)
+        throw "genomeobj.datasets{} missing";
+      let ds;
+      for (const k in genome.datasets) {
+        if (k.toLowerCase() == req.query.dsname.toLowerCase()) {
+          ds = genome.datasets[k];
+          break;
+        }
+      }
+      if (!ds)
+        throw "invalid dsname";
+      if (ds.isMds3) {
+        return res.send({ ds: mds3_init.client_copy(ds) });
+      }
+      if (ds.isMds) {
+        return res.send({ ds: mds_clientcopy(ds) });
+      }
+      return res.send({ ds: copy_legacyDataset(ds) });
+    } catch (e) {
+      res.send({ error: e.message || e });
+    }
+  };
+}
+function mds_clientcopy(ds) {
+  const ds2 = {
+    isMds: true,
+    noHandleOnClient: ds.noHandleOnClient,
+    label: ds.label,
+    version: ds.version,
+    annotationsampleset2matrix: ds.annotationsampleset2matrix,
+    mutationAttribute: ds.mutationAttribute,
+    locusAttribute: ds.locusAttribute,
+    alleleAttribute: ds.alleleAttribute,
+    // these are quick fixes and should be deleted later
+    hide_genotypedownload: ds.hide_genotypedownload,
+    hide_phewas: ds.hide_phewas,
+    sample2bam: ds.sample2bam
+  };
+  if (ds.queries) {
+    ds2.queries = {};
+  }
+  if (ds.track) {
+    ds2.track = mds2_init.client_copy(ds);
+  }
+  if (ds.singlesamplemutationjson) {
+    ds2.singlesamplemutationjson = 1;
+  }
+  if (ds.gene2mutcount) {
+    ds2.gene2mutcount = true;
+    ds2.mutCountType = ds.gene2mutcount.mutationTypes;
+  }
+  if (ds.assayAvailability) {
+    ds2.assayAvailability = 1;
+  }
+  if (ds.cohort && ds.cohort.sampleAttribute) {
+    const toclient = {};
+    for (const k in ds.cohort.sampleAttribute.attributes) {
+      const a = ds.cohort.sampleAttribute.attributes[k];
+      if (!a.clientnoshow)
+        toclient[k] = a;
+    }
+    ds2.sampleAttribute = { attributes: toclient };
+  }
+  if (ds.cohort) {
+    if (ds.cohort.termdb) {
+      ds2.termdb = {
+        selectCohort: ds.cohort.termdb.selectCohort
+      };
+    }
+    if (ds.cohort.attributes && ds.cohort.attributes.defaulthidden) {
+      ds2.cohortHiddenAttr = ds.cohort.attributes.defaulthidden;
+    }
+    if (ds.cohort.survivalplot) {
+      ds2.survivalplot = {
+        samplegroupattrlst: ds.cohort.survivalplot.samplegroupattrlst,
+        plots: []
+      };
+      for (const k in ds.cohort.survivalplot.plots) {
+        const p = ds.cohort.survivalplot.plots[k];
+        ds2.survivalplot.plots.push({
+          key: k,
+          name: p.name,
+          timelabel: p.timelabel
+        });
+      }
+    }
+    if (ds.cohort.mutation_signature) {
+      const sets = {};
+      for (const k in ds.cohort.mutation_signature.sets) {
+        const s = ds.cohort.mutation_signature.sets[k];
+        sets[k] = {
+          name: s.name,
+          signatures: s.signatures
+        };
+      }
+      ds2.mutation_signature = { sets };
+    }
+  }
+  for (const k in ds.queries) {
+    const q = ds.queries[k];
+    const clientquery = {
+      // revealed to client
+      name: q.name,
+      hideforthemoment: q.hideforthemoment
+      // hide track not ready to show on client
+    };
+    if (q.istrack) {
+      clientquery.istrack = true;
+      clientquery.type = q.type;
+      clientquery.isfull = q.isfull;
+      if (q.nochr != void 0) {
+        clientquery.nochr = q.nochr;
+      }
+      if (q.infoFilter) {
+        clientquery.infoFilter = q.infoFilter;
+      }
+      if (q.readcountCutoff) {
+        clientquery.readcountCutoff = q.readcountCutoff;
+      }
+      if (q.valueLabel) {
+        clientquery.valueLabel = q.valueLabel;
+      }
+      if (q.valueCutoff) {
+        clientquery.valueCutoff = q.valueCutoff;
+      }
+      if (q.bplengthUpperLimit) {
+        clientquery.bplengthUpperLimit = q.bplengthUpperLimit;
+      }
+      if (q.segmeanValueCutoff) {
+        clientquery.segmeanValueCutoff = q.segmeanValueCutoff;
+      }
+      if (q.lohLengthUpperLimit) {
+        clientquery.lohLengthUpperLimit = q.lohLengthUpperLimit;
+      }
+      if (q.type == common.tkt.mdssvcnv) {
+        if (q.groupsamplebyattr) {
+          clientquery.groupsamplebyattr = q.groupsamplebyattr;
+        }
+        clientquery.multihidelabel_fusion = q.multihidelabel_fusion;
+        clientquery.multihidelabel_sv = q.multihidelabel_sv;
+        clientquery.multihidelabel_vcf = q.multihidelabel_vcf;
+        clientquery.showfullmode = q.showfullmode;
+        clientquery.legend_vorigin = q.legend_vorigin;
+        clientquery.no_loh = q.no_loh;
+        if (q.expressionrank_querykey) {
+          const e = ds.queries[q.expressionrank_querykey];
+          clientquery.checkexpressionrank = {
+            querykey: q.expressionrank_querykey,
+            datatype: e.datatype
+          };
+          if (e.boxplotbysamplegroup && e.boxplotbysamplegroup.additionals) {
+            const lst = [];
+            if (e.boxplotbysamplegroup.attributes)
+              lst.push(e.boxplotbysamplegroup.attributes.map((i) => i.label).join(", "));
+            for (const i of e.boxplotbysamplegroup.additionals)
+              lst.push(i.label);
+            clientquery.checkexpressionrank.boxplotgroupers = lst;
+          }
+        }
+        if (q.vcf_querykey) {
+          clientquery.checkvcf = {
+            querykey: q.vcf_querykey,
+            info: ds.queries[q.vcf_querykey].info,
+            format: {}
+          };
+          for (const tk of ds.queries[q.vcf_querykey].tracks) {
+            if (tk.format) {
+              for (const k2 in tk.format) {
+                clientquery.checkvcf.format[k2] = tk.format[k2];
+              }
+            }
+          }
+        }
+      }
+    } else if (q.isgenenumeric) {
+      clientquery.isgenenumeric = true;
+      clientquery.datatype = q.datatype;
+      clientquery.no_ase = q.no_ase;
+    } else {
+      continue;
+    }
+    ds2.queries[k] = clientquery;
+  }
+  return ds2;
+}
+function copy_legacyDataset(ds) {
+  const ds2 = {
+    noHandleOnClient: ds.noHandleOnClient,
+    sampleselectable: ds.sampleselectable,
+    label: ds.label,
+    dsinfo: ds.dsinfo,
+    stratify: ds.stratify,
+    cohort: ds.cohort,
+    vcfinfofilter: ds.vcfinfofilter,
+    info2table: ds.info2table,
+    info2singletable: ds.info2singletable,
+    url4variant: ds.url4variant,
+    itemlabelname: ds.itemlabelname
+  };
+  if (ds.snvindel_attributes) {
+    ds2.snvindel_attributes = [];
+    for (const at of ds.snvindel_attributes) {
+      const rep = {};
+      for (const k in at) {
+        if (k == "lst") {
+          rep.lst = [];
+          for (const e of at.lst) {
+            const rep2 = {};
+            for (const k2 in e)
+              rep2[k2] = e[k2];
+            rep.lst.push(rep2);
+          }
+        } else {
+          rep[k] = at[k];
+        }
+      }
+      ds2.snvindel_attributes.push(rep);
+    }
+  }
+  if (ds.snvindel_legend) {
+    ds2.snvindel_legend = ds.snvindel_legend;
+  }
+  const vcfinfo = {};
+  let hasvcf = false;
+  for (const q of ds.queries) {
+    if (q.vcf) {
+      hasvcf = true;
+      vcfinfo[q.vcf.vcfid] = q.vcf;
+    }
+  }
+  if (hasvcf) {
+    ds2.id2vcf = vcfinfo;
+  }
+  return ds2;
+}
+export {
+  api
+};

package/routes/dsdata.js

@@ -0,0 +1,127 @@
+import path from "path";
+import { spawn } from "child_process";
+import serverconfig from "#src/serverconfig.js";
+import * as common from "#shared/common.js";
+const api = {
+  // route endpoint
+  // - no need for trailing slash
+  // - should be a noun (method is based on HTTP GET, POST, etc)
+  // - don't add 'Data' as response is assumed to be data
+  endpoint: "dsdata",
+  methods: {
+    get: {
+      init,
+      request: {
+        typeId: "any"
+      },
+      response: {
+        typeId: "any"
+      }
+    },
+    post: {
+      alternativeFor: "get",
+      init
+    }
+  }
+};
+function init({ genomes }) {
+  return async function handle_dsdata(req, res) {
+    try {
+      if (!genomes[req.query.genome])
+        throw "invalid genome";
+      if (!req.query.dsname)
+        throw ".dsname missing";
+      const ds = genomes[req.query.genome].datasets[req.query.dsname];
+      if (!ds)
+        throw "invalid dsname";
+      const data = [];
+      for (const query of ds.queries) {
+        if (req.query.expressiononly && !query.isgeneexpression) {
+          continue;
+        }
+        if (req.query.noexpression && query.isgeneexpression) {
+          continue;
+        }
+        if (query.dsblocktracklst) {
+          continue;
+        }
+        if (query.vcffile) {
+          const d = await handle_dsdata_vcf(query, req);
+          data.push(d);
+          continue;
+        }
+        if (query.makequery) {
+          const d = handle_dsdata_makequery(ds, query, req, genomes);
+          data.push(d);
+          continue;
+        }
+        throw "unknow type from one of ds.queries[]";
+      }
+      res.send({ data });
+    } catch (e) {
+      if (e.stack)
+        console.log(e.stack);
+      res.send({ error: e.message || e });
+    }
+  };
+}
+function handle_dsdata_makequery(ds, query, req, genomes) {
+  if (req.query.isoform) {
+    if (genomes[req.query.genome].genomicNameRegexp.test(req.query.isoform))
+      return;
+  }
+  const [sqlstr, values] = query.makequery(req.query);
+  if (!sqlstr) {
+    return;
+  }
+  const rows = ds.newconn.prepare(sqlstr).all(values);
+  let lst;
+  if (query.tidy) {
+    lst = rows.map((i) => query.tidy(i));
+  } else {
+    lst = rows;
+  }
+  const result = {};
+  if (query.isgeneexpression) {
+    result.lst = lst;
+    result.isgeneexpression = true;
+    result.config = query.config;
+    for (const q2 of ds.queries) {
+      if (!q2.dsblocktracklst)
+        continue;
+      for (const tk of q2.dsblocktracklst) {
+        if (tk.type == common.tkt.junction) {
+          result.config.dsjunctiontk = tk;
+        }
+      }
+    }
+  } else {
+    result.lst = lst;
+  }
+  return result;
+}
+function handle_dsdata_vcf(query, req) {
+  const par = [
+    path.join(serverconfig.tpmasterdir, query.vcffile),
+    (query.vcf.nochr ? req.query.range.chr.replace("chr", "") : req.query.range.chr) + ":" + req.query.range.start + "-" + req.query.range.stop
+  ];
+  return new Promise((resolve, reject) => {
+    const ps = spawn(serverconfig.tabix, par);
+    const out = [], out2 = [];
+    ps.stdout.on("data", (i) => out.push(i));
+    ps.stderr.on("data", (i) => out2.push(i));
+    ps.on("close", (code) => {
+      const e = out2.join("").trim();
+      if (e != "")
+        reject("error querying vcf file");
+      const tmp = out.join("").trim();
+      resolve({
+        lines: tmp == "" ? [] : tmp.split("\n"),
+        vcfid: query.vcf.vcfid
+      });
+    });
+  });
+}
+export {
+  api
+};