@sjcrh/proteinpaint-server 2.83.0 → 2.85.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sjcrh/proteinpaint-server",
-  "version": "2.83.0",
+  "version": "2.85.0",
   "type": "module",
   "description": "a genomics visualization tool for exploring a cohort's genotype and phenotype data",
   "main": "src/app.js",
@@ -28,7 +28,6 @@
     "dedup": "./dedupjs.sh",
     "//todo": "refactor or deprecate the scripts below",
     "pretest": "tsc && ./test/pretest.js",
-    "prepare": "ts-patch install",
     "pretest:type": "npm run checkers",
     "pretest:integration": "tsc",
     "test:integration": "echo 'TODO: server integration tests'",
@@ -43,7 +42,7 @@
     "@babel/preset-env": "^7.9.6",
     "@babel/preset-typescript": "^7.21.4",
     "@babel/register": "^7.14.5",
-    "@sjcrh/proteinpaint-types": "2.83.0",
+    "@sjcrh/proteinpaint-types": "2.85.0",
     "@types/node": "^20.11.24",
     "@types/tough-cookie": "^4.0.5",
     "@typescript-eslint/eslint-plugin": "^5.60.0",
@@ -58,13 +57,14 @@
     "ts-patch": "^3.0.2",
     "tsx": "^4.7.1",
     "typedoc": "^0.24.8",
-    "typescript": "^5.0.3",
+    "typedoc-plugin-missing-exports": "^2.0.1",
+    "typescript": "^5.6.3",
     "typia": "^4.1.14"
   },
   "dependencies": {
-    "@sjcrh/augen": "2.46.0",
-    "@sjcrh/proteinpaint-rust": "2.81.5",
-    "@sjcrh/proteinpaint-shared": "2.83.0",
+    "@sjcrh/augen": "2.85.0",
+    "@sjcrh/proteinpaint-rust": "2.84.0",
+    "@sjcrh/proteinpaint-shared": "2.85.0",
     "better-sqlite3": "^9.4.1",
     "body-parser": "^1.15.2",
     "canvas": "~2.11.2",
@@ -85,8 +85,7 @@
     "node-fetch": "^2.6.1",
     "partjson": "^0.58.2",
     "tiny-async-pool": "^1.2.0",
-    "tough-cookie": "^4.1.4",
-    "typedoc-plugin-missing-exports": "^2.0.1"
+    "tough-cookie": "^4.1.4"
   },
   "repository": {
     "type": "git",

package/routes/_template_.js

@@ -1,28 +1,31 @@
+import { snpPayload } from "#types";
 const api = {
   // route endpoint
   // - no need for trailing slash
-  // - should be a noun (method is based on HTTP GET, POST, etc)
+  // - should be a noun
   // - don't add 'Data' as response is assumed to be data
-  endpoint: "...",
+  // - don't prefix with `get`, such as `/getmyroute`, the method is already indicated via HTTP GET
+  endpoint: "/myroute",
   methods: {
     get: {
-      init,
-      request: {
-        typeId: "any"
-      },
-      response: {
-        typeId: "any"
-      }
+      ...snpPayload,
+      // this would "spread"/copy-over payload key-values from snpPayload, such as {request: {typeId}, examples: []}
+      init
     },
     post: {
-      alternativeFor: "get",
+      ...snpPayload,
+      // repeat for post method, since PP routes are mostly read-only and POST can handle bigger payloads
       init
     }
+    // !!! DO NOT USE expressjs 'all' method shortcut !!!
+    // it will initialize 20+ methods includng HEAD which can break expected HTTP response
   }
 };
 function init({ genomes }) {
   return async function(req, res) {
-    console.log(genomes, req, res);
+    const q = req.query;
+    console.log(genomes[q.genome]);
+    res.send({});
   };
 }
 export {

package/routes/brainImaging.js

@@ -1,18 +1,19 @@
 import fs from "fs";
 import path from "path";
 import serverconfig from "#src/serverconfig.js";
+import { brainImagingPayload } from "#types";
 import { spawn } from "child_process";
+import { getData } from "../src/termdb.matrix.js";
 const api = {
   endpoint: "brainImaging",
   methods: {
     get: {
-      init,
-      request: {
-        typeId: "GetBrainImagingRequest"
-      },
-      response: {
-        typeId: "GetBrainImagingResponse"
-      }
+      ...brainImagingPayload,
+      init
+    },
+    post: {
+      ...brainImagingPayload,
+      init
     }
   }
 };
@@ -26,15 +27,26 @@ function init({ genomes }) {
       const ds = g.datasets[query.dslabel];
       if (!ds)
         throw "invalid dataset name";
-      const brainImage = await getBrainImage(query, genomes);
-      res.send({ brainImage });
+      let plane, index;
+      if (query.l) {
+        plane = "L";
+        index = query.l;
+      } else if (query.f) {
+        plane = "F";
+        index = query.f;
+      } else {
+        plane = "T";
+        index = query.t;
+      }
+      const brainImage = await getBrainImage(query, genomes, plane, index);
+      res.send({ brainImage, plane });
     } catch (e) {
       console.log(e);
       res.status(404).send("Sample brain image not found");
     }
   };
 }
-async function getBrainImage(query, genomes) {
+async function getBrainImage(query, genomes, plane, index) {
   const ds = genomes[query.genome].datasets[query.dslabel];
   const q = ds.queries.NIdata;
   const key = query.refKey;
@@ -42,58 +54,89 @@ async function getBrainImage(query, genomes) {
     const refFile = path.join(serverconfig.tpmasterdir, q[key].referenceFile);
     const dirPath = path.join(serverconfig.tpmasterdir, q[key].samples);
     const files = fs.readdirSync(dirPath).filter((file) => file.endsWith(".nii") && fs.statSync(path.join(dirPath, file)).isFile());
-    if (query.samplesOnly) {
-      const sampleNames = files.map((name) => name.split(".nii")[0]);
-      if (q[key].sampleColumns) {
-        const samples = {};
-        for (const s of sampleNames) {
-          const annoForOneS = { sample: s };
-          const sid = ds.cohort.termdb.q.sampleName2id(s);
-          for (const term of q[key].sampleColumns) {
-            const v = ds.cohort.termdb.q.getSample2value(term.termid, sid);
-            if (v[0]) {
-              annoForOneS[term.termid] = v[0].value;
-            }
-          }
-          samples[s] = annoForOneS;
-        }
-        return Object.values(samples);
+    const terms = [];
+    const divideByTW = query.divideByTW;
+    const overlayTW = query.overlayTW;
+    if (divideByTW)
+      terms.push(divideByTW);
+    if (overlayTW)
+      terms.push(overlayTW);
+    const selectedSampleNames = query.selectedSampleFileNames.map((s) => s.split(".nii")[0]);
+    const data = await getData({ terms }, ds, q.genome);
+    const divideByCat = {};
+    for (const sampleName of selectedSampleNames) {
+      const sampleId = ds.sampleName2Id.get(sampleName);
+      const sampleData = data.samples[sampleId];
+      const samplePath = path.join(dirPath, sampleName) + ".nii";
+      const divideCategory = divideByTW ? sampleData[divideByTW.$id].value : "default";
+      const overlayCategory = overlayTW ? sampleData[overlayTW.$id].value : "default";
+      if (!divideByCat[divideCategory])
+        divideByCat[divideCategory] = {};
+      if (!divideByCat[divideCategory][overlayCategory])
+        divideByCat[divideCategory][overlayCategory] = {
+          samples: [],
+          color: overlayTW?.term?.values?.[overlayCategory]?.color || "red"
+        };
+      divideByCat[divideCategory][overlayCategory].samples.push(samplePath);
+    }
+    const lengths = [];
+    for (const dcategory in divideByCat)
+      for (const category in divideByCat[dcategory]) {
+        const samples = divideByCat[dcategory][category].samples;
+        lengths.push(samples.length);
       }
-      return sampleNames;
+    const maxLength = Math.max(...lengths);
+    const brainImageDict = {};
+    for (const dcategory in divideByCat) {
+      let catNum = 0;
+      const filesByCat = divideByCat[dcategory];
+      for (const category in filesByCat)
+        catNum += filesByCat[category].samples.length;
+      const url = await generateBrainImage(refFile, plane, index, maxLength, JSON.stringify(filesByCat));
+      brainImageDict[dcategory] = { url, catNum };
     }
-    return new Promise((resolve, reject) => {
-      const filePaths = query.selectedSampleFileNames.map((file) => path.join(dirPath, file));
-      const cmd = [
-        `${serverconfig.binpath}/../python/src/plotBrainImaging.py`,
-        refFile,
-        query.l,
-        query.f,
-        query.t,
-        ...filePaths
-      ];
-      const ps = spawn(serverconfig.python, cmd);
-      const imgData = [];
-      ps.stdout.on("data", (data) => {
-        imgData.push(data);
-      });
-      ps.stderr.on("data", (data) => {
-        console.error(`stderr: ${data}`);
-        reject(new Error(`Python script filed: ${data}`));
-      });
-      ps.on("close", (code) => {
-        if (code === 0) {
-          const imageBuffer = Buffer.concat(imgData);
-          const base64Data = imageBuffer.toString("base64");
-          const imgUrl = `data:image/png;base64,${base64Data}`;
-          resolve(imgUrl);
-        } else {
-          reject(new Error(`Python script exited with code ${code}`));
-        }
-      });
-    });
+    return brainImageDict;
   } else {
     throw "no reference or sample files";
   }
+  function getFilesByCat(tw) {
+    const filesByCat = {};
+    for (const [key2, value] of Object.entries(tw.term.values)) {
+      filesByCat[key2] = { samples: [], color: value.color || "red" };
+    }
+    return filesByCat;
+  }
+}
+async function generateBrainImage(refFile, plane, index, maxLength, filesJson) {
+  return new Promise((resolve, reject) => {
+    const cmd = [
+      `${serverconfig.binpath}/../python/src/plotBrainImaging.py`,
+      refFile,
+      plane,
+      index,
+      maxLength,
+      filesJson
+    ];
+    const ps = spawn(serverconfig.python, cmd);
+    const imgData = [];
+    ps.stdout.on("data", (data) => {
+      imgData.push(data);
+    });
+    ps.stderr.on("data", (data) => {
+      console.error(`stderr: ${data}`);
+      reject(new Error(`Python script filed: ${data}`));
+    });
+    ps.on("close", (code) => {
+      if (code === 0) {
+        const imageBuffer = Buffer.concat(imgData);
+        const base64Data = imageBuffer.toString("base64");
+        const imgUrl = `data:image/png;base64,${base64Data}`;
+        resolve(imgUrl);
+      } else {
+        reject(new Error(`Python script exited with code ${code}`));
+      }
+    });
+  });
 }
 export {
   api

package/routes/brainImagingSamples.js

@@ -0,0 +1,120 @@
+import fs from "fs";
+import path from "path";
+import serverconfig from "#src/serverconfig.js";
+import { spawn } from "child_process";
+const api = {
+  endpoint: "brainImagingSamples",
+  methods: {
+    get: {
+      init,
+      request: {
+        typeId: "BrainImagingSamplesRequest"
+      },
+      response: {
+        typeId: "BrainImagingSamplesResponse"
+      }
+    }
+  }
+};
+function init({ genomes }) {
+  return async (req, res) => {
+    try {
+      const query = req.query;
+      const g = genomes[query.genome];
+      if (!g)
+        throw "invalid genome name";
+      const ds = g.datasets[query.dslabel];
+      if (!ds)
+        throw "invalid dataset name";
+      const samples = await getBrainImageSamples(query, genomes);
+      res.send({ samples });
+    } catch (e) {
+      console.log(e);
+      res.status(404).send("Sample brain image not found");
+    }
+  };
+}
+async function getBrainImageSamples(query, genomes) {
+  const ds = genomes[query.genome].datasets[query.dslabel];
+  const q = ds.queries.NIdata;
+  const key = query.refKey;
+  if (q[key].referenceFile && q[key].samples) {
+    const dirPath = path.join(serverconfig.tpmasterdir, q[key].samples);
+    const files = fs.readdirSync(dirPath).filter((file) => file.endsWith(".nii") && fs.statSync(path.join(dirPath, file)).isFile());
+    const sampleNames = files.map((name) => name.split(".nii")[0]);
+    if (q[key].sampleColumns) {
+      const samples = {};
+      for (const s of sampleNames) {
+        const annoForOneS = { sample: s };
+        const sid = ds.cohort.termdb.q.sampleName2id(s);
+        for (const term of q[key].sampleColumns) {
+          const v = ds.cohort.termdb.q.getSample2value(term.termid, sid);
+          if (v[0]) {
+            annoForOneS[term.termid] = v[0].value;
+          }
+        }
+        samples[s] = annoForOneS;
+      }
+      return Object.values(samples);
+    }
+    return sampleNames.map((name) => ({ sample: name }));
+  } else {
+    throw "no reference or sample files";
+  }
+}
+async function validate_query_NIdata(ds) {
+  const q = ds.queries.NIdata;
+  if (!q || !serverconfig.features?.showBrainImaging)
+    return;
+  for (const key in q) {
+    if (q[key].referenceFile && q[key].samples) {
+      q[key].get = async (sampleName, l, f, t) => {
+        const refFile = path.join(serverconfig.tpmasterdir, q[key].referenceFile);
+        const sampleFile = path.join(serverconfig.tpmasterdir, q[key].samples, sampleName);
+        try {
+          await fs.promises.stat(sampleFile);
+        } catch (e) {
+          if (e.code == "EACCES")
+            throw "cannot read file, permission denied";
+          if (e.code == "ENOENT")
+            throw "no data for this sample";
+          throw "failed to load data";
+        }
+        return new Promise((resolve, reject) => {
+          const ps = spawn(serverconfig.python, [
+            `${serverconfig.binpath}/utils/plotBrainImaging.py`,
+            refFile,
+            sampleFile,
+            l,
+            f,
+            t
+          ]);
+          const imgData = [];
+          ps.stdout.on("data", (data) => {
+            imgData.push(data);
+          });
+          ps.stderr.on("data", (data) => {
+            console.error(`stderr: ${data}`);
+            reject(new Error(`Python script filed: ${data}`));
+          });
+          ps.on("close", (code) => {
+            if (code === 0) {
+              const imageBuffer = Buffer.concat(imgData);
+              const base64Data = imageBuffer.toString("base64");
+              const imgUrl = `data:image/png;base64,${base64Data}`;
+              resolve(imgUrl);
+            } else {
+              reject(new Error(`Python script exited with code ${code}`));
+            }
+          });
+        });
+      };
+    } else {
+      throw "no reference or sample files";
+    }
+  }
+}
+export {
+  api,
+  validate_query_NIdata
+};

package/routes/burden.js

@@ -1,3 +1,4 @@
+import { burdenPayload } from "#types";
 import run_R from "#src/run_R.js";
 import path from "path";
 import serverconfig from "#src/serverconfig.js";
@@ -5,68 +6,35 @@ const api = {
   endpoint: "burden",
   methods: {
     get: {
-      init({ genomes }) {
-        return async (req, res) => {
-          try {
-            const genome = genomes[req.query.genome];
-            if (!genome)
-              throw `invalid q.genome=${req.query.genome}`;
-            const q = req.query;
-            const ds = genome.datasets[q.dslabel];
-            if (!ds)
-              throw `invalid q.genome=${req.query.dslabel}`;
-            if (!ds.cohort.cumburden?.files)
-              throw `missing ds.cohort.cumburden.files`;
-            const estimates = await getBurdenEstimates(req, ds);
-            const { keys, rows } = formatPayload(estimates);
-            res.send({ status: "ok", keys, rows });
-          } catch (e) {
-            res.send({ status: "error", error: e.message || e });
-          }
-        };
-      },
-      request: {
-        typeId: "BurdenRequest"
-      },
-      response: {
-        typeId: "BurdenResponse"
-      },
-      examples: [
-        {
-          request: {
-            body: {
-              genome: "hg38",
-              // TODO: !!! use hg38-test and TermdbTest !!!
-              dslabel: "SJLife",
-              diaggrp: 5,
-              sex: 1,
-              white: 1,
-              agedx: 1,
-              bleo: 0,
-              etop: 0,
-              cisp: 0,
-              carbo: 0,
-              steriod: 0,
-              vcr: 0,
-              hdmtx: 0,
-              itmt: 0,
-              ced: 0,
-              dox: 0,
-              heart: 0,
-              brain: 0,
-              abd: 0,
-              pelvis: 0,
-              chest: 0
-            }
-          },
-          response: {
-            header: { status: 200 }
-          }
-        }
-      ]
+      init,
+      ...burdenPayload
+    },
+    post: {
+      init,
+      ...burdenPayload
     }
   }
 };
+function init({ genomes }) {
+  return async function handler(req, res) {
+    try {
+      const genome = genomes[req.query.genome];
+      if (!genome)
+        throw `invalid q.genome=${req.query.genome}`;
+      const q = req.query;
+      const ds = genome.datasets[q.dslabel];
+      if (!ds)
+        throw `invalid q.genome=${req.query.dslabel}`;
+      if (!ds.cohort.cumburden?.files)
+        throw `missing ds.cohort.cumburden.files`;
+      const estimates = await getBurdenEstimates(req, ds);
+      const { keys, rows } = formatPayload(estimates);
+      res.send({ status: "ok", keys, rows });
+    } catch (e) {
+      res.send({ status: "error", error: e.message || e });
+    }
+  };
+}
 async function getBurdenEstimates(q, ds) {
   for (const k in q.query) {
     q.query[k] = Number(q.query[k]);

package/routes/dataset.js

@@ -1,49 +1,41 @@
 import * as mds2_init from "#src/mds2.init.js";
 import * as mds3_init from "#src/mds3.init.js";
 import * as common from "#shared/common.js";
+import { datasetPayload } from "#types";
 const api = {
   endpoint: "getDataset",
   // should rename to simply 'dataset', method is based on HTTP method
   methods: {
     get: {
       init,
-      request: {
-        typeId: "any"
-      },
-      response: {
-        typeId: "any"
-      }
+      ...datasetPayload
     },
     post: {
-      alternativeFor: "get",
-      init
+      init,
+      ...datasetPayload
     }
   }
 };
 function init({ genomes }) {
   return function(req, res) {
     try {
-      const genome = genomes[req.query.genome];
+      const q = req.query;
+      const genome = genomes[q.genome];
       if (!genome)
         throw "unknown genome";
       if (!genome.datasets)
         throw "genomeobj.datasets{} missing";
       let ds;
       for (const k in genome.datasets) {
-        if (k.toLowerCase() == req.query.dsname.toLowerCase()) {
+        if (k.toLowerCase() == q.dsname.toLowerCase()) {
           ds = genome.datasets[k];
           break;
         }
       }
       if (!ds)
         throw "invalid dsname";
-      if (ds.isMds3) {
-        return res.send({ ds: mds3_init.client_copy(ds) });
-      }
-      if (ds.isMds) {
-        return res.send({ ds: mds_clientcopy(ds) });
-      }
-      return res.send({ ds: copy_legacyDataset(ds) });
+      const copy = ds.isMds3 ? mds3_init.client_copy(ds) : ds.isMds ? mds_clientcopy(ds) : copy_legacyDataset(ds);
+      return res.send({ ds: copy });
     } catch (e) {
       res.send({ error: e.message || e });
     }

package/routes/dsdata.js

@@ -2,6 +2,7 @@ import path from "path";
 import { spawn } from "child_process";
 import serverconfig from "#src/serverconfig.js";
 import * as common from "#shared/common.js";
+import { dsDataPayload } from "#types";
 const api = {
   // route endpoint
   // - no need for trailing slash
@@ -10,16 +11,11 @@ const api = {
   endpoint: "dsdata",
   methods: {
     get: {
-      init,
-      request: {
-        typeId: "any"
-      },
-      response: {
-        typeId: "any"
-      }
+      ...dsDataPayload,
+      init
     },
     post: {
-      alternativeFor: "get",
+      ...dsDataPayload,
       init
     }
   }
@@ -27,19 +23,20 @@ const api = {
 function init({ genomes }) {
   return async function handle_dsdata(req, res) {
     try {
-      if (!genomes[req.query.genome])
+      const q = req.query;
+      if (!genomes[q.genome])
         throw "invalid genome";
-      if (!req.query.dsname)
+      if (!q.dsname)
         throw ".dsname missing";
-      const ds = genomes[req.query.genome].datasets[req.query.dsname];
+      const ds = genomes[q.genome].datasets[q.dsname];
       if (!ds)
         throw "invalid dsname";
       const data = [];
       for (const query of ds.queries) {
-        if (req.query.expressiononly && !query.isgeneexpression) {
+        if (q.expressiononly && !query.isgeneexpression) {
           continue;
         }
-        if (req.query.noexpression && query.isgeneexpression) {
+        if (q.noexpression && query.isgeneexpression) {
           continue;
         }
         if (query.dsblocktracklst) {
@@ -110,7 +107,7 @@ function handle_dsdata_vcf(query, req) {
     const out = [], out2 = [];
     ps.stdout.on("data", (i) => out.push(i));
     ps.stderr.on("data", (i) => out2.push(i));
-    ps.on("close", (code) => {
+    ps.on("close", () => {
       const e = out2.join("").trim();
       if (e != "")
         reject("error querying vcf file");