@sjcrh/proteinpaint-server 2.83.0 → 2.84.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sjcrh/proteinpaint-server",
-  "version": "2.83.0",
+  "version": "2.84.0",
   "type": "module",
   "description": "a genomics visualization tool for exploring a cohort's genotype and phenotype data",
   "main": "src/app.js",
@@ -43,7 +43,7 @@
     "@babel/preset-env": "^7.9.6",
     "@babel/preset-typescript": "^7.21.4",
     "@babel/register": "^7.14.5",
-    "@sjcrh/proteinpaint-types": "2.83.0",
+    "@sjcrh/proteinpaint-types": "2.84.0",
     "@types/node": "^20.11.24",
     "@types/tough-cookie": "^4.0.5",
     "@typescript-eslint/eslint-plugin": "^5.60.0",
@@ -58,12 +58,13 @@
     "ts-patch": "^3.0.2",
     "tsx": "^4.7.1",
     "typedoc": "^0.24.8",
-    "typescript": "^5.0.3",
+    "typedoc-plugin-missing-exports": "^2.0.1",
+    "typescript": "^5.6.3",
     "typia": "^4.1.14"
   },
   "dependencies": {
     "@sjcrh/augen": "2.46.0",
-    "@sjcrh/proteinpaint-rust": "2.81.5",
+    "@sjcrh/proteinpaint-rust": "2.84.0",
     "@sjcrh/proteinpaint-shared": "2.83.0",
     "better-sqlite3": "^9.4.1",
     "body-parser": "^1.15.2",
@@ -85,8 +86,7 @@
     "node-fetch": "^2.6.1",
     "partjson": "^0.58.2",
     "tiny-async-pool": "^1.2.0",
-    "tough-cookie": "^4.1.4",
-    "typedoc-plugin-missing-exports": "^2.0.1"
+    "tough-cookie": "^4.1.4"
   },
   "repository": {
     "type": "git",

package/routes/brainImaging.js

@@ -2,6 +2,7 @@ import fs from "fs";
 import path from "path";
 import serverconfig from "#src/serverconfig.js";
 import { spawn } from "child_process";
+import { getData } from "../src/termdb.matrix.js";
 const api = {
   endpoint: "brainImaging",
   methods: {
@@ -13,6 +14,15 @@ const api = {
       response: {
         typeId: "GetBrainImagingResponse"
       }
+    },
+    post: {
+      init,
+      request: {
+        typeId: "GetBrainImagingRequest"
+      },
+      response: {
+        typeId: "GetBrainImagingResponse"
+      }
     }
   }
 };
@@ -26,15 +36,26 @@ function init({ genomes }) {
       const ds = g.datasets[query.dslabel];
       if (!ds)
         throw "invalid dataset name";
-      const brainImage = await getBrainImage(query, genomes);
-      res.send({ brainImage });
+      let plane, index;
+      if (query.l) {
+        plane = "L";
+        index = query.l;
+      } else if (query.f) {
+        plane = "F";
+        index = query.f;
+      } else {
+        plane = "T";
+        index = query.t;
+      }
+      const brainImage = await getBrainImage(query, genomes, plane, index);
+      res.send({ brainImage, plane });
     } catch (e) {
       console.log(e);
       res.status(404).send("Sample brain image not found");
     }
   };
 }
-async function getBrainImage(query, genomes) {
+async function getBrainImage(query, genomes, plane, index) {
   const ds = genomes[query.genome].datasets[query.dslabel];
   const q = ds.queries.NIdata;
   const key = query.refKey;
@@ -42,58 +63,89 @@ async function getBrainImage(query, genomes) {
     const refFile = path.join(serverconfig.tpmasterdir, q[key].referenceFile);
     const dirPath = path.join(serverconfig.tpmasterdir, q[key].samples);
     const files = fs.readdirSync(dirPath).filter((file) => file.endsWith(".nii") && fs.statSync(path.join(dirPath, file)).isFile());
-    if (query.samplesOnly) {
-      const sampleNames = files.map((name) => name.split(".nii")[0]);
-      if (q[key].sampleColumns) {
-        const samples = {};
-        for (const s of sampleNames) {
-          const annoForOneS = { sample: s };
-          const sid = ds.cohort.termdb.q.sampleName2id(s);
-          for (const term of q[key].sampleColumns) {
-            const v = ds.cohort.termdb.q.getSample2value(term.termid, sid);
-            if (v[0]) {
-              annoForOneS[term.termid] = v[0].value;
-            }
-          }
-          samples[s] = annoForOneS;
-        }
-        return Object.values(samples);
+    const terms = [];
+    const divideByTW = query.divideByTW;
+    const overlayTW = query.overlayTW;
+    if (divideByTW)
+      terms.push(divideByTW);
+    if (overlayTW)
+      terms.push(overlayTW);
+    const selectedSampleNames = query.selectedSampleFileNames.map((s) => s.split(".nii")[0]);
+    const data = await getData({ terms }, ds, q.genome);
+    const divideByCat = {};
+    for (const sampleName of selectedSampleNames) {
+      const sampleId = ds.sampleName2Id.get(sampleName);
+      const sampleData = data.samples[sampleId];
+      const samplePath = path.join(dirPath, sampleName) + ".nii";
+      const divideCategory = divideByTW ? sampleData[divideByTW.$id].value : "default";
+      const overlayCategory = overlayTW ? sampleData[overlayTW.$id].value : "default";
+      if (!divideByCat[divideCategory])
+        divideByCat[divideCategory] = {};
+      if (!divideByCat[divideCategory][overlayCategory])
+        divideByCat[divideCategory][overlayCategory] = {
+          samples: [],
+          color: overlayTW?.term?.values?.[overlayCategory]?.color || "red"
+        };
+      divideByCat[divideCategory][overlayCategory].samples.push(samplePath);
+    }
+    const lengths = [];
+    for (const dcategory in divideByCat)
+      for (const category in divideByCat[dcategory]) {
+        const samples = divideByCat[dcategory][category].samples;
+        lengths.push(samples.length);
       }
-      return sampleNames;
+    const maxLength = Math.max(...lengths);
+    const brainImageDict = {};
+    for (const dcategory in divideByCat) {
+      let catNum = 0;
+      const filesByCat = divideByCat[dcategory];
+      for (const category in filesByCat)
+        catNum += filesByCat[category].samples.length;
+      const url = await generateBrainImage(refFile, plane, index, maxLength, JSON.stringify(filesByCat));
+      brainImageDict[dcategory] = { url, catNum };
     }
-    return new Promise((resolve, reject) => {
-      const filePaths = query.selectedSampleFileNames.map((file) => path.join(dirPath, file));
-      const cmd = [
-        `${serverconfig.binpath}/../python/src/plotBrainImaging.py`,
-        refFile,
-        query.l,
-        query.f,
-        query.t,
-        ...filePaths
-      ];
-      const ps = spawn(serverconfig.python, cmd);
-      const imgData = [];
-      ps.stdout.on("data", (data) => {
-        imgData.push(data);
-      });
-      ps.stderr.on("data", (data) => {
-        console.error(`stderr: ${data}`);
-        reject(new Error(`Python script filed: ${data}`));
-      });
-      ps.on("close", (code) => {
-        if (code === 0) {
-          const imageBuffer = Buffer.concat(imgData);
-          const base64Data = imageBuffer.toString("base64");
-          const imgUrl = `data:image/png;base64,${base64Data}`;
-          resolve(imgUrl);
-        } else {
-          reject(new Error(`Python script exited with code ${code}`));
-        }
-      });
-    });
+    return brainImageDict;
   } else {
     throw "no reference or sample files";
   }
+  function getFilesByCat(tw) {
+    const filesByCat = {};
+    for (const [key2, value] of Object.entries(tw.term.values)) {
+      filesByCat[key2] = { samples: [], color: value.color || "red" };
+    }
+    return filesByCat;
+  }
+}
+async function generateBrainImage(refFile, plane, index, maxLength, filesJson) {
+  return new Promise((resolve, reject) => {
+    const cmd = [
+      `${serverconfig.binpath}/../python/src/plotBrainImaging.py`,
+      refFile,
+      plane,
+      index,
+      maxLength,
+      filesJson
+    ];
+    const ps = spawn(serverconfig.python, cmd);
+    const imgData = [];
+    ps.stdout.on("data", (data) => {
+      imgData.push(data);
+    });
+    ps.stderr.on("data", (data) => {
+      console.error(`stderr: ${data}`);
+      reject(new Error(`Python script filed: ${data}`));
+    });
+    ps.on("close", (code) => {
+      if (code === 0) {
+        const imageBuffer = Buffer.concat(imgData);
+        const base64Data = imageBuffer.toString("base64");
+        const imgUrl = `data:image/png;base64,${base64Data}`;
+        resolve(imgUrl);
+      } else {
+        reject(new Error(`Python script exited with code ${code}`));
+      }
+    });
+  });
 }
 export {
   api

package/routes/brainImagingSamples.js

@@ -0,0 +1,120 @@
+import fs from "fs";
+import path from "path";
+import serverconfig from "#src/serverconfig.js";
+import { spawn } from "child_process";
+const api = {
+  endpoint: "brainImagingSamples",
+  methods: {
+    get: {
+      init,
+      request: {
+        typeId: "GetBrainImagingSamplesRequest"
+      },
+      response: {
+        typeId: "GetBrainImagingSamplesResponse"
+      }
+    }
+  }
+};
+function init({ genomes }) {
+  return async (req, res) => {
+    try {
+      const query = req.query;
+      const g = genomes[query.genome];
+      if (!g)
+        throw "invalid genome name";
+      const ds = g.datasets[query.dslabel];
+      if (!ds)
+        throw "invalid dataset name";
+      const samples = await getBrainImageSamples(query, genomes);
+      res.send({ samples });
+    } catch (e) {
+      console.log(e);
+      res.status(404).send("Sample brain image not found");
+    }
+  };
+}
+async function getBrainImageSamples(query, genomes) {
+  const ds = genomes[query.genome].datasets[query.dslabel];
+  const q = ds.queries.NIdata;
+  const key = query.refKey;
+  if (q[key].referenceFile && q[key].samples) {
+    const dirPath = path.join(serverconfig.tpmasterdir, q[key].samples);
+    const files = fs.readdirSync(dirPath).filter((file) => file.endsWith(".nii") && fs.statSync(path.join(dirPath, file)).isFile());
+    const sampleNames = files.map((name) => name.split(".nii")[0]);
+    if (q[key].sampleColumns) {
+      const samples = {};
+      for (const s of sampleNames) {
+        const annoForOneS = { sample: s };
+        const sid = ds.cohort.termdb.q.sampleName2id(s);
+        for (const term of q[key].sampleColumns) {
+          const v = ds.cohort.termdb.q.getSample2value(term.termid, sid);
+          if (v[0]) {
+            annoForOneS[term.termid] = v[0].value;
+          }
+        }
+        samples[s] = annoForOneS;
+      }
+      return Object.values(samples);
+    }
+    return sampleNames.map((name) => ({ sample: name }));
+  } else {
+    throw "no reference or sample files";
+  }
+}
+async function validate_query_NIdata(ds, genome) {
+  const q = ds.queries.NIdata;
+  if (!q || !serverconfig.features?.showBrainImaging)
+    return;
+  for (const key in q) {
+    if (q[key].referenceFile && q[key].samples) {
+      q[key].get = async (sampleName, l, f, t) => {
+        const refFile = path.join(serverconfig.tpmasterdir, q[key].referenceFile);
+        const sampleFile = path.join(serverconfig.tpmasterdir, q[key].samples, sampleName);
+        try {
+          await fs.promises.stat(sampleFile);
+        } catch (e) {
+          if (e.code == "EACCES")
+            throw "cannot read file, permission denied";
+          if (e.code == "ENOENT")
+            throw "no data for this sample";
+          throw "failed to load data";
+        }
+        return new Promise((resolve, reject) => {
+          const ps = spawn(serverconfig.python, [
+            `${serverconfig.binpath}/utils/plotBrainImaging.py`,
+            refFile,
+            sampleFile,
+            l,
+            f,
+            t
+          ]);
+          const imgData = [];
+          ps.stdout.on("data", (data) => {
+            imgData.push(data);
+          });
+          ps.stderr.on("data", (data) => {
+            console.error(`stderr: ${data}`);
+            reject(new Error(`Python script filed: ${data}`));
+          });
+          ps.on("close", (code) => {
+            if (code === 0) {
+              const imageBuffer = Buffer.concat(imgData);
+              const base64Data = imageBuffer.toString("base64");
+              const imgUrl = `data:image/png;base64,${base64Data}`;
+              resolve(imgUrl);
+            } else {
+              reject(new Error(`Python script exited with code ${code}`));
+            }
+          });
+        });
+      };
+    } else {
+      throw "no reference or sample files";
+    }
+  }
+}
+export {
+  api,
+  validate_query_NIdata
+};

package/routes/termdb.boxplot.js

@@ -43,14 +43,18 @@ function init({ genomes }) {
       const sampleType = `All ${data.sampleType?.plural_name || "samples"}`;
       const key2values = /* @__PURE__ */ new Map();
       const overlayTerm = q.divideTw;
-      for (const [key, val] of Object.entries(data.samples)) {
+      for (const val of Object.values(data.samples)) {
         const value = val[q.tw.$id];
         if (!Number.isFinite(value?.value))
           continue;
+        if (q.tw.term.values?.[value.value]?.uncomputable)
+          continue;
         if (overlayTerm) {
           if (!val[overlayTerm?.$id])
             continue;
           const value2 = val[overlayTerm.$id];
+          if (overlayTerm.term?.values?.[value2.key]?.uncomputable)
+            continue;
           if (!key2values.has(value2.key))
             key2values.set(value2.key, []);
           key2values.get(value2.key).push(value.value);
@@ -64,57 +68,52 @@ function init({ genomes }) {
       let absMin, absMax, maxLabelLgth;
       for (const [key, values] of sortKey2values(data, key2values, overlayTerm)) {
         const sortedValues = values.sort((a, b) => a - b);
-        if (!absMin || sortedValues[0] < absMin)
+        if (absMin === null || absMin === void 0 || sortedValues[0] < absMin)
           absMin = sortedValues[0];
-        if (!absMax || sortedValues[sortedValues.length - 1] > absMax)
+        if (absMax === null || absMax === void 0 || sortedValues[sortedValues.length - 1] > absMax)
           absMax = sortedValues[sortedValues.length - 1];
         const vs = sortedValues.map((v) => {
           const value = { value: v };
           return value;
         });
+        const _plot = {
+          // label,
+          // values,
+          // plotValueCount: values.length
+          boxplot: boxplot_getvalue(vs),
+          min: sortedValues[0],
+          max: sortedValues[sortedValues.length - 1]
+        };
         if (overlayTerm) {
           let label = overlayTerm?.term?.values?.[key]?.label || key;
           label = `${label}, n=${values.length}`;
-          if (!maxLabelLgth || label.length > maxLabelLgth.length)
+          if (!maxLabelLgth || label.length > maxLabelLgth)
             maxLabelLgth = label.length;
-          const plot = {
-            // label,
-            // values,
+          const plot = Object.assign(_plot, {
             seriesId: key,
-            color: overlayTerm?.term?.values?.[key]?.color || null,
-            boxplot: boxplot_getvalue(vs),
-            //Need sd and mean?
-            // plotValueCount: values.length,
-            min: sortedValues[0],
-            max: sortedValues[sortedValues.length - 1]
-          };
+            color: overlayTerm?.term?.values?.[key]?.color || null
+          });
           plot.boxplot.label = label;
           plots.push(plot);
         } else {
           const label = `${sampleType}, n=${values.length}`;
           if (!maxLabelLgth || label.length > maxLabelLgth.length)
             maxLabelLgth = label.length;
-          const plot = {
-            // label,
-            // values,
-            // plotValueCount: values.length,
-            boxplot: boxplot_getvalue(vs),
-            min: sortedValues[0],
-            max: sortedValues[sortedValues.length - 1]
-          };
-          plot.boxplot.label = label;
-          plots.push(plot);
+          _plot.boxplot.label = label;
+          plots.push(_plot);
         }
       }
-      data.plots = plots;
-      data.absMin = absMin;
-      data.absMax = absMax;
-      data.maxLabelLgth = maxLabelLgth;
-      res.send(data);
+      const returnData = {
+        absMin,
+        absMax,
+        maxLabelLgth,
+        plots
+      };
+      res.send(returnData);
     } catch (e) {
       res.send({ error: e?.message || e });
       if (e instanceof Error && e.stack)
-        console.log(e);
+        console.error(e);
     }
   };
 }