@sjcrh/proteinpaint-server 2.109.1-0 → 2.110.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sjcrh/proteinpaint-server",
-  "version": "2.109.1-0",
+  "version": "2.110.0",
   "type": "module",
   "description": "a genomics visualization tool for exploring a cohort's genotype and phenotype data",
   "main": "src/app.js",
@@ -17,7 +17,7 @@
     "dev": "npm run start",
     "prestart": "tsx emitImports.js dev > server.js",
     "start": "tsx watch . /start.js",
-    "test:unit": "tsx emitImports.js unit > serverTests.js && tsx serverTests.js && rm -rf ./cache",
+    "test:unit": "tsx emitImports.js unit > serverTests.js && c8 tsx serverTests.js && rm -rf ./cache",
     "getconf": "../build/getConfigProp.js",
     "doc": "../augen/build.sh routes shared/types/routes shared/checkers ../public/docs/server",
     "mjs": "esbuild \"$DIR/*.ts\" --platform=node --outdir=\"$DIR\" --format=esm",
@@ -46,6 +46,7 @@
     "@types/tough-cookie": "^4.0.5",
     "@typescript-eslint/eslint-plugin": "^8.13.0",
     "babel-loader": "^8.2.2",
+    "c8": "^10.1.3",
     "esbuild": "^0.19.12",
     "glob": "^10.4.5",
     "node-watch": "^0.7.1",
@@ -59,9 +60,9 @@
   },
   "dependencies": {
     "@sjcrh/augen": "2.109.1-0",
-    "@sjcrh/proteinpaint-rust": "2.99.0",
-    "@sjcrh/proteinpaint-shared": "2.109.1-0",
-    "@sjcrh/proteinpaint-types": "2.109.1-0",
+    "@sjcrh/proteinpaint-rust": "2.110.0",
+    "@sjcrh/proteinpaint-shared": "2.109.1",
+    "@sjcrh/proteinpaint-types": "2.110.0",
     "@types/express": "^5.0.0",
     "@types/express-session": "^1.18.1",
     "better-sqlite3": "^9.4.1",
@@ -82,7 +83,7 @@
     "ky": "^1.2.1",
     "lazy": "^1.0.11",
     "micromatch": "^4.0.5",
-    "minimatch": "^3.1.2",
+    "minimatch": "^10.0.1",
     "node-fetch": "^2.6.1",
     "partjson": "^0.58.2",
     "redis": "^4.7.0",

package/routes/gdc.mafBuild.js

@@ -1,6 +1,6 @@
 import ky from "ky";
 import { joinUrl } from "#shared/joinUrl.js";
-import { run_rust_stream } from "@sjcrh/proteinpaint-rust";
+import { stream_rust } from "@sjcrh/proteinpaint-rust";
 import { gdcMafPayload } from "#types/checkers";
 import { maxTotalSizeCompressed } from "./gdc.maf.ts";
 import { mayLog } from "#src/helpers.ts";
@@ -46,19 +46,26 @@ async function buildMaf(q, res, ds) {
     host: joinUrl(host.rest, "data")
     // must use the /data/ endpoint from current host
   };
-  const rustStream = run_rust_stream("gdcmaf", JSON.stringify(arg));
-  res.setHeader("Content-Type", "application/octet-stream");
-  res.setHeader("Content-Disposition", "attachment; filename=cohort.maf.gz");
-  rustStream.pipe(res);
-  rustStream.on("end", () => {
+  const boundary = "GDC_MAF_MULTIPART_BOUNDARY";
+  res.setHeader("content-type", `multipart/mixed; boundary=${boundary}`);
+  res.write(`--${boundary}`);
+  res.write("\ncontent-disposition: attachment; filename=cohort.maf.gz");
+  res.write("\ncontent-type: application/octet-stream\n\n");
+  res.flush();
+  const rustStream = stream_rust("gdcmaf", JSON.stringify(arg), emitJson);
+  rustStream.pipe(res, { end: false });
+  function emitJson(data, end = true) {
+    res.write(`
+--${boundary}`);
+    res.write("\ncontent-type: application/json");
+    const json = typeof data === "string" ? data : JSON.stringify(data || { ok: true, status: "ok" });
+    res.write("\n\n" + json);
+    res.write(`
+--${boundary}--`);
     mayLog("rust gdcmaf", Date.now() - t0);
-    res.end();
-  });
-  rustStream.on("error", (err) => {
-    console.error(err);
-    res.statusCode = 500;
-    res.end("Internal Server Error");
-  });
+    if (end)
+      res.end();
+  }
 }
 async function getFileLstUnderSizeLimit(lst, host) {
   if (lst.length == 0)

package/routes/termdb.DE.js

@@ -1,3 +1,4 @@
+import fs from "fs";
 import path from "path";
 import { diffExpPayload } from "#types/checkers";
 import { run_rust } from "@sjcrh/proteinpaint-rust";
@@ -188,6 +189,7 @@ async function run_DE(param, ds, term_results, term_results2) {
     control: cases_string,
     data_type: "do_DE",
     input_file: q.file,
+    cachedir: serverconfig.cachedir,
     min_count: param.min_count,
     min_total_count: param.min_total_count,
     storage_type: param.storage_type
@@ -207,25 +209,45 @@ async function run_DE(param, ds, term_results, term_results2) {
     }
   }
   const sample_size_limit = 8;
-  let result;
   if (group1names.length <= sample_size_limit && group2names.length <= sample_size_limit || param.method == "edgeR") {
     if (param.method == "edgeR") {
       expression_input.VarGenes = param.VarGenes;
     }
-    const time1 = (/* @__PURE__ */ new Date()).valueOf();
-    result = JSON.parse(
+    const time12 = (/* @__PURE__ */ new Date()).valueOf();
+    const result2 = JSON.parse(
       await run_R(path.join(serverconfig.binpath, "utils", "edge.R"), JSON.stringify(expression_input))
     );
-    mayLog("Time taken to run edgeR:", Date.now() - time1, "ms");
+    mayLog("Time taken to run edgeR:", Date.now() - time12, "ms");
     param.method = "edgeR";
-  } else {
-    const time1 = (/* @__PURE__ */ new Date()).valueOf();
-    result = JSON.parse(await run_rust("DEanalysis", JSON.stringify(expression_input)));
-    mayLog("Time taken to run rust DE pipeline:", Date.now() - time1, "ms");
-    param.method = "wilcoxon";
+    const ql_imagePath = path.join(serverconfig.cachedir, result2.edgeR_ql_image_name[0]);
+    mayLog("ql_imagePath:", ql_imagePath);
+    const mds_imagePath = path.join(serverconfig.cachedir, result2.edgeR_mds_image_name[0]);
+    mayLog("mds_imagePath:", mds_imagePath);
+    const mds_base64Image = await readFileAndDelete(mds_imagePath);
+    const ql_base64Image = await readFileAndDelete(ql_imagePath);
+    return {
+      data: result2.gene_data,
+      sample_size1,
+      sample_size2,
+      method: param.method,
+      ql_image: ql_base64Image,
+      // QL fit image
+      mds_image: mds_base64Image
+      // MDS image
+    };
   }
+  const time1 = (/* @__PURE__ */ new Date()).valueOf();
+  const result = JSON.parse(await run_rust("DEanalysis", JSON.stringify(expression_input)));
+  mayLog("Time taken to run rust DE pipeline:", Date.now() - time1, "ms");
+  param.method = "wilcoxon";
   return { data: result, sample_size1, sample_size2, method: param.method };
 }
+async function readFileAndDelete(file) {
+  const data = await fs.promises.readFile(file);
+  fs.unlink(file, () => {
+  });
+  return Buffer.from(data).toString("base64");
+}
 export {
   api
 };

package/routes/termdb.boxplot.js

@@ -3,6 +3,7 @@ import { getData } from "../src/termdb.matrix.js";
 import { boxplot_getvalue } from "../src/utils.js";
 import { sortKey2values } from "./termdb.violin.ts";
 import { roundValueAuto } from "#shared/roundValue.js";
+const minSampleSize = 5;
 const api = {
   endpoint: "termdb/boxplot",
   methods: {
@@ -124,7 +125,7 @@ function setHiddenPlots(term, plots) {
   return plots;
 }
 function setDescrStats(boxplot, sortedValues) {
-  if (sortedValues.length < 5)
+  if (sortedValues.length < minSampleSize)
     return [{ id: "total", label: "Total", value: sortedValues.length }];
   const mean = sortedValues.reduce((s2, i) => s2 + i, 0) / sortedValues.length;
   let s = 0;

package/routes/termdb.cluster.js

@@ -10,6 +10,7 @@ import { getResult as getResultGene } from "#src/gene.js";
 import { TermTypes, NUMERIC_DICTIONARY_TERM } from "#shared/terms.js";
 import { getData } from "#src/termdb.matrix.js";
 import { termType2label } from "#shared/terms.js";
+import { mayLog } from "#src/helpers.ts";
 const api = {
   endpoint: "termdb/cluster",
   methods: {
@@ -74,8 +75,10 @@ async function getResult(q, ds, genome) {
     ;
     ({ term2sample2value, byTermId, bySampleId } = await ds.queries[q.dataType].get(_q));
   }
+  const removedHierClusterTerms = [];
   for (const [term, obj] of term2sample2value) {
     if (Object.keys(obj).length === 0) {
+      removedHierClusterTerms.push(term);
       term2sample2value.delete(term);
       delete byTermId[term];
     }
@@ -88,9 +91,11 @@ async function getResult(q, ds, genome) {
   }
   const t = Date.now();
   const clustering = await doClustering(term2sample2value, q, Object.keys(bySampleId).length);
-  if (serverconfig.debugmode)
-    console.log("clustering done:", Date.now() - t, "ms");
-  return { clustering, byTermId, bySampleId };
+  mayLog("clustering done:", Date.now() - t, "ms");
+  const result = { clustering, byTermId, bySampleId };
+  if (removedHierClusterTerms.length)
+    result.removedHierClusterTerms = removedHierClusterTerms;
+  return result;
 }
 async function getNumericDictTermAnnotation(q, ds, genome) {
   const getDataArgs = {

package/routes/termdb.singleSampleMutation.js

@@ -46,14 +46,7 @@ async function validate_query_singleSampleMutation(ds, genome) {
     gdcValidate_query_singleSampleMutation(ds, genome);
   } else if (_q.src == "native") {
     _q.get = async (q) => {
-      let fileName = q.sample;
-      if (ds.cohort?.termdb?.q?.sampleName2id) {
-        fileName = ds.cohort.termdb.q.sampleName2id(q.sample);
-        if (fileName == void 0) {
-          return [];
-        }
-      }
-      const file = path.join(serverconfig.tpmasterdir, _q.folder, fileName.toString());
+      const file = path.join(serverconfig.tpmasterdir, _q.folder, q.sample);
       try {
         await fs.promises.stat(file);
       } catch (e) {

package/routes/termdb.violin.js

@@ -8,6 +8,7 @@ import summaryStats from "#shared/descriptive.stats.js";
 import { isNumericTerm } from "#shared/terms.js";
 import { getBinsDensity } from "#shared/violin.bins.js";
 import { numericBins, parseValues } from "./termdb.boxplot.ts";
+const minSampleSize = 5;
 const api = {
   endpoint: "termdb/violin",
   methods: {
@@ -193,7 +194,7 @@ function createCanvasImg(q, result, ds) {
     ctx.strokeStyle = "rgba(0,0,0,0.8)";
     ctx.lineWidth = q.strokeWidth / q.devicePixelRatio;
     ctx.globalAlpha = 0.5;
-    ctx.fillStyle = "#ffe6e6";
+    ctx.fillStyle = plot.values.length <= minSampleSize ? "black" : "#ffe6e6";
     if (q.devicePixelRatio != 1) {
       ctx.scale(q.devicePixelRatio, q.devicePixelRatio);
     }
@@ -220,7 +221,7 @@ function createCanvasImg(q, result, ds) {
         ctx.stroke();
       });
     plot.src = canvas.toDataURL();
-    plot.density = getBinsDensity(axisScale, plot, q.isKDE, q.ticks);
+    plot.density = getBinsDensity(plot, q.isKDE, q.ticks);
     plot.summaryStats = summaryStats(plot.values).values;
     delete plot.values;
   }