@sjcrh/proteinpaint-server 2.109.1 → 2.111.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sjcrh/proteinpaint-server",
-  "version": "2.109.1",
+  "version": "2.111.0",
   "type": "module",
   "description": "a genomics visualization tool for exploring a cohort's genotype and phenotype data",
   "main": "src/app.js",
@@ -60,9 +60,9 @@
   },
   "dependencies": {
     "@sjcrh/augen": "2.109.1-0",
-    "@sjcrh/proteinpaint-rust": "2.99.0",
-    "@sjcrh/proteinpaint-shared": "2.109.1",
-    "@sjcrh/proteinpaint-types": "2.109.1",
+    "@sjcrh/proteinpaint-rust": "2.111.0",
+    "@sjcrh/proteinpaint-shared": "2.111.0",
+    "@sjcrh/proteinpaint-types": "2.111.0",
     "@types/express": "^5.0.0",
     "@types/express-session": "^1.18.1",
     "better-sqlite3": "^9.4.1",

package/routes/correlationVolcano.js

@@ -3,7 +3,6 @@ import { getData } from "../src/termdb.matrix.js";
 import run_R from "../src/run_R.js";
 import serverconfig from "../src/serverconfig.js";
 import { mayLog } from "#src/helpers.ts";
-import { stdDev } from "#shared/violin.bins.js";
 import path from "path";
 const minArrayLength = 3;
 const minSD = 0.05;
@@ -134,7 +133,18 @@ function validate_correlationVolcano(ds) {
     throw "unknown cv.variables.type";
   }
 }
+function mean(data) {
+  return data.reduce((sum, value) => sum + value, 0) / data.length;
+}
+function stdDev(data) {
+  const meanValue = mean(data);
+  const squaredDifferences = data.map((value) => Math.pow(value - meanValue, 2));
+  const variance = squaredDifferences.reduce((sum, value) => sum + value, 0) / data.length;
+  return Math.sqrt(variance);
+}
 export {
   api,
+  mean,
+  stdDev,
   validate_correlationVolcano
 };

package/routes/gdc.mafBuild.js

@@ -1,6 +1,6 @@
 import ky from "ky";
 import { joinUrl } from "#shared/joinUrl.js";
-import { run_rust_stream } from "@sjcrh/proteinpaint-rust";
+import { stream_rust } from "@sjcrh/proteinpaint-rust";
 import { gdcMafPayload } from "#types/checkers";
 import { maxTotalSizeCompressed } from "./gdc.maf.ts";
 import { mayLog } from "#src/helpers.ts";
@@ -46,19 +46,26 @@ async function buildMaf(q, res, ds) {
     host: joinUrl(host.rest, "data")
     // must use the /data/ endpoint from current host
   };
-  const rustStream = run_rust_stream("gdcmaf", JSON.stringify(arg));
-  res.setHeader("Content-Type", "application/octet-stream");
-  res.setHeader("Content-Disposition", "attachment; filename=cohort.maf.gz");
-  rustStream.pipe(res);
-  rustStream.on("end", () => {
+  const boundary = "GDC_MAF_MULTIPART_BOUNDARY";
+  res.setHeader("content-type", `multipart/mixed; boundary=${boundary}`);
+  res.write(`--${boundary}`);
+  res.write("\ncontent-disposition: attachment; filename=cohort.maf.gz");
+  res.write("\ncontent-type: application/octet-stream\n\n");
+  res.flush();
+  const rustStream = stream_rust("gdcmaf", JSON.stringify(arg), emitJson);
+  rustStream.pipe(res, { end: false });
+  function emitJson(data, end = true) {
+    res.write(`
+--${boundary}`);
+    res.write("\ncontent-type: application/json");
+    const json = typeof data === "string" ? data : JSON.stringify(data || { ok: true, status: "ok" });
+    res.write("\n\n" + json);
+    res.write(`
+--${boundary}--`);
     mayLog("rust gdcmaf", Date.now() - t0);
-    res.end();
-  });
-  rustStream.on("error", (err) => {
-    console.error(err);
-    res.statusCode = 500;
-    res.end("Internal Server Error");
-  });
+    if (end)
+      res.end();
+  }
 }
 async function getFileLstUnderSizeLimit(lst, host) {
   if (lst.length == 0)

package/routes/termdb.DE.js

@@ -1,3 +1,4 @@
+import fs from "fs";
 import path from "path";
 import { diffExpPayload } from "#types/checkers";
 import { run_rust } from "@sjcrh/proteinpaint-rust";
@@ -6,6 +7,7 @@ import { get_ds_tdb } from "../src/termdb.js";
 import run_R from "../src/run_R.js";
 import { mayLog } from "#src/helpers.ts";
 import serverconfig from "../src/serverconfig.js";
+import imagesize from "image-size";
 const api = {
   endpoint: "DEanalysis",
   methods: {
@@ -58,6 +60,8 @@ function init({ genomes }) {
           throw term_results2.error;
       }
       const results = await run_DE(req.query, ds, term_results, term_results2);
+      if (!results || !results.data)
+        throw "No data [termdb.DE.ts init()]";
       res.send(results);
     } catch (e) {
       res.send({ status: "error", error: e.message || e });
@@ -188,6 +192,7 @@ async function run_DE(param, ds, term_results, term_results2) {
     control: cases_string,
     data_type: "do_DE",
     input_file: q.file,
+    cachedir: serverconfig.cachedir,
     min_count: param.min_count,
     min_total_count: param.min_total_count,
     storage_type: param.storage_type
@@ -207,25 +212,50 @@ async function run_DE(param, ds, term_results, term_results2) {
     }
   }
   const sample_size_limit = 8;
-  let result;
   if (group1names.length <= sample_size_limit && group2names.length <= sample_size_limit || param.method == "edgeR") {
-    if (param.method == "edgeR") {
-      expression_input.VarGenes = param.VarGenes;
-    }
-    const time1 = (/* @__PURE__ */ new Date()).valueOf();
-    result = JSON.parse(
+    const time12 = (/* @__PURE__ */ new Date()).valueOf();
+    const result2 = JSON.parse(
       await run_R(path.join(serverconfig.binpath, "utils", "edge.R"), JSON.stringify(expression_input))
     );
-    mayLog("Time taken to run edgeR:", Date.now() - time1, "ms");
+    mayLog("Time taken to run edgeR:", Date.now() - time12, "ms");
     param.method = "edgeR";
-  } else {
-    const time1 = (/* @__PURE__ */ new Date()).valueOf();
-    result = JSON.parse(await run_rust("DEanalysis", JSON.stringify(expression_input)));
-    mayLog("Time taken to run rust DE pipeline:", Date.now() - time1, "ms");
-    param.method = "wilcoxon";
+    const ql_imagePath = path.join(serverconfig.cachedir, result2.edgeR_ql_image_name[0]);
+    mayLog("ql_imagePath:", ql_imagePath);
+    const mds_imagePath = path.join(serverconfig.cachedir, result2.edgeR_mds_image_name[0]);
+    mayLog("mds_imagePath:", mds_imagePath);
+    await readFileAndDelete(mds_imagePath, "mds_image", result2);
+    await readFileAndDelete(ql_imagePath, "ql_image", result2);
+    return {
+      data: result2.gene_data,
+      sample_size1,
+      sample_size2,
+      method: param.method,
+      ql_image: result2.ql_image,
+      // QL fit image
+      mds_image: result2.mds_image
+      // MDS image
+    };
   }
+  const time1 = (/* @__PURE__ */ new Date()).valueOf();
+  const result = JSON.parse(await run_rust("DEanalysis", JSON.stringify(expression_input)));
+  mayLog("Time taken to run rust DE pipeline:", Date.now() - time1, "ms");
+  param.method = "wilcoxon";
   return { data: result, sample_size1, sample_size2, method: param.method };
 }
+async function readFileAndDelete(file, key, response) {
+  const plot = await fs.promises.readFile(file);
+  const plotBuffer = Buffer.from(plot).toString("base64");
+  const obj = {
+    src: `data:image/png;base64,${plotBuffer}`,
+    size: imagesize(file),
+    key
+  };
+  response[key] = obj;
+  fs.unlink(file, (err) => {
+    if (err)
+      throw err;
+  });
+}
 export {
   api
 };

package/routes/termdb.boxplot.js

@@ -3,6 +3,7 @@ import { getData } from "../src/termdb.matrix.js";
 import { boxplot_getvalue } from "../src/utils.js";
 import { sortKey2values } from "./termdb.violin.ts";
 import { roundValueAuto } from "#shared/roundValue.js";
+const minSampleSize = 5;
 const api = {
   endpoint: "termdb/boxplot",
   methods: {
@@ -124,7 +125,7 @@ function setHiddenPlots(term, plots) {
   return plots;
 }
 function setDescrStats(boxplot, sortedValues) {
-  if (sortedValues.length < 5)
+  if (sortedValues.length < minSampleSize)
     return [{ id: "total", label: "Total", value: sortedValues.length }];
   const mean = sortedValues.reduce((s2, i) => s2 + i, 0) / sortedValues.length;
   let s = 0;

package/routes/termdb.violin.js

@@ -6,8 +6,11 @@ import { createCanvas } from "canvas";
 import { getOrderedLabels } from "../src/termdb.barchart.js";
 import summaryStats from "#shared/descriptive.stats.js";
 import { isNumericTerm } from "#shared/terms.js";
-import { getBinsDensity } from "#shared/violin.bins.js";
 import { numericBins, parseValues } from "./termdb.boxplot.ts";
+import serverconfig from "../src/serverconfig.js";
+import run_R from "../src/run_R.js";
+import path from "path";
+const minSampleSize = 5;
 const api = {
   endpoint: "termdb/violin",
   methods: {
@@ -73,7 +76,7 @@ async function trigger_getViolinPlotData(q, ds, genome) {
   const valuesObject = divideValues(q, data, sampleType);
   const result = setResponse(valuesObject, data, q, sampleType);
   await getWilcoxonData(q.divideTw, result);
-  createCanvasImg(q, result, ds);
+  await createCanvasImg(q, result, ds);
   return result;
 }
 async function getWilcoxonData(divideTw, result) {
@@ -167,7 +170,7 @@ function setResponse(valuesObject, data, q, sampleType) {
   };
   return result;
 }
-function createCanvasImg(q, result, ds) {
+async function createCanvasImg(q, result, ds) {
   if (!q.radius)
     q.radius = 5;
   if (q.radius <= 0)
@@ -177,6 +180,10 @@ function createCanvasImg(q, result, ds) {
   if (!q.strokeWidth)
     q.strokeWidth = 0.2;
   const refSize = q.radius * 4;
+  const plot2Values = {};
+  for (const plot of result.plots)
+    plot2Values[plot.label] = plot.values;
+  const densities = await getDensities(plot2Values);
   let axisScale;
   const useLog = q.unit == "log";
   if (useLog) {
@@ -188,12 +195,14 @@ function createCanvasImg(q, result, ds) {
   const scaledRadius = q.radius / q.devicePixelRatio;
   const arcEndAngle = scaledRadius * Math.PI;
   for (const plot of result.plots) {
+    plot.density = densities[plot.label];
     const canvas = createCanvas(width, height);
     const ctx = canvas.getContext("2d");
-    ctx.strokeStyle = "rgba(0,0,0,0.8)";
+    const symbolOpacity = plot.density.densityMax == 0 ? 1 : 0.8;
+    ctx.strokeStyle = `rgba(0,0,0,${symbolOpacity})`;
     ctx.lineWidth = q.strokeWidth / q.devicePixelRatio;
-    ctx.globalAlpha = 0.5;
-    ctx.fillStyle = "#ffe6e6";
+    ctx.globalAlpha = symbolOpacity;
+    ctx.fillStyle = plot.values.length <= minSampleSize ? "black" : "#ffe6e6";
     if (q.devicePixelRatio != 1) {
       ctx.scale(q.devicePixelRatio, q.devicePixelRatio);
     }
@@ -220,13 +229,43 @@ function createCanvasImg(q, result, ds) {
         ctx.stroke();
       });
     plot.src = canvas.toDataURL();
-    plot.density = getBinsDensity(plot, q.isKDE, q.ticks);
     plot.summaryStats = summaryStats(plot.values).values;
-    delete plot.values;
   }
 }
+async function getDensity(values) {
+  const result = await getDensities({ plot: values });
+  return result.plot;
+}
+async function getDensities(plot2Values) {
+  const densityScript = path.join(serverconfig.binpath, "utils", "density.R");
+  const plot2Density = JSON.parse(await run_R(densityScript, JSON.stringify(plot2Values)));
+  const densities = {};
+  for (const plot in plot2Density) {
+    const result = plot2Density[plot];
+    const bins = [];
+    let densityMin = Infinity;
+    let densityMax = -Infinity;
+    let xMin = Infinity;
+    let xMax = -Infinity;
+    for (const [i, x] of Object.entries(result.x)) {
+      const density2 = result.y[i];
+      xMin = Math.min(xMin, x);
+      xMax = Math.max(xMax, x);
+      densityMin = Math.min(densityMin, density2);
+      densityMax = Math.max(densityMax, density2);
+      bins.push({ x0: x, density: density2 });
+    }
+    bins.unshift({ x0: xMin, density: densityMin });
+    bins.push({ x0: xMax, density: densityMin });
+    const density = { bins, densityMin, densityMax, minvalue: xMin, maxvalue: xMax };
+    densities[plot] = density;
+  }
+  return densities;
+}
 export {
   api,
+  getDensities,
+  getDensity,
   getWilcoxonData,
   sortKey2values,
   trigger_getViolinPlotData