@sjcrh/proteinpaint-server 2.111.0 → 2.112.1-0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sjcrh/proteinpaint-server",
-  "version": "2.111.0",
+  "version": "2.112.1-0",
   "type": "module",
   "description": "a genomics visualization tool for exploring a cohort's genotype and phenotype data",
   "main": "src/app.js",
@@ -60,9 +60,9 @@
   },
   "dependencies": {
     "@sjcrh/augen": "2.109.1-0",
-    "@sjcrh/proteinpaint-rust": "2.111.0",
-    "@sjcrh/proteinpaint-shared": "2.111.0",
-    "@sjcrh/proteinpaint-types": "2.111.0",
+    "@sjcrh/proteinpaint-rust": "2.112.0",
+    "@sjcrh/proteinpaint-shared": "2.112.1-0",
+    "@sjcrh/proteinpaint-types": "2.112.1-0",
     "@types/express": "^5.0.0",
     "@types/express-session": "^1.18.1",
     "better-sqlite3": "^9.4.1",

package/routes/correlationVolcano.js

@@ -4,6 +4,7 @@ import run_R from "../src/run_R.js";
 import serverconfig from "../src/serverconfig.js";
 import { mayLog } from "#src/helpers.ts";
 import path from "path";
+import { getStdDev } from "#shared/descriptive.stats.js";
 const minArrayLength = 3;
 const minSD = 0.05;
 const api = {
@@ -70,7 +71,7 @@ async function compute(q, ds, genome) {
   const [acceptedVariables, skippedVariables] = Array.from(vtid2array.values()).reduce(
     ([accepted, skipped], t) => {
       const grterThanOne = t.v1.length > minArrayLength && t.v2.length > minArrayLength;
-      const significantSD = stdDev(t.v1) > minSD && stdDev(t.v2) > minSD;
+      const significantSD = getStdDev(t.v1) > minSD && getStdDev(t.v2) > minSD;
       const v = grterThanOne && significantSD ? accepted : skipped;
       if (v === accepted)
         accepted.push(t);
@@ -133,18 +134,7 @@ function validate_correlationVolcano(ds) {
     throw "unknown cv.variables.type";
   }
 }
-function mean(data) {
-  return data.reduce((sum, value) => sum + value, 0) / data.length;
-}
-function stdDev(data) {
-  const meanValue = mean(data);
-  const squaredDifferences = data.map((value) => Math.pow(value - meanValue, 2));
-  const variance = squaredDifferences.reduce((sum, value) => sum + value, 0) / data.length;
-  return Math.sqrt(variance);
-}
 export {
   api,
-  mean,
-  stdDev,
   validate_correlationVolcano
 };

package/routes/gdc.mafBuild.js

@@ -46,22 +46,59 @@ async function buildMaf(q, res, ds) {
     host: joinUrl(host.rest, "data")
     // must use the /data/ endpoint from current host
   };
-  const boundary = "GDC_MAF_MULTIPART_BOUNDARY";
-  res.setHeader("content-type", `multipart/mixed; boundary=${boundary}`);
+  const boundary = "------------------------GDC-MAF-BUILD";
+  res.setHeader("Content-Type", `multipart/form-data; boundary=${boundary}`);
   res.write(`--${boundary}`);
-  res.write("\ncontent-disposition: attachment; filename=cohort.maf.gz");
-  res.write("\ncontent-type: application/octet-stream\n\n");
+  res.write('\r\nContent-Disposition: form-data; name="gzfile"; filename="cohort.maf.gz"');
+  res.write("\r\nContent-Type: application/gzip\r\n\r\n");
   res.flush();
-  const rustStream = stream_rust("gdcmaf", JSON.stringify(arg), emitJson);
-  rustStream.pipe(res, { end: false });
+  try {
+    const streams = stream_rust("gdcmaf", JSON.stringify(arg), emitJson);
+    if (streams) {
+      const { rustStream, endStream } = streams;
+      res.on("close", () => {
+        if (res.writableEnded)
+          return;
+        try {
+          console.log("\n-- forced res.end() ---\n");
+          res.end();
+        } catch (e) {
+          console.log("error with forced res.end()", e);
+        }
+        try {
+          endStream();
+        } catch (e) {
+          console.log("error in calling endStream()", e);
+        }
+      });
+      rustStream.pipe(res, { end: false }).on("error", (e) => {
+        console.log("rustStream.pipe().on(error)", e);
+      }).on("end", () => {
+        if (res.writableEnded)
+          return;
+        console.log("rustStream.on(end), trigger res.end()");
+        res.end();
+      });
+    } else {
+      emitJson({ error: "server error: undefined rustStream" });
+    }
+  } catch (e) {
+    console.log("error calling stream_rust(gdcmaf)", e);
+  }
   function emitJson(data, end = true) {
-    res.write(`
+    if (res.writableEnded)
+      return;
+    if (data) {
+      res.write(`\r
 --${boundary}`);
-    res.write("\ncontent-type: application/json");
-    const json = typeof data === "string" ? data : JSON.stringify(data || { ok: true, status: "ok" });
-    res.write("\n\n" + json);
-    res.write(`
---${boundary}--`);
+      res.write('\r\nContent-Disposition: form-data; name="errors"');
+      res.write("\r\nContent-Type: application/x-jsonlines");
+      const json = typeof data === "string" ? data : JSON.stringify(data || { ok: true, status: "ok" });
+      res.write("\r\n\r\n" + json);
+    }
+    res.write(`\r
+--${boundary}--\r
+`);
     mayLog("rust gdcmaf", Date.now() - t0);
     if (end)
       res.end();

package/routes/termdb.DE.js

@@ -8,6 +8,7 @@ import run_R from "../src/run_R.js";
 import { mayLog } from "#src/helpers.ts";
 import serverconfig from "../src/serverconfig.js";
 import imagesize from "image-size";
+import { get_header_txt } from "#src/utils.js";
 const api = {
   endpoint: "DEanalysis",
   methods: {
@@ -230,10 +231,12 @@ async function run_DE(param, ds, term_results, term_results2) {
       sample_size1,
       sample_size2,
       method: param.method,
-      ql_image: result2.ql_image,
-      // QL fit image
-      mds_image: result2.mds_image
-      // MDS image
+      images: [
+        result2.mds_image,
+        // MDS image
+        result2.ql_image
+        // QL fit image
+      ]
     };
   }
   const time1 = (/* @__PURE__ */ new Date()).valueOf();
@@ -256,6 +259,38 @@ async function readFileAndDelete(file, key, response) {
       throw err;
   });
 }
+async function validate_query_rnaseqGeneCount(ds, genome) {
+  const q = ds.queries.rnaseqGeneCount;
+  if (!q)
+    return;
+  if (!q.file)
+    throw "unknown data type for rnaseqGeneCount";
+  q.file = path.join(serverconfig.tpmasterdir, q.file);
+  {
+    let samples = [];
+    if (ds.queries.rnaseqGeneCount.storage_type == "text") {
+      samples = (await get_header_txt(q.file, null)).split("	").slice(4);
+    } else if (ds.queries.rnaseqGeneCount.storage_type == "HDF5") {
+      const get_samples_from_hdf5 = {
+        input_file: q.file,
+        data_type: "get_samples"
+      };
+      const time1 = (/* @__PURE__ */ new Date()).valueOf();
+      const result = await run_rust("DEanalysis", JSON.stringify(get_samples_from_hdf5));
+      const time2 = (/* @__PURE__ */ new Date()).valueOf();
+      samples = result.split(",");
+    } else
+      throw "unknown storage type:" + ds.queries.rnaseqGeneCount.storage_type;
+    q.allSampleSet = new Set(samples);
+    const unknownSamples = [];
+    for (const n of q.allSampleSet) {
+      if (!ds.cohort.termdb.q.sampleName2id(n))
+        unknownSamples.push(n);
+    }
+    console.log(q.allSampleSet.size, `rnaseqGeneCount samples from ${ds.label}`);
+  }
+}
 export {
-  api
+  api,
+  validate_query_rnaseqGeneCount
 };

package/routes/termdb.boxplot.js

@@ -3,6 +3,7 @@ import { getData } from "../src/termdb.matrix.js";
 import { boxplot_getvalue } from "../src/utils.js";
 import { sortKey2values } from "./termdb.violin.ts";
 import { roundValueAuto } from "#shared/roundValue.js";
+import { getMean, getVariance } from "#shared/descriptive.stats.js";
 const minSampleSize = 5;
 const api = {
   endpoint: "termdb/boxplot",
@@ -127,14 +128,9 @@ function setHiddenPlots(term, plots) {
 function setDescrStats(boxplot, sortedValues) {
   if (sortedValues.length < minSampleSize)
     return [{ id: "total", label: "Total", value: sortedValues.length }];
-  const mean = sortedValues.reduce((s2, i) => s2 + i, 0) / sortedValues.length;
-  let s = 0;
-  for (const v of sortedValues) {
-    s += Math.pow(v - mean, 2);
-  }
-  const sd = Math.sqrt(s / (sortedValues.length - 1));
-  const squareDiffs = sortedValues.map((x) => (x - mean) ** 2).reduce((a, b) => a + b, 0);
-  const variance = squareDiffs / (sortedValues.length - 1);
+  const mean = getMean(sortedValues);
+  const variance = getVariance(sortedValues);
+  const sd = Math.sqrt(variance);
   return [
     { id: "total", label: "Total", value: sortedValues.length },
     { id: "min", label: "Minimum", value: roundValueAuto(sortedValues[0], true) },

package/routes/termdb.config.js

@@ -148,12 +148,23 @@ function addNonDictionaryQueries(c, ds, genome) {
     q2.defaultBlock2GeneMode = q.defaultBlock2GeneMode;
   if (q.snvindel) {
     q2.snvindel = {
-      allowSNPs: q.snvindel.allowSNPs
+      allowSNPs: q.snvindel.allowSNPs,
+      // details{} lists default method for computing variants, can be modified and is part of state
+      // some of the stuff here are to provide user-selectable choices
+      // e.g. computing methods, info fields, populations.
+      details: q.snvindel.details,
+      populations: q.snvindel.populations
     };
   }
+  if (q.trackLst) {
+    q2.trackLst = q.trackLst;
+  }
   if (q.svfusion) {
     q2.svfusion = {};
   }
+  if (q.ld) {
+    q2.ld = JSON.parse(JSON.stringify(q.ld));
+  }
   if (q.cnv) {
     q2.cnv = {};
     for (const k of [

package/routes/termdb.descrstats.js

@@ -1,5 +1,5 @@
 import { descrStatsPayload } from "#types/checkers";
-import Summarystats from "#shared/descriptive.stats.js";
+import { summaryStats } from "#shared/descriptive.stats.js";
 import { getData } from "#src/termdb.matrix.js";
 const api = {
   endpoint: "termdb/descrstats",
@@ -48,7 +48,7 @@ function init({ genomes }) {
         values.push(Number(value));
       }
       if (values.length) {
-        result = Summarystats(values);
+        result = summaryStats(values);
       } else {
         result = {};
       }

package/routes/termdb.violin.js

@@ -1,10 +1,10 @@
 import { violinPayload } from "#types/checkers";
-import { scaleLinear, scaleLog } from "d3";
+import { scaleLinear, scaleLog, extent } from "d3";
 import { run_rust } from "@sjcrh/proteinpaint-rust";
 import { getData } from "../src/termdb.matrix.js";
 import { createCanvas } from "canvas";
 import { getOrderedLabels } from "../src/termdb.barchart.js";
-import summaryStats from "#shared/descriptive.stats.js";
+import { summaryStats } from "#shared/descriptive.stats.js";
 import { isNumericTerm } from "#shared/terms.js";
 import { numericBins, parseValues } from "./termdb.boxplot.ts";
 import serverconfig from "../src/serverconfig.js";
@@ -183,7 +183,7 @@ async function createCanvasImg(q, result, ds) {
   const plot2Values = {};
   for (const plot of result.plots)
     plot2Values[plot.label] = plot.values;
-  const densities = await getDensities(plot2Values);
+  const densities = await getDensities(plot2Values, result.min, result.max);
   let axisScale;
   const useLog = q.unit == "log";
   if (useLog) {
@@ -196,13 +196,14 @@ async function createCanvasImg(q, result, ds) {
   const arcEndAngle = scaledRadius * Math.PI;
   for (const plot of result.plots) {
     plot.density = densities[plot.label];
+    const noDensityRendered = plot.density.densityMax == 0;
     const canvas = createCanvas(width, height);
     const ctx = canvas.getContext("2d");
-    const symbolOpacity = plot.density.densityMax == 0 ? 1 : 0.8;
+    const symbolOpacity = noDensityRendered ? 1 : 0.6;
     ctx.strokeStyle = `rgba(0,0,0,${symbolOpacity})`;
     ctx.lineWidth = q.strokeWidth / q.devicePixelRatio;
     ctx.globalAlpha = symbolOpacity;
-    ctx.fillStyle = plot.values.length <= minSampleSize ? "black" : "#ffe6e6";
+    ctx.fillStyle = "#ffe6e6";
     if (q.devicePixelRatio != 1) {
       ctx.scale(q.devicePixelRatio, q.devicePixelRatio);
     }
@@ -233,12 +234,13 @@ async function createCanvasImg(q, result, ds) {
   }
 }
 async function getDensity(values) {
-  const result = await getDensities({ plot: values });
+  const [min, max] = extent(values);
+  const result = await getDensities({ plot: values }, min, max);
   return result.plot;
 }
-async function getDensities(plot2Values) {
+async function getDensities(plot2Values, min, max) {
   const densityScript = path.join(serverconfig.binpath, "utils", "density.R");
-  const plot2Density = JSON.parse(await run_R(densityScript, JSON.stringify(plot2Values)));
+  const plot2Density = JSON.parse(await run_R(densityScript, JSON.stringify({ plot2Values, min, max })));
   const densities = {};
   for (const plot in plot2Density) {
     const result = plot2Density[plot];

package/routes/tileserver.js

@@ -1,8 +1,12 @@
 import { tilePayload } from "#types/checkers";
 import ky from "ky";
+import { TileServerShardingAlgorithm } from "#src/sharding/TileServerShardingAlgorithm.ts";
+import { ShardManager } from "#src/sharding/ShardManager.ts";
+import SessionManager from "#src/wsisessions/SessionManager.ts";
 import serverconfig from "#src/serverconfig.js";
+import path from "path";
 const api = {
-  endpoint: `tileserver/layer/slide/:sampleId/zoomify/:TileGroup/:z-:x-:y@1x.jpg`,
+  endpoint: `tileserver/layer/slide/:sessionId/zoomify/:TileGroup/:z-:x-:y@1x.jpg`,
   methods: {
     get: {
       ...tilePayload,
@@ -14,12 +18,40 @@ const api = {
     }
   }
 };
-function init() {
+function init({ genomes }) {
   return async (req, res) => {
     try {
-      const { sampleId, TileGroup, z, x, y } = req.params;
-      const url = `${serverconfig.tileServerURL}/tileserver/layer/slide/${sampleId}/zoomify/${TileGroup}/${z}-${x}-${y}@1x.jpg`;
+      const { sessionId, TileGroup, z, x, y } = req.params;
+      const query = req.query;
+      const wsiImage = query.wsi_image;
+      if (!wsiImage)
+        throw new Error("No wsi_image param provided");
+      const dslabel = query.dslabel;
+      if (!dslabel)
+        throw new Error("No dslabel param provided");
+      const g = genomes[query.genome];
+      if (!g)
+        throw new Error("Invalid genome name");
+      const ds = g.datasets[query.dslabel];
+      if (!ds)
+        throw new Error("Invalid dataset name");
+      const sampleId = query.sample_id;
+      if (!sampleId)
+        throw new Error("Invalid sample id");
+      const mount = serverconfig.features?.tileserver?.mount;
+      if (!mount)
+        throw new Error("No mount available for TileServer");
+      const wsiImagePath = path.join(`${mount}/${ds.queries.WSImages.imageBySampleFolder}/${sampleId}`, wsiImage);
+      if (!wsiImage)
+        throw new Error("Invalid wsi_image");
+      const shardManager = ShardManager.getInstance();
+      const tileServer = await shardManager.shardingAlgorithmsMap?.get(TileServerShardingAlgorithm.TILE_SERVER_SHARDING_KEY)?.getShard(wsiImagePath);
+      if (!tileServer) {
+        throw new Error("No tile server");
+      }
+      const url = `${tileServer.url}/tileserver/layer/slide/${sessionId}/zoomify/${TileGroup}/${z}-${x}-${y}@1x.jpg`;
       const response = await ky.get(url, { timeout: 12e4 });
+      await SessionManager.getInstance().updateSession(wsiImagePath);
       const buffer = await response.arrayBuffer();
       res.status(response.status).send(Buffer.from(buffer));
     } catch (error) {

package/routes/wsimages.js

@@ -1,12 +1,13 @@
 import ky from "ky";
 import qs from "qs";
 import path from "path";
-import serverconfig from "#src/serverconfig.js";
 import { CookieJar } from "tough-cookie";
 import { promisify } from "util";
 import { wsImagesPayload } from "#types/checkers";
-import SessionManager, { SessionData } from "../src/wsisessions/SessionManager.ts";
-import crypto from "crypto";
+import SessionManager from "../src/wsisessions/SessionManager.ts";
+import { ShardManager } from "#src/sharding/ShardManager.ts";
+import { TileServerShardingAlgorithm } from "#src/sharding/TileServerShardingAlgorithm.ts";
+import serverconfig from "#src/serverconfig.js";
 const routePath = "wsimages";
 const api = {
   endpoint: `${routePath}`,
@@ -40,28 +41,20 @@ function init({ genomes }) {
       const cookieJar = new CookieJar();
       const setCookie = promisify(cookieJar.setCookie.bind(cookieJar));
       const getCookieString = promisify(cookieJar.getCookieString.bind(cookieJar));
-      let sessionManager;
-      let sessionData;
-      let userSessionId = void 0;
-      if (serverconfig.redis) {
-        sessionManager = SessionManager.getInstance(serverconfig.redis.url);
-        userSessionId = crypto.createHash("sha256").update(crypto.randomBytes(32).toString("hex")).digest("hex");
-        sessionData = await sessionManager.getSession(wsimage);
-      }
-      const sessionId = sessionData ? sessionData.imageSessionId : await getSessionId(cookieJar, getCookieString, setCookie, wsimage, ds, sampleId);
-      if (serverconfig.redis && sessionManager) {
-        await manageUserSession(sessionManager, sessionData, wsimage, userSessionId, sessionId);
-      }
-      const getWsiImageResponse = await getWsiImageDimensions(sessionId, getCookieString);
+      const mount = serverconfig.features?.tileserver?.mount;
+      if (!mount)
+        throw new Error("No mount available for TileServer");
+      const wsiImagePath = path.join(`${mount}/${ds.queries.WSImages.imageBySampleFolder}/${sampleId}`, wsimage);
+      const sessionId = await getSessionId(cookieJar, getCookieString, setCookie, wsiImagePath);
+      const getWsiImageResponse = await getWsiImageDimensions(sessionId, getCookieString, wsiImagePath);
       const payload = {
         status: "ok",
         wsiSessionId: sessionId,
-        browserImageInstanceId: serverconfig.redis ? userSessionId : void 0,
         slide_dimensions: getWsiImageResponse.slide_dimensions
       };
       res.status(200).json(payload);
     } catch (e) {
-      console.error(e);
+      console.warn(e);
       res.status(500).send({
         status: "error",
         error: e.message || e
@@ -69,21 +62,28 @@ function init({ genomes }) {
     }
   };
 }
-async function getSessionId(cookieJar, getCookieString, setCookie, wsimage, ds, sampleId) {
-  await ky.get(`${serverconfig.tileServerURL}/tileserver/session_id`, {
+async function getSessionId(cookieJar, getCookieString, setCookie, wsimage) {
+  const sessionManager = SessionManager.getInstance();
+  const invalidateResult = await sessionManager.syncAndInvalidateSessions(wsimage);
+  if (!invalidateResult)
+    throw new Error("Session invalidation failed");
+  const sessionData = await sessionManager.getSession(wsimage);
+  if (sessionData) {
+    return sessionData.imageSessionId;
+  }
+  const tileServer = await sessionManager.getTileServerShard(wsimage);
+  if (!tileServer)
+    throw new Error("No TileServer shard available");
+  await ky.get(`${tileServer.url}/tileserver/session_id`, {
     timeout: 5e4,
     hooks: getHooks(cookieJar, getCookieString, setCookie)
   });
-  const cookieString = await getCookieString(`${serverconfig.tileServerURL}/tileserver/session_id`);
+  const cookieString = await getCookieString(`${tileServer.url}/tileserver/session_id`);
   const sessionId = cookieString.match(/session_id=([^;]*)/)?.[1];
   if (!sessionId)
     throw new Error("session_id not found");
-  const sampleWsiTileServer = path.join(
-    `${serverconfig.tileServerMount}/${ds.queries.WSImages.imageBySampleFolder}/${sampleId}`,
-    wsimage
-  );
-  const data = qs.stringify({ slide_path: sampleWsiTileServer });
-  await ky.put(`${serverconfig.tileServerURL}/tileserver/slide`, {
+  const data = qs.stringify({ slide_path: wsimage });
+  await ky.put(`${tileServer.url}/tileserver/slide`, {
     body: data,
     timeout: 5e4,
     headers: {
@@ -92,19 +92,16 @@ async function getSessionId(cookieJar, getCookieString, setCookie, wsimage, ds,
     },
     hooks: getHooks(cookieJar, getCookieString, setCookie)
   });
+  await sessionManager.setSession(wsimage, sessionId, tileServer);
   return sessionId;
 }
-async function manageUserSession(sessionManager, sessionData, wsimage, userId, sessionId) {
-  if (!sessionData) {
-    await sessionManager.setSession(wsimage, new SessionData(sessionId, [userId]));
-  } else if (!sessionData.userSessionIds || !sessionData.userSessionIds.includes(userId)) {
-    sessionData.userSessionIds = sessionData.userSessionIds || [];
-    sessionData.userSessionIds.push(userId);
-    await sessionManager.setSession(wsimage, sessionData);
+async function getWsiImageDimensions(sessionId, getCookieString, wsimage) {
+  const shardManager = ShardManager.getInstance();
+  const tileServer = await shardManager.shardingAlgorithmsMap?.get(TileServerShardingAlgorithm.TILE_SERVER_SHARDING_KEY)?.getShard(wsimage);
+  if (!tileServer) {
+    throw new Error("No tile server");
   }
-}
-async function getWsiImageDimensions(sessionId, getCookieString) {
-  return await ky.get(`${serverconfig.tileServerURL}/tileserver/slide`, {
+  return await ky.get(`${tileServer.url}/tileserver/slide`, {
     timeout: 12e4,
     hooks: {
       beforeRequest: [