@sjcrh/proteinpaint-server 2.146.4-1 → 2.147.1

package/dataset/termdb.test.js

@@ -287,10 +287,7 @@ function termdb_test_default() {
         }
       },
       cnv: {
-        file: "files/hg38/TermdbTest/TermdbTest_CNV_gene.gz",
-        cnvMaxLength: 2e7,
-        cnvGainCutoff: 0.1,
-        cnvLossCutoff: -0.1
+        file: "files/hg38/TermdbTest/TermdbTest_CNV_gene.gz"
       },
       singleSampleMutation: {
         src: "native",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sjcrh/proteinpaint-server",
-  "version": "2.146.4-1",
+  "version": "2.147.1",
   "type": "module",
   "description": "a genomics visualization tool for exploring a cohort's genotype and phenotype data",
   "main": "src/app.js",
@@ -64,8 +64,8 @@
     "@sjcrh/proteinpaint-python": "2.146.0",
     "@sjcrh/proteinpaint-r": "2.146.4-0",
     "@sjcrh/proteinpaint-rust": "2.146.4-1",
-    "@sjcrh/proteinpaint-shared": "2.146.4-1",
-    "@sjcrh/proteinpaint-types": "2.146.4-1",
+    "@sjcrh/proteinpaint-shared": "2.147.1",
+    "@sjcrh/proteinpaint-types": "2.147.1",
     "@types/express": "^5.0.0",
     "@types/express-session": "^1.18.1",
     "better-sqlite3": "^9.4.1",

package/routes/aiProjectSelectedWSImages.js

@@ -1,6 +1,3 @@
-import path from "path";
-import fs from "fs";
-import serverconfig from "#src/serverconfig.js";
 import { aiProjectSelectedWSImagesResponsePayload } from "#types/checkers";
 import { getDbConnection } from "#src/aiHistoDBConnection.ts";
 const api = {
@@ -37,31 +34,12 @@ function init({ genomes }) {
           wsimage.uncertainty = ds.queries?.WSImages?.uncertainty;
           wsimage.activePatchColor = ds.queries?.WSImages?.activePatchColor;
           if (ds.queries.WSImages.getWSIPredictionPatches) {
-            const predictionsFile = await ds.queries.WSImages.getWSIPredictionPatches(projectId, wsimageFilename);
-            const mount = serverconfig.features?.tileserver?.mount;
-            if (!mount) throw new Error("No mount available for TileServer");
-            const predictionsFilePath = path.join(mount, ds.queries.WSImages.aiToolImageFolder, predictionsFile[0]);
-            const predictionsData = JSON.parse(fs.readFileSync(predictionsFilePath, "utf8"));
-            wsimage.predictions = predictionsData.features.map((d) => {
-              const featClass = wsimage.classes?.find((f) => f.id == d.properties.class)?.label;
-              return {
-                zoomCoordinates: d.properties.zoomCoordinates,
-                uncertainty: d.properties.uncertainty,
-                class: featClass
-              };
-            });
+            const predictions = await ds.queries.WSImages.getWSIPredictionPatches(projectId, wsimageFilename);
+            wsimage.predictions = predictions;
           }
           wsimages.push(wsimage);
         }
       }
-      if (ds.queries.WSImages.getWSIPredictionOverlay) {
-        for (const wsimage of wsimages) {
-          const predictionOverlay = await ds.queries.WSImages.getWSIPredictionOverlay(wsimage.filename);
-          if (predictionOverlay) {
-            wsimage.predictionLayers = [predictionOverlay];
-          }
-        }
-      }
       res.send({ wsimages });
     } catch (e) {
       console.log(e);

package/routes/aiProjectTrainModel.js

@@ -0,0 +1,61 @@
+import { aiProjectTrainModelPayload } from "#types/checkers";
+import { TileServerSessionsHandler } from "#src/wsisessions/TileServerSessionsHandler.ts";
+import SessionManager from "#src/wsisessions/SessionManager.ts";
+const api = {
+  endpoint: "aiProjectTrainModel",
+  methods: {
+    get: {
+      ...aiProjectTrainModelPayload,
+      init
+    },
+    post: {
+      ...aiProjectTrainModelPayload,
+      init
+    }
+  }
+};
+function init({ genomes }) {
+  return async function(req, res) {
+    try {
+      const query = req.query;
+      console.log(query);
+      const g = genomes[query.genome];
+      if (!g) throw "invalid genome name";
+      const ds = g.datasets[query.dslabel];
+      if (!ds) throw "invalid dataset name";
+      if (typeof ds.queries.WSImages.retrainModel == "function") {
+        await ds.queries.WSImages.retrainModel(query.projectId);
+        try {
+          const handler = new TileServerSessionsHandler();
+          const sessionMgr = SessionManager.getInstance();
+          const keySessions = await sessionMgr.getAllKeyValues();
+          const sessions = keySessions.map((kv) => kv?.sessionData).filter(Boolean);
+          if (sessions && sessions.length) {
+            await handler.resetSessions(sessions);
+            await Promise.all(keySessions.map((kv) => sessionMgr.deleteSession(kv.key)));
+          }
+        } catch (err) {
+          console.warn("TileServerSessionsHandler error:", err);
+        }
+      } else {
+        res.status(500).send({
+          status: "error",
+          error: "No retraining script defined"
+        });
+      }
+      res.status(200).send({
+        status: "ok",
+        message: `Retraining model completed`
+      });
+    } catch (e) {
+      console.warn(e);
+      res.status(500).send({
+        status: "error",
+        error: e.message || e
+      });
+    }
+  };
+}
+export {
+  api
+};

package/routes/termdb.cohort.summary.js

@@ -2,6 +2,7 @@ import { termdbCohortSummaryPayload } from "#types/checkers";
 import { get_ds_tdb } from "#src/termdb.js";
 import { mayCopyFromCookie } from "#src/utils.js";
 import { get_samples } from "#src/termdb.sql.js";
+import { authApi } from "#src/auth.js";
 const api = {
   endpoint: "termdb/cohort/summary",
   methods: {
@@ -19,6 +20,7 @@ function init({ genomes }) {
       const genome = genomes[q.genome];
       if (!genome) throw "invalid genome";
       const [ds] = get_ds_tdb(genome, q);
+      authApi.mayAdjustFilter(req.query, ds, []);
       let count;
       if (q.filter?.lst?.length) {
         const samples = await get_samples(q, ds);

package/routes/termdb.filterTermValues.js

@@ -55,7 +55,8 @@ function getList(samplesPerFilter, filtersData, tw, showAll) {
   const sampleValues = Array.from(new Set(annotations));
   const filteredValues = [];
   for (const value of values) {
-    const label = value.label.replace(/["']/g, "");
+    let label = value.label.replace(/["']/g, "");
+    if (label.length > 50) label = label.slice(0, 47) + "...";
     const disabled = !sampleValues.includes(value.key || value.label);
     if (!showAll && disabled) continue;
     filteredValues.push({ value: value.key || value.label, label, disabled });

package/routes/termdb.profileFormScores.js

@@ -1,5 +1,5 @@
 import { ProfileFormScoresPayload } from "#types/checkers";
-import { getData } from "../src/termdb.matrix.js";
+import { getScoresData } from "./termdb.profileScores.js";
 const api = {
   endpoint: "termdb/profileFormScores",
   methods: {
@@ -28,56 +28,24 @@ function init({ genomes }) {
   };
 }
 async function getScoresDict(query, ds) {
-  if (!query.filterByUserSites) query.__protected__.ignoredTermIds.push(query.facilityTW.term.id);
   const terms = [...query.scoreTerms, query.facilityTW];
   if (query.scScoreTerms) terms.push(...query.scScoreTerms);
-  const data = await getData(
-    {
-      terms,
-      filter: query.site || !query.isAggregate ? void 0 : query.filter,
-      //if isRadarFacility and site is specified, do not apply the filter
-      __protected__: query.__protected__
-    },
-    ds
-  );
-  const lst = Object.values(data.samples);
-  let sites = lst.map((s) => {
-    return { label: data.refs.bySampleId[s.sample].label, value: s.sample };
-  });
-  sites = lst.map((s) => {
-    return { label: data.refs.bySampleId[s.sample].label, value: s.sample };
-  });
-  if (query.userSites) {
-    sites = sites.filter((s) => query.userSites.includes(s.label));
-    if (lst.length == 1 && !sites.length) {
-      const siteId = lst[0].sample;
-      const site = data.refs.bySampleId[siteId].label;
-      const hospital2 = query.facilityTW.term.values[site].label;
-      throw `The access to the hospital ${hospital2} is denied`;
-    }
-  }
-  let sitesSelected = [];
-  if (query.site) sitesSelected = [query.site];
-  else if (!query.isAggregate) sitesSelected = [sites[0].value];
-  else sitesSelected = query.sites;
-  const sampleData = sitesSelected?.length == 1 ? data.samples[sitesSelected[0]] : null;
-  let samples = sampleData ? [sampleData] : Object.values(data.samples);
-  if (sitesSelected?.length > 0) samples = samples.filter((s) => sitesSelected.includes(s.sample));
+  const data = await getScoresData(query, ds, terms);
   const term2Score = {};
   for (const d of query.scoreTerms) {
     const getDictFunc = (sample) => getDict(d.$id, sample);
-    const percents = getPercentsDict(getDictFunc, samples);
+    const percents = getPercentsDict(getDictFunc, data.samples);
     term2Score[d.term.id] = percents;
   }
   if (query.scScoreTerms)
     for (const d of query.scScoreTerms) {
-      const percents = getSCPercentsDict(d, samples);
+      const percents = getSCPercentsDict(d, data.samples);
       term2Score[d.term.id] = percents;
     }
-  const facilityValue = sampleData?.[query.facilityTW.$id];
+  const facilityValue = data.sampleData?.[query.facilityTW.$id];
   const termValue = query.facilityTW.term.values[facilityValue?.value];
   const hospital = termValue?.label || termValue?.key;
-  return { term2Score, sites, hospital, n: sampleData ? 1 : samples.length };
+  return { term2Score, sites: data.sites, hospital, n: data.sampleData ? 1 : data.samples.length };
 }
 function getDict(key, sample) {
   if (!sample[key]) return null;

package/routes/termdb.profileScores.js

@@ -27,52 +27,61 @@ function init({ genomes }) {
     }
   };
 }
-async function getScores(query, ds) {
+async function getScoresData(query, ds, terms) {
   if (!query.filterByUserSites) query.__protected__.ignoredTermIds.push(query.facilityTW.term.id);
-  const terms = [query.facilityTW];
-  for (const term of query.scoreTerms) {
-    terms.push(term.score);
-    if (term.maxScore?.term) {
-      terms.push(term.maxScore);
-    }
-  }
+  const { clientAuthResult, activeCohort } = query.__protected__;
+  const userSites = clientAuthResult[activeCohort].sites;
   const data = await getData(
     {
       terms,
-      filter: query.site || !query.isAggregate ? void 0 : query.filter,
-      //if site is specified, do not apply the filter that is for the aggregation
+      filter: query.filter,
       __protected__: query.__protected__
     },
     ds
   );
+  if (data.error) throw data.error;
   const lst = Object.values(data.samples);
   let sites = lst.map((s) => {
-    return { label: data.refs.bySampleId[s.sample].label, value: s.sample };
+    let label = query.facilityTW.term.values[s[query.facilityTW.$id].value]?.label || s[query.facilityTW.$id].value;
+    if (label.length > 50) label = label.slice(0, 47) + "...";
+    return {
+      value: s[query.facilityTW.$id].value,
+      label
+    };
+  });
+  if (userSites) {
+    sites = sites.filter((s) => userSites.includes(s.value));
+  }
+  sites.sort((a, b) => {
+    return a.label.localeCompare(b.label);
   });
-  if (query.userSites) {
-    sites = sites.filter((s) => query.userSites.includes(s.label));
-    if (lst.length == 1 && !sites.length) {
-      const siteId = lst[0].sample;
-      const site = data.refs.bySampleId[siteId].label;
-      const hospital2 = query.facilityTW.term.values[site].label;
-      throw `The access to the hospital ${hospital2} is denied`;
+  const samples = Object.values(data.samples);
+  let sampleData, site;
+  if ("facilitySite" in query) {
+    const facilitySite = query.facilitySite || sites[0].value;
+    const index = lst.findIndex((s) => s[query.facilityTW.$id].value == facilitySite);
+    sampleData = lst[index];
+    site = sites.find((s) => s.value == facilitySite);
+  } else if (sites.length == 1) {
+    sampleData = data.samples[sites[0].value];
+    site = sites[0];
+  }
+  return { samples, sampleData, sites, site };
+}
+async function getScores(query, ds) {
+  const terms = [query.facilityTW];
+  for (const term of query.scoreTerms) {
+    terms.push(term.score);
+    if (term.maxScore?.term) {
+      terms.push(term.maxScore);
     }
   }
-  let sitesSelected = [];
-  if (query.site) sitesSelected = [query.site];
-  else if (!query.isAggregate) sitesSelected = [sites[0].value];
-  else sitesSelected = query.sites;
-  const sampleData = sitesSelected?.length == 1 ? data.samples[sitesSelected[0]] : null;
-  let samples = Object.values(data.samples);
-  if (sitesSelected?.length > 0) samples = samples.filter((s) => sitesSelected.includes(s.sample));
+  const data = await getScoresData(query, ds, terms);
   const term2Score = {};
   for (const d of query.scoreTerms) {
-    term2Score[d.score.term.id] = getPercentage(d, samples, sampleData);
+    term2Score[d.score.term.id] = getPercentage(d, data.samples, data.sampleData);
   }
-  const facilityValue = sampleData?.[query.facilityTW.$id];
-  const termValue = query.facilityTW.term.values[facilityValue?.value];
-  const hospital = termValue?.label || termValue?.key;
-  return { term2Score, sites, hospital, n: sampleData ? 1 : samples.length };
+  return { term2Score, sites: data.sites, site: data.site, n: data.sampleData ? 1 : data.samples.length };
 }
 function getPercentage(d, samples, sampleData) {
   if (!d) return null;
@@ -91,5 +100,6 @@ function getPercentage(d, samples, sampleData) {
   }
 }
 export {
-  api
+  api,
+  getScoresData
 };

package/routes/termdb.violin.js

@@ -30,7 +30,7 @@ function init({ genomes }) {
       if (!g) throw "invalid genome name";
       const ds = g.datasets?.[q.dslabel];
       if (!ds) throw "invalid ds";
-      data = await trigger_getViolinPlotData(q, ds);
+      data = await getViolin(q, ds);
     } catch (e) {
       data = { error: e?.message || e };
       if (e instanceof Error && e.stack) console.log(e);
@@ -38,7 +38,7 @@ function init({ genomes }) {
     res.send(data);
   };
 }
-async function trigger_getViolinPlotData(q, ds) {
+async function getViolin(q, ds) {
   if (typeof q.tw?.term != "object" || typeof q.tw?.q != "object") throw "q.tw not of {term,q}";
   const term = q.tw.term;
   if (!q.tw.q.mode) q.tw.q.mode = "continuous";
@@ -175,7 +175,7 @@ async function createCanvasImg(q, result, ds) {
   if (q.radius <= 0) throw "q.radius is not a number";
   else q.radius = +q.radius;
   if (!q.strokeWidth) q.strokeWidth = 0.2;
-  const refSize = q.radius * 4;
+  const isH = q.orientation == "horizontal";
   for (const k of Object.keys(result.charts)) {
     const chart = result.charts[k];
     const plot2Values = {};
@@ -184,46 +184,44 @@ async function createCanvasImg(q, result, ds) {
     let axisScale;
     const useLog = q.unit == "log";
     if (useLog) {
-      axisScale = scaleLog().base(ds.cohort.termdb.logscaleBase2 ? 2 : 10).domain([result.min, result.max]).range(q.orientation === "horizontal" ? [0, q.svgw] : [q.svgw, 0]);
+      axisScale = scaleLog().base(ds.cohort.termdb.logscaleBase2 ? 2 : 10).domain([result.min, result.max]).range(isH ? [0, q.svgw] : [q.svgw, 0]);
     } else {
-      axisScale = scaleLinear().domain([result.min, result.max]).range(q.orientation === "horizontal" ? [0, q.svgw] : [q.svgw, 0]);
+      axisScale = scaleLinear().domain([result.min, result.max]).range(isH ? [0, q.svgw] : [q.svgw, 0]);
     }
-    const [width, height] = q.orientation == "horizontal" ? [q.svgw * q.devicePixelRatio, refSize * q.devicePixelRatio] : [refSize * q.devicePixelRatio, q.svgw * q.devicePixelRatio];
-    const scaledRadius = q.radius / q.devicePixelRatio;
-    const arcEndAngle = scaledRadius * Math.PI;
+    const [width, height] = isH ? [q.svgw * q.devicePixelRatio, q.radius * q.devicePixelRatio] : [q.radius * q.devicePixelRatio, q.svgw * q.devicePixelRatio];
     for (const plot of chart.plots) {
       plot.density = densities[plot.label];
-      const noDensityRendered = plot.density.densityMax == 0;
       const canvas = createCanvas(width, height);
       const ctx = canvas.getContext("2d");
-      const symbolOpacity = noDensityRendered ? 1 : 0.6;
-      ctx.strokeStyle = `rgba(0,0,0,${symbolOpacity})`;
-      ctx.lineWidth = q.strokeWidth / q.devicePixelRatio;
-      ctx.globalAlpha = symbolOpacity;
-      ctx.fillStyle = "#ffe6e6";
-      if (q.devicePixelRatio != 1) {
-        ctx.scale(q.devicePixelRatio, q.devicePixelRatio);
-      }
-      if (q.datasymbol === "rug")
+      if (q.devicePixelRatio != 1) ctx.scale(q.devicePixelRatio, q.devicePixelRatio);
+      ctx.strokeStyle = "black";
+      ctx.lineWidth = 1;
+      if (q.datasymbol === "rug") {
+        ctx.globalAlpha = 0.8;
         plot.values.forEach((i) => {
+          const s = axisScale(i);
           ctx.beginPath();
-          if (q.orientation == "horizontal") {
-            ctx.moveTo(+axisScale(i), 0);
-            ctx.lineTo(+axisScale(i), scaledRadius * 2);
+          if (isH) {
+            ctx.moveTo(s, 0);
+            ctx.lineTo(s, q.radius);
           } else {
-            ctx.moveTo(0, +axisScale(i));
-            ctx.lineTo(scaledRadius * 2, +axisScale(i));
+            ctx.moveTo(0, s);
+            ctx.lineTo(q.radius, s);
           }
           ctx.stroke();
         });
-      else if (q.datasymbol === "bean")
+      } else if (q.datasymbol === "bean") {
+        ctx.globalAlpha = 0.6;
+        ctx.fillStyle = "#ffe6e6";
         plot.values.forEach((i) => {
+          const s = axisScale(i);
           ctx.beginPath();
-          if (q.orientation === "horizontal") ctx.arc(+axisScale(i), q.radius, scaledRadius, 0, arcEndAngle);
-          else ctx.arc(q.radius, +axisScale(i), scaledRadius, 0, arcEndAngle);
+          if (isH) ctx.arc(s, q.radius / 2, q.radius / 2, 0, 2 * Math.PI);
+          else ctx.arc(q.radius / 2, s, q.radius / 2, 0, 2 * Math.PI);
           ctx.fill();
           ctx.stroke();
         });
+      }
       plot.src = canvas.toDataURL();
       plot.summaryStats = getDescrStats(plot.values);
     }
@@ -264,6 +262,5 @@ export {
   getDensities,
   getDensity,
   getWilcoxonData,
-  sortPlot2Values,
-  trigger_getViolinPlotData
+  sortPlot2Values
 };

package/routes/wsimages.js

@@ -40,7 +40,7 @@ function init({ genomes }) {
       const setCookie = promisify(cookieJar.setCookie.bind(cookieJar));
       const getCookieString = promisify(cookieJar.getCookieString.bind(cookieJar));
       const wsiImagePath = await getWSImagePath(ds, wSImagesRequest);
-      const session = await getSessionId(ds, cookieJar, getCookieString, setCookie, wsiImagePath);
+      const session = await getSessionId(ds, cookieJar, getCookieString, setCookie, wsiImagePath, aiProjectId);
       const getWsiImageResponse = await getWsiImageDimensions(
         session.imageSessionId,
         getCookieString,
@@ -74,7 +74,7 @@ async function getWSImagePath(ds, wSImagesRequest) {
     return path.join(`${mount}/${ds.queries.WSImages.aiToolImageFolder}/`, wSImagesRequest.wsimage);
   }
 }
-async function getSessionId(ds, cookieJar, getCookieString, setCookie, wsimage) {
+async function getSessionId(ds, cookieJar, getCookieString, setCookie, wsimage, projectId) {
   const sessionManager = SessionManager.getInstance();
   const invalidateResult = await sessionManager.syncAndInvalidateSessions(wsimage);
   if (!invalidateResult) throw new Error("Session invalidation failed");
@@ -102,55 +102,39 @@ async function getSessionId(ds, cookieJar, getCookieString, setCookie, wsimage)
     },
     hooks: getHooks(cookieJar, getCookieString, setCookie)
   });
-  if (ds.queries.WSImages.getWSIPredictionOverlay) {
-    const predictionOverlay = await ds.queries.WSImages.getWSIPredictionOverlay(wsimage);
-    if (predictionOverlay) {
+  if (ds.queries.WSImages.getPredictionLayers) {
+    const predictionLayers = await ds.queries.WSImages.getPredictionLayers(projectId, wsimage);
+    if (predictionLayers) {
       const mount = serverconfig.features?.tileserver?.mount;
-      const overlayFilePath = path.join(`${mount}/${ds.queries.WSImages.aiToolImageFolder}/`, predictionOverlay);
-      const annotationsData = qs.stringify({
-        overlay_path: overlayFilePath
-      });
-      const layerNumber = await ky.put(`${tileServer.url}/tileserver/overlay`, {
-        body: annotationsData,
-        timeout: 5e4,
-        headers: {
-          "Content-Type": "application/x-www-form-urlencoded",
-          Cookie: `session_id=${sessionId}`
-        },
-        hooks: getHooks(cookieJar, getCookieString, setCookie)
-      }).json();
-      const overlay = {
-        layerNumber,
-        predictionOverlayType: "Prediction"
+      const resolveFilename = (key) => {
+        if (!predictionLayers) return void 0;
+        if (predictionLayers instanceof Map) return predictionLayers.get(key) ?? void 0;
+        return predictionLayers[key] ?? void 0;
       };
-      overlays.push(overlay);
-    }
-  }
-  if (ds.queries.WSImages.getWSIUncertaintyOverlay) {
-    const uncertaintyOverlay = await ds.queries.WSImages.getWSIUncertaintyOverlay(wsimage);
-    if (uncertaintyOverlay) {
-      const mount = serverconfig.features?.tileserver?.mount;
-      const uncertaintyOverlayFilePath = path.join(
-        `${mount}/${ds.queries.WSImages.aiToolImageFolder}/`,
-        uncertaintyOverlay
-      );
-      const uncertaintyData = qs.stringify({
-        overlay_path: uncertaintyOverlayFilePath
-      });
-      const layerNumber = await ky.put(`${tileServer.url}/tileserver/overlay`, {
-        body: uncertaintyData,
-        timeout: 5e4,
-        headers: {
-          "Content-Type": "application/x-www-form-urlencoded",
-          Cookie: `session_id=${sessionId}`
-        },
-        hooks: getHooks(cookieJar, getCookieString, setCookie)
-      }).json();
-      const overlay = {
-        layerNumber,
-        predictionOverlayType: "Uncertainty"
+      const pushOverlayForKey = async (key, predictionOverlayType) => {
+        const filename = resolveFilename(key);
+        if (!filename) return;
+        const overlayFilePath = path.join(`${mount}/${ds.queries.WSImages.aiToolImageFolder}/`, filename);
+        const annotationsData = qs.stringify({
+          overlay_path: overlayFilePath
+        });
+        const layerNumber = await ky.put(`${tileServer.url}/tileserver/overlay`, {
+          body: annotationsData,
+          timeout: 5e4,
+          headers: {
+            "Content-Type": "application/x-www-form-urlencoded",
+            Cookie: `session_id=${sessionId}`
+          },
+          hooks: getHooks(cookieJar, getCookieString, setCookie)
+        }).json();
+        const overlay = {
+          layerNumber,
+          predictionOverlayType
+        };
+        overlays.push(overlay);
       };
-      overlays.push(overlay);
+      await pushOverlayForKey("Prediction", "Prediction");
+      await pushOverlayForKey("Uncertainty", "Uncertainty");
     }
   }
   const sessionData = await sessionManager.setSession(wsimage, sessionId, tileServer, overlays);