@sjcrh/proteinpaint-server 2.145.1 → 2.146.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sjcrh/proteinpaint-server",
-  "version": "2.145.1",
+  "version": "2.146.0",
   "type": "module",
   "description": "a genomics visualization tool for exploring a cohort's genotype and phenotype data",
   "main": "src/app.js",
@@ -61,11 +61,11 @@
   },
   "dependencies": {
     "@sjcrh/augen": "2.143.0",
-    "@sjcrh/proteinpaint-python": "2.144.0",
+    "@sjcrh/proteinpaint-python": "2.146.0",
     "@sjcrh/proteinpaint-r": "2.145.1-0",
-    "@sjcrh/proteinpaint-rust": "2.145.1",
-    "@sjcrh/proteinpaint-shared": "2.145.1-0",
-    "@sjcrh/proteinpaint-types": "2.145.1-1",
+    "@sjcrh/proteinpaint-rust": "2.146.0",
+    "@sjcrh/proteinpaint-shared": "2.145.2",
+    "@sjcrh/proteinpaint-types": "2.146.0",
     "@types/express": "^5.0.0",
     "@types/express-session": "^1.18.1",
     "better-sqlite3": "^9.4.1",
@@ -82,9 +82,7 @@
     "got": "^14.2.0",
     "image-size": "^0.5.5",
     "jsonwebtoken": "^9.0.0",
-    "jstat": "^1.9.3",
     "ky": "^1.2.1",
-    "lazy": "^1.0.11",
     "micromatch": "^4.0.5",
     "minimatch": "^10.0.1",
     "node-fetch": "^2.6.1",

package/routes/aiProjectSelectedWSImages.js

@@ -51,9 +51,6 @@ function init({ genomes }) {
               };
             });
           }
-          if (ds.queries.WSImages.makeGeoJson) {
-            await ds.queries.WSImages.makeGeoJson(projectId, wsimageFilename);
-          }
           wsimages.push(wsimage);
         }
       }

package/routes/termdb.boxplot.js

@@ -2,9 +2,7 @@ import { boxplotPayload } from "#types/checkers";
 import { getData } from "../src/termdb.matrix.js";
 import { boxplot_getvalue } from "../src/utils.js";
 import { sortPlot2Values } from "./termdb.violin.ts";
-import { roundValueAuto } from "#shared/roundValue.js";
-import { getMean, getVariance } from "#shared/descriptive.stats.js";
-const minSampleSize = 5;
+import { summaryStats, getDescriptiveStats, summaryStatsFromStats } from "#shared/descriptive.stats.js";
 const api = {
   endpoint: "termdb/boxplot",
   methods: {
@@ -31,6 +29,9 @@ function init({ genomes }) {
       if (q.divideTw) terms.push(q.divideTw);
       const data = await getData({ filter: q.filter, filter0: q.filter0, terms, __protected__: q.__protected__ }, ds);
       if (data.error) throw data.error;
+      const samples = Object.values(data.samples);
+      const values = samples.map((s) => s?.[q.tw.$id]?.value).filter((v) => typeof v === "number");
+      const statsAllSummary = summaryStats(values).values;
       const sampleType = `All ${data.sampleType?.plural_name || "samples"}`;
       const overlayTerm = q.overlayTw;
       const divideTerm = q.divideTw;
@@ -45,24 +46,30 @@ function init({ genomes }) {
       if (!absMin && absMin !== 0) throw "absMin is undefined [termdb.boxplot init()]";
       if (!absMax && absMax !== 0) throw "absMax is undefined [termdb.boxplot init()]";
       const charts = {};
+      let outlierMin = Number.POSITIVE_INFINITY, outlierMax = Number.NEGATIVE_INFINITY;
       for (const [chart, plot2values] of chart2plot2values) {
         const plots = [];
-        for (const [key, values] of sortPlot2Values(data, plot2values, overlayTerm)) {
-          const sortedValues = values.sort((a, b) => a - b);
+        for (const [key, values2] of sortPlot2Values(data, plot2values, overlayTerm)) {
+          const sortedValues = values2.sort((a, b) => a - b);
           const vs = sortedValues.map((v) => {
             const value = { value: v };
             return value;
           });
-          const boxplot = boxplot_getvalue(vs);
+          const stats = getDescriptiveStats(sortedValues);
+          if (q.removeOutliers) {
+            outlierMin = Math.min(outlierMin, stats.outlierMin);
+            outlierMax = Math.max(outlierMax, stats.outlierMax);
+          }
+          const descrStats = summaryStatsFromStats(stats, true).values;
+          const boxplot = boxplot_getvalue(vs, q.removeOutliers);
           if (!boxplot) throw "boxplot_getvalue failed [termdb.boxplot init()]";
-          const descrStats = setDescrStats(boxplot, sortedValues);
           const _plot = {
             boxplot,
             descrStats
           };
           if (overlayTerm) {
             const _key = overlayTerm?.term?.values?.[key]?.label || key;
-            const plotLabel = `${_key}, n=${values.length}`;
+            const plotLabel = `${_key}, n=${values2.length}`;
             const overlayBins = numericBins(overlayTerm, data);
             const plot = Object.assign(_plot, {
               color: overlayTerm?.term?.values?.[key]?.color || null,
@@ -73,7 +80,7 @@ function init({ genomes }) {
             plot.boxplot.label = plotLabel;
             plots.push(plot);
           } else {
-            const plotLabel = `${sampleType}, n=${values.length}`;
+            const plotLabel = `${sampleType}, n=${values2.length}`;
             const plot = Object.assign(_plot, {
               key: sampleType
             });
@@ -89,10 +96,11 @@ function init({ genomes }) {
         charts[chart] = { chartId: chart, plots };
       }
       const returnData = {
-        absMin,
-        absMax,
+        absMin: q.removeOutliers ? outlierMin : absMin,
+        absMax: q.removeOutliers ? outlierMax : absMax,
         charts,
-        uncomputableValues: setUncomputableValues(uncomputableValues)
+        uncomputableValues: setUncomputableValues(uncomputableValues),
+        descrStats: statsAllSummary
       };
       res.send(returnData);
     } catch (e) {
@@ -114,24 +122,6 @@ function setHiddenPlots(term, plots) {
   }
   return plots;
 }
-function setDescrStats(boxplot, sortedValues) {
-  if (sortedValues.length < minSampleSize) return [{ id: "total", label: "Total", value: sortedValues.length }];
-  const mean = getMean(sortedValues);
-  const variance = getVariance(sortedValues);
-  const sd = Math.sqrt(variance);
-  return [
-    { id: "total", label: "Total", value: sortedValues.length },
-    { id: "min", label: "Minimum", value: roundValueAuto(sortedValues[0], true) },
-    { id: "p25", label: "1st quartile", value: roundValueAuto(boxplot.p25, true) },
-    { id: "median", label: "Median", value: roundValueAuto(boxplot.p50, true) },
-    { id: "mean", label: "Mean", value: roundValueAuto(mean, true) },
-    { id: "p75", label: "3rd quartile", value: roundValueAuto(boxplot.p75, true) },
-    { id: "max", label: "Maximum", value: roundValueAuto(sortedValues[sortedValues.length - 1], true) },
-    { id: "sd", label: "Standard deviation", value: isNaN(sd) ? null : roundValueAuto(sd, true) },
-    { id: "variance", label: "Variance", value: roundValueAuto(variance, true) },
-    { id: "iqr", label: "Inter-quartile range", value: roundValueAuto(boxplot.iqr, true) }
-  ];
-}
 function setUncomputableValues(values) {
   if (Object.entries(values)?.length) {
     return Object.entries(values).map(([label, v]) => ({ label, value: v }));

package/routes/termdb.chat.js

@@ -0,0 +1,71 @@
+import { ChatPayload } from "#types/checkers";
+import { run_rust } from "@sjcrh/proteinpaint-rust";
+import serverconfig from "../src/serverconfig.js";
+import { mayLog } from "#src/helpers.ts";
+const api = {
+  endpoint: "termdb/chat",
+  methods: {
+    get: {
+      ...ChatPayload,
+      init
+    },
+    post: {
+      ...ChatPayload,
+      init
+    }
+  }
+};
+function init({ genomes }) {
+  return async (req, res) => {
+    const q = req.query;
+    try {
+      const g = genomes[q.genome];
+      if (!g) throw "invalid genome";
+      const ds = g.datasets?.[q.dslabel];
+      if (!ds) throw "invalid dslabel";
+      let apilink;
+      let comp_model_name;
+      let embedding_model_name;
+      if (serverconfig.llm_backend == "SJ") {
+        apilink = serverconfig.sj_apilink;
+        comp_model_name = serverconfig.sj_comp_model_name;
+        embedding_model_name = serverconfig.sj_embedding_model_name;
+      } else if (serverconfig.llm_backend == "ollama") {
+        apilink = serverconfig.ollama_apilink;
+        comp_model_name = serverconfig.ollama_comp_model_name;
+        embedding_model_name = serverconfig.ollama_embedding_model_name;
+      } else {
+        throw "llm_backend either needs to be 'SJ' or 'ollama'";
+      }
+      const chatbot_input = {
+        // Just hardcoding variables here, these will later be defined in more appropriate places
+        user_input: q.prompt,
+        apilink,
+        dataset_db: serverconfig.tpmasterdir + "/" + ds.cohort.db.file,
+        comp_model_name,
+        embedding_model_name,
+        llm_backend_name: serverconfig.llm_backend
+        // The type of backend (engine) used for running the embedding and completion model. Currently "SJ" and "Ollama" are supported
+      };
+      const time1 = (/* @__PURE__ */ new Date()).valueOf();
+      const ai_output_data = await run_rust("aichatbot", JSON.stringify(chatbot_input));
+      const time2 = (/* @__PURE__ */ new Date()).valueOf();
+      mayLog("Time taken to run rust AI chatbot:", time2 - time1, "ms");
+      let ai_output_json = "";
+      for (const line of ai_output_data.split("\n")) {
+        if (line.startsWith("final_output:") == true) {
+          ai_output_json = JSON.parse(line.replace("final_output:", ""));
+        } else {
+          mayLog(line);
+        }
+      }
+      res.send(ai_output_json);
+    } catch (e) {
+      if (e.stack) console.log(e.stack);
+      res.send({ error: e?.message || e });
+    }
+  };
+}
+export {
+  api
+};

package/routes/termdb.cohort.summary.js

@@ -1,6 +1,7 @@
 import { termdbCohortSummaryPayload } from "#types/checkers";
 import { get_ds_tdb } from "#src/termdb.js";
 import { mayCopyFromCookie } from "#src/utils.js";
+import { get_samples } from "#src/termdb.sql.js";
 const api = {
   endpoint: "termdb/cohort/summary",
   methods: {
@@ -18,7 +19,11 @@ function init({ genomes }) {
       const genome = genomes[q.genome];
       if (!genome) throw "invalid genome";
       const [ds] = get_ds_tdb(genome, q);
-      const count = ds.cohort.termdb.q?.getCohortSampleCount?.(q.cohort) || 1;
+      let count;
+      if (ds.cohort.termdb.getAdditionalFilter) {
+        const samples = await get_samples(q, ds);
+        count = samples.length;
+      } else count = ds.cohort.termdb.q?.getCohortSampleCount?.(q.cohort) || 1;
       res.send({ count });
     } catch (e) {
       res.send({ error: e.message || e });

package/routes/termdb.config.js

@@ -134,15 +134,9 @@ function addNonDictionaryQueries(c, ds, genome) {
     };
     if (q.snvindel.byisoform?.processTwsInOneQuery) q2.snvindel.byisoform = { processTwsInOneQuery: true };
   }
-  if (q.trackLst) {
-    q2.trackLst = q.trackLst;
-  }
   if (q.svfusion) {
     q2.svfusion = {};
   }
-  if (q.ld) {
-    q2.ld = JSON.parse(JSON.stringify(q.ld));
-  }
   if (q.cnv) {
     q2.cnv = {};
     for (const k of [
@@ -177,6 +171,15 @@ function addNonDictionaryQueries(c, ds, genome) {
   if (q.geneExpression) {
     q2.geneExpression = { unit: q.geneExpression.unit };
   }
+  if (q.ld) {
+    q2.ld = structuredClone(q.ld);
+  }
+  if (q.trackLst) {
+    q2.trackLst = q.trackLst;
+  }
+  if (q.chat) {
+    q2.chat = {};
+  }
   if (q.NIdata && serverconfig.features.showBrainImaging) {
     q2.NIdata = {};
     for (const k in q.NIdata) {

package/routes/termdb.violin.js

@@ -56,6 +56,10 @@ async function trigger_getViolinPlotData(q, ds) {
     },
     ds
   );
+  const samples = Object.values(data.samples);
+  let values = samples.map((s) => s?.[q.tw.$id]?.value).filter((v) => typeof v === "number");
+  if (q.unit == "log") values = values.filter((v) => v > 0);
+  const descrStats = summaryStats(values).values;
   const sampleType = `All ${data.sampleType?.plural_name || "samples"}`;
   if (data.error) throw data.error;
   if (q.overlayTw && data.refs.byTermId[q.overlayTw.$id]) {
@@ -71,6 +75,7 @@ async function trigger_getViolinPlotData(q, ds) {
   const result = setResponse(valuesObject, data, q);
   if (q.overlayTw) await getWilcoxonData(result);
   await createCanvasImg(q, result, ds);
+  result["descrStats"] = descrStats;
   return result;
 }
 async function getWilcoxonData(result) {

package/routes/wsimages.js

@@ -40,7 +40,7 @@ function init({ genomes }) {
       const setCookie = promisify(cookieJar.setCookie.bind(cookieJar));
       const getCookieString = promisify(cookieJar.getCookieString.bind(cookieJar));
       const wsiImagePath = await getWSImagePath(ds, wSImagesRequest);
-      const session = await getSessionId(ds, cookieJar, getCookieString, setCookie, wsiImagePath, aiProjectId);
+      const session = await getSessionId(ds, cookieJar, getCookieString, setCookie, wsiImagePath);
       const getWsiImageResponse = await getWsiImageDimensions(
         session.imageSessionId,
         getCookieString,
@@ -74,7 +74,7 @@ async function getWSImagePath(ds, wSImagesRequest) {
     return path.join(`${mount}/${ds.queries.WSImages.aiToolImageFolder}/`, wSImagesRequest.wsimage);
   }
 }
-async function getSessionId(ds, cookieJar, getCookieString, setCookie, wsimage, aiProjectId) {
+async function getSessionId(ds, cookieJar, getCookieString, setCookie, wsimage) {
   const sessionManager = SessionManager.getInstance();
   const invalidateResult = await sessionManager.syncAndInvalidateSessions(wsimage);
   if (!invalidateResult) throw new Error("Session invalidation failed");
@@ -154,44 +154,6 @@ async function getSessionId(ds, cookieJar, getCookieString, setCookie, wsimage,
     }
   }
   const sessionData = await sessionManager.setSession(wsimage, sessionId, tileServer, overlays);
-  if (ds.queries.WSImages.getWSIPredictionPatches) {
-    const predictionPatches = await ds.queries.WSImages.getWSIPredictionPatches(aiProjectId, wsimage);
-    if (!predictionPatches) throw new Error("No prediction files found");
-    const mount = serverconfig.features?.tileserver?.mount;
-    if (!mount) throw new Error("No mount available for TileServer");
-    if (predictionPatches.length > 0) {
-      for (const predictionPatch of predictionPatches) {
-        const predictionFilePath = path.join(`${mount}/${ds.queries.WSImages.aiToolImageFolder}/`, predictionPatch);
-        const predictionsData = qs.stringify({
-          overlay_path: predictionFilePath
-        });
-        await ky.put(`${tileServer.url}/tileserver/overlay`, {
-          body: predictionsData,
-          timeout: 5e4,
-          headers: {
-            "Content-Type": "application/x-www-form-urlencoded",
-            Cookie: `session_id=${sessionId}`
-          },
-          hooks: getHooks(cookieJar, getCookieString, setCookie)
-        });
-      }
-      const cmapData = qs.stringify({
-        cmap: JSON.stringify({
-          keys: ["prediction", "annotation"],
-          values: [ds.queries.WSImages.predictionColor, ds.queries.WSImages.annotationsColor]
-        })
-      });
-      await ky.put(`${tileServer.url}/tileserver/cmap`, {
-        body: cmapData,
-        timeout: 5e4,
-        headers: {
-          "Content-Type": "application/x-www-form-urlencoded",
-          Cookie: `session_id=${sessionId}`
-        },
-        hooks: getHooks(cookieJar, getCookieString, setCookie)
-      });
-    }
-  }
   return sessionData;
 }
 async function getWsiImageDimensions(sessionId, getCookieString, wsimage) {