@sjcrh/proteinpaint-server 2.184.0 → 2.185.0

package/dataset/protected.test.js

@@ -9,6 +9,11 @@ function protected_test_default() {
       canAccess: data.count >= minSize
     };
   };
+  ds.demoJwtInput = {
+    user: {
+      datasets: ["ABC", "XYZ"]
+    }
+  };
   return ds;
 }
 export {

package/dataset/termdb.test.js

@@ -417,7 +417,7 @@ function termdb_test_default() {
           src: "native",
           plots: [
             {
-              name: "scRNA",
+              name: "UMAP",
               folder: "files/hg38/TermdbTest/scrna/umap",
               fileSuffix: "_umap.txt",
               colorColumns: [

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sjcrh/proteinpaint-server",
-  "version": "2.184.0",
+  "version": "2.185.0",
   "type": "module",
   "description": "a genomics visualization tool for exploring a cohort's genotype and phenotype data",
   "main": "src/app.js",
@@ -62,11 +62,11 @@
   },
   "dependencies": {
     "@sjcrh/augen": "2.181.1",
-    "@sjcrh/proteinpaint-python": "2.183.2-0",
+    "@sjcrh/proteinpaint-python": "2.185.0",
     "@sjcrh/proteinpaint-r": "2.181.0",
-    "@sjcrh/proteinpaint-rust": "2.183.0",
-    "@sjcrh/proteinpaint-shared": "2.183.0",
-    "@sjcrh/proteinpaint-types": "2.184.0",
+    "@sjcrh/proteinpaint-rust": "2.185.0",
+    "@sjcrh/proteinpaint-shared": "2.185.0",
+    "@sjcrh/proteinpaint-types": "2.185.0",
     "@types/express": "^5.0.0",
     "@types/express-session": "^1.18.1",
     "better-sqlite3": "^12.4.1",

package/routes/brainImagingSamples.js

@@ -1,6 +1,7 @@
 import fs from "fs";
 import path from "path";
 import serverconfig from "#src/serverconfig.js";
+import { getData } from "#src/termdb.matrix.js";
 const api = {
   endpoint: "brainImagingSamples",
   methods: {
@@ -40,14 +41,24 @@ async function getBrainImageSamples(query, genomes) {
     const files = fs.readdirSync(dirPath).filter((file) => file.endsWith(".nii") && fs.statSync(path.join(dirPath, file)).isFile());
     const sampleNames = files.map((name) => name.split(".nii")[0]);
     if (q[key].sampleColumns) {
+      const terms = q[key].sampleColumns.map((term) => ({
+        $id: term.termid,
+        term: { id: term.termid },
+        q: {}
+      }));
+      const data = await getData({ terms }, ds);
+      if (data.error) throw data.error;
       const samples = {};
       for (const s of sampleNames) {
         const annoForOneS = { sample: s };
         const sid = ds.cohort.termdb.q.sampleName2id(s);
-        for (const term of q[key].sampleColumns) {
-          const v = ds.cohort.termdb.q.getSample2value(term.termid, sid);
-          if (v[0]) {
-            annoForOneS[term.termid] = v[0].value;
+        const sampleData = data.samples?.[sid];
+        if (sampleData) {
+          for (const term of q[key].sampleColumns) {
+            const v = sampleData[term.termid];
+            if (v?.value !== void 0) {
+              annoForOneS[term.termid] = v.value;
+            }
           }
         }
         samples[s] = annoForOneS;

package/routes/genesetEnrichment.js

@@ -1,4 +1,5 @@
 import { genesetEnrichmentPayload } from "#types/checkers";
+import crypto from "crypto";
 import fs from "fs";
 import path from "path";
 import serverconfig from "#src/serverconfig.js";
@@ -6,6 +7,7 @@ import { run_python } from "@sjcrh/proteinpaint-python";
 import { run_rust } from "@sjcrh/proteinpaint-rust";
 import { mayLog } from "#src/helpers.ts";
 import { formatElapsedTime } from "#shared";
+import { readCacheFileOrRecompute, stableStringify } from "#src/diffAnalysis.ts";
 const api = {
   endpoint: "genesetEnrichment",
   methods: {
@@ -19,18 +21,17 @@ const api = {
     }
   }
 };
-const cachedir_gsea = path.join(serverconfig.cachedir, "gsea");
 function init({ genomes }) {
   return async (req, res) => {
     try {
       const q = req.query;
       const results = await run_genesetEnrichment_analysis(q, genomes);
       if (!q.geneset_name) {
-        if (typeof results != "object") throw "gsea result is not object";
+        if (typeof results != "object") throw new Error("gsea result is not object");
         res.send(results);
         return;
       }
-      if (typeof results != "string") throw "gsea result is not string";
+      if (typeof results != "string") throw new Error("gsea result is not string");
       res.sendFile(results, (err) => {
         fs.unlink(results, () => {
         });
@@ -45,51 +46,109 @@ function init({ genomes }) {
   };
 }
 async function run_genesetEnrichment_analysis(q, genomes) {
-  if (!genomes[q.genome].termdbs) throw "termdb database is not available for " + q.genome;
+  if (!genomes[q.genome].termdbs) throw new Error("termdb database is not available for " + q.genome);
+  if (q.fetchDE) {
+    const { genes, fold_change } = await resolveGseaGenesAndFoldChange({ q, genomes });
+    return { data: { genes, fold_change } };
+  }
+  if (q.method == "blitzgsea") {
+    return await computeGSEA({ q, genomes });
+  }
+  if (q.method == "cerno") {
+    const { genes, fold_change } = await resolveGseaGenesAndFoldChange({ q, genomes });
+    const genesetenrichment_input = {
+      genes,
+      fold_change,
+      db: genomes[q.genome].termdbs.msigdb.cohort.db.connection.name,
+      geneset_group: q.geneSetGroup,
+      genedb: path.join(serverconfig.tpmasterdir, genomes[q.genome].genedb.dbfile),
+      filter_non_coding_genes: q.filter_non_coding_genes
+    };
+    const time1 = (/* @__PURE__ */ new Date()).valueOf();
+    const gsea_output = JSON.parse(await run_rust("cerno", JSON.stringify(genesetenrichment_input)));
+    mayLog("Time taken to run CERNO:", formatElapsedTime(Date.now() - time1));
+    return gsea_output;
+  }
+  throw new Error("Unknown method:" + q.method);
+}
+async function resolveGseaGenesAndFoldChange({
+  q,
+  genomes
+}) {
+  if (q.cacheId) {
+    if (!q.daRequest) throw new Error("daCacheMissing");
+    const result = await readCacheFileOrRecompute({ daRequest: q.daRequest, genomes });
+    if (result.cacheId !== q.cacheId) throw new Error("cacheId does not match daRequest");
+    return {
+      genes: result.geneData.map((g) => g.gene_name),
+      fold_change: result.geneData.map((g) => g.fold_change)
+    };
+  }
+  if (!q.genes || !q.fold_change) throw new Error("requires genes and fold_change when cacheId is absent");
+  return { genes: q.genes, fold_change: q.fold_change };
+}
+const pendingTracker_gsea = /* @__PURE__ */ new Map();
+const pendingTracker_img = /* @__PURE__ */ new Map();
+async function computeGSEA({
+  q,
+  genomes
+}) {
+  const { genes, fold_change } = await resolveGseaGenesAndFoldChange({ q, genomes });
+  const gseaComputeArg = {
+    genes,
+    fold_change,
+    geneSetGroup: q.geneSetGroup,
+    num_permutations: q.num_permutations,
+    filter_non_coding_genes: q.filter_non_coding_genes
+  };
+  const key = stableStringify(gseaComputeArg);
+  const pickle_file = crypto.createHash("sha256").update(key).digest("hex").slice(0, 32) + ".pkl";
+  const tracker = q.geneset_name ? pendingTracker_img : pendingTracker_gsea;
+  const inFlight = tracker.get(pickle_file);
+  if (inFlight) return inFlight;
+  const work = runGseaPython({ q, genomes, gseaComputeArg, pickle_file });
+  tracker.set(pickle_file, work);
+  return work.finally(() => tracker.delete(pickle_file));
+}
+async function runGseaPython({
+  q,
+  genomes,
+  gseaComputeArg,
+  pickle_file
+}) {
+  const cachedir_gsea = path.join(serverconfig.cachedir, "gsea");
   const genesetenrichment_input = {
-    genes: q.genes,
-    fold_change: q.fold_change,
+    genes: gseaComputeArg.genes,
+    fold_change: gseaComputeArg.fold_change,
     db: genomes[q.genome].termdbs.msigdb.cohort.db.connection.name,
-    // For now msigdb has been added, but later databases other than msigdb may be used
-    geneset_group: q.geneSetGroup,
+    geneset_group: gseaComputeArg.geneSetGroup,
     genedb: path.join(serverconfig.tpmasterdir, genomes[q.genome].genedb.dbfile),
-    filter_non_coding_genes: q.filter_non_coding_genes
+    filter_non_coding_genes: gseaComputeArg.filter_non_coding_genes,
+    cachedir: cachedir_gsea,
+    pickle_file,
+    geneset_name: q.geneset_name,
+    num_permutations: gseaComputeArg.num_permutations
   };
-  let gsea_output;
-  if (q.method == "blitzgsea") {
-    genesetenrichment_input.cachedir = cachedir_gsea;
-    genesetenrichment_input.pickle_file = q.pickle_file;
-    genesetenrichment_input.geneset_name = q.geneset_name;
-    genesetenrichment_input.num_permutations = q.num_permutations;
-    const time1 = (/* @__PURE__ */ new Date()).valueOf();
-    gsea_output = await run_python("gsea.py", "/" + JSON.stringify(genesetenrichment_input));
-    mayLog("Time taken to run blitzgsea:", formatElapsedTime(Date.now() - time1));
-    let result;
-    let data_found = false;
-    let image_found = false;
-    for (const line of gsea_output.split("\n")) {
-      if (line.startsWith("result: ")) {
-        result = JSON.parse(line.replace("result: ", ""));
-        data_found = true;
-      } else if (line.startsWith("image: ")) {
-        result = JSON.parse(line.replace("image: ", ""));
-        image_found = true;
-      } else {
-        mayLog(line);
-      }
+  const time1 = (/* @__PURE__ */ new Date()).valueOf();
+  const gsea_output = await run_python("gsea.py", "/" + JSON.stringify(genesetenrichment_input));
+  mayLog("Time taken to run blitzgsea:", formatElapsedTime(Date.now() - time1));
+  let result;
+  let data_found = false;
+  let image_found = false;
+  for (const line of gsea_output.split("\n")) {
+    if (line.startsWith("result: ")) {
+      result = JSON.parse(line.replace("result: ", ""));
+      data_found = true;
+    } else if (line.startsWith("image: ")) {
+      result = JSON.parse(line.replace("image: ", ""));
+      image_found = true;
+    } else {
+      mayLog(line);
     }
-    if (data_found) return result;
-    const image_file_name = path.join(cachedir_gsea, result.image_file);
-    if (image_found) return image_file_name;
-    throw "data or image not found in gsea output; this should not happen";
-  } else if (q.method == "cerno") {
-    const time1 = (/* @__PURE__ */ new Date()).valueOf();
-    gsea_output = JSON.parse(await run_rust("cerno", JSON.stringify(genesetenrichment_input)));
-    mayLog("Time taken to run CERNO:", formatElapsedTime(Date.now() - time1));
-    return gsea_output;
-  } else {
-    throw "Unknown method:" + q.method;
   }
+  if (data_found) return result;
+  if (image_found) return path.join(cachedir_gsea, result.image_file);
+  throw new Error("data or image not found in gsea output; this should not happen");
 }
 export {
   api

package/routes/profile.radar2.js

@@ -0,0 +1,112 @@
+import { ProfileScoresPayload } from "#types/checkers";
+import { getData } from "../src/termdb.matrix.js";
+const api = {
+  endpoint: "termdb/profileRadar2Scores",
+  methods: {
+    get: {
+      ...ProfileScoresPayload,
+      init
+    },
+    post: {
+      ...ProfileScoresPayload,
+      init
+    }
+  }
+};
+function init({ genomes }) {
+  return async (req, res) => {
+    try {
+      const g = genomes[req.query.genome];
+      if (!g) throw "invalid genome name";
+      const ds = g.datasets?.[req.query.dslabel];
+      const result = await getScores(req.query, ds);
+      res.send(result);
+    } catch (e) {
+      console.log(e);
+      res.send({ status: "error", error: e.message || e });
+    }
+  };
+}
+function derivePrefix(query) {
+  const firstScoreId = query.scoreTerms?.[0]?.score?.term?.id;
+  if (firstScoreId?.startsWith("F")) return "F";
+  if (firstScoreId?.startsWith("A")) return "A";
+  for (const entry of query.filter?.lst || []) {
+    const id = entry.tvs?.term?.id;
+    if (id?.startsWith("F")) return "F";
+    if (id?.startsWith("A")) return "A";
+  }
+  throw "cannot determine cohort prefix from scoreTerms or filter term IDs";
+}
+async function getScores(query, ds) {
+  const { activeCohort, clientAuthResult } = query.__protected__;
+  const prefix = derivePrefix(query);
+  const facilityTermId = `${prefix}UNIT`;
+  const facilityTW = { term: { id: facilityTermId }, q: {} };
+  const terms = [facilityTW];
+  for (const t of query.scoreTerms) {
+    terms.push(t.score);
+    if (t.maxScore?.term) terms.push(t.maxScore);
+  }
+  if (!query.filterByUserSites) {
+    query.__protected__.ignoredTermIds.push(facilityTermId);
+  }
+  const cohortAuth = clientAuthResult[activeCohort];
+  const isPublic = !cohortAuth?.role || cohortAuth.role === "public";
+  const userSites = cohortAuth?.sites;
+  const raw = await getData(
+    {
+      terms,
+      filter: query.filter,
+      __protected__: query.__protected__
+    },
+    ds
+  );
+  if (raw.error) throw raw.error;
+  const sampleList = Object.values(raw.samples);
+  let sites = sampleList.map((s) => {
+    const val = s[facilityTW.$id].value;
+    let label = facilityTW.term.values?.[val]?.label || val;
+    if (label.length > 50) label = label.slice(0, 47) + "...";
+    return { value: val, label };
+  });
+  if (userSites && query.filterByUserSites) {
+    sites = sites.filter((s) => userSites.includes(s.value));
+  }
+  sites.sort((a, b) => a.label.localeCompare(b.label));
+  const samples = Object.values(raw.samples);
+  const eligibleSamples = userSites && query.filterByUserSites ? samples.filter((s) => userSites.includes(s[facilityTW.$id].value)) : samples;
+  const term2Score = {};
+  for (const d of query.scoreTerms) {
+    const score = computeMedianPercentage(d, eligibleSamples);
+    if (score !== null) term2Score[d.score.term.id] = score;
+  }
+  return {
+    term2Score,
+    sites: isPublic ? [] : sites,
+    n: eligibleSamples.length
+  };
+}
+function computeMedianPercentage(d, samples) {
+  const percentages = [];
+  for (const s of samples) {
+    const scoreValue = s[d.score.$id]?.value;
+    if (scoreValue == null) continue;
+    let maxScoreValue = null;
+    if (typeof d.maxScore === "number") {
+      maxScoreValue = d.maxScore;
+    } else {
+      maxScoreValue = s[d.maxScore.$id]?.value;
+    }
+    if (maxScoreValue == null || maxScoreValue === 0) continue;
+    percentages.push(scoreValue / maxScoreValue * 100);
+  }
+  if (percentages.length === 0) return null;
+  percentages.sort((a, b) => a - b);
+  const mid = Math.floor(percentages.length / 2);
+  const median = percentages.length % 2 !== 0 ? percentages[mid] : (percentages[mid - 1] + percentages[mid]) / 2;
+  return Math.round(median);
+}
+export {
+  api
+};

package/routes/profile.radarFacility2.js

@@ -0,0 +1,148 @@
+import { ProfileScoresPayload } from "#types/checkers";
+import { getData } from "../src/termdb.matrix.js";
+const api = {
+  endpoint: "termdb/profileRadarFacility2Scores",
+  methods: {
+    get: {
+      ...ProfileScoresPayload,
+      init
+    },
+    post: {
+      ...ProfileScoresPayload,
+      init
+    }
+  }
+};
+function init({ genomes }) {
+  return async (req, res) => {
+    try {
+      const g = genomes[req.query.genome];
+      if (!g) throw "invalid genome name";
+      const ds = g.datasets?.[req.query.dslabel];
+      const result = await getScores(req.query, ds);
+      res.send(result);
+    } catch (e) {
+      console.log(e);
+      res.send({ status: "error", error: e.message || e });
+    }
+  };
+}
+function derivePrefix(query) {
+  const firstScoreId = query.scoreTerms?.[0]?.score?.term?.id;
+  if (firstScoreId?.startsWith("F")) return "F";
+  if (firstScoreId?.startsWith("A")) return "A";
+  for (const entry of query.filter?.lst || []) {
+    const id = entry.tvs?.term?.id;
+    if (id?.startsWith("F")) return "F";
+    if (id?.startsWith("A")) return "A";
+  }
+  throw "cannot determine cohort prefix from scoreTerms or filter term IDs";
+}
+async function getScores(query, ds) {
+  const { activeCohort, clientAuthResult } = query.__protected__;
+  const prefix = derivePrefix(query);
+  const facilityTermId = `${prefix}UNIT`;
+  const facilityTW = { term: { id: facilityTermId }, q: {} };
+  const terms = [facilityTW];
+  for (const t of query.scoreTerms) {
+    terms.push(t.score);
+    if (t.maxScore?.term) terms.push(t.maxScore);
+  }
+  if (!query.filterByUserSites) {
+    query.__protected__.ignoredTermIds.push(facilityTermId);
+  }
+  const cohortAuth = clientAuthResult[activeCohort];
+  const isPublic = !cohortAuth?.role || cohortAuth.role === "public";
+  const isAdmin = cohortAuth?.role === "admin";
+  const userSites = cohortAuth?.sites;
+  const raw = await getData(
+    {
+      terms,
+      filter: query.filter,
+      __protected__: query.__protected__
+    },
+    ds
+  );
+  if (raw.error) throw raw.error;
+  const samples = Object.values(raw.samples);
+  let sites = samples.map((s) => {
+    const val = s[facilityTW.$id].value;
+    let label = facilityTW.term.values?.[val]?.label || val;
+    if (label.length > 50) label = label.slice(0, 47) + "...";
+    return { value: val, label };
+  });
+  if (userSites && !isAdmin) {
+    sites = sites.filter((s) => userSites.includes(s.value));
+  }
+  sites.sort((a, b) => a.label.localeCompare(b.label));
+  const eligibleSamples = userSites && query.filterByUserSites ? samples.filter((s) => userSites.includes(s[facilityTW.$id].value)) : samples;
+  const aggregateScore = {};
+  for (const d of query.scoreTerms) {
+    const score = computeMedianPercentage(d, eligibleSamples);
+    if (score !== null) aggregateScore[d.score.term.id] = score;
+  }
+  let targetSiteValue = null;
+  if (query.facilitySite) {
+    if (isAdmin) targetSiteValue = query.facilitySite;
+    else if (userSites?.includes(query.facilitySite)) targetSiteValue = query.facilitySite;
+  }
+  if (!targetSiteValue && sites.length > 0) targetSiteValue = sites[0].value;
+  let sampleData = void 0;
+  if (!isPublic && targetSiteValue) {
+    const sampleRow = samples.find((s) => s[facilityTW.$id].value == targetSiteValue);
+    if (sampleRow) {
+      const site = sites.find((s) => s.value == targetSiteValue) || {
+        value: targetSiteValue,
+        label: facilityTW.term.values?.[targetSiteValue]?.label || targetSiteValue
+      };
+      const singleSiteScore = {};
+      for (const d of query.scoreTerms) {
+        const percent = computeSinglePercentage(d, sampleRow);
+        if (percent !== null) singleSiteScore[d.score.term.id] = percent;
+      }
+      sampleData = { term2Score: singleSiteScore, site, sites, n: 1 };
+    }
+  }
+  return {
+    term2Score: aggregateScore,
+    sampleData,
+    // Public users: empty (chart should be unreachable anyway via isSupportedChartOverride)
+    sites: isPublic ? [] : sites,
+    n: eligibleSamples.length
+  };
+}
+function computeSinglePercentage(d, sample) {
+  const scoreValue = sample[d.score.$id]?.value;
+  if (scoreValue == null) return null;
+  let maxScoreValue = null;
+  if (typeof d.maxScore === "number") {
+    maxScoreValue = d.maxScore;
+  } else {
+    maxScoreValue = sample[d.maxScore.$id]?.value;
+  }
+  if (maxScoreValue == null || maxScoreValue === 0) return null;
+  return Math.round(scoreValue / maxScoreValue * 100);
+}
+function computeMedianPercentage(d, samples) {
+  const percentages = [];
+  for (const s of samples) {
+    const scoreValue = s[d.score.$id]?.value;
+    if (scoreValue == null) continue;
+    let maxScoreValue = null;
+    if (typeof d.maxScore === "number") {
+      maxScoreValue = d.maxScore;
+    } else {
+      maxScoreValue = s[d.maxScore.$id]?.value;
+    }
+    if (maxScoreValue == null || maxScoreValue === 0) continue;
+    percentages.push(scoreValue / maxScoreValue * 100);
+  }
+  if (percentages.length === 0) return null;
+  percentages.sort((a, b) => a - b);
+  const mid = Math.floor(percentages.length / 2);
+  const median = percentages.length % 2 !== 0 ? percentages[mid] : (percentages[mid - 1] + percentages[mid]) / 2;
+  return Math.round(median);
+}
+export {
+  api
+};

package/routes/saveWSIAnnotation.js

@@ -75,6 +75,27 @@ function validateQuery(ds, connection) {
         };
       }
       const imageId = imageRow.id;
+      const duplicateCheckSql = `
+				SELECT id
+				FROM project_annotations
+				WHERE project_id = ?
+				  AND image_id = ?
+				  AND coordinates = ?
+				LIMIT 1
+			`;
+      const duplicateCheckStmt = connection.prepare(duplicateCheckSql);
+      const duplicateRow = duplicateCheckStmt.get(projectId, imageId, coords);
+      if (duplicateRow) {
+        const deleteDuplicateSql = `
+					DELETE FROM project_annotations
+					WHERE id = ?
+				`;
+        const deleteDuplicateStmt = connection.prepare(deleteDuplicateSql);
+        deleteDuplicateStmt.run(duplicateRow.id);
+        console.log(
+          `Deleted duplicate annotation with id=${duplicateRow.id} for project_id=${projectId}, image_id=${imageId}.`
+        );
+      }
       const insertSql = `
 				INSERT INTO project_annotations (
 					project_id, user_id, coordinates, timestamp, status, class_id, image_id