@sjcrh/proteinpaint-server 2.132.1-0 → 2.132.1-2

package/dataset/termdb.test.js

@@ -7,7 +7,7 @@ function termdb_test_default() {
     isMds3: true,
     isSupportedChartOverride: {
       runChart: () => true,
-      eventCount: () => true
+      frequencyChart: () => true
     },
     cohort: {
       massNav: {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sjcrh/proteinpaint-server",
-  "version": "2.132.1-0",
+  "version": "2.132.1-2",
   "type": "module",
   "description": "a genomics visualization tool for exploring a cohort's genotype and phenotype data",
   "main": "src/app.js",
@@ -61,11 +61,11 @@
   },
   "dependencies": {
     "@sjcrh/augen": "2.121.0",
-    "@sjcrh/proteinpaint-python": "2.118.0",
+    "@sjcrh/proteinpaint-python": "2.132.1-1",
     "@sjcrh/proteinpaint-r": "2.130.0",
     "@sjcrh/proteinpaint-rust": "2.132.1-0",
-    "@sjcrh/proteinpaint-shared": "2.132.0",
-    "@sjcrh/proteinpaint-types": "2.132.1-0",
+    "@sjcrh/proteinpaint-shared": "2.132.1-2",
+    "@sjcrh/proteinpaint-types": "2.132.1-1",
     "@types/express": "^5.0.0",
     "@types/express-session": "^1.18.1",
     "better-sqlite3": "^9.4.1",

package/routes/gdc.grin2.run.js

@@ -1,9 +1,9 @@
 import { runGRIN2Payload } from "#types/checkers";
 import { stream_rust } from "@sjcrh/proteinpaint-rust";
-import { run_R } from "@sjcrh/proteinpaint-r";
 import serverconfig from "#src/serverconfig.js";
 import path from "path";
 import { formatElapsedTime } from "@sjcrh/proteinpaint-shared/time.js";
+import { run_python } from "@sjcrh/proteinpaint-python";
 const api = {
   endpoint: "gdc/runGRIN2",
   methods: {
@@ -22,20 +22,54 @@ function parseJsonlOutput(rustOutput) {
   const allSuccessfulData = [];
   let finalSummary = null;
   let processedFiles = 0;
+  const MAX_RECORDS = 1e5;
+  let totalRecordsProcessed = 0;
+  let isCapReached = false;
   for (const line of lines) {
     const trimmedLine = line.trim();
     if (trimmedLine) {
       try {
         const data = JSON.parse(trimmedLine);
         if (data.type === "data") {
+          if (isCapReached) {
+            console.log(`[GRIN2] Skipping file ${data.case_id} - record cap of ${MAX_RECORDS} already reached`);
+            continue;
+          }
+          const remainingCapacity = MAX_RECORDS - totalRecordsProcessed;
+          const incomingRecords = data.data.length;
+          let recordsToProcess;
+          let recordsProcessedThisFile;
+          if (incomingRecords <= remainingCapacity) {
+            recordsToProcess = data.data;
+            recordsProcessedThisFile = incomingRecords;
+          } else {
+            recordsToProcess = data.data.slice(0, remainingCapacity);
+            recordsProcessedThisFile = remainingCapacity;
+            isCapReached = true;
+            console.log(
+              `[GRIN2] Record cap reached! Processing only ${recordsProcessedThisFile} of ${incomingRecords} records from file ${data.case_id}`
+            );
+          }
           processedFiles++;
-          allSuccessfulData.push(data.data);
+          allSuccessfulData.push(recordsToProcess);
+          totalRecordsProcessed += recordsProcessedThisFile;
+          console.log(totalRecordsProcessed, MAX_RECORDS);
           console.log(
-            `[GRIN2] Processed file ${processedFiles}: ${data.case_id} (${data.data_type}) - ${data.data.length} records`
+            `[GRIN2] Processed file ${processedFiles}: ${data.case_id} (${data.data_type}) - ${recordsProcessedThisFile} records`
           );
+          console.log(`[GRIN2] Total records processed: ${totalRecordsProcessed}/${MAX_RECORDS}`);
+          if (isCapReached) {
+            console.log(
+              `[GRIN2] RECORD CAP REACHED: ${MAX_RECORDS} records processed. Subsequent files will be skipped.`
+            );
+          }
         } else if (data.type === "summary") {
           finalSummary = data;
           console.log(`[GRIN2] Download complete: ${data.successful_files}/${data.total_files} files successful`);
+          if (isCapReached) {
+            console.log(`[GRIN2] Processing stopped due to record cap of ${MAX_RECORDS}`);
+            console.log(`[GRIN2] Total records collected: ${totalRecordsProcessed}`);
+          }
           if (data.failed_files > 0) {
             console.log(`[GRIN2] ${data.failed_files} files failed`);
           }
@@ -127,41 +161,50 @@ function init({ genomes }) {
         throw new Error("Failed to process MAF files: No result from Rust");
       }
       const parsedRustResult = rustResult;
-      let dataForR = [];
+      let dataForPython = [];
       try {
         if (parsedRustResult.successful_data && Array.isArray(parsedRustResult.successful_data)) {
-          dataForR = parsedRustResult.successful_data.flat();
-          console.log(`[GRIN2] Extracted ${dataForR.length} records for R script`);
+          dataForPython = parsedRustResult.successful_data.flat();
+          console.log(`[GRIN2] Extracted ${dataForPython.length} records for python script`);
           console.log(
             `[GRIN2] Success: ${parsedRustResult.summary.successful_files}, Failed: ${parsedRustResult.summary.failed_files}`
           );
         } else {
           console.warn("[GRIN2] Unexpected Rust result format");
-          dataForR = [];
+          dataForPython = [];
         }
       } catch (parseError) {
         console.error("[GRIN2] Error processing Rust result:", parseError);
-        dataForR = [];
+        dataForPython = [];
       }
       const genedbfile = path.join(serverconfig.tpmasterdir, g.genedb.dbfile);
-      const imagefile = path.join(serverconfig.cachedir, `grin2_${Date.now()}_${Math.floor(Math.random() * 1e9)}.png`);
-      const rInput = JSON.stringify({
+      const pyInput = JSON.stringify({
         genedb: genedbfile,
         chromosomelist: g.majorchr,
-        imagefile,
-        lesion: dataForR
+        lesion: dataForPython
         // The mutation string from Rust
       });
-      console.log("[GRIN2] Executing R script...");
+      console.log("[GRIN2] Executing python script...");
       const grin2AnalysisStart = Date.now();
-      const rResult = await run_R("gdcGRIN2.R", rInput, []);
-      console.log("[GRIN2] R execution completed");
+      let pyResult;
+      try {
+        pyResult = await run_python("gdcGRIN2.py", pyInput);
+      } catch (pyError) {
+        console.error("[GRIN2] Python execution failed:", pyError);
+        if (pyError && typeof pyError === "object" && "message" in pyError) {
+          throw new Error(`Python script failed: ${pyError.message}`);
+        } else {
+          throw new Error(`Python script failed: ${String(pyError)}`);
+        }
+      }
+      console.log("[GRIN2] python execution completed");
+      console.log(`[GRIN2] Python stderr: ${pyResult.stderr}`);
       const grin2AnalysisTime = formatElapsedTime(Date.now() - grin2AnalysisStart);
       console.log(`[GRIN2] Rust processing took ${grin2AnalysisTime}`);
       let resultData;
       try {
-        resultData = JSON.parse(rResult);
-        console.log("[GRIN2] Finished R analysis");
+        resultData = JSON.parse(pyResult);
+        console.log("[GRIN2] Finished python analysis");
         const pngImg = resultData.png[0];
         const topGeneTable = resultData.topGeneTable || null;
         const analysisStats = parsedRustResult.summary || {};
@@ -180,7 +223,7 @@ function init({ genomes }) {
           status: "success"
         });
       } catch (parseError) {
-        console.error("[GRIN2] Error parsing R result:", parseError);
+        console.error("[GRIN2] Error parsing python result:", parseError);
       }
     } catch (e) {
       console.error("[GRIN2] Error running analysis:", e);

package/routes/profileFilters.js

@@ -0,0 +1,68 @@
+import { ProfileFiltersPayload } from "#types/checkers";
+import { getData, getSamplesPerFilter } from "../src/termdb.matrix.js";
+const api = {
+  endpoint: "profileFilters",
+  methods: {
+    get: {
+      ...ProfileFiltersPayload,
+      init
+    },
+    post: {
+      ...ProfileFiltersPayload,
+      init
+    }
+  }
+};
+function init({ genomes }) {
+  return async (req, res) => {
+    try {
+      const g = genomes[req.query.genome];
+      if (!g)
+        throw "invalid genome name";
+      const ds = g.datasets?.[req.query.dslabel];
+      getFilters(req.query, ds, g, res);
+    } catch (e) {
+      console.log(e);
+      res.send({ status: "error", error: e.message || e });
+    }
+  };
+}
+function getList(samplesPerFilter, filtersData, tw) {
+  const values = Object.values(tw.term.values);
+  values.sort((v1, v2) => v1.label.localeCompare(v2.label));
+  const twSamples = samplesPerFilter[tw.term.id];
+  const data = [];
+  for (const sample of twSamples) {
+    data.push(filtersData.samples[sample]);
+  }
+  const sampleValues = Array.from(new Set(data.map((sample) => sample[tw.$id]?.value)));
+  for (const value of values) {
+    value.value = value.label;
+    value.disabled = !sampleValues.includes(value.label);
+  }
+  values.unshift({ label: "", value: "" });
+  return values;
+}
+async function getFilters(query, ds, genome, res) {
+  try {
+    const samplesPerFilter = await getSamplesPerFilter(query, ds);
+    const filtersData = await getData(
+      {
+        terms: query.terms
+      },
+      ds,
+      genome
+    );
+    const tw2List = {};
+    for (const tw of query.terms) {
+      tw2List[tw.term.id] = getList(samplesPerFilter, filtersData, tw);
+    }
+    res.send({ ...tw2List });
+  } catch (e) {
+    console.log(e);
+    res.send({ error: e.message || e });
+  }
+}
+export {
+  api
+};

package/routes/profileFormScores.js

@@ -0,0 +1,117 @@
+import { ProfileFormScoresPayload } from "#types/checkers";
+import { getData } from "../src/termdb.matrix.js";
+const api = {
+  endpoint: "profileFormScores",
+  methods: {
+    get: {
+      ...ProfileFormScoresPayload,
+      init
+    },
+    post: {
+      ...ProfileFormScoresPayload,
+      init
+    }
+  }
+};
+function init({ genomes }) {
+  return async (req, res) => {
+    try {
+      const g = genomes[req.query.genome];
+      if (!g)
+        throw "invalid genome name";
+      const ds = g.datasets?.[req.query.dslabel];
+      const result = await getScoresDict(req.query, ds, g);
+      res.send(result);
+    } catch (e) {
+      console.log(e);
+      res.send({ status: "error", error: e.message || e });
+    }
+  };
+}
+async function getScoresDict(query, ds, genome) {
+  const terms = [...query.scoreTerms];
+  if (query.scScoreTerms)
+    terms.push(...query.scScoreTerms);
+  const data = await getData(
+    {
+      terms,
+      filter: query.filter
+      //if isRadarFacility and site is specified, do not apply the filter
+    },
+    ds,
+    genome
+  );
+  const lst = Object.values(data.samples);
+  let sites = lst.map((s) => {
+    return { label: data.refs.bySampleId[s.sample].label, value: s.sample };
+  });
+  sites = lst.map((s) => {
+    return { label: data.refs.bySampleId[s.sample].label, value: s.sample };
+  });
+  if (query.userSites) {
+    sites = sites.filter((s) => query.userSites.includes(s.label));
+  }
+  let userSite;
+  if (query.userSites) {
+    const siteName = query.userSites[0];
+    userSite = ds.sampleName2Id.get(siteName);
+    if (!userSite) {
+      throw `Invalid user site: ${siteName}`;
+    }
+  }
+  const site = query.isAggregate ? query.site : userSite;
+  const sampleData = data.samples[site] || null;
+  const samples = Object.values(data.samples);
+  const term2Score = {};
+  for (const d of query.scoreTerms) {
+    const samples2 = sampleData ? [sampleData] : Object.values(data.samples);
+    const getDictFunc = (sample) => getDict(d.$id, sample);
+    const percents = getPercentsDict(getDictFunc, samples2);
+    term2Score[d.term.id] = percents;
+  }
+  if (query.scScoreTerms)
+    for (const d of query.scScoreTerms) {
+      const samples2 = sampleData ? [sampleData] : Object.values(data.samples);
+      const percents = getSCPercentsDict(d, samples2);
+      term2Score[d.term.id] = percents;
+    }
+  const hospital = sampleData?.[query.facilityTW.$id]?.value;
+  return { term2Score, sites, hospital, n: samples.length };
+}
+function getDict(key, sample) {
+  if (!sample[key])
+    return null;
+  const termData = sample[key].value;
+  return JSON.parse(termData);
+}
+function getPercentsDict(getDictFunc, samples) {
+  const percentageDict = {};
+  for (const sample of samples) {
+    const percents = getDictFunc(sample);
+    if (!percents)
+      continue;
+    for (const key in percents) {
+      const value = percents[key];
+      if (!percentageDict[key])
+        percentageDict[key] = 0;
+      percentageDict[key] += value;
+    }
+  }
+  return percentageDict;
+}
+function getSCPercentsDict(tw, samples) {
+  if (!tw)
+    throw "tw not defined";
+  const percentageDict = {};
+  for (const sample of samples) {
+    const twData = sample[tw.$id];
+    const key = twData?.value;
+    if (!percentageDict[key])
+      percentageDict[key] = 0;
+    percentageDict[key] += 1;
+  }
+  return percentageDict;
+}
+export {
+  api
+};

package/routes/profileScores.js

@@ -0,0 +1,101 @@
+import { ProfileScoresPayload } from "#types/checkers";
+import { getData } from "../src/termdb.matrix.js";
+const api = {
+  endpoint: "profileScores",
+  methods: {
+    get: {
+      ...ProfileScoresPayload,
+      init
+    },
+    post: {
+      ...ProfileScoresPayload,
+      init
+    }
+  }
+};
+function init({ genomes }) {
+  return async (req, res) => {
+    try {
+      const g = genomes[req.query.genome];
+      if (!g)
+        throw "invalid genome name";
+      const ds = g.datasets?.[req.query.dslabel];
+      const result = await getScores(req.query, ds, g);
+      res.send(result);
+    } catch (e) {
+      console.log(e);
+      res.send({ status: "error", error: e.message || e });
+    }
+  };
+}
+async function getScores(query, ds, genome) {
+  const isRadarFacility = query.isRadarFacility;
+  const terms = [query.facilityTW];
+  for (const term of query.scoreTerms) {
+    terms.push(term.score);
+    if (term.maxScore) {
+      terms.push(term.maxScore);
+    }
+  }
+  const data = await getData(
+    {
+      terms,
+      filter: isRadarFacility && query.site ? void 0 : query.filter
+      //if isRadarFacility and site is specified, do not apply the filter
+    },
+    ds,
+    genome
+  );
+  const lst = Object.values(data.samples);
+  let sites = lst.map((s) => {
+    return { label: data.refs.bySampleId[s.sample].label, value: s.sample };
+  });
+  sites = lst.map((s) => {
+    return { label: data.refs.bySampleId[s.sample].label, value: s.sample };
+  });
+  if (query.userSites) {
+    sites = sites.filter((s) => query.userSites.includes(s.label));
+  }
+  let userSite;
+  if (query.userSites) {
+    const siteName = query.userSites[0];
+    userSite = ds.sampleName2Id.get(siteName);
+    if (!userSite) {
+      throw `Invalid user site: ${siteName}`;
+    }
+  }
+  let site;
+  if (isRadarFacility)
+    site = query.site;
+  else
+    site = query.isAggregate ? query.site : userSite;
+  const sampleData = data.samples[site] || null;
+  const samples = Object.values(data.samples);
+  const term2Score = {};
+  for (const d of query.scoreTerms) {
+    term2Score[d.score.term.id] = getPercentage(d, samples, sampleData);
+  }
+  const hospital = sampleData?.[query.facilityTW.$id]?.value;
+  return { term2Score, sites, hospital, n: samples.length };
+}
+function getPercentage(d, samples, sampleData) {
+  if (!d)
+    return null;
+  const isAggregate = sampleData == null;
+  if (isAggregate) {
+    const maxScore = d.maxScore.term ? samples[0]?.[d.maxScore.$id]?.value : d.maxScore;
+    const scores = samples.map((sample) => sample[d.score.$id]?.value / maxScore * 100);
+    scores.sort((s1, s2) => s1 - s2);
+    const middle = Math.floor(scores.length / 2);
+    const score = scores.length % 2 !== 0 ? scores[middle] : (scores[middle - 1] + scores[middle]) / 2;
+    return Math.round(score);
+  } else {
+    const score = sampleData[d.score.$id]?.value;
+    const maxScore = d.maxScore.term ? sampleData[d.maxScore.$id]?.value : d.maxScore;
+    const percentage = score / maxScore * 100;
+    return Math.round(percentage);
+  }
+}
+export {
+  api
+};