@sjcrh/proteinpaint-server 2.133.5 → 2.135.0

package/dataset/termdb.test.js

@@ -7,7 +7,8 @@ function termdb_test_default() {
     isMds3: true,
     isSupportedChartOverride: {
       runChart: () => true,
-      frequencyChart: () => true
+      frequencyChart: () => true,
+      report: () => true
     },
     cohort: {
       massNav: {
@@ -106,6 +107,76 @@ function termdb_test_default() {
           settings: {
             coxDisclaimer: "This is a test disclaimer for the cox regression analysis."
           }
+        },
+        plotConfigByCohort: {
+          default: {
+            report: {
+              sections: [
+                {
+                  name: "Demographics",
+                  plots: [
+                    {
+                      chartType: "barchart",
+                      settings: { barchart: { colorBars: true } },
+                      term: {
+                        id: "agedx"
+                      }
+                    },
+                    {
+                      chartType: "barchart",
+                      term: { id: "sex" },
+                      settings: { barchart: { colorBars: true } }
+                    },
+                    {
+                      chartType: "barchart",
+                      term: { id: "genetic_race" },
+                      settings: { barchart: { colorBars: true } }
+                    }
+                  ]
+                },
+                {
+                  name: "Diagnosis",
+                  plots: [
+                    {
+                      chartType: "barchart",
+                      term: { id: "diaggrp" },
+                      settings: { barchart: { colorBars: true } }
+                    }
+                  ]
+                },
+                {
+                  name: "Treatment",
+                  plots: [
+                    {
+                      chartType: "barchart",
+                      term: { id: "hrtavg" },
+                      settings: { barchart: { colorBars: true } }
+                    },
+                    {
+                      chartType: "barchart",
+                      term: { id: "aaclassic_5" },
+                      settings: { barchart: { colorBars: true } }
+                    }
+                  ]
+                },
+                {
+                  name: "Survival",
+                  plots: [
+                    {
+                      chartType: "survival",
+                      term: { id: "efs" },
+                      term2: { id: "diaggrp" }
+                    },
+                    {
+                      chartType: "survival",
+                      term: { id: "os" },
+                      term2: { id: "diaggrp" }
+                    }
+                  ]
+                }
+              ]
+            }
+          }
         }
       },
       scatterplots: {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sjcrh/proteinpaint-server",
-  "version": "2.133.5",
+  "version": "2.135.0",
   "type": "module",
   "description": "a genomics visualization tool for exploring a cohort's genotype and phenotype data",
   "main": "src/app.js",
@@ -61,11 +61,11 @@
   },
   "dependencies": {
     "@sjcrh/augen": "2.121.0",
-    "@sjcrh/proteinpaint-python": "2.133.0",
+    "@sjcrh/proteinpaint-python": "2.135.0",
     "@sjcrh/proteinpaint-r": "2.130.0",
-    "@sjcrh/proteinpaint-rust": "2.133.0",
-    "@sjcrh/proteinpaint-shared": "2.133.5",
-    "@sjcrh/proteinpaint-types": "2.133.5",
+    "@sjcrh/proteinpaint-rust": "2.135.0",
+    "@sjcrh/proteinpaint-shared": "2.135.0",
+    "@sjcrh/proteinpaint-types": "2.135.0",
     "@types/express": "^5.0.0",
     "@types/express-session": "^1.18.1",
     "better-sqlite3": "^9.4.1",

package/routes/{profileFilters.js → filterTermValues.js}

@@ -1,14 +1,14 @@
-import { ProfileFiltersPayload } from "#types/checkers";
+import { FilterTermValuesPayload } from "#types/checkers";
 import { getData, getSamplesPerFilter } from "../src/termdb.matrix.js";
 const api = {
-  endpoint: "profileFilters",
+  endpoint: "filterTermValues",
   methods: {
     get: {
-      ...ProfileFiltersPayload,
+      ...FilterTermValuesPayload,
       init
     },
     post: {
-      ...ProfileFiltersPayload,
+      ...FilterTermValuesPayload,
       init
     }
   }
@@ -38,9 +38,11 @@ function getList(samplesPerFilter, filtersData, tw) {
   const sampleValues = Array.from(new Set(data.map((sample) => sample[tw.$id]?.value)));
   for (const value of values) {
     value.value = value.label;
-    value.disabled = !sampleValues.includes(value.label);
+    const label = value.label;
+    value.disabled = !sampleValues.includes(label);
   }
   values.unshift({ label: "", value: "" });
+  values.sort((a, b) => a.label.localeCompare(b.label));
   return values;
 }
 async function getFilters(query, ds, genome, res) {

package/routes/gdc.grin2.run.js

@@ -3,6 +3,7 @@ import { stream_rust } from "@sjcrh/proteinpaint-rust";
 import serverconfig from "#src/serverconfig.js";
 import path from "path";
 import { run_python } from "@sjcrh/proteinpaint-python";
+import { mayLog } from "#src/helpers.ts";
 const api = {
   endpoint: "gdc/runGRIN2",
   methods: {
@@ -16,6 +17,120 @@ const api = {
     }
   }
 };
+function init({ genomes }) {
+  return async (req, res) => {
+    try {
+      await runGrin2(genomes, req, res);
+    } catch (e) {
+      console.error("[GRIN2] Error stack:", e.stack);
+      const errorResponse = {
+        status: "error",
+        error: e.message || String(e)
+      };
+      res.status(500).send(errorResponse);
+    }
+  };
+}
+async function runGrin2(genomes, req, res) {
+  const g = genomes.hg38;
+  if (!g)
+    throw "hg38 missing";
+  const ds = g.datasets.GDC;
+  if (!ds)
+    throw "hg38 GDC missing";
+  const parsedRequest = req.query;
+  const rustInput = {
+    caseFiles: parsedRequest.caseFiles,
+    mafOptions: parsedRequest.mafOptions,
+    cnvOptions: parsedRequest.cnvOptions,
+    chromosomes: []
+  };
+  const pyInput = {
+    genedb: path.join(serverconfig.tpmasterdir, g.genedb.dbfile),
+    chromosomelist: {},
+    lesion: []
+  };
+  for (const c in g.majorchr) {
+    if (ds.queries.singleSampleMutation?.discoPlot?.skipChrM) {
+      if (c.toLowerCase() == "chrm")
+        continue;
+    }
+    rustInput.chromosomes.push(c);
+    pyInput.chromosomelist[c] = g.majorchr[c];
+  }
+  let rustOutput = "";
+  let buffer = "";
+  const downloadStartTime = Date.now();
+  const streamResult = stream_rust("gdcGRIN2", JSON.stringify(rustInput), (errors) => {
+    if (errors) {
+      throw new Error(`Rust process failed: ${errors}`);
+    }
+  });
+  if (!streamResult) {
+    throw new Error("Failed to start Rust streaming process");
+  }
+  for await (const chunk of streamResult.rustStream) {
+    const chunkStr = chunk.toString();
+    rustOutput += chunkStr;
+    buffer += chunkStr;
+    const lines = buffer.split("\n");
+    buffer = lines.pop() || "";
+    for (const line of lines) {
+      const trimmedLine = line.trim();
+      if (trimmedLine) {
+        try {
+          const data = JSON.parse(trimmedLine);
+          if (data.type === "summary") {
+            mayLog(`[GRIN2] Download complete: ${data.successful_files}/${data.total_files} files successful`);
+            if (data.failed_files > 0) {
+              mayLog(`[GRIN2] ${data.failed_files} files failed`);
+            }
+          }
+        } catch (_parseError) {
+        }
+      }
+    }
+  }
+  mayLog("[GRIN2] Rust execution completed");
+  const downloadTime = Date.now() - downloadStartTime;
+  const downloadTimeToPrint = Math.round(downloadTime / 1e3);
+  mayLog(`[GRIN2] Rust processing took ${downloadTimeToPrint}`);
+  const rustResult = parseJsonlOutput(rustOutput);
+  if (!rustResult) {
+    throw new Error("Failed to process MAF files: No result from Rust");
+  }
+  const parsedRustResult = rustResult;
+  if (parsedRustResult.successful_data && Array.isArray(parsedRustResult.successful_data)) {
+    pyInput.lesion = parsedRustResult.successful_data.flat();
+    mayLog(`[GRIN2] Extracted ${pyInput.lesion.length} records for python script`);
+    mayLog(
+      `[GRIN2] Success: ${parsedRustResult.summary.successful_files}, Failed: ${parsedRustResult.summary.failed_files}`
+    );
+  } else {
+    throw "Unexpected Rust result format";
+  }
+  const grin2AnalysisStart = Date.now();
+  const pyResult = await run_python("gdcGRIN2.py", JSON.stringify(pyInput));
+  mayLog(`[GRIN2] Python stderr: ${pyResult.stderr}`);
+  const grin2AnalysisTime = Date.now() - grin2AnalysisStart;
+  const grin2AnalysisTimeToPrint = Math.round(grin2AnalysisTime / 1e3);
+  mayLog(`[GRIN2] Python processing took ${grin2AnalysisTimeToPrint}`);
+  const resultData = JSON.parse(pyResult);
+  const pngImg = resultData.png[0];
+  const topGeneTable = resultData.topGeneTable || null;
+  const totalProcessTime = downloadTimeToPrint + grin2AnalysisTimeToPrint;
+  return res.json({
+    pngImg,
+    topGeneTable,
+    rustResult: parsedRustResult,
+    timing: {
+      rustProcessingTime: downloadTimeToPrint,
+      grin2ProcessingTime: grin2AnalysisTimeToPrint,
+      totalTime: totalProcessTime
+    },
+    status: "success"
+  });
+}
 function parseJsonlOutput(rustOutput) {
   const lines = rustOutput.trim().split("\n");
   const allSuccessfulData = [];
@@ -31,7 +146,7 @@ function parseJsonlOutput(rustOutput) {
         const data = JSON.parse(trimmedLine);
         if (data.type === "data") {
           if (isCapReached) {
-            console.log(`[GRIN2] Skipping file ${data.case_id} - record cap of ${MAX_RECORDS} already reached`);
+            mayLog(`[GRIN2] Skipping file ${data.case_id} - record cap of ${MAX_RECORDS} already reached`);
             continue;
           }
           const remainingCapacity = MAX_RECORDS - totalRecordsProcessed;
@@ -45,32 +160,29 @@ function parseJsonlOutput(rustOutput) {
             recordsToProcess = data.data.slice(0, remainingCapacity);
             recordsProcessedThisFile = remainingCapacity;
             isCapReached = true;
-            console.log(
+            mayLog(
               `[GRIN2] Record cap reached! Processing only ${recordsProcessedThisFile} of ${incomingRecords} records from file ${data.case_id}`
             );
           }
           processedFiles++;
           allSuccessfulData.push(recordsToProcess);
           totalRecordsProcessed += recordsProcessedThisFile;
-          console.log(totalRecordsProcessed, MAX_RECORDS);
-          console.log(
+          mayLog(
             `[GRIN2] Processed file ${processedFiles}: ${data.case_id} (${data.data_type}) - ${recordsProcessedThisFile} records`
           );
-          console.log(`[GRIN2] Total records processed: ${totalRecordsProcessed}/${MAX_RECORDS}`);
+          mayLog(`[GRIN2] Total records processed: ${totalRecordsProcessed}/${MAX_RECORDS}`);
           if (isCapReached) {
-            console.log(
-              `[GRIN2] RECORD CAP REACHED: ${MAX_RECORDS} records processed. Subsequent files will be skipped.`
-            );
+            mayLog(`[GRIN2] RECORD CAP REACHED: ${MAX_RECORDS} records processed. Subsequent files will be skipped.`);
           }
         } else if (data.type === "summary") {
           finalSummary = data;
-          console.log(`[GRIN2] Download complete: ${data.successful_files}/${data.total_files} files successful`);
+          mayLog(`[GRIN2] Download complete: ${data.successful_files}/${data.total_files} files successful`);
           if (isCapReached) {
-            console.log(`[GRIN2] Processing stopped due to record cap of ${MAX_RECORDS}`);
-            console.log(`[GRIN2] Total records collected: ${totalRecordsProcessed}`);
+            mayLog(`[GRIN2] Processing stopped due to record cap of ${MAX_RECORDS}`);
+            mayLog(`[GRIN2] Total records collected: ${totalRecordsProcessed}`);
           }
           if (data.failed_files > 0) {
-            console.log(`[GRIN2] ${data.failed_files} files failed`);
+            mayLog(`[GRIN2] ${data.failed_files} files failed`);
           }
         }
       } catch (parseError) {
@@ -86,6 +198,7 @@ function parseJsonlOutput(rustOutput) {
     successful_data: allSuccessfulData,
     failed_files: finalSummary.errors || [],
     summary: {
+      type: "summary",
       total_files: finalSummary.total_files,
       successful_files: finalSummary.successful_files,
       failed_files: finalSummary.failed_files,
@@ -93,148 +206,11 @@ function parseJsonlOutput(rustOutput) {
       filtered_records: finalSummary.filtered_records || 0,
       filtered_maf_records: finalSummary.filtered_maf_records || 0,
       filtered_cnv_records: finalSummary.filtered_cnv_records || 0,
-      filtered_records_by_case: finalSummary.filtered_records_by_case || {},
       included_maf_records: finalSummary.included_maf_records || 0,
-      included_cnv_records: finalSummary.included_cnv_records || 0
-    }
-  };
-}
-function init({ genomes }) {
-  return async (req, res) => {
-    try {
-      console.log("[GRIN2] Validating genome configuration");
-      const g = genomes.hg38;
-      if (!g)
-        throw "hg38 missing";
-      const ds = g.datasets.GDC;
-      if (!ds)
-        throw "hg38 GDC missing";
-      console.log(`[GRIN2] Request received:`, JSON.stringify(req.query));
-      const parsedRequest = req.query;
-      console.log(`[GRIN2] Parsed request: ${JSON.stringify(parsedRequest)}`);
-      const rustInput = JSON.stringify({
-        caseFiles: parsedRequest.caseFiles,
-        mafOptions: parsedRequest.mafOptions,
-        cnvOptions: parsedRequest.cnvOptions
-      });
-      console.log(`[GRIN2] Rust input: ${rustInput}`);
-      console.log("[GRIN2] Executing Rust function with streaming...");
-      let rustOutput = "";
-      let buffer = "";
-      const downloadStartTime = Date.now();
-      const streamResult = stream_rust("gdcGRIN2", rustInput, (errors) => {
-        if (errors) {
-          throw new Error(`Rust process failed: ${errors}`);
-        }
-      });
-      if (!streamResult) {
-        throw new Error("Failed to start Rust streaming process");
-      }
-      for await (const chunk of streamResult.rustStream) {
-        const chunkStr = chunk.toString();
-        rustOutput += chunkStr;
-        buffer += chunkStr;
-        const lines = buffer.split("\n");
-        buffer = lines.pop() || "";
-        for (const line of lines) {
-          const trimmedLine = line.trim();
-          if (trimmedLine) {
-            try {
-              const data = JSON.parse(trimmedLine);
-              if (data.type === "summary") {
-                console.log(`[GRIN2] Download complete: ${data.successful_files}/${data.total_files} files successful`);
-                if (data.failed_files > 0) {
-                  console.log(`[GRIN2] ${data.failed_files} files failed`);
-                }
-              }
-            } catch (_parseError) {
-            }
-          }
-        }
-      }
-      console.log("[GRIN2] Rust execution completed");
-      const downloadTime = Date.now() - downloadStartTime;
-      const downloadTimeToPrint = Math.round(downloadTime / 1e3);
-      console.log(`[GRIN2] Rust processing took ${downloadTimeToPrint}`);
-      const rustResult = parseJsonlOutput(rustOutput);
-      if (!rustResult) {
-        throw new Error("Failed to process MAF files: No result from Rust");
-      }
-      const parsedRustResult = rustResult;
-      let dataForPython = [];
-      try {
-        if (parsedRustResult.successful_data && Array.isArray(parsedRustResult.successful_data)) {
-          dataForPython = parsedRustResult.successful_data.flat();
-          console.log(`[GRIN2] Extracted ${dataForPython.length} records for python script`);
-          console.log(
-            `[GRIN2] Success: ${parsedRustResult.summary.successful_files}, Failed: ${parsedRustResult.summary.failed_files}`
-          );
-        } else {
-          console.warn("[GRIN2] Unexpected Rust result format");
-          dataForPython = [];
-        }
-      } catch (parseError) {
-        console.error("[GRIN2] Error processing Rust result:", parseError);
-        dataForPython = [];
-      }
-      const genedbfile = path.join(serverconfig.tpmasterdir, g.genedb.dbfile);
-      const pyInput = JSON.stringify({
-        genedb: genedbfile,
-        chromosomelist: g.majorchr,
-        lesion: dataForPython
-        // The mutation string from Rust
-      });
-      console.log("[GRIN2] Executing python script...");
-      const grin2AnalysisStart = Date.now();
-      let pyResult;
-      try {
-        pyResult = await run_python("gdcGRIN2.py", pyInput);
-      } catch (pyError) {
-        console.error("[GRIN2] Python execution failed:", pyError);
-        if (pyError && typeof pyError === "object" && "message" in pyError) {
-          throw new Error(`Python script failed: ${pyError.message}`);
-        } else {
-          throw new Error(`Python script failed: ${String(pyError)}`);
-        }
-      }
-      console.log("[GRIN2] python execution completed");
-      console.log(`[GRIN2] Python stderr: ${pyResult.stderr}`);
-      const grin2AnalysisTime = Date.now() - grin2AnalysisStart;
-      const grin2AnalysisTimeToPrint = Math.round(grin2AnalysisTime / 1e3);
-      console.log(`[GRIN2] Python processing took ${grin2AnalysisTimeToPrint}`);
-      let resultData;
-      try {
-        resultData = JSON.parse(pyResult);
-        console.log("[GRIN2] Finished python analysis");
-        const pngImg = resultData.png[0];
-        const topGeneTable = resultData.topGeneTable || null;
-        const analysisStats = parsedRustResult.summary || {};
-        const totalProcessTime = downloadTimeToPrint + grin2AnalysisTimeToPrint;
-        console.log("[GRIN2] Total GRIN2 processing time:", totalProcessTime);
-        return res.json({
-          pngImg,
-          topGeneTable,
-          rustResult: parsedRustResult,
-          analysisStats,
-          timing: {
-            rustProcessingTime: downloadTimeToPrint,
-            grin2ProcessingTime: grin2AnalysisTimeToPrint,
-            totalTime: totalProcessTime
-          },
-          status: "success"
-        });
-      } catch (parseError) {
-        console.error("[GRIN2] Error parsing python result:", parseError);
-      }
-    } catch (e) {
-      console.error("[GRIN2] Error running analysis:", e);
-      console.error("[GRIN2] Error stack:", e.stack);
-      const errorResponse = {
-        status: "error",
-        error: e.message || String(e)
-      };
-      console.log(`[GRIN2] Sending error response: ${JSON.stringify(errorResponse)}`);
-      res.status(500).send(errorResponse);
+      included_cnv_records: finalSummary.included_cnv_records || 0,
+      filtered_records_by_case: finalSummary.filtered_records_by_case || {},
+      hyper_mutator_records: finalSummary.hyper_mutator_records || {},
+      skippedChromosomes: finalSummary.skipped_chromosomes || {}
     }
   };
 }

package/routes/profileFormScores.js

@@ -35,7 +35,7 @@ async function getScoresDict(query, ds, genome) {
   const data = await getData(
     {
       terms,
-      filter: query.filter
+      filter: query.site || !query.isAggregate ? void 0 : query.filter
       //if isRadarFacility and site is specified, do not apply the filter
     },
     ds,
@@ -48,19 +48,14 @@ async function getScoresDict(query, ds, genome) {
   sites = lst.map((s) => {
     return { label: data.refs.bySampleId[s.sample].label, value: s.sample };
   });
-  if (query.userSites) {
-    sites = sites.filter((s) => query.userSites.includes(s.label));
-  }
-  let userSite;
-  if (query.userSites) {
-    const siteName = query.userSites[0];
-    userSite = ds.sampleName2Id.get(siteName);
-    if (!userSite) {
-      throw `Invalid user site: ${siteName}`;
-    }
-  }
-  const site = query.isAggregate ? query.site : userSite;
-  const sampleData = data.samples[site] || null;
+  let sitesSelected = [];
+  if (query.site)
+    sitesSelected = [query.site];
+  else if (!query.isAggregate)
+    sitesSelected = [sites[0].value];
+  else
+    sitesSelected = query.sites;
+  const sampleData = sitesSelected?.length == 1 ? data.samples[sitesSelected[0]] : null;
   const samples = Object.values(data.samples);
   const term2Score = {};
   for (const d of query.scoreTerms) {
@@ -76,7 +71,7 @@ async function getScoresDict(query, ds, genome) {
       term2Score[d.term.id] = percents;
     }
   const hospital = sampleData?.[query.facilityTW.$id]?.value;
-  return { term2Score, sites, hospital, n: samples.length };
+  return { term2Score, sites, hospital, n: sampleData ? 1 : samples.length };
 }
 function getDict(key, sample) {
   if (!sample[key])

package/routes/profileScores.js

@@ -29,7 +29,6 @@ function init({ genomes }) {
   };
 }
 async function getScores(query, ds, genome) {
-  const isRadarFacility = query.isRadarFacility;
   const terms = [query.facilityTW];
   for (const term of query.scoreTerms) {
     terms.push(term.score);
@@ -40,8 +39,8 @@ async function getScores(query, ds, genome) {
   const data = await getData(
     {
       terms,
-      filter: isRadarFacility && query.site ? void 0 : query.filter
-      //if isRadarFacility and site is specified, do not apply the filter
+      filter: query.site || !query.isAggregate ? void 0 : query.filter
+      //if site is specified, do not apply the filter that is for the aggregation
     },
     ds,
     genome
@@ -56,27 +55,23 @@ async function getScores(query, ds, genome) {
   if (query.userSites) {
     sites = sites.filter((s) => query.userSites.includes(s.label));
   }
-  let userSite;
-  if (query.userSites) {
-    const siteName = query.userSites[0];
-    userSite = ds.sampleName2Id.get(siteName);
-    if (!userSite) {
-      throw `Invalid user site: ${siteName}`;
-    }
-  }
-  let site;
-  if (isRadarFacility)
-    site = query.site;
+  let sitesSelected = [];
+  if (query.site)
+    sitesSelected = [query.site];
+  else if (!query.isAggregate)
+    sitesSelected = [sites[0].value];
   else
-    site = query.isAggregate ? query.site : userSite;
-  const sampleData = data.samples[site] || null;
-  const samples = Object.values(data.samples);
+    sitesSelected = query.sites;
+  const sampleData = sitesSelected?.length == 1 ? data.samples[sitesSelected[0]] : null;
+  let samples = Object.values(data.samples);
+  if (sitesSelected?.length > 0)
+    samples = samples.filter((s) => sitesSelected.includes(s.sample));
   const term2Score = {};
   for (const d of query.scoreTerms) {
     term2Score[d.score.term.id] = getPercentage(d, samples, sampleData);
   }
   const hospital = sampleData?.[query.facilityTW.$id]?.value;
-  return { term2Score, sites, hospital, n: samples.length };
+  return { term2Score, sites, hospital, n: sampleData ? 1 : samples.length };
 }
 function getPercentage(d, samples, sampleData) {
   if (!d)