@sjcrh/proteinpaint-server 2.124.1 → 2.126.0

package/README.md

@@ -1,4 +1,4 @@
-# Proteinpaint Server
+# ProteinPaint Server
 
 The data server backend for the ProteinPaint application
 
@@ -6,26 +6,38 @@ The data server backend for the ProteinPaint application
 
 The client dependencies should be installed as a workspace, follow the README at the project root.
 
-You can either:
-- use a docker container for development, see ../build/dev/README.md
-- or install the [system depedencies](https://docs.google.com/document/d/1tkEHG_vYtT-OifPV-tlPeWQUMsEd3aWAKf5ExOT8G34/edit#heading=h.jy5sdrb1zkut)
-as listed in the [installation instructions](https://docs.google.com/document/d/1tkEHG_vYtT-OifPV-tlPeWQUMsEd3aWAKf5ExOT8G34/edit#heading=h.6nxua6c3ik9l).
+You can either use a docker container for development (../build/dev/README.md), or build from source.
+See https://github.com/stjude/proteinpaint/wiki.
 
 
-## Develop
+## Serverconfig
+
+`server/serverconfig.json` is used:
+- when running any server or test scripts from the pp/server directory
+- if `${process.cwd()/serverconfig.json` does not exist wherever `@sjcrh/proteinpaint-server` is called (if can give example of this usecase)
 
-From the proteinpaint/server/ directory:
+If no `test:unit` code uses serverconfig, then it would have been okay to not have `server/serverconfig.json`.
+However, it's safer to simply have that file available just in case any imported server code uses serverconfig
+and runs as part of test:unit, and that's why `server/emitImports.js` creates one if it doesn't exist by
+copying `container/ci/serverconfig.json`.
 
-`npm run dev` rebundles backend code
 
-`npm start` runs the proteinpaint server
+## Develop
 
+The local development environment is usually triggered following one of these:
+- follow the `Develop` section in [proteinpaint/README.md](https://github.com/stjude/proteinpaint/blob/master/README.md) (preferred)
+- `npm start` from the `server` dir
 
 ## Test
 
-```bash
-npm test
-```
+To run both type checks and test:unit: `npm run test`.
+
+To run type check only: `npx tsc`.
+
+To run unit tests only: `npm run test:unit`
+
+To run specific test file: `npx tsx path/to/spec.ts` 
+
 
 ## Build
 

package/dataset/termdb.test.js

@@ -6,7 +6,8 @@ function termdb_test_default() {
   return {
     isMds3: true,
     isSupportedChartOverride: {
-      runChart: () => true
+      runChart: () => true,
+      eventCount: () => true
     },
     cohort: {
       massNav: {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sjcrh/proteinpaint-server",
-  "version": "2.124.1",
+  "version": "2.126.0",
   "type": "module",
   "description": "a genomics visualization tool for exploring a cohort's genotype and phenotype data",
   "main": "src/app.js",
@@ -16,8 +16,9 @@
   "scripts": {
     "dev": "npm run start",
     "prestart": "tsx emitImports.js dev > server.js",
-    "start": "tsx watch . /start.js",
+    "start": "tsx watch ./start.js",
     "test:unit": "tsx emitImports.js unit > serverTests.js && c8 tsx serverTests.js && rm -rf ./cache",
+    "test": "tsc && npm run test:unit",
     "precombined:coverage": "tsx emitImports.js unit > serverTests.js",
     "combined:coverage": "coverageKey=test c8 --all --src=proteinpaint/server --experimental-monocart -r=v8 -r=html -r=json -r=markdown-summary -r=markdown-details -o=./.coverage tsx ./coverage.js & ",
     "postcombined:coverage": "rm -rf ./cache",
@@ -29,13 +30,7 @@
     "build": "./build.sh",
     "prepack": "npm run build",
     "postpack": "./dedupjs.sh",
-    "dedup": "./dedupjs.sh",
-    "//todo": "refactor or deprecate the scripts below",
-    "pretest": "tsc && ./test/pretest.js",
-    "pretest:type": "npm run checkers",
-    "pretest:integration": "tsc",
-    "test:integration": "echo 'TODO: server integration tests'",
-    "test:tsc": "tsc --esModuleInterop --noEmit --allowImportingTsExtensions ./shared/types/test/*.type.spec.ts"
+    "dedup": "./dedupjs.sh"
   },
   "author": "",
   "license": "SEE LICENSE IN ./LICENSE",
@@ -67,10 +62,10 @@
   "dependencies": {
     "@sjcrh/augen": "2.121.0",
     "@sjcrh/proteinpaint-python": "2.118.0",
-    "@sjcrh/proteinpaint-r": "2.123.0",
-    "@sjcrh/proteinpaint-rust": "2.124.0",
-    "@sjcrh/proteinpaint-shared": "2.124.0",
-    "@sjcrh/proteinpaint-types": "2.124.0",
+    "@sjcrh/proteinpaint-r": "2.126.0",
+    "@sjcrh/proteinpaint-rust": "2.126.0",
+    "@sjcrh/proteinpaint-shared": "2.126.0",
+    "@sjcrh/proteinpaint-types": "2.126.0",
     "@types/express": "^5.0.0",
     "@types/express-session": "^1.18.1",
     "better-sqlite3": "^9.4.1",

package/routes/gdcGRIN2list.js

@@ -0,0 +1,121 @@
+import { gdcMafPayload } from "#types/checkers";
+import ky from "ky";
+import { joinUrl } from "#shared/joinUrl.js";
+import serverconfig from "#src/serverconfig.js";
+const maxFileNumber = 1e3;
+const allowedWorkflowType = "Aliquot Ensemble Somatic Variant Merging and Masking";
+const maxTotalSizeCompressed = serverconfig.features.gdcMafMaxFileSize || 4e8;
+const api = {
+  endpoint: "gdc/GRIN2list",
+  methods: {
+    get: {
+      ...gdcMafPayload,
+      init
+    },
+    post: {
+      ...gdcMafPayload,
+      init
+    }
+  }
+};
+function init({ genomes }) {
+  return async (req, res) => {
+    try {
+      const g = genomes.hg38;
+      if (!g)
+        throw "hg38 missing";
+      const ds = g.datasets.GDC;
+      if (!ds)
+        throw "hg38 GDC missing";
+      const payload = await listMafFiles(req.query, ds);
+      res.send(payload);
+    } catch (e) {
+      res.send({ status: "error", error: e.message || e });
+    }
+  };
+}
+async function listMafFiles(q, ds) {
+  const dataFormatFilter = {
+    op: "and",
+    content: [{ op: "=", content: { field: "data_format", value: "MAF" } }]
+  };
+  const filters = {
+    op: "and",
+    content: [
+      dataFormatFilter,
+      { op: "=", content: { field: "experimental_strategy", value: q.experimentalStrategy } },
+      { op: "=", content: { field: "analysis.workflow_type", value: allowedWorkflowType } },
+      { op: "=", content: { field: "access", value: "open" } }
+    ]
+  };
+  const case_filters = { op: "and", content: [] };
+  if (q.filter0) {
+    case_filters.content.push(q.filter0);
+  }
+  const { host } = ds.getHostHeaders(q);
+  const body = {
+    filters,
+    size: maxFileNumber,
+    fields: [
+      "id",
+      "file_size",
+      "cases.project.project_id",
+      // for display only
+      "cases.case_id",
+      // case uuid for making case url link to portal
+      "cases.submitter_id",
+      // used when listing all cases & files
+      "cases.samples.tissue_type",
+      "cases.samples.tumor_descriptor"
+    ].join(",")
+  };
+  if (case_filters.content.length)
+    body.case_filters = case_filters;
+  const response = await ky.post(joinUrl(host.rest, "files"), { timeout: false, json: body });
+  if (!response.ok)
+    throw `HTTP Error: ${response.status} ${response.statusText}`;
+  const re = await response.json();
+  if (!Number.isInteger(re.data?.pagination?.total))
+    throw "re.data.pagination.total is not int";
+  if (!Array.isArray(re.data?.hits))
+    throw "re.data.hits[] not array";
+  const files = [];
+  for (const h of re.data.hits) {
+    const c = h.cases?.[0];
+    if (!c)
+      throw "h.cases[0] missing";
+    const file = {
+      id: h.id,
+      project_id: c.project.project_id,
+      file_size: h.file_size,
+      case_submitter_id: c.submitter_id,
+      case_uuid: c.case_id,
+      sample_types: []
+    };
+    if (c.samples) {
+      let normalTypeName;
+      for (const { tumor_descriptor, tissue_type } of c.samples) {
+        if (tissue_type == "Normal") {
+          normalTypeName = (tumor_descriptor == "Not Applicable" ? "" : tumor_descriptor + " ") + tissue_type;
+          continue;
+        }
+        file.sample_types.push(tumor_descriptor + " " + tissue_type);
+      }
+      if (normalTypeName)
+        file.sample_types.push(normalTypeName);
+    }
+    file.sample_types = [...new Set(file.sample_types)];
+    files.push(file);
+  }
+  files.sort((a, b) => b.file_size - a.file_size);
+  const result = {
+    files,
+    filesTotal: re.data.pagination.total,
+    maxTotalSizeCompressed
+  };
+  return result;
+}
+export {
+  api,
+  maxTotalSizeCompressed
+};

package/routes/gdcGRIN2run.js

@@ -0,0 +1,96 @@
+import { runGRIN2Payload } from "#types/checkers";
+import { run_rust } from "@sjcrh/proteinpaint-rust";
+import { run_R } from "@sjcrh/proteinpaint-r";
+import serverconfig from "#src/serverconfig.js";
+import path from "path";
+const api = {
+  endpoint: "gdc/runGRIN2",
+  methods: {
+    get: {
+      ...runGRIN2Payload,
+      init
+    },
+    post: {
+      ...runGRIN2Payload,
+      init
+    }
+  }
+};
+function init({ genomes }) {
+  return async (req, res) => {
+    try {
+      console.log("[GRIN2] Validating genome configuration");
+      const g = genomes.hg38;
+      if (!g)
+        throw "hg38 missing";
+      const ds = g.datasets.GDC;
+      if (!ds)
+        throw "hg38 GDC missing";
+      const caseFiles = req.query;
+      console.log(`[GRIN2] Request received: ${JSON.stringify(caseFiles)}`);
+      if (!caseFiles) {
+        throw "Missing or invalid cases data";
+      }
+      try {
+        console.log("[GRIN2] Calling Rust for file processing...");
+        const rustInput = JSON.stringify(caseFiles);
+        console.log("[GRIN2] Executing Rust function...");
+        const rustResult = await run_rust("gdcGRIN2", rustInput);
+        console.log("[GRIN2] Rust execution completed");
+        console.log(`[GRIN2] Rust result type: ${typeof rustResult}`);
+        if (!rustResult) {
+          throw new Error("Failed to process MAF files: No result from Rust");
+        }
+        let parsedRustResult;
+        try {
+          parsedRustResult = typeof rustResult === "string" ? JSON.parse(rustResult) : rustResult;
+          console.log(`[GRIN2] Parsed Rust result: ${JSON.stringify(parsedRustResult).substring(0, 200)}...`);
+        } catch (parseError) {
+          console.error("[GRIN2] Error parsing Rust result:", parseError);
+        }
+        const genedbfile = path.join(serverconfig.tpmasterdir, g.genedb.dbfile);
+        const imagefile = path.join(serverconfig.cachedir, `grin2_${Date.now()}_${Math.floor(Math.random() * 1e9)}.png`);
+        const rInput = JSON.stringify({
+          genedb: genedbfile,
+          chromosomelist: g.majorchr,
+          imagefile,
+          lesion: rustResult
+          // The mutation string from Rust
+        });
+        console.log(`R input: ${rInput}`);
+        const parsedInput = JSON.parse(rInput);
+        console.log("Parsed lesion data type:", typeof parsedInput.lesion);
+        console.log(
+          "Parsed lesion data length:",
+          typeof parsedInput.lesion === "string" ? parsedInput.lesion.length : "not a string"
+        );
+        console.log("[GRIN2] Executing R script...");
+        const rResult = await run_R("gdcGRIN2.R", rInput, []);
+        console.log(`[GRIN2] R execution completed, result: ${rResult}`);
+        let resultData;
+        try {
+          resultData = JSON.parse(rResult);
+          console.log("[GRIN2] Finished R analysis");
+          const pngImg = resultData.png[0];
+          return res.json({ pngImg });
+        } catch (parseError) {
+          console.error("[GRIN2] Error parsing R result:", parseError);
+          console.log("[GRIN2] Raw R result:", rResult);
+        }
+      } finally {
+      }
+    } catch (e) {
+      console.error("[GRIN2] Error running analysis:", e);
+      console.error("[GRIN2] Error stack:", e.stack);
+      const errorResponse = {
+        status: "error",
+        error: e.message || String(e)
+      };
+      console.log(`[GRIN2] Sending error response: ${JSON.stringify(errorResponse)}`);
+      res.status(500).send(errorResponse);
+    }
+  };
+}
+export {
+  api
+};

package/routes/genesetEnrichment.js

@@ -3,8 +3,8 @@ import fs from "fs";
 import path from "path";
 import serverconfig from "#src/serverconfig.js";
 import { run_python } from "@sjcrh/proteinpaint-python";
+import { run_rust } from "@sjcrh/proteinpaint-rust";
 import { mayLog } from "#src/helpers.ts";
-import { DeleteCacheFiles } from "#src/DeleteCacheFiles.ts";
 const api = {
   endpoint: "genesetEnrichment",
   methods: {
@@ -18,6 +18,7 @@ const api = {
     }
   }
 };
+const cachedir_gsea = path.join(serverconfig.cachedir, "gsea");
 function init({ genomes }) {
   return async (req, res) => {
     try {
@@ -48,26 +49,27 @@ function init({ genomes }) {
 async function run_genesetEnrichment_analysis(q, genomes) {
   if (!genomes[q.genome].termdbs)
     throw "termdb database is not available for " + q.genome;
-  const cache = new DeleteCacheFiles({
-    cachedir: serverconfig.cachedir_gsea,
-    fileExtensions: [".pkl"],
-    maxSize: 1e6
-  });
-  await cache.mayResetCacheCheckTimeout(cache.checkWait);
   const genesetenrichment_input = {
     genes: q.genes,
     fold_change: q.fold_change,
     db: genomes[q.genome].termdbs.msigdb.cohort.db.connection.name,
     // For now msigdb has been added, but later databases other than msigdb may be used
     geneset_group: q.geneSetGroup,
-    cachedir: serverconfig.cachedir_gsea,
-    geneset_name: q.geneset_name,
-    pickle_file: q.pickle_file,
-    num_permutations: q.num_permutations,
     genedb: path.join(serverconfig.tpmasterdir, genomes[q.genome].genedb.dbfile),
     filter_non_coding_genes: q.filter_non_coding_genes
   };
-  const gsea_output = await run_python("gsea.py", "/" + JSON.stringify(genesetenrichment_input));
+  let gsea_output;
+  if (q.method == "blitzgsea") {
+    genesetenrichment_input.cachedir = cachedir_gsea;
+    genesetenrichment_input.pickle_file = q.pickle_file;
+    genesetenrichment_input.geneset_name = q.geneset_name;
+    genesetenrichment_input.num_permutations = q.num_permutations;
+    gsea_output = await run_python("gsea.py", "/" + JSON.stringify(genesetenrichment_input));
+  } else if (q.method == "cerno") {
+    gsea_output = await run_rust("cerno", JSON.stringify(genesetenrichment_input));
+  } else {
+    throw "Unknown method:" + q.method;
+  }
   let result;
   let data_found = false;
   let image_found = false;
@@ -85,7 +87,7 @@ async function run_genesetEnrichment_analysis(q, genomes) {
   if (data_found)
     return result;
   if (image_found)
-    return path.join(serverconfig.cachedir_gsea, result.image_file);
+    return path.join(cachedir_gsea, result.image_file);
   throw "data or image not found in gsea output; this should not happen";
 }
 export {

package/routes/termdb.cluster.js

@@ -36,7 +36,7 @@ function init({ genomes }) {
       const ds = g.datasets[q.dslabel];
       if (!ds)
         throw "invalid dataset name";
-      if (ds.__gdc && !ds.__gdc.doneCaching)
+      if (!ds.__gdc?.doneCaching)
         throw "The server has not finished caching the case IDs: try again in about 2 minutes.";
       if ([TermTypes.GENE_EXPRESSION, TermTypes.METABOLITE_INTENSITY, NUMERIC_DICTIONARY_TERM].includes(q.dataType)) {
         if (!ds.queries?.[q.dataType] && q.dataType !== NUMERIC_DICTIONARY_TERM)

package/routes/termdb.config.js

@@ -171,6 +171,7 @@ function addNonDictionaryQueries(c, ds, genome) {
       "cnvMaxLength",
       "cnvGainCutoff",
       "cnvLossCutoff",
+      "cnvCutoffsByGene",
       "absoluteValueRenderMax",
       "gainColor",
       "lossColor"