@sjcrh/proteinpaint-server 2.117.0 → 2.118.1-1

package/dataset/termdb.test.js

@@ -2,246 +2,245 @@ import serverconfig from "@sjcrh/proteinpaint-server/src/serverconfig.js";
 import * as path from "path";
 import { existsSync, unlinkSync, symlinkSync, access, constants } from "fs";
 copyDataFilesFromRepo2Tp();
-var termdb_test_default = {
-  isMds3: true,
-  cohort: {
-    massNav: {
-      tabs: {
-        about: {
-          dataRelease: {
-            version: "?",
-            link: "testLink"
-          },
-          additionalInfo: "<a href=testLink>Tutorial</a> <a href=testLink>Get help</a>"
+function termdb_test_default() {
+  return {
+    isMds3: true,
+    cohort: {
+      massNav: {
+        tabs: {
+          about: {
+            dataRelease: {
+              version: "?",
+              link: "testLink"
+            },
+            additionalInfo: "<a href=testLink>Tutorial</a> <a href=testLink>Get help</a>"
+          }
         }
-      }
-    },
-    db: {
-      file: "files/hg38/TermdbTest/db"
-    },
-    termdb: {
-      allowedTermTypes: ["geneVariant"],
-      displaySampleIds: true,
-      // allow to display sample-level data
-      timeUnit: "years",
-      minTimeSinceDx: 5,
-      // enrollment in sjlife requires 5 years since cancer diagnosis
-      ageEndOffset: 274e-5,
-      // number of years to offset ending age of patients
-      // for cox outcome with timeScale='age'
-      // 1 day (i.e. 1/365 or 0.00274) needs to be added
-      // to age_end to prevent age_end = age_start (which
-      // would cause regression analysis to fail in R)
-      cohortStartTimeMsg: "5 years post cancer diagnosis",
-      selectCohort: {
-        // wrap term.id into a term json object so as to use it in tvs;
-        // the term is not required to exist in termdb
-        // term.id is specific to this dataset, should not use literally in client/server code but always through a variable
-        term: {
-          id: "subcohort",
-          type: "multivalue"
-        },
-        prompt: "Select a cohort and test the plots.",
-        values: [
-          // <ul><li> for items, with a radio button for each.
-          {
-            keys: ["ABC"],
-            label: "ABC Cohort (ABC)",
-            shortLabel: "ABC",
-            isdefault: true
+      },
+      db: {
+        file: "files/hg38/TermdbTest/db"
+      },
+      termdb: {
+        allowedTermTypes: ["geneVariant"],
+        displaySampleIds: true,
+        // allow to display sample-level data
+        timeUnit: "years",
+        minTimeSinceDx: 5,
+        // enrollment in sjlife requires 5 years since cancer diagnosis
+        ageEndOffset: 274e-5,
+        // number of years to offset ending age of patients
+        // for cox outcome with timeScale='age'
+        // 1 day (i.e. 1/365 or 0.00274) needs to be added
+        // to age_end to prevent age_end = age_start (which
+        // would cause regression analysis to fail in R)
+        cohortStartTimeMsg: "5 years post cancer diagnosis",
+        selectCohort: {
+          // wrap term.id into a term json object so as to use it in tvs;
+          // the term is not required to exist in termdb
+          // term.id is specific to this dataset, should not use literally in client/server code but always through a variable
+          term: {
+            id: "subcohort",
+            type: "multivalue"
           },
-          {
-            keys: ["XYZ"],
-            label: "XYZ cohort (XYZ)",
-            shortLabel: "XYZ"
+          prompt: "Select a cohort and test the plots.",
+          values: [
+            // <ul><li> for items, with a radio button for each.
+            {
+              keys: ["ABC"],
+              label: "ABC Cohort (ABC)",
+              shortLabel: "ABC",
+              isdefault: true
+            },
+            {
+              keys: ["XYZ"],
+              label: "XYZ cohort (XYZ)",
+              shortLabel: "XYZ"
+            },
+            {
+              keys: ["ABC", "XYZ"],
+              label: "Combined ABC+XYZ*",
+              shortLabel: "ABC+XYZ"
+              // show note under label in smaller text size
+            }
+          ],
+          asterisk: "*fineprint on an item"
+        },
+        dataDownloadCatch: {
+          helpLink: "https://university.stjude.cloud/docs/visualization-community/data-download/",
+          missingAccess: {
+            message: "You are missing approval to one or more of the required datasets. Please go to <a target=_blank href='MISSING-ACCESS-LINK'>Genomics Platform Data Browser</a> to request access. For more information, please see this <a target=_blank href='https://university.stjude.cloud/docs/visualization-community/data-download/'>tutorial.</a>",
+            links: {
+              sjlife: "https://platform.stjude.cloud/data/cohorts?selected_tags=SJC-DS-1002",
+              ccss: "https://platform.stjude.cloud/data/cohorts?selected_tags=SJC-DS-1005",
+              "sjlife,ccss": "https://platform.stjude.cloud/data/cohorts?selected_tags=SJC-DS-1002,SJC-DS-1005",
+              fake: "https://platform.stjude.cloud/data/cohorts?selected_tags=SJC-DS-1002"
+            }
           },
-          {
-            keys: ["ABC", "XYZ"],
-            label: "Combined ABC+XYZ*",
-            shortLabel: "ABC+XYZ"
-            // show note under label in smaller text size
+          jwt: {
+            "Invalid token": "https://university.stjude.cloud/docs/visualization-community/data-download/"
           }
-        ],
-        asterisk: "*fineprint on an item"
-      },
-      dataDownloadCatch: {
-        helpLink: "https://university.stjude.cloud/docs/visualization-community/data-download/",
-        missingAccess: {
-          message: "You are missing approval to one or more of the required datasets. Please go to <a target=_blank href='MISSING-ACCESS-LINK'>Genomics Platform Data Browser</a> to request access. For more information, please see this <a target=_blank href='https://university.stjude.cloud/docs/visualization-community/data-download/'>tutorial.</a>",
-          links: {
-            sjlife: "https://platform.stjude.cloud/data/cohorts?selected_tags=SJC-DS-1002",
-            ccss: "https://platform.stjude.cloud/data/cohorts?selected_tags=SJC-DS-1005",
-            "sjlife,ccss": "https://platform.stjude.cloud/data/cohorts?selected_tags=SJC-DS-1002,SJC-DS-1005",
-            fake: "https://platform.stjude.cloud/data/cohorts?selected_tags=SJC-DS-1002"
+        },
+        matrix: {
+          settings: {
+            ignoreCnvValues: true
           }
         },
-        jwt: {
-          "Invalid token": "https://university.stjude.cloud/docs/visualization-community/data-download/"
+        termid2totalsize2: {},
+        regression: {
+          settings: {
+            coxDisclaimer: "This is a test disclaimer for the cox regression analysis."
+          }
         }
       },
-      matrix: {
-        settings: {
-          ignoreCnvValues: true
-        }
+      scatterplots: {
+        plots: [
+          {
+            name: "TermdbTest TSNE",
+            dimension: 2,
+            file: "files/hg38/TermdbTest/tnse.txt",
+            colorTW: { id: "diaggrp" }
+          }
+        ]
       },
-      termid2totalsize2: {},
-      regression: {
-        settings: {
-          coxDisclaimer: "This is a test disclaimer for the cox regression analysis."
-        }
+      matrixplots: {
+        plots: [
+          {
+            name: "Matrix plot",
+            file: "files/hg38/TermdbTest/TermdbTest_matrix.json"
+          }
+        ]
       }
     },
-    scatterplots: {
-      plots: [
-        {
-          name: "TermdbTest TSNE",
-          dimension: 2,
-          file: "files/hg38/TermdbTest/tnse.txt",
-          colorTW: { id: "diaggrp" }
-        }
-      ]
-    },
-    matrixplots: {
-      plots: [
-        {
-          name: "Matrix plot",
-          file: "files/hg38/TermdbTest/TermdbTest_matrix.json"
+    customTwQByType: {
+      // dataset-specific termsetting configs on certain term types
+      geneVariant: {
+        default: { cnvGainCutoff: 0.1, cnvLossCutoff: -0.1, cnvMaxLength: 0 },
+        byGene: {
+          // key is term.name, thus possible to use non-gene names e.g. C19MC
+          MYCN: { cnvGainCutoff: 0.5, cnvLossCutoff: -0.1, cnvMaxLength: 0 }
         }
-      ]
-    }
-  },
-  customTwQByType: {
-    // dataset-specific termsetting configs on certain term types
-    geneVariant: {
-      default: { cnvGainCutoff: 0.1, cnvLossCutoff: -0.1, cnvMaxLength: 0 },
-      byGene: {
-        // key is term.name, thus possible to use non-gene names e.g. C19MC
-        MYCN: { cnvGainCutoff: 0.5, cnvLossCutoff: -0.1, cnvMaxLength: 0 }
-      }
-    }
-  },
-  variant2samples: {
-    variantkey: "ssm_id",
-    // required, tells client to return ssm_id for identifying variants
-    // list of term ids as sample details
-    twLst: [
-      { id: "sex", q: {} },
-      { id: "diaggrp", q: {} },
-      { id: "agedx", q: {} }
-    ],
-    // small list of terms for sunburst rings
-    sunburst_twLst: [{ id: "sex", q: {} }]
-  },
-  queries: {
-    snvindel: {
-      forTrack: true,
-      byrange: {
-        bcffile: "files/hg38/TermdbTest/TermdbTest.bcf.gz"
-      },
-      skewerRim: {
-        type: "format",
-        formatKey: "origin",
-        rim1value: "germline",
-        noRimValue: "somatic"
       }
     },
-    svfusion: {
-      byrange: {
-        file: "files/hg38/TermdbTest/TermdbTest_Fusion.gz"
-      }
+    variant2samples: {
+      variantkey: "ssm_id",
+      // required, tells client to return ssm_id for identifying variants
+      // list of term ids as sample details
+      twLst: [
+        { id: "sex", q: {} },
+        { id: "diaggrp", q: {} },
+        { id: "agedx", q: {} }
+      ],
+      // small list of terms for sunburst rings
+      sunburst_twLst: [{ id: "sex", q: {} }]
     },
-    cnv: {
-      byrange: {
-        src: "native",
+    queries: {
+      snvindel: {
+        forTrack: true,
+        byrange: {
+          bcffile: "files/hg38/TermdbTest/TermdbTest.bcf.gz"
+        },
+        skewerRim: {
+          type: "format",
+          formatKey: "origin",
+          rim1value: "germline",
+          noRimValue: "somatic"
+        }
+      },
+      svfusion: {
+        byrange: {
+          file: "files/hg38/TermdbTest/TermdbTest_Fusion.gz"
+        }
+      },
+      cnv: {
         file: "files/hg38/TermdbTest/TermdbTest_CNV_gene.gz"
+      },
+      /*
+      on the fly cnv calls from gene body probe signals are no longer used
+      probe2cnv:{
+      	file: 'files/hg19/pnet/PNET.probesignals.gz'
       }
-    },
-    /*
-    on the fly cnv calls from gene body probe signals are no longer used
-    probe2cnv:{
-    	file: 'files/hg19/pnet/PNET.probesignals.gz'
-    }
-    */
-    singleSampleMutation: {
-      src: "native",
-      sample_id_key: "sample_id",
-      folder: "files/hg38/TermdbTest/mutationpersample/"
-    },
-    singleSampleGenomeQuantification: {
-      // to show genome-wide quantification plot for a sample
-      MethylationArray: {
-        description: "Genome-wide copy number variation based on tumor/normal methylation array, averaged at genomic bins.",
-        min: -1.2,
-        max: 1.2,
-        sample_id_key: "sample_id",
-        folder: "files/hg38/TermdbTest/methylationArrayNormalSubtracted_bins/",
-        // binned data, not probe-level
-        positiveColor: "#a35069",
-        negativeColor: "#5051a3",
-        singleSampleGbtk: "methylationProbeSignal"
-      }
-    },
-    singleSampleGbtk: {
-      methylationProbeSignal: {
-        description: "Probe signals from methylation array.",
-        min: -1.2,
-        max: 1.2,
+      */
+      singleSampleMutation: {
+        src: "native",
         sample_id_key: "sample_id",
-        folder: "files/hg38/TermdbTest/methylationArrayNormalSubtracted/"
-        // probe-level data, only for locus view as genome browser track
-        // each file is 1.gz and 1.gz.tbi, bedgraph format
+        folder: "files/hg38/TermdbTest/mutationpersample/"
+      },
+      singleSampleGenomeQuantification: {
+        // to show genome-wide quantification plot for a sample
+        MethylationArray: {
+          description: "Genome-wide copy number variation based on tumor/normal methylation array, averaged at genomic bins.",
+          min: -1.2,
+          max: 1.2,
+          sample_id_key: "sample_id",
+          folder: "files/hg38/TermdbTest/methylationArrayNormalSubtracted_bins/",
+          // binned data, not probe-level
+          positiveColor: "#a35069",
+          negativeColor: "#5051a3",
+          singleSampleGbtk: "methylationProbeSignal"
+        }
+      },
+      singleSampleGbtk: {
+        methylationProbeSignal: {
+          description: "Probe signals from methylation array.",
+          min: -1.2,
+          max: 1.2,
+          sample_id_key: "sample_id",
+          folder: "files/hg38/TermdbTest/methylationArrayNormalSubtracted/"
+          // probe-level data, only for locus view as genome browser track
+          // each file is 1.gz and 1.gz.tbi, bedgraph format
+        }
+      },
+      geneExpression: {
+        src: "native",
+        hdf5File: true,
+        file: "files/hg38/TermdbTest/TermdbTest.fpkm.matrix.h5"
+      },
+      topVariablyExpressedGenes: {
+        src: "native"
+      },
+      WSImages: {
+        type: "H&E",
+        imageBySampleFolder: "files/hg38/TermdbTest/wsimages"
       }
     },
-    geneExpression: {
-      src: "native",
-      hdf5File: true,
-      file: "files/hg38/TermdbTest/TermdbTest.fpkm.matrix.h5"
-    },
-    topVariablyExpressedGenes: {
-      src: "native"
-    },
-    WSImages: {
-      type: "H&E",
-      imageBySampleFolder: "files/hg38/TermdbTest/wsimages"
-    }
-  },
-  assayAvailability: {
-    // term used below must be annotated on samples rather than patients(root). otherwise matrix will pull wrong samples for geneVariant term
-    byDt: {
-      // snvindel, differentiating sample origin
-      1: {
-        byOrigin: {
-          germline: {
-            term_id: "assayavailability_germline",
-            label: "Germline",
-            // human readable label of this origin
-            yes: { value: ["1"] },
-            no: { value: ["2"] }
-          },
-          somatic: {
-            term_id: "wgs_curated",
-            label: "Somatic",
-            yes: { value: ["1"] },
-            no: { value: ["0"] }
-            // the category doesn't exist in termdb but is still supplied since somatic.no{} is required
+    assayAvailability: {
+      // term used below must be annotated on samples rather than patients(root). otherwise matrix will pull wrong samples for geneVariant term
+      byDt: {
+        // snvindel, differentiating sample origin
+        1: {
+          byOrigin: {
+            germline: {
+              term_id: "assayavailability_germline",
+              label: "Germline",
+              // human readable label of this origin
+              yes: { value: ["1"] },
+              no: { value: ["2"] }
+            },
+            somatic: {
+              term_id: "wgs_curated",
+              label: "Somatic",
+              yes: { value: ["1"] },
+              no: { value: ["0"] }
+              // the category doesn't exist in termdb but is still supplied since somatic.no{} is required
+            }
           }
+        },
+        // fusion
+        2: {
+          term_id: "assayavailability_fusion",
+          yes: { value: ["1"] },
+          no: { value: ["2"] }
+        },
+        // cnv
+        4: {
+          term_id: "assayavailability_cnv",
+          yes: { value: ["1"] },
+          no: { value: ["2"] }
         }
-      },
-      // fusion
-      2: {
-        term_id: "assayavailability_fusion",
-        yes: { value: ["1"] },
-        no: { value: ["2"] }
-      },
-      // cnv
-      4: {
-        term_id: "assayavailability_cnv",
-        yes: { value: ["1"] },
-        no: { value: ["2"] }
       }
     }
-  }
-};
+  };
+}
 function copyDataFilesFromRepo2Tp() {
   if (existsSync("/home/root/pp"))
     return;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sjcrh/proteinpaint-server",
-  "version": "2.117.0",
+  "version": "2.118.1-1",
   "type": "module",
   "description": "a genomics visualization tool for exploring a cohort's genotype and phenotype data",
   "main": "src/app.js",
@@ -66,9 +66,10 @@
   },
   "dependencies": {
     "@sjcrh/augen": "2.116.0",
+    "@sjcrh/proteinpaint-python": "2.118.0",
     "@sjcrh/proteinpaint-rust": "2.117.0",
-    "@sjcrh/proteinpaint-shared": "2.117.0",
-    "@sjcrh/proteinpaint-types": "2.117.0",
+    "@sjcrh/proteinpaint-shared": "2.118.0",
+    "@sjcrh/proteinpaint-types": "2.118.1-1",
     "@types/express": "^5.0.0",
     "@types/express-session": "^1.18.1",
     "better-sqlite3": "^9.4.1",

package/routes/brainImaging.js

@@ -1,10 +1,10 @@
 import path from "path";
 import serverconfig from "#src/serverconfig.js";
 import { brainImagingPayload } from "#types/checkers";
-import { spawn } from "child_process";
 import { getData } from "../src/termdb.matrix.js";
 import { isNumericTerm } from "@sjcrh/proteinpaint-shared/terms.js";
 import { getColors } from "#shared/common.js";
+import { run_python } from "@sjcrh/proteinpaint-python";
 const api = {
   endpoint: "brainImaging",
   methods: {
@@ -124,7 +124,20 @@ async function getBrainImage(query, genomes, plane, index) {
         if (!legend[category])
           legend[category] = { color: filesByCat[category].color, maxLength };
       }
-      const url = await generateBrainImage(refFile, plane, index, maxLength, JSON.stringify(filesByCat));
+      const arg = {
+        refFile,
+        plane,
+        index,
+        maxLength,
+        filesByCat: JSON.stringify(filesByCat)
+      };
+      let url;
+      try {
+        url = await run_python("plotBrainImaging.py", JSON.stringify(arg));
+      } catch (error) {
+        const errmsg = "Error running Python script:" + error;
+        throw new Error(errmsg);
+      }
       brainImageDict[dcategory] = { url, catNum };
     }
     if (query.legendFilter) {
@@ -141,30 +154,6 @@ async function getBrainImage(query, genomes, plane, index) {
     throw "no reference or sample files";
   }
 }
-async function generateBrainImage(refFile, plane, index, maxLength, filesJson) {
-  return new Promise((resolve, reject) => {
-    const cmd = [`${serverconfig.binpath}/utils/plotBrainImaging.py`, refFile, plane, index, maxLength, filesJson];
-    const ps = spawn(serverconfig.python, cmd);
-    const imgData = [];
-    ps.stdout.on("data", (data) => {
-      imgData.push(data);
-    });
-    ps.stderr.on("data", (data) => {
-      console.error(`stderr: ${data}`);
-      reject(new Error(`Python script filed: ${data}`));
-    });
-    ps.on("close", (code) => {
-      if (code === 0) {
-        const imageBuffer = Buffer.concat(imgData);
-        const base64Data = imageBuffer.toString("base64");
-        const imgUrl = `data:image/png;base64,${base64Data}`;
-        resolve(imgUrl);
-      } else {
-        reject(new Error(`Python script exited with code ${code}`));
-      }
-    });
-  });
-}
 export {
   api
 };

package/routes/brainImagingSamples.js

@@ -1,7 +1,6 @@
 import fs from "fs";
 import path from "path";
 import serverconfig from "#src/serverconfig.js";
-import { spawn } from "child_process";
 const api = {
   endpoint: "brainImagingSamples",
   methods: {
@@ -62,59 +61,6 @@ async function getBrainImageSamples(query, genomes) {
     throw "no reference or sample files";
   }
 }
-async function validate_query_NIdata(ds) {
-  const q = ds.queries.NIdata;
-  if (!q || !serverconfig.features?.showBrainImaging)
-    return;
-  for (const key in q) {
-    if (q[key].referenceFile && q[key].samples) {
-      q[key].get = async (sampleName, l, f, t) => {
-        const refFile = path.join(serverconfig.tpmasterdir, q[key].referenceFile);
-        const sampleFile = path.join(serverconfig.tpmasterdir, q[key].samples, sampleName);
-        try {
-          await fs.promises.stat(sampleFile);
-        } catch (e) {
-          if (e.code == "EACCES")
-            throw "cannot read file, permission denied";
-          if (e.code == "ENOENT")
-            throw "no data for this sample";
-          throw "failed to load data";
-        }
-        return new Promise((resolve, reject) => {
-          const ps = spawn(serverconfig.python, [
-            `${serverconfig.binpath}/utils/plotBrainImaging.py`,
-            refFile,
-            sampleFile,
-            l,
-            f,
-            t
-          ]);
-          const imgData = [];
-          ps.stdout.on("data", (data) => {
-            imgData.push(data);
-          });
-          ps.stderr.on("data", (data) => {
-            console.error(`stderr: ${data}`);
-            reject(new Error(`Python script filed: ${data}`));
-          });
-          ps.on("close", (code) => {
-            if (code === 0) {
-              const imageBuffer = Buffer.concat(imgData);
-              const base64Data = imageBuffer.toString("base64");
-              const imgUrl = `data:image/png;base64,${base64Data}`;
-              resolve(imgUrl);
-            } else {
-              reject(new Error(`Python script exited with code ${code}`));
-            }
-          });
-        });
-      };
-    } else {
-      throw "no reference or sample files";
-    }
-  }
-}
 export {
-  api,
-  validate_query_NIdata
+  api
 };