@sjcrh/proteinpaint-server 2.203.1 → 2.205.0

package/dataset/termdb.test.js

@@ -412,8 +412,8 @@ function termdb_test_default() {
       },
       singleCell: {
         samples: {
-          sampleColumns: [{ termid: "sex" }],
-          extraSampleTabLabel: "sex"
+          sampleColumns: [{ termid: "diaggrp" }],
+          extraSampleTabLabel: "diaggrp"
         },
         data: {
           sameLegend: true,
@@ -437,14 +437,22 @@ function termdb_test_default() {
           folder: "files/hg38/TermdbTest/scrna/geneExpHdf5"
         }
       },
-      WSImages: {
-        type: "H&E",
-        imageBySampleFolder: "files/hg38/TermdbTest/wsimages"
-      },
-      // w2 plot: each sample with images is a subfolder of this directory,
-      // holding that sample's .svs files (folder/<sample>/<fileName>)
+      // w2 plot. folder: spatial images, one per subfolder of the sample's
+      // directory (folder/<sample>/<imageName>/), files inside found by the
+      // *FileSuffix fields. wsiFolder: plain whole-slide images, laid out as
+      // wsiFolder/<sample>/<imageName>/<slide file>
       w2: {
-        folder: "files/hg38/TermdbTest/wsimages"
+        folder: "files/hg38/TermdbTest/spatial",
+        wsiFolder: "files/hg38/TermdbTest/wsimages",
+        // .ome.tif extension is required for the JPEG-2000 OME-TIFF reader
+        // in wsi_tile.py open_slide() to engage
+        tiffFileSuffix: "morphology.ome.tif",
+        nucleusBoundariesFileSuffix: "nucleus_boundaries.csv",
+        cellBoundariesFileSuffix: "cell_boundaries.csv",
+        geneExpressionFileSuffix: "gene_expression.h5",
+        // default viewer settings, overridable in the burger menu
+        geneExpression: "PTPRC",
+        annotationLevel: 1
       },
       trackLst: {
         jsonFile: "files/hg38/TermdbTest/trackLst/facet.json",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sjcrh/proteinpaint-server",
-  "version": "2.203.1",
+  "version": "2.205.0",
   "type": "module",
   "description": "a genomics visualization tool for exploring a cohort's genotype and phenotype data",
   "main": "src/app.js",
@@ -46,9 +46,8 @@
     "@typescript-eslint/eslint-plugin": "^8.13.0",
     "babel-loader": "^8.2.2",
     "c8": "^10.1.3",
-    "esbuild": "^0.25.9",
+    "esbuild": "^0.28.1",
     "monocart-coverage-reports": "^2.12.1",
-    "node-notifier": "^9.0.1",
     "node-watch": "^0.7.1",
     "nodemon": "^3.1.10",
     "prettier": "^2.8.8",
@@ -57,33 +56,29 @@
     "typescript": "^5.6.3"
   },
   "dependencies": {
-    "@sjcrh/augen": "2.190.0",
-    "@sjcrh/proteinpaint-python": "2.203.0",
-    "@sjcrh/proteinpaint-r": "2.203.0",
-    "@sjcrh/proteinpaint-rust": "2.203.1",
-    "@sjcrh/proteinpaint-shared": "2.203.1",
-    "@sjcrh/proteinpaint-types": "2.203.1",
+    "@sjcrh/augen": "2.204.0",
+    "@sjcrh/proteinpaint-python": "2.204.0",
+    "@sjcrh/proteinpaint-r": "2.204.0",
+    "@sjcrh/proteinpaint-rust": "2.204.0",
+    "@sjcrh/proteinpaint-shared": "2.205.0",
+    "@sjcrh/proteinpaint-types": "2.205.0",
     "@types/express": "^5.0.0",
     "@types/express-session": "^1.18.1",
     "better-sqlite3": "^12.4.1",
     "body-parser": "^1.15.2",
     "canvas": "~3.2.0",
     "compression": "^1.6.2",
-    "connect-redis": "^6.1.3",
     "cookie-parser": "^1.4.5",
     "d3": "^7.6.1",
     "deep-object-diff": "^1.1.0",
     "express": "^4.17.1",
     "express-basic-auth": "^1.1.5",
     "express-session": "^1.18.1",
-    "got": "^14.2.0",
     "jsonwebtoken": "^9.0.3",
     "ky": "^1.2.1",
     "micromatch": "^4.0.5",
     "minimatch": "^10.0.1",
-    "node-fetch": "^2.6.1",
     "partjson": "^0.58.2",
-    "redis": "^4.7.0",
     "skia-canvas": "~3.0.8",
     "tiny-async-pool": "^1.2.0",
     "tough-cookie": "^4.1.4"

package/routes/brainImaging.js

@@ -1,6 +1,8 @@
+import fs from "fs";
 import path from "path";
 import serverconfig from "#src/serverconfig.js";
 import { getData } from "../src/termdb.matrix.js";
+import { filterSampleNamesByAccess } from "#src/termdb.sql.js";
 import { isNumericTerm } from "#shared";
 import { getColors } from "#shared";
 import { run_python } from "@sjcrh/proteinpaint-python";
@@ -13,10 +15,10 @@ function init({ genomes }) {
       const ds = g.datasets[query.dslabel];
       if (!ds) throw "invalid dataset name";
       let plane, index;
-      if (query.l) {
+      if (query.l != void 0) {
         plane = "L";
         index = query.l;
-      } else if (query.f) {
+      } else if (query.f != void 0) {
         plane = "F";
         index = query.f;
       } else {
@@ -27,7 +29,7 @@ function init({ genomes }) {
       res.send({ brainImage, plane, legend });
     } catch (e) {
       console.log(e);
-      res.status(404).send("Sample brain image not found");
+      res.status(404).send(typeof e == "string" ? e : "Sample brain image not found");
     }
   };
 }
@@ -43,8 +45,14 @@ async function getBrainImage(query, genomes, plane, index) {
     const overlayTW = query.overlayTW;
     if (divideByTW) terms.push(divideByTW);
     if (overlayTW) terms.push(overlayTW);
-    const selectedSampleNames = query.selectedSampleFileNames.map((s) => s.split(".nii")[0]);
-    const data = await getData({ terms }, ds);
+    let selectedSampleNames = query.selectedSampleFileNames.map((s) => s.split(".nii")[0]);
+    const existingFiles = new Set(await fs.promises.readdir(dirPath));
+    selectedSampleNames = selectedSampleNames.filter((s) => existingFiles.has(s + ".nii"));
+    if (!selectedSampleNames.length) throw "no brain imaging data for the requested sample(s)";
+    selectedSampleNames = await filterSampleNamesByAccess(query, ds, selectedSampleNames);
+    if (!selectedSampleNames.length) throw "no selected samples pass the current filter";
+    const data = await getData({ terms, __protected__: query.__protected__ }, ds);
+    if (data.error) throw data.error;
     const divideByCat = {};
     const uniqueOverlayTwCats = /* @__PURE__ */ new Set();
     for (const sampleName of selectedSampleNames) {
@@ -53,11 +61,11 @@ async function getBrainImage(query, genomes, plane, index) {
       const samplePath = path.join(dirPath, sampleName) + ".nii";
       let divideCategory = "default";
       let overlayCategory = "default";
-      if (divideByTW) {
+      if (divideByTW && sampleData) {
         const value = sampleData[divideByTW.$id];
         if (value) divideCategory = divideByTW.term.values?.[value.key]?.label || value.key;
       }
-      if (overlayTW) {
+      if (overlayTW && sampleData) {
         const value = sampleData[overlayTW.$id];
         if (value) {
           overlayCategory = overlayTW.term.values?.[value.key]?.label || value.key;

package/routes/brainImagingSamples.js

@@ -2,6 +2,7 @@ import fs from "fs";
 import path from "path";
 import serverconfig from "#src/serverconfig.js";
 import { getData } from "#src/termdb.matrix.js";
+import { filterSampleNamesByAccess } from "#src/termdb.sql.js";
 function init({ genomes }) {
   return async (req, res) => {
     try {
@@ -14,7 +15,7 @@ function init({ genomes }) {
       res.send({ samples });
     } catch (e) {
       console.log(e);
-      res.status(404).send("Sample brain image not found");
+      res.status(404).send(typeof e == "string" ? e : "Cannot get brain imaging samples");
     }
   };
 }
@@ -24,15 +25,20 @@ async function getBrainImageSamples(query, genomes) {
   const key = query.refKey;
   if (q[key].referenceFile && q[key].samples) {
     const dirPath = path.join(serverconfig.tpmasterdir, q[key].samples);
-    const files = fs.readdirSync(dirPath).filter((file) => file.endsWith(".nii") && fs.statSync(path.join(dirPath, file)).isFile());
-    const sampleNames = files.map((name) => name.split(".nii")[0]);
+    const files = (await fs.promises.readdir(dirPath, { withFileTypes: true })).filter((f) => f.isFile() && f.name.endsWith(".nii")).map((f) => f.name);
+    let sampleNames = files.map((name) => name.split(".nii")[0]);
+    sampleNames = await filterSampleNamesByAccess(query, ds, sampleNames);
+    if (query.samplesOnly) return sampleNames.map((name) => ({ sample: name }));
     if (q[key].sampleColumns) {
-      const terms = q[key].sampleColumns.map((term) => ({
-        $id: term.termid,
-        term: { id: term.termid },
-        q: {}
-      }));
-      const data = await getData({ terms }, ds);
+      const terms = q[key].sampleColumns.map((term) => {
+        const termjson = ds.cohort.termdb.q.termjsonByOneid(term.termid);
+        return {
+          $id: term.termid,
+          term: termjson,
+          q: termjson?.type == "float" || termjson?.type == "integer" ? { mode: "continuous" } : {}
+        };
+      });
+      const data = await getData({ terms, __protected__: query.__protected__ }, ds);
       if (data.error) throw data.error;
       const samples = {};
       for (const s of sampleNames) {

package/routes/termdb.diffMeth.js

@@ -20,9 +20,9 @@ function init({ genomes }) {
         res.send({
           data: {
             [group1Name]: groups.group1names.length,
-            [group2Name]: groups.group2names.length,
-            ...groups.alerts.length ? { alert: groups.alerts.join(" | ") } : {}
-          }
+            [group2Name]: groups.group2names.length
+          },
+          ...groups.alerts.length ? { alert: groups.alerts.join(" | ") } : {}
         });
         return;
       }
@@ -43,12 +43,51 @@ function init({ genomes }) {
     }
   };
 }
-function dmKeyInputs(req) {
+function resolveElementQuery(ds, elementType) {
+  const key = elementType ?? "promoter";
+  const dm = ds?.queries?.dnaMethylation;
+  if (!dm) throw new Error("This dataset does not have methylation data configured.");
+  const q = dm.elements?.[key] ?? (key === "promoter" ? dm.promoter : void 0);
+  if (!q) {
+    const available = [
+      ...Object.keys(dm.elements ?? {}),
+      ...dm.promoter && !dm.elements?.promoter ? ["promoter"] : []
+    ];
+    throw new Error(
+      available.length ? `Unknown element type '${key}'. This dataset offers: ${available.join(", ")}.` : "This dataset does not have element-level methylation data configured."
+    );
+  }
+  if (!q.file) throw new Error(`Methylation matrix file is not configured for element type '${key}'.`);
+  return { key, q };
+}
+function listElementTypes(ds) {
+  const dm = ds?.queries?.dnaMethylation;
+  if (!dm) return [];
+  const out = [];
+  for (const [key, e] of Object.entries(dm.elements ?? {})) {
+    if (e?.file) out.push({ key, label: e.label || key });
+  }
+  if (dm.promoter?.file && !dm.elements?.promoter) {
+    out.unshift({ key: "promoter", label: dm.promoter.label || "Promoters" });
+  }
+  return out;
+}
+function dmKeyInputs(req, imputeMissing) {
   return {
     genome: req.genome,
     dslabel: req.dslabel,
+    /* Which element matrix was tested. Without this field a block request with the
+    same sample groups as an earlier promoter request hashes to the same key and is
+    served the promoter result — wrong rows, wrong coordinates, no error. Defaulting
+    to 'promoter' rather than null keeps pre-existing cache entries valid on deploy. */
+    element_type: req.element_type ?? "promoter",
+    /* Derived from ds config rather than sent by the client, but it still belongs in the
+    key: flipping a dataset's platform changes every p-value, and without it the on-disk
+    cache would keep serving results computed under the old missingness model. */
+    impute_missing: imputeMissing,
     samplelst: canonicalizeSamplelst(req.samplelst),
     min_samples_per_group: req.min_samples_per_group ?? null,
+    exclude_sex_chr: req.exclude_sex_chr ?? null,
     tw: req.tw ?? null,
     tw2: req.tw2 ?? null,
     filter: req.filter ?? null,
@@ -56,26 +95,33 @@ function dmKeyInputs(req) {
   };
 }
 async function getDmCacheResult(req, genomes) {
+  const imputeMissing = genomes?.[req.genome]?.datasets?.[req.dslabel]?.queries?.dnaMethylation?.platform != "wgbs";
   const { result, cacheId } = await cacheOrRecompute({
-    computeArgument: dmKeyInputs(req),
+    computeArgument: dmKeyInputs(req, imputeMissing),
     cacheSubdir: "dm",
     computeFresh: async () => {
       const { ds, term_results, term_results2 } = await resolveDaContext(req, genomes);
-      return runDmFresh(req, ds, term_results, term_results2);
+      return runDmFresh(req, ds, term_results, term_results2, imputeMissing);
     }
   });
   return { result, cacheId };
 }
-async function runDmFresh(param, ds, term_results, term_results2) {
+async function runDmFresh(param, ds, term_results, term_results2, imputeMissing) {
   const groups = await resolveDmSampleGroups(param, ds, term_results, term_results2);
   if (groups.alerts.length) throw new Error(groups.alerts.join(" | "));
-  const q = ds.queries.dnaMethylation.promoter;
+  const { q } = resolveElementQuery(ds, param.element_type);
   const diffMethInput = {
     // Group 1 is control, group 2 is case (same convention as DE).
     case: groups.group2names.join(","),
     control: groups.group1names.join(","),
     input_file: q.file,
-    min_samples_per_group: param.min_samples_per_group
+    // Omitted entirely when the entry does not set it, so an unfiltered request stays
+    // byte-identical to what it was before this field existed.
+    ...q.element_class ? { element_class: q.element_class } : {},
+    min_samples_per_group: param.min_samples_per_group,
+    exclude_sex_chr: param.exclude_sex_chr,
+    // ds-derived, not a user setting: see the platform field on queries.dnaMethylation
+    impute_missing: imputeMissing
   };
   if (param.tw) {
     diffMethInput.conf1 = [...groups.conf1_group2, ...groups.conf1_group1];
@@ -104,12 +150,10 @@ async function resolveDmSampleGroups(param, ds, term_results, term_results2) {
     throw new Error("Group 1 has no samples. Please select at least one sample.");
   if (param.samplelst.groups[1].values?.length < 1)
     throw new Error("Group 2 has no samples. Please select at least one sample.");
-  const q = ds.queries.dnaMethylation?.promoter;
-  if (!q) throw new Error("This dataset does not have promoter-level methylation data configured.");
-  if (!q.file) throw new Error("Promoter methylation data file is not configured for this dataset.");
+  const { q } = resolveElementQuery(ds, param.element_type);
   const g1 = await buildGroupValues(
     param.samplelst.groups[0].values,
-    q,
+    q.allSampleSet,
     ds,
     param.tw,
     param.tw2,
@@ -118,7 +162,7 @@ async function resolveDmSampleGroups(param, ds, term_results, term_results2) {
   );
   const g2 = await buildGroupValues(
     param.samplelst.groups[1].values,
-    q,
+    q.allSampleSet,
     ds,
     param.tw,
     param.tw2,
@@ -146,5 +190,7 @@ async function resolveDmSampleGroups(param, ds, term_results, term_results2) {
 export {
   getDmCacheResult,
   init,
-  resolveDmSampleGroups
+  listElementTypes,
+  resolveDmSampleGroups,
+  resolveElementQuery
 };