@sjcrh/proteinpaint-server 2.135.3-1 → 2.137.0

package/dataset/termdb.test.js

@@ -111,6 +111,7 @@ function termdb_test_default() {
         plotConfigByCohort: {
           default: {
             report: {
+              filterTWs: [{ id: "diaggrp" }],
               sections: [
                 {
                   name: "Demographics",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sjcrh/proteinpaint-server",
-  "version": "2.135.3-1",
+  "version": "2.137.0",
   "type": "module",
   "description": "a genomics visualization tool for exploring a cohort's genotype and phenotype data",
   "main": "src/app.js",
@@ -47,7 +47,6 @@
     "babel-loader": "^8.2.2",
     "c8": "^10.1.3",
     "esbuild": "^0.19.12",
-    "glob": "^10.4.5",
     "monocart-coverage-reports": "^2.12.1",
     "node-notifier": "^9.0.1",
     "node-watch": "^0.7.1",
@@ -60,12 +59,12 @@
     "typescript": "^5.6.3"
   },
   "dependencies": {
-    "@sjcrh/augen": "2.121.0",
+    "@sjcrh/augen": "2.136.0",
     "@sjcrh/proteinpaint-python": "2.135.2-0",
     "@sjcrh/proteinpaint-r": "2.130.0",
-    "@sjcrh/proteinpaint-rust": "2.135.3-1",
-    "@sjcrh/proteinpaint-shared": "2.135.2-0",
-    "@sjcrh/proteinpaint-types": "2.135.2-0",
+    "@sjcrh/proteinpaint-rust": "2.136.0",
+    "@sjcrh/proteinpaint-shared": "2.136.0",
+    "@sjcrh/proteinpaint-types": "2.137.0",
     "@types/express": "^5.0.0",
     "@types/express-session": "^1.18.1",
     "better-sqlite3": "^9.4.1",

package/routes/filterTermValues.js

@@ -35,7 +35,8 @@ function getList(samplesPerFilter, filtersData, tw) {
   for (const sample of twSamples) {
     data.push(filtersData.samples[sample]);
   }
-  const sampleValues = Array.from(new Set(data.map((sample) => sample[tw.$id]?.value)));
+  const annotations = data.filter((s) => s != void 0).map((sample) => sample[tw.$id]?.value);
+  const sampleValues = Array.from(new Set(annotations));
   for (const value of values) {
     value.value = value.label;
     const label = value.label;

package/routes/grin2.js

@@ -0,0 +1,245 @@
+import { GRIN2Payload } from "#types/checkers";
+import serverconfig from "#src/serverconfig.js";
+import path from "path";
+import { run_python } from "@sjcrh/proteinpaint-python";
+import { mayLog } from "#src/helpers.ts";
+import { get_samples } from "#src/termdb.sql.js";
+import { read_file, file_is_readable } from "#src/utils.js";
+import { dtsnvindel, dtcnv, dtfusionrna } from "#shared/common.js";
+const api = {
+  endpoint: "grin2",
+  methods: {
+    get: {
+      ...GRIN2Payload,
+      init
+    },
+    post: {
+      ...GRIN2Payload,
+      init
+    }
+  }
+};
+function init({ genomes }) {
+  return async (req, res) => {
+    try {
+      const request = req.query;
+      const g = genomes[request.genome];
+      if (!g)
+        throw new Error("genome missing");
+      const ds = g.datasets?.[request.dslabel];
+      if (!ds)
+        throw new Error("ds missing");
+      if (!ds.queries?.singleSampleMutation)
+        throw new Error("singleSampleMutation query missing from dataset");
+      const result = await runGrin2(g, ds, request);
+      res.json(result);
+    } catch (e) {
+      console.error("[GRIN2] Error stack:", e.stack);
+      const errorResponse = {
+        status: "error",
+        error: e.message || String(e)
+      };
+      res.status(500).send(errorResponse);
+    }
+  };
+}
+async function runGrin2(g, ds, request) {
+  const startTime = Date.now();
+  mayLog("[GRIN2] Getting samples from cohort filter...");
+  const samples = await get_samples(
+    request.filter,
+    ds,
+    true
+    // must set to true to return sample name to be able to access file. FIXME this can let names revealed to grin2 client, may need to apply access control
+  );
+  const cohortTime = Date.now() - startTime;
+  mayLog(`[GRIN2] Retrieved ${samples.length.toLocaleString()} samples in ${Math.round(cohortTime / 1e3)} seconds`);
+  if (samples.length === 0) {
+    throw new Error("No samples found matching the provided filter criteria");
+  }
+  mayLog("[GRIN2] Processing sample data...");
+  const processingStartTime = Date.now();
+  const { lesionData, processingSummary } = await processSampleData(samples, ds, request);
+  const processingTime = Date.now() - processingStartTime;
+  const processingTimeToPrint = Math.round(processingTime / 1e3);
+  mayLog(`[GRIN2] Data processing took ${processingTimeToPrint} seconds`);
+  mayLog(
+    `[GRIN2] Processing summary: ${processingSummary?.successfulSamples ?? 0}/${processingSummary?.totalSamples ?? samples.length} samples processed successfully`
+  );
+  if (processingSummary && processingSummary.failedSamples > 0) {
+    mayLog(`[GRIN2] Warning: ${processingSummary.failedSamples} samples failed to process`);
+  }
+  if (lesionData.length === 0) {
+    throw new Error("No lesions found after processing all samples. Check filter criteria and input data.");
+  }
+  const pyInput = {
+    genedb: path.join(serverconfig.tpmasterdir, g.genedb.dbfile),
+    chromosomelist: {},
+    lesion: JSON.stringify(lesionData)
+  };
+  for (const c in g.majorchr) {
+    if (ds.queries.singleSampleMutation.discoPlot?.skipChrM) {
+      if (c.toLowerCase() == "chrm")
+        continue;
+    }
+    pyInput.chromosomelist[c] = g.majorchr[c];
+  }
+  mayLog(`[GRIN2] Prepared ${lesionData.length.toLocaleString()} lesions for analysis`);
+  const grin2AnalysisStart = Date.now();
+  const pyResult = await run_python("grin2PpWrapper.py", JSON.stringify(pyInput));
+  if (pyResult.stderr?.trim()) {
+    mayLog(`[GRIN2] Python stderr: ${pyResult.stderr}`);
+    if (pyResult.stderr.includes("ERROR:")) {
+      throw new Error(`Python script error: ${pyResult.stderr}`);
+    }
+  }
+  const grin2AnalysisTime = Date.now() - grin2AnalysisStart;
+  const grin2AnalysisTimeToPrint = Math.round(grin2AnalysisTime / 1e3);
+  mayLog(`[GRIN2] Python processing took ${grin2AnalysisTimeToPrint} seconds`);
+  const resultData = JSON.parse(pyResult.stdout);
+  if (!resultData?.png?.[0]) {
+    throw new Error("Invalid Python output: missing PNG data");
+  }
+  const totalTime = Math.round((Date.now() - startTime) / 1e3);
+  const response = {
+    status: "success",
+    pngImg: resultData.png[0],
+    topGeneTable: resultData.topGeneTable,
+    totalGenes: resultData.totalGenes,
+    showingTop: resultData.showingTop,
+    timing: {
+      processingTime: processingTimeToPrint,
+      grin2Time: grin2AnalysisTimeToPrint,
+      totalTime
+    },
+    processingSummary
+  };
+  return response;
+}
+async function processSampleData(samples, ds, request) {
+  const lesions = [];
+  let lesionId = 1;
+  const processingSummary = {
+    totalSamples: samples.length,
+    successfulSamples: 0,
+    failedSamples: 0,
+    failedFiles: []
+  };
+  mayLog(`[GRIN2] Processing JSON files for ${samples.length.toLocaleString()} samples`);
+  for (const sample of samples) {
+    try {
+      const filepath = path.join(serverconfig.tpmasterdir, ds.queries.singleSampleMutation.folder, sample.name);
+      await file_is_readable(filepath);
+      const mlst = JSON.parse(await read_file(filepath));
+      const sampleLesions = await processSampleMlst(sample.name, mlst, lesionId, request);
+      lesions.push(...sampleLesions);
+      lesionId += sampleLesions.length;
+    } catch (error) {
+      mayLog(
+        `[GRIN2] Error processing sample ${sample.name}: ${typeof error === "object" && error !== null && "message" in error ? error.message : String(error)}`
+      );
+    }
+  }
+  mayLog(`[GRIN2] Total lesions processed: ${lesions.length.toLocaleString()}`);
+  return {
+    lesionData: lesions,
+    processingSummary
+  };
+}
+async function processSampleMlst(sampleName, mlst, startId, request) {
+  const lesions = [];
+  for (const m of mlst) {
+    switch (m.dt) {
+      case dtsnvindel: {
+        if (!request.snvindelOptions)
+          break;
+        const snvIndelLesion = filterAndConvertSnvIndel(sampleName, m, request.snvindelOptions);
+        if (snvIndelLesion)
+          lesions.push(snvIndelLesion);
+        break;
+      }
+      case dtcnv: {
+        if (!request.cnvOptions)
+          break;
+        const cnvLesion = filterAndConvertCnv(sampleName, m, request.cnvOptions);
+        if (cnvLesion)
+          lesions.push(cnvLesion);
+        break;
+      }
+      case dtfusionrna: {
+        if (!request.fusionOptions)
+          break;
+        const fusionLesion = filterAndConvertFusion(sampleName, m, request.fusionOptions);
+        if (fusionLesion)
+          lesions.push(fusionLesion);
+        break;
+      }
+      default:
+        mayLog(`[GRIN2] Unknown data type "${m.dt}" in sample ${sampleName}, skipping entry`);
+        break;
+    }
+  }
+  return lesions;
+}
+function filterAndConvertSnvIndel(sampleName, entry, options) {
+  const opts = {
+    minTotalDepth: options?.minTotalDepth ?? 10,
+    minAltAlleleCount: options?.minAltAlleleCount ?? 2,
+    consequences: options?.consequences ?? [],
+    hyperMutator: options?.hyperMutator ?? 1e3
+  };
+  if (opts.consequences.length > 0 && entry.consequence && !opts.consequences.includes(entry.consequence)) {
+    return null;
+  }
+  return [
+    sampleName,
+    normalizeChromosome(entry.chromosome || entry.chr),
+    String(entry.start || entry.position),
+    String(entry.end || entry.position),
+    "mutation"
+  ];
+}
+function filterAndConvertCnv(sampleName, entry, options) {
+  const opts = {
+    lossThreshold: options?.lossThreshold ?? -0.4,
+    gainThreshold: options?.gainThreshold ?? 0.3,
+    maxSegLength: options?.maxSegLength ?? 0,
+    minSegLength: options?.minSegLength ?? 0,
+    hyperMutator: options?.hyperMutator ?? 500
+  };
+  const isGain = entry.log2Ratio >= opts.gainThreshold;
+  const isLoss = entry.log2Ratio <= opts.lossThreshold;
+  if (!isGain && !isLoss)
+    return null;
+  const lesionType = entry.log2Ratio >= opts.gainThreshold ? "gain" : "loss";
+  return [
+    sampleName,
+    normalizeChromosome(entry.chromosome || entry.chr),
+    String(entry.start || entry.begin),
+    String(entry.end || entry.stop),
+    lesionType
+  ];
+}
+function filterAndConvertFusion(sampleName, entry, options) {
+  const opts = {
+    fusionTypes: options?.fusionTypes ?? ["gene-gene", "gene-intergenic", "readthrough"],
+    minConfidence: options?.minConfidence ?? 0.7
+  };
+  if (entry.fusionType && !opts.fusionTypes.includes(entry.fusionType))
+    return null;
+  if (entry.confidence && entry.confidence < opts.minConfidence)
+    return null;
+  return [
+    sampleName,
+    normalizeChromosome(entry.chromosome || entry.chr),
+    String(entry.start || entry.position),
+    String(entry.end || entry.position),
+    "fusion"
+  ];
+}
+function normalizeChromosome(chrom) {
+  return chrom.startsWith("chr") ? chrom : `chr${chrom}`;
+}
+export {
+  api
+};

package/routes/termdb.categories.js

@@ -51,12 +51,20 @@ async function trigger_getcategories(q, res, tdb, ds, genome) {
   const data = await getData(arg, ds, genome);
   if (data.error)
     throw data.error;
+  const [lst, orderedLabels] = getCategories(data, q, ds, $id);
+  res.send({
+    lst,
+    orderedLabels
+  });
+}
+function getCategories(data, q, ds, $id) {
   const lst = [];
   if (q.tw.term.type == "geneVariant" && q.tw.q.type != "predefined-groupset" && q.tw.q.type != "custom-groupset") {
     const samples = data.samples;
     const dtClassMap = /* @__PURE__ */ new Map();
     if (ds.assayAvailability?.byDt) {
-      for (const [dtType, dtValue] of Object.entries(ds.assayAvailability.byDt)) {
+      for (const [dtType, _dtValue] of Object.entries(ds.assayAvailability.byDt)) {
+        const dtValue = _dtValue;
         if (dtValue.byOrigin) {
           dtClassMap.set(parseInt(dtType), { byOrigin: { germline: {}, somatic: {} } });
         }
@@ -65,6 +73,8 @@ async function trigger_getcategories(q, res, tdb, ds, genome) {
     const sampleCountedFor = /* @__PURE__ */ new Set();
     for (const sampleData of Object.values(samples)) {
       const key = $id;
+      if (!Object.keys(sampleData).includes(key))
+        continue;
       const values = sampleData[key].values;
       sampleCountedFor.clear();
       for (const value of values) {
@@ -126,11 +136,9 @@ async function trigger_getcategories(q, res, tdb, ds, genome) {
   if (orderedLabels.length) {
     lst.sort((a, b) => orderedLabels.indexOf(a.label) - orderedLabels.indexOf(b.label));
   }
-  res.send({
-    lst,
-    orderedLabels
-  });
+  return [lst, orderedLabels];
 }
 export {
-  api
+  api,
+  getCategories
 };

package/routes/termdb.singlecellSamples.js

@@ -51,9 +51,14 @@ async function validate_query_singleCell(ds, genome) {
   const q = ds.queries.singleCell;
   if (!q)
     return;
-  if (typeof q.samples.get != "function") {
+  if (typeof q.samples != "object")
+    throw "singleCell.samples{} not object";
+  if (typeof q.samples.get == "function") {
+  } else {
     validateSamplesNative(q.samples, q.data, ds);
   }
+  if (typeof q.data != "object")
+    throw "singleCell.data{} not object";
   if (q.data.src == "gdcapi") {
     gdc_validate_query_singleCell_data(ds, genome);
   } else if (q.data.src == "native") {
@@ -62,6 +67,8 @@ async function validate_query_singleCell(ds, genome) {
     throw "unknown singleCell.data.src";
   }
   if (q.geneExpression) {
+    if (typeof q.geneExpression != "object")
+      throw "singleCell.geneExpression not object";
     if (q.geneExpression.src == "native") {
       validateGeneExpressionNative(q.geneExpression);
     } else if (q.geneExpression.src == "gdcapi") {
@@ -71,9 +78,13 @@ async function validate_query_singleCell(ds, genome) {
     }
   }
   if (q.DEgenes) {
+    if (typeof q.DEgenes != "object")
+      throw "singleCell.DEgenes not object";
     validate_query_singleCell_DEgenes(ds);
   }
   if (q.images) {
+    if (typeof q.images != "object")
+      throw "singleCell.images not object";
     validateImages(q.images);
   }
 }
@@ -127,68 +138,69 @@ function validateDataNative(D, ds) {
     if (nameSet.has(plot.name))
       throw "duplicate plot.name";
     nameSet.add(plot.name);
+    if (!plot.folder)
+      throw "plot.folder missing";
   }
+  const file2Lines = {};
   D.get = async (q) => {
-    try {
-      const plots = [];
-      let geneExpMap;
-      if (ds.queries.singleCell.geneExpression && q.gene) {
-        geneExpMap = await ds.queries.singleCell.geneExpression.get({ sample: q.sample, gene: q.gene });
-      }
-      const file2Lines = {};
-      for (const plot of D.plots) {
-        if (!q.plots.includes(plot.name))
-          continue;
-        const tsvfile = path.join(
-          serverconfig.tpmasterdir,
-          plot.folder,
-          (q.sample.eID || q.sample.sID) + plot.fileSuffix
-        );
-        if (!file2Lines[tsvfile]) {
-          await file_is_readable(tsvfile);
-          file2Lines[tsvfile] = (await read_file(tsvfile)).trim().split("\n");
-        }
-        const colorColumn = plot.colorColumns.find((c) => c.name == q.colorBy?.[plot.name]) || plot.colorColumns[0];
-        const lines = file2Lines[tsvfile];
-        const expCells = [];
-        const noExpCells = [];
-        for (let i = 1; i < lines.length; i++) {
-          const l = lines[i].split("	");
-          const cellId = lines.length > 3 ? l[0] : void 0, x = Number(l[plot.coordsColumns.x]), y = Number(l[plot.coordsColumns.y]);
-          const category = l[colorColumn?.index] || "";
-          if (!cellId)
-            throw "cell id missing";
-          if (!Number.isFinite(x) || !Number.isFinite(y))
-            throw "x/y not number";
-          const cell = { cellId, x, y, category };
-          if (geneExpMap) {
-            if (geneExpMap[cellId] !== void 0) {
-              cell.geneExp = geneExpMap[cellId];
-              expCells.push(cell);
-            } else {
-              noExpCells.push(cell);
-            }
-          } else
-            noExpCells.push(cell);
+    const plots = [];
+    let geneExpMap;
+    if (ds.queries.singleCell.geneExpression && q.gene) {
+      geneExpMap = await ds.queries.singleCell.geneExpression.get({ sample: q.sample, gene: q.gene });
+    }
+    for (const plot of D.plots) {
+      if (!q.plots.includes(plot.name))
+        continue;
+      const tsvfile = path.join(serverconfig.tpmasterdir, plot.folder, (q.sample.eID || q.sample.sID) + plot.fileSuffix);
+      if (!file2Lines[tsvfile]) {
+        await file_is_readable(tsvfile);
+        const text = await read_file(tsvfile);
+        const lines = text.trim().split("\n");
+        let first = true;
+        const lines2 = [];
+        for (const line of lines) {
+          if (first) {
+            first = false;
+            continue;
+          }
+          lines2.push(line.split("	"));
         }
-        plots.push({
-          name: plot.name,
-          expCells,
-          noExpCells,
-          colorColumns: plot.colorColumns.map((c) => c.name),
-          colorBy: colorColumn?.name,
-          colorMap: colorColumn?.colorMap
-        });
+        file2Lines[tsvfile] = lines2;
       }
-      if (plots.length == 0) {
-        return { nodata: true };
+      const colorColumn = plot.colorColumns.find((c) => c.name == q.colorBy?.[plot.name]) || plot.colorColumns[0];
+      const expCells = [];
+      const noExpCells = [];
+      for (const l of file2Lines[tsvfile]) {
+        const cellId = l[0], x = Number(l[plot.coordsColumns.x]), y = Number(l[plot.coordsColumns.y]);
+        const category = l[colorColumn?.index] || "";
+        if (!cellId)
+          throw "cell id missing";
+        if (!Number.isFinite(x) || !Number.isFinite(y))
+          throw "x/y not number";
+        const cell = { cellId, x, y, category };
+        if (geneExpMap) {
+          if (geneExpMap[cellId] !== void 0) {
+            cell.geneExp = geneExpMap[cellId];
+            expCells.push(cell);
+          } else {
+            noExpCells.push(cell);
+          }
+        } else
+          noExpCells.push(cell);
       }
-      return { plots };
-    } catch (e) {
-      if (e.stack)
-        console.log(e.stack);
-      return { error: e.message || e };
+      plots.push({
+        name: plot.name,
+        expCells,
+        noExpCells,
+        colorColumns: plot.colorColumns.map((c) => c.name),
+        colorBy: colorColumn?.name,
+        colorMap: colorColumn?.colorMap
+      });
+    }
+    if (plots.length == 0) {
+      return { nodata: true };
     }
+    return { plots };
   };
 }
 function validateGeneExpressionNative(G) {