@sjcrh/proteinpaint-server 2.147.2-0 → 2.148.1

package/dataset/termdb.test.js

@@ -39,8 +39,6 @@ function termdb_test_default() {
         displaySampleIds: () => true,
         // allow to display sample-level data
         timeUnit: "years",
-        minTimeSinceDx: 5,
-        // enrollment in sjlife requires 5 years since cancer diagnosis
         ageEndOffset: 274e-5,
         // number of years to offset ending age of patients
         // for cox outcome with timeScale='age'
@@ -108,7 +106,6 @@ function termdb_test_default() {
         plotConfigByCohort: {
           default: {
             report: {
-              filterTWs: [{ id: "diaggrp" }],
               sections: [
                 {
                   name: "Demographics",
@@ -332,7 +329,6 @@ function termdb_test_default() {
       },
       rnaseqGeneCount: {
         storage_type: "HDF5",
-        newformat: true,
         file: "files/hg38/TermdbTest/rnaseq/TermdbTest.geneCounts.new.h5"
       },
       singleCell: {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sjcrh/proteinpaint-server",
-  "version": "2.147.2-0",
+  "version": "2.148.1",
   "type": "module",
   "description": "a genomics visualization tool for exploring a cohort's genotype and phenotype data",
   "main": "src/app.js",
@@ -23,7 +23,7 @@
     "combined:coverage": "coverageKey=test c8 --all --src=proteinpaint/server --experimental-monocart -r=v8 -r=html -r=json -r=markdown-summary -r=markdown-details -o=./.coverage tsx ./coverage.js & ",
     "postcombined:coverage": "rm -rf ./cache",
     "spec:coverage": "tsx test/relevant.js",
-    "getconf": "../build/getConfigProp.js",
+    "getconf": "../build/getConfigProp.cjs",
     "doc": "../augen/build.sh routes shared/types/routes shared/checkers ../public/docs/server",
     "mjs": "esbuild \"$DIR/*.ts\" --platform=node --outdir=\"$DIR\" --format=esm",
     "cjs": "esbuild \"$DIR/*.ts\" --platform=node --outdir=\"$DIR\" --format=cjs",
@@ -63,9 +63,9 @@
     "@sjcrh/augen": "2.143.0",
     "@sjcrh/proteinpaint-python": "2.146.0",
     "@sjcrh/proteinpaint-r": "2.146.4-0",
-    "@sjcrh/proteinpaint-rust": "2.146.4-1",
+    "@sjcrh/proteinpaint-rust": "2.148.1",
     "@sjcrh/proteinpaint-shared": "2.147.1",
-    "@sjcrh/proteinpaint-types": "2.147.2-0",
+    "@sjcrh/proteinpaint-types": "2.148.1",
     "@types/express": "^5.0.0",
     "@types/express-session": "^1.18.1",
     "better-sqlite3": "^9.4.1",

package/routes/aiProjectSelectedWSImages.js

@@ -35,7 +35,11 @@ function init({ genomes }) {
           wsimage.activePatchColor = ds.queries?.WSImages?.activePatchColor;
           if (ds.queries.WSImages.getWSIPredictionPatches) {
             const predictions = await ds.queries.WSImages.getWSIPredictionPatches(projectId, wsimageFilename);
-            wsimage.predictions = predictions;
+            const classMap = new Map((wsimage.classes || []).map((c) => [c.id, c.label]));
+            wsimage.predictions = (predictions || []).map((p) => {
+              const label = classMap.get(p.class) ?? p.class;
+              return { ...p, class: label };
+            });
           }
           wsimages.push(wsimage);
         }

package/routes/aiProjectTrainModel.js

@@ -42,6 +42,7 @@ function init({ genomes }) {
           status: "error",
           error: "No retraining script defined"
         });
+        return;
       }
       res.status(200).send({
         status: "ok",

package/routes/grin2.js

@@ -6,6 +6,7 @@ import { mayLog } from "#src/helpers.ts";
 import { get_samples } from "#src/termdb.sql.js";
 import { read_file, file_is_readable } from "#src/utils.js";
 import { dtsnvindel, dtcnv, dtfusionrna } from "#shared/common.js";
+const MAX_LESIONS_PER_TYPE = 5e4;
 const api = {
   endpoint: "grin2",
   methods: {
@@ -23,7 +24,6 @@ function init({ genomes }) {
   return async (req, res) => {
     try {
       const request = req.query;
-      console.log("[GRIN2] request:", request);
       const g = genomes[request.genome];
       if (!g) throw new Error("genome missing");
       const ds = g.datasets?.[request.dslabel];
@@ -43,7 +43,6 @@ function init({ genomes }) {
 }
 async function runGrin2(g, ds, request) {
   const startTime = Date.now();
-  mayLog("[GRIN2] Getting samples from cohort filter...");
   const samples = await get_samples(
     request,
     ds,
@@ -55,16 +54,16 @@ async function runGrin2(g, ds, request) {
   if (samples.length === 0) {
     throw new Error("No samples found matching the provided filter criteria");
   }
-  mayLog("[GRIN2] Processing sample data...");
+  const tracker = getLesionTracker(request);
   const processingStartTime = Date.now();
-  const { lesions, processingSummary } = await processSampleData(samples, ds, request);
+  const { lesions, processingSummary } = await processSampleData(samples, ds, request, tracker);
   const processingTime = Date.now() - processingStartTime;
   const processingTimeToPrint = Math.round(processingTime / 1e3);
   mayLog(`[GRIN2] Data processing took ${processingTimeToPrint} seconds`);
   mayLog(
-    `[GRIN2] Processing summary: ${processingSummary?.successfulSamples ?? 0}/${processingSummary?.totalSamples ?? samples.length} samples processed successfully`
+    `[GRIN2] Processing summary: ${processingSummary?.processedSamples ?? 0}/${processingSummary?.totalSamples ?? samples.length} samples processed successfully`
   );
-  if (processingSummary && processingSummary.failedSamples > 0) {
+  if (processingSummary?.failedSamples !== void 0 && processingSummary.failedSamples > 0) {
     mayLog(`[GRIN2] Warning: ${processingSummary.failedSamples} samples failed to process`);
   }
   if (lesions.length === 0) {
@@ -79,14 +78,12 @@ async function runGrin2(g, ds, request) {
     width: request.width,
     height: request.height
   };
-  mayLog("[GRIN2] Prepared input for Python script:", { ...pyInput });
   for (const c in g.majorchr) {
     if (ds.queries.singleSampleMutation.discoPlot?.skipChrM) {
       if (c.toLowerCase() == "chrm") continue;
     }
     pyInput.chromosomelist[c] = g.majorchr[c];
   }
-  mayLog(`[GRIN2] Prepared ${lesions.length.toLocaleString()} lesions for analysis`);
   const grin2AnalysisStart = Date.now();
   const pyResult = await run_python("grin2PpWrapper.py", JSON.stringify(pyInput));
   if (pyResult.stderr?.trim()) {
@@ -119,75 +116,137 @@ async function runGrin2(g, ds, request) {
   };
   return response;
 }
-async function processSampleData(samples, ds, request) {
+function getLesionTracker(req) {
+  const currentTypes = [];
+  if (req.snvindelOptions) currentTypes.push(dtsnvindel);
+  if (req.cnvOptions) currentTypes.push(dtcnv);
+  if (req.fusionOptions) currentTypes.push(dtfusionrna);
+  const track = /* @__PURE__ */ new Map();
+  for (const t of currentTypes) track.set(t, { count: 0 });
+  return track;
+}
+function allTypesCapped(tracker) {
+  for (const value of tracker.values()) {
+    if (value.count < MAX_LESIONS_PER_TYPE) return false;
+  }
+  return true;
+}
+async function processSampleData(samples, ds, request, tracker) {
   const lesions = [];
-  let lesionId = 1;
   const processingSummary = {
     totalSamples: samples.length,
-    successfulSamples: 0,
+    processedSamples: 0,
     failedSamples: 0,
-    failedFiles: []
+    failedFiles: [],
+    totalLesions: 0,
+    processedLesions: 0,
+    unprocessedSamples: 0
   };
-  mayLog(`[GRIN2] Processing JSON files for ${samples.length.toLocaleString()} samples`);
-  for (const sample of samples) {
+  outer: for (let i = 0; i < samples.length; i++) {
+    if (allTypesCapped(tracker)) {
+      const remaining = samples.length - i;
+      if (remaining > 0) processingSummary.unprocessedSamples += remaining;
+      mayLog("[GRIN2] All enabled per-type caps reached; stopping early.");
+      break outer;
+    }
+    const sample = samples[i];
+    const filepath = path.join(serverconfig.tpmasterdir, ds.queries.singleSampleMutation.folder, sample.name);
     try {
-      const filepath = path.join(serverconfig.tpmasterdir, ds.queries.singleSampleMutation.folder, sample.name);
       await file_is_readable(filepath);
       const mlst = JSON.parse(await read_file(filepath));
-      const { sampleLesions } = await processSampleMlst(sample.name, mlst, lesionId, request);
+      const { sampleLesions } = await processSampleMlst(sample.name, mlst, request, tracker);
       lesions.push(...sampleLesions);
-      lesionId += sampleLesions.length;
-      processingSummary.successfulSamples++;
+      processingSummary.processedSamples += 1;
+      processingSummary.totalLesions += sampleLesions.length;
+      if (allTypesCapped(tracker)) {
+        const remaining = samples.length - 1 - i;
+        if (remaining > 0) processingSummary.unprocessedSamples += remaining;
+        mayLog("[GRIN2] All enabled per-type caps reached; stopping early.");
+        break outer;
+      }
     } catch (error) {
-      processingSummary.failedSamples++;
-      processingSummary.failedFiles?.push(sample.name);
+      processingSummary.failedSamples += 1;
+      processingSummary.failedFiles.push({
+        sampleName: sample.name,
+        filePath: filepath,
+        error: error instanceof Error ? error.message || "Unknown error" : String(error)
+      });
+      mayLog(`[GRIN2] Error processing sample ${sample.name}`);
+    }
+  }
+  processingSummary.processedLesions = lesions.length;
+  for (const [type, info] of tracker.entries()) {
+    if (info.count >= MAX_LESIONS_PER_TYPE) {
+      let label;
+      switch (type) {
+        case dtsnvindel:
+          label = "mutation";
+          break;
+        case dtcnv:
+          label = "CNV (gain/loss)";
+          break;
+        case dtfusionrna:
+          label = "fusion";
+          break;
+        default:
+          label = `type ${type}`;
+          break;
+      }
       mayLog(
-        `[GRIN2] Error processing sample ${sample.name}: ${typeof error === "object" && error !== null && "message" in error ? error.message : String(error)}`
+        `[GRIN2] Warning: ${label} lesions reached the per-type cap of ${MAX_LESIONS_PER_TYPE.toLocaleString()}. No further ${label} lesions were processed.`
       );
     }
   }
-  mayLog(`[GRIN2] Total lesions processed: ${lesions.length.toLocaleString()}`);
-  return {
-    lesions,
-    processingSummary
-  };
+  return { lesions, processingSummary };
 }
-async function processSampleMlst(sampleName, mlst, startId, request) {
-  const lesions = [];
+async function processSampleMlst(sampleName, mlst, request, tracker) {
+  const sampleLesions = [];
   for (const m of mlst) {
     switch (m.dt) {
       case dtsnvindel: {
         if (!request.snvindelOptions) break;
-        const snvIndelLesion = filterAndConvertSnvIndel(sampleName, m, request.snvindelOptions);
-        if (snvIndelLesion) {
-          lesions.push(snvIndelLesion);
+        const entry = tracker.get(dtsnvindel);
+        if (entry && entry.count >= MAX_LESIONS_PER_TYPE) {
+          break;
+        }
+        const les = filterAndConvertSnvIndel(sampleName, m, request.snvindelOptions);
+        if (les && entry) {
+          entry.count++;
+          sampleLesions.push(les);
         }
         break;
       }
       case dtcnv: {
         if (!request.cnvOptions) break;
-        const cnvLesion = filterAndConvertCnv(sampleName, m, request.cnvOptions);
-        if (cnvLesion) {
-          lesions.push(cnvLesion);
+        const cnv = tracker.get(dtcnv);
+        if (cnv && cnv.count >= MAX_LESIONS_PER_TYPE) {
+          break;
+        }
+        const les = filterAndConvertCnv(sampleName, m, request.cnvOptions);
+        if (les && cnv) {
+          cnv.count++;
+          sampleLesions.push(les);
         }
         break;
       }
       case dtfusionrna: {
         if (!request.fusionOptions) break;
-        const fusionLesion = filterAndConvertFusion(sampleName, m, request.fusionOptions);
-        if (fusionLesion) {
-          lesions.push(fusionLesion);
+        const fusion = tracker.get(dtfusionrna);
+        if (fusion && fusion.count >= MAX_LESIONS_PER_TYPE) {
+          break;
+        }
+        const les = filterAndConvertFusion(sampleName, m, request.fusionOptions);
+        if (les && fusion) {
+          fusion.count++;
+          sampleLesions.push(les);
         }
         break;
       }
-      default: {
+      default:
         break;
-      }
     }
   }
-  return {
-    sampleLesions: lesions
-  };
+  return { sampleLesions };
 }
 function filterAndConvertSnvIndel(sampleName, entry, options) {
   if (!options?.consequences) {

package/routes/termdb.DE.js

@@ -204,12 +204,7 @@ async function run_DE(param, ds, term_results, term_results2) {
   const sample_size_limit = 8;
   if (group1names.length <= sample_size_limit && group2names.length <= sample_size_limit || param.method == "edgeR" || param.method == "limma") {
     const time12 = (/* @__PURE__ */ new Date()).valueOf();
-    let result2;
-    if (q.newformat) {
-      result2 = JSON.parse(await run_R("edge_newh5.R", JSON.stringify(expression_input)));
-    } else {
-      result2 = JSON.parse(await run_R("edge.R", JSON.stringify(expression_input)));
-    }
+    const result2 = JSON.parse(await run_R("edge_newh5.R", JSON.stringify(expression_input)));
     mayLog("Time taken to run edgeR:", formatElapsedTime(Date.now() - time12));
     param.method = "edgeR";
     const ql_imagePath = path.join(serverconfig.cachedir, result2.edgeR_ql_image_name[0]);
@@ -259,30 +254,18 @@ async function validate_query_rnaseqGeneCount(ds) {
     if (ds.queries.rnaseqGeneCount.storage_type == "text") {
       samples = (await get_header_txt(q.file, null)).split("	").slice(4);
     } else if (ds.queries.rnaseqGeneCount.storage_type == "HDF5") {
-      if (q.newformat) {
-        const get_samples_from_hdf5 = {
-          hdf5_file: q.file,
-          validate: true
-        };
-        const time1 = (/* @__PURE__ */ new Date()).valueOf();
-        const result = await run_rust("readH5", JSON.stringify(get_samples_from_hdf5));
-        const time2 = (/* @__PURE__ */ new Date()).valueOf();
-        mayLog("Time taken to query gene expression:", time2 - time1, "ms");
-        const vr = JSON.parse(result);
-        if (vr.status !== "success") throw vr.message;
-        if (!Array.isArray(vr.samples)) throw "HDF5 file has no samples, please check file.";
-        samples = vr.samples;
-      } else {
-        const get_samples_from_hdf5 = {
-          input_file: q.file,
-          data_type: "get_samples"
-        };
-        const time1 = (/* @__PURE__ */ new Date()).valueOf();
-        const result = await run_rust("DEanalysis", JSON.stringify(get_samples_from_hdf5));
-        const time2 = (/* @__PURE__ */ new Date()).valueOf();
-        mayLog("Time taken to query gene expression:", time2 - time1, "ms");
-        samples = result.split(",");
-      }
+      const get_samples_from_hdf5 = {
+        hdf5_file: q.file,
+        validate: true
+      };
+      const time1 = (/* @__PURE__ */ new Date()).valueOf();
+      const result = await run_rust("readH5", JSON.stringify(get_samples_from_hdf5));
+      const time2 = (/* @__PURE__ */ new Date()).valueOf();
+      mayLog("Time taken to query gene expression:", time2 - time1, "ms");
+      const vr = JSON.parse(result);
+      if (vr.status !== "success") throw vr.message;
+      if (!Array.isArray(vr.samples)) throw "HDF5 file has no samples, please check file.";
+      samples = vr.samples;
     } else throw "unknown storage type:" + ds.queries.rnaseqGeneCount.storage_type;
     q.allSampleSet = new Set(samples);
     const unknownSamples = [];