@sjcrh/proteinpaint-server 2.188.1 → 2.190.0

package/routes/genesetEnrichment.js

@@ -1,5 +1,3 @@
-import { genesetEnrichmentPayload } from "#types/checkers";
-import crypto from "crypto";
 import fs from "fs";
 import path from "path";
 import serverconfig from "#src/serverconfig.js";
@@ -7,21 +5,10 @@ import { run_python } from "@sjcrh/proteinpaint-python";
 import { run_rust } from "@sjcrh/proteinpaint-rust";
 import { mayLog } from "#src/helpers.ts";
 import { formatElapsedTime } from "#shared";
-import { readCacheFileOrRecompute, stableStringify } from "#src/diffAnalysis.ts";
+import { getDeCacheResult } from "../routes/termdb.DE.ts";
+import { getDmCacheResult } from "../routes/termdb.diffMeth.ts";
+import { cacheOrRecompute } from "#src/utils/cacheOrRecompute.ts";
 import { get_ds_tdb } from "#src/termdb.js";
-const api = {
-  endpoint: "genesetEnrichment",
-  methods: {
-    get: {
-      ...genesetEnrichmentPayload,
-      init
-    },
-    post: {
-      ...genesetEnrichmentPayload,
-      init
-    }
-  }
-};
 function init({ genomes }) {
   return async (req, res) => {
     try {
@@ -45,7 +32,7 @@ function init({ genomes }) {
         }
       });
     } catch (e) {
-      res.send({ status: "error", error: e.message || e });
+      res.status(e.status || 500).send({ status: "error", error: e.message || e, code: e.code });
       if (e.stack) console.log(e.stack);
     }
   };
@@ -57,7 +44,8 @@ async function run_genesetEnrichment_analysis(q, genomes) {
     return { data: { genes, fold_change } };
   }
   if (q.method == "blitzgsea") {
-    return await computeGSEA({ q, genomes });
+    if (q.geneset_name) return await computeGseaImage({ q, genomes });
+    return await computeGseaInitial({ q, genomes });
   }
   if (q.method == "cerno") {
     const { genes, fold_change } = await resolveGseaGenesAndFoldChange({ q, genomes });
@@ -76,30 +64,137 @@ async function run_genesetEnrichment_analysis(q, genomes) {
   }
   throw new Error("Unknown method:" + q.method);
 }
+function gseaKeyInputs(q, genes, fold_change) {
+  return {
+    genes,
+    fold_change,
+    geneSetGroup: q.geneSetGroup,
+    num_permutations: q.num_permutations,
+    filter_non_coding_genes: q.filter_non_coding_genes
+  };
+}
+async function getGseaCacheResult({
+  q,
+  genomes
+}) {
+  const { genes, fold_change } = await resolveGseaGenesAndFoldChange({ q, genomes });
+  const cacheArg = gseaKeyInputs(q, genes, fold_change);
+  const { result, cacheId } = await cacheOrRecompute({
+    computeArgument: cacheArg,
+    cacheSubdir: "gsea",
+    computeFresh: async () => {
+      const { table, pickleB64 } = await runGseaPythonForTable({ q, genomes, cacheArg });
+      const cacheResult = { table, pickleB64 };
+      return cacheResult;
+    }
+  });
+  return { result, cacheId };
+}
+async function computeGseaInitial({
+  q,
+  genomes
+}) {
+  const { result } = await getGseaCacheResult({ q, genomes });
+  return result.table;
+}
+const pendingImageRequests = /* @__PURE__ */ new Map();
+async function computeGseaImage({
+  q,
+  genomes
+}) {
+  const { result, cacheId } = await getGseaCacheResult({ q, genomes });
+  const dedupKey = `${cacheId}:${q.geneset_name}`;
+  const inFlight = pendingImageRequests.get(dedupKey);
+  if (inFlight) return inFlight;
+  const work = runGseaPythonForImage({
+    q,
+    genomes,
+    pickleB64: result.pickleB64
+  }).finally(() => pendingImageRequests.delete(dedupKey));
+  pendingImageRequests.set(dedupKey, work);
+  return work;
+}
+async function runGseaPythonForTable({
+  q,
+  genomes,
+  cacheArg
+}) {
+  const pyInput = buildPyInput({ ...q, geneset_name: void 0 }, genomes, cacheArg);
+  const time1 = (/* @__PURE__ */ new Date()).valueOf();
+  const gsea_output = await run_python("gsea.py", "/" + JSON.stringify(pyInput));
+  mayLog("Time taken to run blitzgsea:", formatElapsedTime(Date.now() - time1));
+  for (const line of gsea_output.split("\n")) {
+    if (line.startsWith("result: ")) {
+      const parsed = JSON.parse(line.replace("result: ", ""));
+      if (parsed?.error) throw new Error(parsed.error);
+      const { pickle_b64, ...table } = parsed;
+      if (!pickle_b64) throw new Error("gsea.py result missing pickle_b64");
+      return { table, pickleB64: pickle_b64 };
+    }
+    mayLog(line);
+  }
+  throw new Error("gsea.py did not emit a result line on the initial path");
+}
+async function runGseaPythonForImage({
+  q,
+  genomes,
+  pickleB64
+}) {
+  const { genes, fold_change } = await resolveGseaGenesAndFoldChange({ q, genomes });
+  const cacheArg = gseaKeyInputs(q, genes, fold_change);
+  const pyInput = buildPyInput(q, genomes, cacheArg, pickleB64);
+  const time1 = (/* @__PURE__ */ new Date()).valueOf();
+  const gsea_output = await run_python("gsea.py", "/" + JSON.stringify(pyInput));
+  mayLog("Time taken to render gsea image:", formatElapsedTime(Date.now() - time1));
+  for (const line of gsea_output.split("\n")) {
+    if (line.startsWith("image: ")) {
+      const parsed = JSON.parse(line.replace("image: ", ""));
+      return path.join(serverconfig.cachedir, "gsea", parsed.image_file);
+    }
+    if (line.startsWith("result: ")) {
+      const parsed = JSON.parse(line.replace("result: ", ""));
+      if (parsed?.error) return { error: parsed.error };
+    }
+    mayLog(line);
+  }
+  throw new Error("gsea.py did not emit an image line on the detail path");
+}
+function buildPyInput(q, genomes, cacheArg, pickleB64) {
+  return {
+    genes: cacheArg.genes,
+    fold_change: cacheArg.fold_change,
+    db: genomes[q.genome].termdbs.msigdb.cohort.db.connection.name,
+    geneset_group: cacheArg.geneSetGroup,
+    genedb: path.join(serverconfig.tpmasterdir, genomes[q.genome].genedb.dbfile),
+    filter_non_coding_genes: cacheArg.filter_non_coding_genes,
+    cachedir: path.join(serverconfig.cachedir, "gsea"),
+    geneset_name: q.geneset_name,
+    num_permutations: cacheArg.num_permutations,
+    ...pickleB64 ? { pickle_b64: pickleB64 } : {}
+  };
+}
 async function resolveGseaGenesAndFoldChange({
   q,
   genomes
 }) {
   if (q.cacheId) {
     if (!q.daRequest) throw new Error("daCacheMissing");
-    const result = await readCacheFileOrRecompute({
-      daRequest: q.daRequest,
-      genomes
-    });
-    if (result.cacheId !== q.cacheId) throw new Error("cacheId does not match daRequest");
-    if (result.kind === "DE") {
+    const kind = q.daRequest.kind;
+    if (kind !== "DE" && kind !== "DM") throw new Error('daRequest.kind must be "DE" or "DM"');
+    if (kind === "DE") {
+      const { result: result2, cacheId: cacheId2 } = await getDeCacheResult(q.daRequest, genomes);
+      if (cacheId2 !== q.cacheId) throw new Error("cacheId does not match daRequest");
       return {
-        genes: result.geneData.map((g) => g.gene_name),
-        fold_change: result.geneData.map((g) => g.fold_change)
+        genes: result2.geneRows.map((g) => g.gene_name),
+        fold_change: result2.geneRows.map((g) => g.fold_change)
       };
     }
-    if (result.kind === "DM") {
-      return {
-        genes: result.promoterData.map((p) => p.gene_name),
-        fold_change: result.promoterData.map((p) => p.fold_change)
-      };
-    }
-    throw new Error(`unexpected result kind: ${result.kind}`);
+    const { result, cacheId } = await getDmCacheResult(q.daRequest, genomes);
+    if (cacheId !== q.cacheId) throw new Error("cacheId does not match daRequest");
+    return {
+      genes: result.promoterRows.map((p) => p.gene_name),
+      fold_change: result.promoterRows.map((p) => p.fold_change)
+    };
   }
   if (q.dapParams) {
     const genome = genomes[q.genome];
@@ -126,69 +221,6 @@ async function resolveGseaGenesAndFoldChange({
   if (!q.genes || !q.fold_change) throw new Error("requires genes and fold_change when cacheId is absent");
   return { genes: q.genes, fold_change: q.fold_change };
 }
-const pendingTracker_gsea = /* @__PURE__ */ new Map();
-const pendingTracker_img = /* @__PURE__ */ new Map();
-async function computeGSEA({
-  q,
-  genomes
-}) {
-  const { genes, fold_change } = await resolveGseaGenesAndFoldChange({ q, genomes });
-  const gseaComputeArg = {
-    genes,
-    fold_change,
-    geneSetGroup: q.geneSetGroup,
-    num_permutations: q.num_permutations,
-    filter_non_coding_genes: q.filter_non_coding_genes
-  };
-  const key = stableStringify(gseaComputeArg);
-  const pickle_file = crypto.createHash("sha256").update(key).digest("hex").slice(0, 32) + ".pkl";
-  const tracker = q.geneset_name ? pendingTracker_img : pendingTracker_gsea;
-  const inFlight = tracker.get(pickle_file);
-  if (inFlight) return inFlight;
-  const work = runGseaPython({ q, genomes, gseaComputeArg, pickle_file });
-  tracker.set(pickle_file, work);
-  return work.finally(() => tracker.delete(pickle_file));
-}
-async function runGseaPython({
-  q,
-  genomes,
-  gseaComputeArg,
-  pickle_file
-}) {
-  const cachedir_gsea = path.join(serverconfig.cachedir, "gsea");
-  const genesetenrichment_input = {
-    genes: gseaComputeArg.genes,
-    fold_change: gseaComputeArg.fold_change,
-    db: genomes[q.genome].termdbs.msigdb.cohort.db.connection.name,
-    geneset_group: gseaComputeArg.geneSetGroup,
-    genedb: path.join(serverconfig.tpmasterdir, genomes[q.genome].genedb.dbfile),
-    filter_non_coding_genes: gseaComputeArg.filter_non_coding_genes,
-    cachedir: cachedir_gsea,
-    pickle_file,
-    geneset_name: q.geneset_name,
-    num_permutations: gseaComputeArg.num_permutations
-  };
-  const time1 = (/* @__PURE__ */ new Date()).valueOf();
-  const gsea_output = await run_python("gsea.py", "/" + JSON.stringify(genesetenrichment_input));
-  mayLog("Time taken to run blitzgsea:", formatElapsedTime(Date.now() - time1));
-  let result;
-  let data_found = false;
-  let image_found = false;
-  for (const line of gsea_output.split("\n")) {
-    if (line.startsWith("result: ")) {
-      result = JSON.parse(line.replace("result: ", ""));
-      data_found = true;
-    } else if (line.startsWith("image: ")) {
-      result = JSON.parse(line.replace("image: ", ""));
-      image_found = true;
-    } else {
-      mayLog(line);
-    }
-  }
-  if (data_found) return result;
-  if (image_found) return path.join(cachedir_gsea, result.image_file);
-  throw new Error("data or image not found in gsea output; this should not happen");
-}
 export {
-  api
+  init
 };

package/routes/grin2.js

@@ -1,4 +1,3 @@
-import { GRIN2Payload } from "#types/checkers";
 import serverconfig from "#src/serverconfig.js";
 import path from "path";
 import { run_python } from "@sjcrh/proteinpaint-python";
@@ -9,28 +8,15 @@ import { get_samples } from "#src/termdb.sql.js";
 import { read_file, file_is_readable } from "#src/utils.js";
 import { dtsnvindel, dtcnv, dtfusionrna, dtsv, dt2lesion, optionToDt, formatElapsedTime } from "#shared";
 import { mayFilterByMaf } from "#src/mds3.init.js";
-import crypto from "crypto";
+import { cacheOrRecompute } from "#src/utils/cacheOrRecompute.ts";
 import { promisify } from "node:util";
 import { exec as execCallback } from "node:child_process";
 const MAX_LESIONS = serverconfig.features.grin2maxLesions || 25e4;
 const GRIN2_MEMORY_BUDGET_MB = 950;
-const GRIN2_CONCURRENCY_LIMIT = 10;
+const GRIN2_CONCURRENCY_LIMIT = 5;
 const MEMORY_BASE_MB = 260;
 const MEMORY_PER_1K_LESIONS = 2.4;
 const MIN_LESIONS = 5e4;
-const api = {
-  endpoint: "grin2",
-  methods: {
-    get: {
-      ...GRIN2Payload,
-      init
-    },
-    post: {
-      ...GRIN2Payload,
-      init
-    }
-  }
-};
 function init({ genomes }) {
   return async (req, res) => {
     const signal = req.query.__abortSignal;
@@ -51,9 +37,10 @@ function init({ genomes }) {
       console.error("[GRIN2] Error stack:", e.stack);
       const errorResponse = {
         status: "error",
-        error: e.message || String(e)
+        error: e.message || String(e),
+        code: e.code
       };
-      res.status(500).send(errorResponse);
+      res.status(e.status || 500).send(errorResponse);
     }
   };
 }
@@ -117,11 +104,6 @@ async function runGrin2WithLimit(g, ds, request, signal) {
     mayLog(`[GRIN2] Analysis complete. Active jobs: ${activeGrin2Jobs}/${GRIN2_CONCURRENCY_LIMIT}`);
   }
 }
-function generateCacheFileName() {
-  const randomHex = crypto.randomBytes(16).toString("hex");
-  const cacheFileName = `grin2_results_${randomHex}.txt`;
-  return path.join(serverconfig.cachedir, "grin2", cacheFileName);
-}
 function buildLesionTypeMap(availableOptions) {
   const lesionTypeMap = {};
   for (const option of availableOptions) {
@@ -143,59 +125,22 @@ function getCnvLesionType(isGain) {
   return lesionType.lesionType;
 }
 async function runGrin2(g, ds, request, signal) {
-  const startTime = Date.now();
-  const samples = await get_samples(
-    request,
-    ds,
-    true
-    // must set to true to return sample name to be able to access file. FIXME this can let names revealed to grin2 client, may need to apply access control
-  );
-  const cohortTime = Date.now() - startTime;
-  mayLog(`[GRIN2] Retrieved ${samples.length.toLocaleString()} samples in ${formatElapsedTime(cohortTime)}`);
-  if (samples.length === 0) {
-    throw new Error("No samples found matching the provided filter criteria");
-  }
-  const processingStartTime = Date.now();
-  const { lesions, processing } = await processSampleData(samples, ds, request);
-  if (!processing) throw new Error("Processing summary is missing");
-  const processingTime = Date.now() - processingStartTime;
-  mayLog(`[GRIN2] Data processing took ${formatElapsedTime(processingTime)}`);
-  mayLog(
-    `[GRIN2] Processing summary: ${processing?.processedSamples ?? 0}/${processing?.totalSamples ?? samples.length} samples processed successfully`
-  );
-  if (processing?.failedSamples !== void 0 && processing.failedSamples > 0) {
-    mayLog(`[GRIN2] Warning: ${processing.failedSamples} samples failed to process`);
-  }
-  if (lesions.length === 0) {
-    throw new Error("No lesions found after processing all samples. Check filter criteria and input data.");
-  }
-  const availableDataTypes = Object.keys(optionToDt).filter((key) => key in request);
-  const pyInput = {
-    genedb: path.join(serverconfig.tpmasterdir, g.genedb.dbfile),
-    chromosomelist: {},
-    lesion: JSON.stringify(lesions),
-    cacheFileName: generateCacheFileName(),
-    maxGenesToShow: request.maxGenesToShow,
-    lesionTypeMap: buildLesionTypeMap(availableDataTypes)
-  };
+  const chromosomelist = {};
   for (const c in g.majorchr) {
-    pyInput.chromosomelist[c] = g.majorchr[c];
-  }
-  const grin2AnalysisStart = Date.now();
-  const pyResult = await run_python("grin2PpWrapper.py", JSON.stringify(pyInput), { signal });
-  if (pyResult.stderr?.trim()) {
-    mayLog(`[GRIN2] Python stderr: ${pyResult.stderr}`);
-    if (pyResult.stderr.includes("ERROR:")) {
-      throw new Error(`Python script error: ${pyResult.stderr}`);
-    }
-  }
-  const grin2AnalysisTime = Date.now() - grin2AnalysisStart;
-  mayLog(`[GRIN2] Python processing took ${formatElapsedTime(grin2AnalysisTime)}`);
-  const resultData = JSON.parse(pyResult);
+    chromosomelist[c] = g.majorchr[c];
+  }
+  const {
+    result: cacheResult,
+    cacheFile,
+    freshCompute,
+    processingTime,
+    grin2AnalysisTime
+  } = await getGrin2CacheResult(request, g, ds, chromosomelist, signal);
+  const { resultData, processing } = cacheResult;
   const rustInput = {
-    file: resultData.cacheFileName,
+    file: cacheFile,
     type: "grin2",
-    chrSizes: pyInput.chromosomelist,
+    chrSizes: chromosomelist,
     plot_width: request.width,
     plot_height: request.height,
     device_pixel_ratio: request.devicePixelRatio,
@@ -224,8 +169,8 @@ async function runGrin2(g, ds, request, signal) {
       });
     });
     for (const [type, data] of Object.entries(processing.lesionCounts.byType)) {
-      const { count, samples: samples2 } = data;
-      lesionTypeRows.push([typeLabels[type] || type, `${count.toLocaleString()} (${samples2} samples)`]);
+      const { count, samples } = data;
+      lesionTypeRows.push([typeLabels[type] || type, `${count.toLocaleString()} (${samples} samples)`]);
     }
   }
   const capWarningRows = [];
@@ -238,6 +183,7 @@ async function runGrin2(g, ds, request, signal) {
   }
   const response = {
     status: "success",
+    fromCache: !freshCompute,
     pngImg: manhattanPlotData.png,
     plotData: manhattanPlotData.plot_data,
     topGeneTable: resultData.topGeneTable,
@@ -248,7 +194,7 @@ async function runGrin2(g, ds, request, signal) {
           rows: [
             ["Total Genes", resultData.totalGenes.toLocaleString()],
             ["Showing Top", resultData.showingTop.toLocaleString()],
-            ["Cache File Name", resultData.cacheFileName],
+            ["Cache File Name", cacheFile],
             ["Total Samples", processing.totalSamples.toLocaleString()],
             ["Processed Samples", processing.processedSamples.toLocaleString()],
             ["Unprocessed Samples", (processing.unprocessedSamples ?? 0).toLocaleString()],
@@ -276,8 +222,8 @@ async function runGrin2(g, ds, request, signal) {
         {
           name: "Timing",
           rows: [
-            ["Processing", formatElapsedTime(processingTime)],
-            ["GRIN2", formatElapsedTime(grin2AnalysisTime)],
+            ["Processing", freshCompute ? formatElapsedTime(processingTime) : "cached"],
+            ["GRIN2", freshCompute ? formatElapsedTime(grin2AnalysisTime) : "cached"],
             ["Plotting", formatElapsedTime(manhattanPlotTime)],
             ["Total", formatElapsedTime(totalTime)],
             ...capWarningRows
@@ -288,6 +234,91 @@ async function runGrin2(g, ds, request, signal) {
   };
   return response;
 }
+function grin2KeyInputs(req) {
+  return {
+    genome: req.genome,
+    dslabel: req.dslabel,
+    filter: req.filter ?? null,
+    snvindelOptions: req.snvindelOptions ?? null,
+    cnvOptions: req.cnvOptions ?? null,
+    fusionOptions: req.fusionOptions ?? null,
+    svOptions: req.svOptions ?? null,
+    maxGenesToShow: req.maxGenesToShow ?? null
+  };
+}
+async function getGrin2CacheResult(request, g, ds, chromosomelist, signal) {
+  let processingTime = 0;
+  let grin2AnalysisTime = 0;
+  let freshCompute = false;
+  const {
+    result,
+    cacheId,
+    cacheFilePath: cacheFile
+  } = await cacheOrRecompute({
+    computeArgument: grin2KeyInputs(request),
+    cacheSubdir: "grin2",
+    computeFresh: async () => {
+      freshCompute = true;
+      const out = await runGrin2Fresh(request, g, ds, chromosomelist, signal);
+      processingTime = out.processingTime;
+      grin2AnalysisTime = out.grin2AnalysisTime;
+      return out.cacheResult;
+    }
+  });
+  return { result, cacheId, cacheFile, freshCompute, processingTime, grin2AnalysisTime };
+}
+async function runGrin2Fresh(request, g, ds, chromosomelist, signal) {
+  const startTime = Date.now();
+  const samples = await get_samples(
+    request,
+    ds,
+    true
+    // must set to true to return sample name to be able to access file. FIXME this can let names revealed to grin2 client, may need to apply access control
+  );
+  const cohortTime = Date.now() - startTime;
+  mayLog(`[GRIN2] Retrieved ${samples.length.toLocaleString()} samples in ${formatElapsedTime(cohortTime)}`);
+  if (samples.length === 0) {
+    throw new Error("No samples found matching the provided filter criteria");
+  }
+  const processingStartTime = Date.now();
+  const { lesions, processing } = await processSampleData(samples, ds, request);
+  if (!processing) throw new Error("Processing summary is missing");
+  const processingTime = Date.now() - processingStartTime;
+  mayLog(`[GRIN2] Data processing took ${formatElapsedTime(processingTime)}`);
+  mayLog(
+    `[GRIN2] Processing summary: ${processing?.processedSamples ?? 0}/${processing?.totalSamples ?? samples.length} samples processed successfully`
+  );
+  if (processing?.failedSamples !== void 0 && processing.failedSamples > 0) {
+    mayLog(`[GRIN2] Warning: ${processing.failedSamples} samples failed to process`);
+  }
+  if (lesions.length === 0) {
+    throw new Error("No lesions found after processing all samples. Check filter criteria and input data.");
+  }
+  const availableDataTypes = Object.keys(optionToDt).filter((key) => key in request);
+  const pyInput = {
+    genedb: path.join(serverconfig.tpmasterdir, g.genedb.dbfile),
+    chromosomelist,
+    lesion: JSON.stringify(lesions),
+    maxGenesToShow: request.maxGenesToShow,
+    lesionTypeMap: buildLesionTypeMap(availableDataTypes)
+  };
+  const grin2AnalysisStart = Date.now();
+  const pyResult = await run_python("grin2PpWrapper.py", JSON.stringify(pyInput), { signal });
+  if (pyResult.stderr?.trim()) {
+    mayLog(`[GRIN2] Python stderr: ${pyResult.stderr}`);
+    if (pyResult.stderr.includes("ERROR:")) {
+      throw new Error(`Python script error: ${pyResult.stderr}`);
+    }
+  }
+  const grin2AnalysisTime = Date.now() - grin2AnalysisStart;
+  mayLog(`[GRIN2] Python processing took ${formatElapsedTime(grin2AnalysisTime)}`);
+  const resultData = JSON.parse(pyResult);
+  const cacheResult = {
+    resultData,
+    processing
+  };
+  return { cacheResult, processingTime, grin2AnalysisTime };
+}
 async function processSampleData(samples, ds, request) {
   const lesions = [];
   const maxLesions = await getMaxLesions();
@@ -506,5 +537,5 @@ function filterAndConvertSV(sampleName, entry, _options) {
   return lesionA;
 }
 export {
-  api
+  init
 };

package/routes/saveWSIAnnotation.js

@@ -1,16 +1,5 @@
-import { saveWSIAnnotationPayload } from "#types/checkers";
-import { FlagStatus, SelectionPrefixes, checkSelectionType } from "#shared";
+import { FlagStatus, SelectionPrefixes, checkSelectionType } from "#types";
 import { getDbConnection } from "#src/aiHistoDBConnection.ts";
-const routePath = "saveWSIAnnotation";
-const api = {
-  endpoint: `${routePath}`,
-  methods: {
-    post: {
-      ...saveWSIAnnotationPayload,
-      init
-    }
-  }
-};
 function init({ genomes }) {
   return async (req, res) => {
     try {
@@ -162,6 +151,6 @@ function validateQuery(ds, connection) {
   };
 }
 export {
-  api,
+  init,
   validate_query_saveWSIAnnotation
 };