@sjcrh/proteinpaint-server 2.190.2-0 → 2.191.0

package/routes/grin2.js

@@ -1,541 +0,0 @@
-import serverconfig from "#src/serverconfig.js";
-import path from "path";
-import { run_python } from "@sjcrh/proteinpaint-python";
-import { run_rust } from "@sjcrh/proteinpaint-rust";
-import { mayLog } from "#src/helpers.ts";
-import os from "os";
-import { get_samples } from "#src/termdb.sql.js";
-import { read_file, file_is_readable } from "#src/utils.js";
-import { dtsnvindel, dtcnv, dtfusionrna, dtsv, dt2lesion, optionToDt, formatElapsedTime } from "#shared";
-import { mayFilterByMaf } from "#src/mds3.init.js";
-import { cacheOrRecompute } from "#src/utils/cacheOrRecompute.ts";
-import { promisify } from "node:util";
-import { exec as execCallback } from "node:child_process";
-const MAX_LESIONS = serverconfig.features.grin2maxLesions || 25e4;
-const GRIN2_MEMORY_BUDGET_MB = 950;
-const GRIN2_CONCURRENCY_LIMIT = 5;
-const MEMORY_BASE_MB = 260;
-const MEMORY_PER_1K_LESIONS = 2.4;
-const MIN_LESIONS = 5e4;
-function init({ genomes }) {
-  return async (req, res) => {
-    const signal = req.query.__abortSignal;
-    try {
-      const request = req.query;
-      const g = genomes[request.genome];
-      if (!g) throw new Error("genome missing");
-      const ds = g.datasets?.[request.dslabel];
-      if (!ds) throw new Error("ds missing");
-      if (!ds.queries?.singleSampleMutation) throw new Error("singleSampleMutation query missing from dataset");
-      const result = await runGrin2WithLimit(g, ds, request, signal);
-      res.json(result);
-    } catch (e) {
-      if (signal?.aborted) {
-        mayLog("[GRIN2] Analysis aborted due to client disconnect");
-        return;
-      }
-      console.error("[GRIN2] Error stack:", e.stack);
-      const errorResponse = {
-        status: "error",
-        error: e.message || String(e),
-        code: e.code
-      };
-      res.status(e.status || 500).send(errorResponse);
-    }
-  };
-}
-const exec = promisify(execCallback);
-async function getAvailableMemoryMB() {
-  try {
-    if (process.platform === "darwin") {
-      const { stdout } = await exec("vm_stat");
-      const output = stdout.toString();
-      const headerLine = output.split("\n")[0] || "";
-      const pageSizeMatch = headerLine.match(/page size of\s+(\d+)\s+bytes/i);
-      const pageSize = pageSizeMatch ? parseInt(pageSizeMatch[1], 10) : 16384;
-      const freeMatch = output.match(/Pages free:\s+(\d+)/);
-      const inactiveMatch = output.match(/Pages inactive:\s+(\d+)/);
-      const freePages = freeMatch ? parseInt(freeMatch[1], 10) : 0;
-      const inactivePages = inactiveMatch ? parseInt(inactiveMatch[1], 10) : 0;
-      return (freePages + inactivePages) * pageSize / (1024 * 1024);
-    } else {
-      const { stdout } = await exec("free -m");
-      const output = stdout.toString();
-      const lines = output.split("\n");
-      const memLine = lines.find((l) => l.startsWith("Mem:"));
-      if (memLine) {
-        const parts = memLine.split(/\s+/);
-        return parseInt(parts[6]);
-      }
-    }
-  } catch (e) {
-    mayLog(`[GRIN2] Memory check failed, using fallback: ${e}`);
-  }
-  return os.freemem() / (1024 * 1024);
-}
-async function getMaxLesions() {
-  const availableMemoryMB = await getAvailableMemoryMB();
-  mayLog(`[GRIN2] Available system memory: ${availableMemoryMB.toFixed(0)} MB`);
-  if (availableMemoryMB < GRIN2_MEMORY_BUDGET_MB * 2) {
-    const reducedBudget = availableMemoryMB * 0.4;
-    mayLog(`[GRIN2] Reducing lesion cap due to memory constraints. New budget: ${reducedBudget.toFixed(2)} MB`);
-    const calculated = Math.floor((reducedBudget - MEMORY_BASE_MB) / MEMORY_PER_1K_LESIONS) * 1e3;
-    mayLog(`[GRIN2] Calculated lesion cap based on memory: ${calculated.toLocaleString()}`);
-    return Math.max(MIN_LESIONS, Math.min(MAX_LESIONS, calculated));
-  }
-  return MAX_LESIONS;
-}
-let activeGrin2Jobs = 0;
-async function runGrin2WithLimit(g, ds, request, signal) {
-  if (activeGrin2Jobs >= GRIN2_CONCURRENCY_LIMIT) {
-    const error = new Error(
-      `GRIN2 analysis queue is full (${GRIN2_CONCURRENCY_LIMIT} concurrent analyses). Please try again in a few minutes.`
-    );
-    error.status = 429;
-    error.statusCode = 429;
-    throw error;
-  }
-  activeGrin2Jobs++;
-  mayLog(`[GRIN2] Starting analysis. Active jobs: ${activeGrin2Jobs}/${GRIN2_CONCURRENCY_LIMIT}`);
-  try {
-    return await runGrin2(g, ds, request, signal);
-  } finally {
-    activeGrin2Jobs--;
-    mayLog(`[GRIN2] Analysis complete. Active jobs: ${activeGrin2Jobs}/${GRIN2_CONCURRENCY_LIMIT}`);
-  }
-}
-function buildLesionTypeMap(availableOptions) {
-  const lesionTypeMap = {};
-  for (const option of availableOptions) {
-    const dt = optionToDt[option];
-    if (!dt || !dt2lesion[dt]) continue;
-    dt2lesion[dt].lesionTypes.forEach((lt) => {
-      lesionTypeMap[lt.lesionType] = lt.name;
-    });
-  }
-  return lesionTypeMap;
-}
-function getCnvLesionType(isGain) {
-  const cnvConfig = dt2lesion[dtcnv];
-  const targetName = isGain ? "Gain" : "Loss";
-  const lesionType = cnvConfig.lesionTypes.find((lt) => lt.name === targetName);
-  if (!lesionType) {
-    throw new Error(`CNV lesion type '${targetName}' not found`);
-  }
-  return lesionType.lesionType;
-}
-async function runGrin2(g, ds, request, signal) {
-  const chromosomelist = {};
-  for (const c in g.majorchr) {
-    chromosomelist[c] = g.majorchr[c];
-  }
-  const {
-    result: cacheResult,
-    cacheFile,
-    freshCompute,
-    processingTime,
-    grin2AnalysisTime
-  } = await getGrin2CacheResult(request, g, ds, chromosomelist, signal);
-  const { resultData, processing } = cacheResult;
-  const rustInput = {
-    file: cacheFile,
-    type: "grin2",
-    chrSizes: chromosomelist,
-    plot_width: request.width,
-    plot_height: request.height,
-    device_pixel_ratio: request.devicePixelRatio,
-    png_dot_radius: request.pngDotRadius,
-    lesion_type_colors: request.lesionTypeColors,
-    q_value_threshold: request.qValueThreshold,
-    max_capped_points: request.maxCappedPoints,
-    hard_cap: request.hardCap,
-    bin_size: request.binSize
-  };
-  const manhattanPlotStart = Date.now();
-  const rsResult = await run_rust("manhattan_plot", JSON.stringify(rustInput), [], { signal });
-  const manhattanPlotTime = Date.now() - manhattanPlotStart;
-  mayLog(`[GRIN2] Manhattan plot generation took ${formatElapsedTime(manhattanPlotTime)}`);
-  const manhattanPlotData = JSON.parse(rsResult);
-  if (!manhattanPlotData?.png) {
-    throw new Error("Invalid Rust output: missing PNG data");
-  }
-  const totalTime = processingTime + grin2AnalysisTime + manhattanPlotTime;
-  const lesionTypeRows = [];
-  if (processing.lesionCounts?.byType) {
-    const typeLabels = {};
-    Object.values(dt2lesion).forEach((config) => {
-      config.lesionTypes.forEach((lt) => {
-        typeLabels[lt.lesionType] = lt.name;
-      });
-    });
-    for (const [type, data] of Object.entries(processing.lesionCounts.byType)) {
-      const { count, samples } = data;
-      lesionTypeRows.push([typeLabels[type] || type, `${count.toLocaleString()} (${samples} samples)`]);
-    }
-  }
-  const capWarningRows = [];
-  const expectedToProcess = processing.totalSamples - processing.failedSamples;
-  if (processing.processedSamples < expectedToProcess) {
-    capWarningRows.push([
-      "Note",
-      `Lesion cap of ${processing.lesionCap?.toLocaleString()} was reached before all samples could be processed. Analysis ran on ${processing.processedSamples.toLocaleString()} of ${expectedToProcess.toLocaleString()} samples.`
-    ]);
-  }
-  const response = {
-    status: "success",
-    fromCache: !freshCompute,
-    pngImg: manhattanPlotData.png,
-    plotData: manhattanPlotData.plot_data,
-    topGeneTable: resultData.topGeneTable,
-    stats: {
-      lst: [
-        {
-          name: "GRIN2 Processing Summary",
-          rows: [
-            ["Total Genes", resultData.totalGenes.toLocaleString()],
-            ["Showing Top", resultData.showingTop.toLocaleString()],
-            ["Cache File Name", cacheFile],
-            ["Total Samples", processing.totalSamples.toLocaleString()],
-            ["Processed Samples", processing.processedSamples.toLocaleString()],
-            ["Unprocessed Samples", (processing.unprocessedSamples ?? 0).toLocaleString()],
-            ["Failed Samples", processing.failedSamples.toLocaleString()],
-            ["Failed Files", (processing.failedFiles?.length ?? 0).toLocaleString()],
-            ["Total Lesions", processing.totalLesions.toLocaleString()],
-            ["Processed Lesions", processing.processedLesions.toLocaleString()]
-          ]
-        },
-        {
-          name: "Lesion Counts",
-          rows: lesionTypeRows
-        },
-        {
-          name: "Memory Usage",
-          rows: [
-            ["Start", `${resultData.memory?.start} MB`],
-            ["After prep", `${resultData.memory?.after_prep} MB`],
-            ["After overlaps", `${resultData.memory?.after_overlaps} MB`],
-            ["After counts", `${resultData.memory?.after_counts} MB`],
-            ["After stats", `${resultData.memory?.after_stats} MB`],
-            ["Peak", `${resultData.memory?.peak} MB`]
-          ]
-        },
-        {
-          name: "Timing",
-          rows: [
-            ["Processing", freshCompute ? formatElapsedTime(processingTime) : "cached"],
-            ["GRIN2", freshCompute ? formatElapsedTime(grin2AnalysisTime) : "cached"],
-            ["Plotting", formatElapsedTime(manhattanPlotTime)],
-            ["Total", formatElapsedTime(totalTime)],
-            ...capWarningRows
-          ]
-        }
-      ]
-    }
-  };
-  return response;
-}
-function grin2KeyInputs(req) {
-  return {
-    genome: req.genome,
-    dslabel: req.dslabel,
-    filter: req.filter ?? null,
-    snvindelOptions: req.snvindelOptions ?? null,
-    cnvOptions: req.cnvOptions ?? null,
-    fusionOptions: req.fusionOptions ?? null,
-    svOptions: req.svOptions ?? null,
-    maxGenesToShow: req.maxGenesToShow ?? null
-  };
-}
-async function getGrin2CacheResult(request, g, ds, chromosomelist, signal) {
-  let processingTime = 0;
-  let grin2AnalysisTime = 0;
-  let freshCompute = false;
-  const {
-    result,
-    cacheId,
-    cacheFilePath: cacheFile
-  } = await cacheOrRecompute({
-    computeArgument: grin2KeyInputs(request),
-    cacheSubdir: "grin2",
-    computeFresh: async () => {
-      freshCompute = true;
-      const out = await runGrin2Fresh(request, g, ds, chromosomelist, signal);
-      processingTime = out.processingTime;
-      grin2AnalysisTime = out.grin2AnalysisTime;
-      return out.cacheResult;
-    }
-  });
-  return { result, cacheId, cacheFile, freshCompute, processingTime, grin2AnalysisTime };
-}
-async function runGrin2Fresh(request, g, ds, chromosomelist, signal) {
-  const startTime = Date.now();
-  const samples = await get_samples(
-    request,
-    ds,
-    true
-    // must set to true to return sample name to be able to access file. FIXME this can let names revealed to grin2 client, may need to apply access control
-  );
-  const cohortTime = Date.now() - startTime;
-  mayLog(`[GRIN2] Retrieved ${samples.length.toLocaleString()} samples in ${formatElapsedTime(cohortTime)}`);
-  if (samples.length === 0) {
-    throw new Error("No samples found matching the provided filter criteria");
-  }
-  const processingStartTime = Date.now();
-  const { lesions, processing } = await processSampleData(samples, ds, request);
-  if (!processing) throw new Error("Processing summary is missing");
-  const processingTime = Date.now() - processingStartTime;
-  mayLog(`[GRIN2] Data processing took ${formatElapsedTime(processingTime)}`);
-  mayLog(
-    `[GRIN2] Processing summary: ${processing?.processedSamples ?? 0}/${processing?.totalSamples ?? samples.length} samples processed successfully`
-  );
-  if (processing?.failedSamples !== void 0 && processing.failedSamples > 0) {
-    mayLog(`[GRIN2] Warning: ${processing.failedSamples} samples failed to process`);
-  }
-  if (lesions.length === 0) {
-    throw new Error("No lesions found after processing all samples. Check filter criteria and input data.");
-  }
-  const availableDataTypes = Object.keys(optionToDt).filter((key) => key in request);
-  const pyInput = {
-    genedb: path.join(serverconfig.tpmasterdir, g.genedb.dbfile),
-    chromosomelist,
-    lesion: JSON.stringify(lesions),
-    maxGenesToShow: request.maxGenesToShow,
-    lesionTypeMap: buildLesionTypeMap(availableDataTypes)
-  };
-  const grin2AnalysisStart = Date.now();
-  const pyResult = await run_python("grin2PpWrapper.py", JSON.stringify(pyInput), { signal });
-  if (pyResult.stderr?.trim()) {
-    mayLog(`[GRIN2] Python stderr: ${pyResult.stderr}`);
-    if (pyResult.stderr.includes("ERROR:")) {
-      throw new Error(`Python script error: ${pyResult.stderr}`);
-    }
-  }
-  const grin2AnalysisTime = Date.now() - grin2AnalysisStart;
-  mayLog(`[GRIN2] Python processing took ${formatElapsedTime(grin2AnalysisTime)}`);
-  const resultData = JSON.parse(pyResult);
-  const cacheResult = {
-    resultData,
-    processing
-  };
-  return { cacheResult, processingTime, grin2AnalysisTime };
-}
-async function processSampleData(samples, ds, request) {
-  const lesions = [];
-  const maxLesions = await getMaxLesions();
-  mayLog(`[GRIN2] Max lesions for this run: ${maxLesions.toLocaleString()}`);
-  const samplesPerType = /* @__PURE__ */ new Map();
-  const enabledTypes = [];
-  if (request.snvindelOptions) enabledTypes.push(dtsnvindel);
-  if (request.cnvOptions) enabledTypes.push(dtcnv);
-  if (request.fusionOptions) enabledTypes.push(dtfusionrna);
-  if (request.svOptions) enabledTypes.push(dtsv);
-  for (const type of enabledTypes) {
-    samplesPerType.set(type, /* @__PURE__ */ new Set());
-  }
-  const processing = {
-    totalSamples: samples.length,
-    processedSamples: 0,
-    failedSamples: 0,
-    failedFiles: [],
-    totalLesions: 0,
-    processedLesions: 0,
-    unprocessedSamples: 0
-  };
-  for (let i = 0; i < samples.length; i++) {
-    if (lesions.length >= maxLesions) {
-      const remaining = samples.length - i;
-      if (remaining > 0) processing.unprocessedSamples += remaining;
-      mayLog(`[GRIN2] Overall lesion cap (${maxLesions}) reached; stopping early.`);
-      break;
-    }
-    const sample = samples[i];
-    const filepath = path.join(serverconfig.tpmasterdir, ds.queries.singleSampleMutation.folder, sample.name);
-    try {
-      await file_is_readable(filepath);
-      const mlst = JSON.parse(await read_file(filepath));
-      const { sampleLesions, contributedTypes } = processSampleMlst(sample.name, mlst, request);
-      const skipChrM = ds.queries.singleSampleMutation.discoPlot?.skipChrM;
-      const filteredLesions = skipChrM ? sampleLesions.filter((lesion) => lesion[1].toLowerCase() !== "chrm") : sampleLesions;
-      const remainingCapacity = maxLesions - lesions.length;
-      const lesionsToAdd = filteredLesions.slice(0, remainingCapacity);
-      lesions.push(...lesionsToAdd);
-      for (const type of contributedTypes) {
-        samplesPerType.get(type)?.add(sample.name);
-      }
-      processing.processedSamples += 1;
-      processing.totalLesions += filteredLesions.length;
-    } catch (error) {
-      processing.failedSamples += 1;
-      processing.failedFiles.push({
-        sampleName: sample.name,
-        filePath: filepath,
-        error: error instanceof Error ? error.message || "Unknown error" : String(error)
-      });
-      mayLog(`[GRIN2] Error processing sample ${sample.name}`);
-    }
-  }
-  processing.processedLesions = lesions.length;
-  processing.lesionCap = maxLesions;
-  const lesionCounts = {
-    total: lesions.length,
-    byType: {}
-  };
-  const lesionTypeCounts = {};
-  for (const lesion of lesions) {
-    const lesionType = lesion[4];
-    lesionTypeCounts[lesionType] = (lesionTypeCounts[lesionType] || 0) + 1;
-  }
-  for (const type of enabledTypes) {
-    const sampleCount = samplesPerType.get(type)?.size || 0;
-    const dtConfig = dt2lesion[type];
-    if (!dtConfig) continue;
-    dtConfig.lesionTypes.forEach((lt) => {
-      lesionCounts.byType[lt.lesionType] = {
-        count: lesionTypeCounts[lt.lesionType] || 0,
-        samples: sampleCount
-      };
-    });
-  }
-  processing.lesionCounts = lesionCounts;
-  return { lesions, processing };
-}
-function processSampleMlst(sampleName, mlst, request) {
-  const sampleLesions = [];
-  const contributedTypes = /* @__PURE__ */ new Set();
-  for (const m of mlst) {
-    switch (m.dt) {
-      case dtsnvindel: {
-        if (!request.snvindelOptions) break;
-        const les = filterAndConvertSnvIndel(sampleName, m, request.snvindelOptions);
-        if (les) {
-          sampleLesions.push(les);
-          contributedTypes.add(dtsnvindel);
-        }
-        break;
-      }
-      case dtcnv: {
-        if (!request.cnvOptions) break;
-        const les = filterAndConvertCnv(sampleName, m, request.cnvOptions);
-        if (les) {
-          sampleLesions.push(les);
-          contributedTypes.add(dtcnv);
-        }
-        break;
-      }
-      case dtfusionrna: {
-        if (!request.fusionOptions) break;
-        const les = filterAndConvertFusion(sampleName, m, request.fusionOptions);
-        if (les) {
-          if (Array.isArray(les[0])) {
-            for (const lesion of les) {
-              sampleLesions.push(lesion);
-            }
-          } else {
-            sampleLesions.push(les);
-          }
-          contributedTypes.add(dtfusionrna);
-        }
-        break;
-      }
-      case dtsv: {
-        if (!request.svOptions) break;
-        const les = filterAndConvertSV(sampleName, m, request.svOptions);
-        if (les) {
-          if (Array.isArray(les[0])) {
-            for (const lesion of les) {
-              sampleLesions.push(lesion);
-            }
-          } else {
-            sampleLesions.push(les);
-          }
-          contributedTypes.add(dtsv);
-        }
-        break;
-      }
-      default:
-        break;
-    }
-  }
-  return { sampleLesions, contributedTypes };
-}
-function filterAndConvertSnvIndel(sampleName, entry, options) {
-  if (!options?.consequences) {
-    return null;
-  }
-  if (options.consequences.length > 0 && entry.class && !options.consequences.includes(entry.class)) {
-    return null;
-  }
-  if (!Number.isInteger(entry.pos)) {
-    return null;
-  }
-  if (options.mafFilter?.lst?.length) {
-    if (!Array.isArray(entry.vafs)) return null;
-    const copy = { dt: dtsnvindel };
-    for (const v of entry.vafs) {
-      copy[v.id] = v.refCount + "," + v.altCount;
-    }
-    try {
-      if (!mayFilterByMaf(options.mafFilter, copy)) return null;
-    } catch (e) {
-      mayLog("mayFilterByMaf() crashed on a snvindel " + (e instanceof Error ? e.message : String(e)));
-      return null;
-    }
-  }
-  const start = entry.pos;
-  const end = entry.pos;
-  return [sampleName, entry.chr, start, end, dt2lesion[dtsnvindel].lesionTypes[0].lesionType];
-}
-function filterAndConvertCnv(sampleName, entry, options) {
-  if (!options || options.gainThreshold === void 0 || options.lossThreshold === void 0 || options.maxSegLength === void 0) {
-    return null;
-  }
-  if (!Number.isInteger(entry.start)) {
-    return null;
-  }
-  if (!Number.isInteger(entry.stop)) {
-    return null;
-  }
-  if (options.maxSegLength > 0 && entry.stop - entry.start > options.maxSegLength) {
-    return null;
-  }
-  const isGain = entry.value >= options.gainThreshold;
-  const isLoss = entry.value <= options.lossThreshold;
-  if (!isGain && !isLoss) return null;
-  const lesionType = getCnvLesionType(isGain);
-  const start = entry.start;
-  const end = entry.stop;
-  return [sampleName, entry.chr, start, end, lesionType];
-}
-function filterAndConvertFusion(sampleName, entry, _options) {
-  if (!entry.chrA || entry.posA === void 0) {
-    return null;
-  }
-  const startA = entry.posA;
-  const endA = entry.posA;
-  const lesionA = [sampleName, entry.chrA, startA, endA, dt2lesion[dtfusionrna].lesionTypes[0].lesionType];
-  if (entry.chrB && entry.posB !== void 0) {
-    const startB = entry.posB;
-    const endB = entry.posB;
-    const lesionB = [sampleName, entry.chrB, startB, endB, dt2lesion[dtfusionrna].lesionTypes[0].lesionType];
-    return [lesionA, lesionB];
-  }
-  return lesionA;
-}
-function filterAndConvertSV(sampleName, entry, _options) {
-  if (!entry.chrA || entry.posA === void 0) {
-    return null;
-  }
-  const startA = entry.posA;
-  const endA = entry.posA;
-  const lesionA = [sampleName, entry.chrA, startA, endA, dt2lesion[dtsv].lesionTypes[0].lesionType];
-  if (entry.chrB && entry.posB !== void 0) {
-    const startB = entry.posB;
-    const endB = entry.posB;
-    const lesionB = [sampleName, entry.chrB, startB, endB, dt2lesion[dtsv].lesionTypes[0].lesionType];
-    return [lesionA, lesionB];
-  }
-  return lesionA;
-}
-export {
-  init
-};