@sjcrh/proteinpaint-server 2.192.0 → 2.193.0

package/routes/gdc.grin2.list.js

@@ -1,238 +0,0 @@
-import ky from "ky";
-import { joinUrl } from "#shared/joinUrl.js";
-import serverconfig from "#src/serverconfig.js";
-import { mayLog } from "#src/helpers.ts";
-const mafMaxFileNumber = 1e3, cnvMaxFileNumber = 1e3;
-const allowedWorkflowType = "Aliquot Ensemble Somatic Variant Merging and Masking";
-const maxFileSizeAllowed = 1e6;
-const maxTotalSizeCompressed = serverconfig.features.gdcMafMaxFileSize || 4e8;
-function init({ genomes }) {
-  return async (req, res) => {
-    try {
-      const g = genomes.hg38;
-      if (!g) throw "hg38 missing";
-      const ds = g.datasets?.GDC;
-      if (!ds) throw "hg38 GDC missing";
-      console.log("GRIN2 List Route Received:", JSON.stringify(req.query, null, 2));
-      const result = {};
-      if (req.query.mafOptions) {
-        result.mafFiles = {
-          files: [],
-          filesTotal: 0,
-          maxTotalSizeCompressed: 0,
-          fileCounts: { maf: 0 }
-        };
-        await listMafFiles(req.query, result, ds);
-      }
-      if (req.query.cnvOptions) {
-        result.cnvFiles = {
-          files: [],
-          maxTotalSizeCompressed
-        };
-        await listCnvFiles(req.query, result, ds);
-      }
-      res.send(result);
-    } catch (e) {
-      if (e.stack) console.log(e.stack);
-      res.send({ status: "error", error: e.message || e });
-    }
-  };
-}
-async function listMafFiles(q, result, ds) {
-  if (!q.mafOptions) return;
-  if (!result.mafFiles) {
-    throw new Error("result.mafFiles must be initialized before calling listMafFiles");
-  }
-  const filters = {
-    op: "and",
-    content: [
-      { op: "=", content: { field: "data_format", value: "MAF" } },
-      { op: "=", content: { field: "experimental_strategy", value: q.mafOptions.experimentalStrategy } },
-      { op: "=", content: { field: "analysis.workflow_type", value: allowedWorkflowType } },
-      { op: "=", content: { field: "access", value: "open" } }
-    ]
-  };
-  const case_filters = { op: "and", content: [] };
-  if (q.filter0) {
-    case_filters.content.push(q.filter0);
-  }
-  const { host, headers } = ds.getHostHeaders(q);
-  const body = {
-    filters,
-    size: mafMaxFileNumber,
-    fields: [
-      "id",
-      "file_size",
-      "cases.project.project_id",
-      "cases.case_id",
-      "cases.submitter_id",
-      "cases.samples.tissue_type",
-      "cases.samples.tumor_descriptor"
-    ].join(",")
-  };
-  if (case_filters.content.length) body.case_filters = case_filters;
-  const response = await ky.post(joinUrl(host.rest, "files"), { timeout: false, headers, json: body });
-  if (!response.ok) throw `HTTP Error: ${response.status} ${response.statusText}`;
-  const re = await response.json();
-  if (!Number.isInteger(re.data?.pagination?.total)) throw "re.data.pagination.total is not int";
-  if (!Array.isArray(re.data?.hits)) throw "re.data.hits[] not array";
-  const files = [];
-  const filteredFiles = [];
-  for (const h of re.data.hits) {
-    const c = h.cases?.[0];
-    if (!c) throw "h.cases[0] missing";
-    if (h.file_size >= maxFileSizeAllowed) {
-      filteredFiles.push({
-        fileId: h.id,
-        fileSize: h.file_size,
-        reason: `File size (${h.file_size} bytes) exceeds maximum allowed size (${maxFileSizeAllowed} bytes)`
-      });
-      mayLog(
-        `File ${h.id} with a size of ${h.file_size} bytes is larger then the allowed file size. It is excluded from the list.
-If you want to include it, please increase the maxFileSizeAllowed in the code.`
-      );
-      continue;
-    }
-    const file = {
-      id: h.id,
-      project_id: c.project.project_id,
-      file_size: h.file_size,
-      case_submitter_id: c.submitter_id,
-      case_uuid: c.case_id,
-      sample_types: []
-    };
-    if (c.samples) {
-      let normalTypeName;
-      for (const { tumor_descriptor, tissue_type } of c.samples) {
-        if (tissue_type == "Normal") {
-          normalTypeName = (tumor_descriptor == "Not Applicable" ? "" : tumor_descriptor + " ") + tissue_type;
-          continue;
-        }
-        file.sample_types.push(tumor_descriptor + " " + tissue_type);
-      }
-      if (normalTypeName) file.sample_types.push(normalTypeName);
-    }
-    file.sample_types = [...new Set(file.sample_types)];
-    files.push(file);
-  }
-  const filesByCase = /* @__PURE__ */ new Map();
-  for (const file of files) {
-    const caseId = file.case_submitter_id;
-    if (!filesByCase.has(caseId)) {
-      filesByCase.set(caseId, []);
-    }
-    filesByCase.get(caseId).push(file);
-  }
-  const deduplicatedFiles = [];
-  let duplicatesRemoved = 0;
-  const caseDetails = [];
-  for (const [caseId, caseFiles] of filesByCase) {
-    if (caseFiles.length > 1) {
-      caseFiles.sort((a, b) => b.file_size - a.file_size);
-      deduplicatedFiles.push(caseFiles[0]);
-      duplicatesRemoved += caseFiles.length - 1;
-      caseDetails.push({
-        caseName: caseId,
-        fileCount: caseFiles.length,
-        keptFileSize: caseFiles[0].file_size
-      });
-      mayLog(`Case ${caseId}: Found ${caseFiles.length} MAF files, keeping largest (${caseFiles[0].file_size} bytes)`);
-    } else {
-      deduplicatedFiles.push(caseFiles[0]);
-    }
-  }
-  if (duplicatesRemoved > 0) {
-    mayLog(
-      `GRIN2 MAF deduplication: Removed ${duplicatesRemoved} duplicate files, kept ${deduplicatedFiles.length} unique cases`
-    );
-  }
-  deduplicatedFiles.sort((a, b) => b.file_size - a.file_size);
-  result.mafFiles.files.push(...deduplicatedFiles);
-  result.mafFiles.filesTotal = re.data.pagination.total;
-  if (result.mafFiles.fileCounts) {
-    result.mafFiles.fileCounts.maf = files.length;
-  }
-  result.mafFiles.deduplicationStats = {
-    originalFileCount: files.length,
-    deduplicatedFileCount: deduplicatedFiles.length,
-    duplicatesRemoved,
-    caseDetails,
-    filteredFiles
-  };
-}
-async function listCnvFiles(q, result, ds) {
-  if (!q.cnvOptions) {
-    console.log("No cnvOptions provided, returning empty cnvFiles");
-    return;
-  }
-  const case_filters = { op: "and", content: [] };
-  if (q.filter0) {
-    case_filters.content.push(q.filter0);
-  }
-  const body = {
-    size: cnvMaxFileNumber,
-    fields: [
-      "cases.samples.tissue_type",
-      "cases.project.project_id",
-      "cases.submitter_id",
-      "cases.case_id",
-      "data_type",
-      "file_id",
-      "file_size",
-      "data_format",
-      "experimental_strategy",
-      "analysis.workflow_type"
-    ].join(","),
-    filters: {
-      op: "in",
-      content: {
-        field: "data_type",
-        value: ["Copy Number Segment", "Masked Copy Number Segment", "Allele-specific Copy Number Segment"]
-      }
-    }
-  };
-  if (case_filters.content.length) body.case_filters = case_filters;
-  const { host, headers } = ds.getHostHeaders(q);
-  try {
-    const re = await ky.post(joinUrl(host.rest, "files"), { timeout: false, headers, json: body }).json();
-    console.log("API Response:", {
-      hits: re.data?.hits?.length || 0,
-      firstHit: re.data?.hits?.[0]
-    });
-    if (!Array.isArray(re.data.hits)) {
-      throw new Error("API response data.hits is not an array");
-    }
-    const cnvFiles = [];
-    for (const h of re.data.hits) {
-      if (h.data_format != "TXT") {
-        continue;
-      }
-      if (!h.analysis?.workflow_type) throw "h.analysis.workflow_type missing";
-      const c = h.cases?.[0];
-      if (!c) throw "h.cases[0] missing";
-      if (h.data_type == "Allele-specific Copy Number Segment") {
-      } else if (h.data_type == "Masked Copy Number Segment" || h.data_type == "Copy Number Segment" && h.analysis.workflow_type != "DNACopy") {
-        const file = {
-          id: h.file_id || h.id,
-          // Handle both field names
-          project_id: c.project?.project_id || "unknown",
-          // Safe access with fallback
-          file_size: h.file_size,
-          case_submitter_id: c.submitter_id,
-          case_uuid: c.case_id,
-          sample_types: c.samples?.map((s) => s.tissue_type).filter(Boolean) || []
-        };
-        cnvFiles.push(file);
-      }
-    }
-    result.cnvFiles = { files: cnvFiles, maxTotalSizeCompressed };
-    console.log(`Successfully processed ${cnvFiles.length} CNV files`);
-  } catch (error) {
-    console.error("Error fetching CNV files:", error);
-    result.cnvFiles = { files: [], maxTotalSizeCompressed };
-  }
-}
-export {
-  init,
-  maxTotalSizeCompressed
-};

package/routes/gdc.grin2.run.js

@@ -1,127 +0,0 @@
-import { run_rust } from "@sjcrh/proteinpaint-rust";
-import serverconfig from "#src/serverconfig.js";
-import path from "path";
-import { run_python } from "@sjcrh/proteinpaint-python";
-import { mayLog } from "#src/helpers.ts";
-const MAX_RECORD = 1e5;
-function init({ genomes }) {
-  return async (req, res) => {
-    try {
-      await runGrin2(genomes, req, res);
-    } catch (e) {
-      console.error("[GRIN2] Error stack:", e.stack);
-      const errorResponse = {
-        status: "error",
-        error: e.message || String(e)
-      };
-      res.status(500).send(errorResponse);
-    }
-  };
-}
-async function runGrin2(genomes, req, res) {
-  const g = genomes.hg38;
-  if (!g) throw "hg38 missing";
-  const ds = g.datasets.GDC;
-  if (!ds) throw "hg38 GDC missing";
-  const parsedRequest = req.query;
-  const rustInput = {
-    caseFiles: parsedRequest.caseFiles,
-    mafOptions: parsedRequest.mafOptions,
-    cnvOptions: parsedRequest.cnvOptions,
-    chromosomes: [],
-    max_record: MAX_RECORD
-  };
-  const pyInput = {
-    genedb: path.join(serverconfig.tpmasterdir, g.genedb.dbfile),
-    chromosomelist: {},
-    lesion: "",
-    devicePixelRatio: parsedRequest.devicePixelRatio || 2,
-    plot_width: parsedRequest.plot_width || 1e3,
-    plot_height: parsedRequest.plot_height || 400,
-    pngDotRadius: parsedRequest.pngDotRadius || 2
-  };
-  for (const c in g.majorchr) {
-    if (ds.queries.singleSampleMutation?.discoPlot?.skipChrM) {
-      if (c.toLowerCase() == "chrm") continue;
-    }
-    rustInput.chromosomes.push(c);
-    pyInput.chromosomelist[c] = g.majorchr[c];
-  }
-  const downloadStartTime = Date.now();
-  const rustOutput = await run_rust("gdcGRIN2", JSON.stringify(rustInput));
-  mayLog("[GRIN2] Rust execution completed");
-  const downloadTime = Date.now() - downloadStartTime;
-  const downloadTimeToPrint = Math.round(downloadTime / 1e3);
-  mayLog(`[GRIN2] Rust processing took ${downloadTimeToPrint} seconds`);
-  const parsedRustResult = parseRustOutput(rustOutput);
-  if (!parsedRustResult) {
-    throw new Error("Failed to process files: No result from Rust");
-  }
-  if (parsedRustResult.successful_data !== void 0 && parsedRustResult.successful_data !== null) {
-    pyInput.lesion = parsedRustResult.successful_data;
-    mayLog(`[GRIN2] Extracted ${parsedRustResult.successful_data.length.toLocaleString()} characters for python script`);
-    mayLog(
-      `[GRIN2] Success: ${parsedRustResult.summary.successful_files.toLocaleString()}, Failed: ${parsedRustResult.summary.failed_files.toLocaleString()}`
-    );
-  } else {
-    throw "No successful data returned from Rust processing";
-  }
-  const grin2AnalysisStart = Date.now();
-  const pyResult = await run_python("grin2PpWrapperGDC.py", JSON.stringify(pyInput));
-  if (pyResult.stderr?.trim()) {
-    mayLog(`[GRIN2] Python stderr: ${pyResult.stderr}`);
-  }
-  const grin2AnalysisTime = Date.now() - grin2AnalysisStart;
-  const grin2AnalysisTimeToPrint = Math.round(grin2AnalysisTime / 1e3);
-  mayLog(`[GRIN2] Python processing took ${grin2AnalysisTimeToPrint} seconds`);
-  const resultData = JSON.parse(pyResult);
-  const pngImg = resultData.png[0];
-  const topGeneTable = resultData.topGeneTable || null;
-  const totalProcessTime = downloadTimeToPrint + grin2AnalysisTimeToPrint;
-  return res.json({
-    resultData,
-    pngImg,
-    topGeneTable,
-    rustResult: parsedRustResult,
-    timing: {
-      rustProcessingTime: downloadTimeToPrint,
-      grin2ProcessingTime: grin2AnalysisTimeToPrint,
-      totalTime: totalProcessTime
-    },
-    status: "success"
-  });
-}
-function parseRustOutput(rustOutput) {
-  try {
-    const rustResult = JSON.parse(rustOutput);
-    if (!rustResult.grin2lesion || !rustResult.summary) {
-      throw new Error("Invalid Rust output: missing lesion data or summary");
-    }
-    mayLog(`[GRIN2] Parsed Rust output successfully`);
-    return {
-      successful_data: rustResult.grin2lesion,
-      failed_files: rustResult.summary.errors || [],
-      summary: {
-        type: "summary",
-        total_files: rustResult.summary.total_files,
-        successful_files: rustResult.summary.successful_files,
-        failed_files: rustResult.summary.failed_files,
-        errors: rustResult.summary.errors || [],
-        filtered_records: rustResult.summary.filtered_records || 0,
-        filtered_maf_records: rustResult.summary.filtered_maf_records || 0,
-        filtered_cnv_records: rustResult.summary.filtered_cnv_records || 0,
-        included_maf_records: rustResult.summary.included_maf_records || 0,
-        included_cnv_records: rustResult.summary.included_cnv_records || 0,
-        filtered_records_by_case: rustResult.summary.filtered_records_by_case || {},
-        hyper_mutator_records: rustResult.summary.hyper_mutator_records || {},
-        excluded_by_max_record: rustResult.summary.excluded_by_max_record || {},
-        skippedChromosomes: rustResult.summary.skipped_chromosomes || {}
-      }
-    };
-  } catch (error) {
-    throw new Error(`Failed to parse Rust output: ${error instanceof Error ? error.message : "Unknown error"}`);
-  }
-}
-export {
-  init
-};