@sjcrh/proteinpaint-rust 2.133.0 → 2.135.2-0

package/package.json

@@ -1,5 +1,5 @@
 {
-	"version": "2.133.0",
+	"version": "2.135.2-0",
 	"name": "@sjcrh/proteinpaint-rust",
 	"type": "module",
 	"description": "Rust-based utilities for proteinpaint",

package/src/gdcGRIN2.rs

@@ -1,19 +1,34 @@
 /*
-  This script downloads cohort maf files from GDC and gracefully handles timeout and other possible errors related to GDC api processing for use by the client file summary div
+  This script can either download cohort maf/cnv files from GDC or read them from local files, with default behavior being to download from GDC. It gracefully handles timeout and other possible errors related to GDC API processing or file reading for use by the client file summary div.
 
   Key improvements:
   1. Graceful error handling - individual file failures don't stop the entire process
   2. Better timeout handling with retries
   3. More detailed error reporting
   4. Continues processing even when some files fail
+  5. Added chromosome filtering
+  6. Supports reading from local files with --from-file flag
+
+  Command-line arguments:
+  - --from-file: Read data from local files instead of downloading from GDC
 
   Input JSON:
     caseFiles
     mafOptions: For SNVindel filtering
+    cnvOptions: For CNV filtering
+    chromosomes: chromosomes will be included:[]
+
   Output mutations as JSON array.
+    {
+        grin2lesion:str,
+        summary:{}
+    }
 
   Example of usage:
-    echo '{"caseFiles": {"MP2PRT-PATFJE": {"maf": "26ea7b6f-8bc4-4e83-ace1-2125b493a361"},"MP2PRT-PAPIGD": {"maf": "653d7458-f4af-4328-a1ce-3bbf22a2e347"}, "TCGA-CG-4300": { "cnv":"46372ec2-ff79-4d07-b375-9ba8a12c11f3", "maf":"c09b208d-2e7b-4116-9580-27f20f4c7e67"}},"mafOptions": {"minTotalDepth": 100,"minAltAlleleCount": 20,"hyperMutator":8000,"consequences":["missense_variant","frameshift_variant"]}, "cnvOptions":{"lossThreshold":-1, "gainThreshold": 1.5, "segLength":2000000, "hyperMutator":8000}}' | ./target/release/gdcGRIN2
+    echo '{"caseFiles": {"MP2PRT-PATFJE": {"maf": "26ea7b6f-8bc4-4e83-ace1-2125b493a361"},"MP2PRT-PAPIGD": {"maf": "653d7458-f4af-4328-a1ce-3bbf22a2e347"}, "TCGA-CG-4300": { "cnv":"46372ec2-ff79-4d07-b375-9ba8a12c11f3", "maf":"c09b208d-2e7b-4116-9580-27f20f4c7e67"}},"mafOptions": {"minTotalDepth": 10,"minAltAlleleCount": 2,"hyperMutator":8000,"consequences":["missense_variant","frameshift_variant"]}, "cnvOptions":{"lossThreshold":-0.4, "gainThreshold": 0.3, "segLength":2000000, "hyperMutator":500}, "chromosomes":["chr1","chr2","chr3"], "max_record": 100000}' | ./target/release/gdcGRIN2
+  Example of usage (read from local files):
+    echo '{"caseFiles": {"MP2PRT-PATFJE": {"maf": "26ea7b6f-8bc4-4e83-ace1-2125b493a361"},"MP2PRT-PAPIGD": {"maf": "653d7458-f4af-4328-a1ce-3bbf22a2e347"}, "TCGA-CG-4300": { "cnv":"46372ec2-ff79-4d07-b375-9ba8a12c11f3", "maf":"c09b208d-2e7b-4116-9580-27f20f4c7e67"}},"mafOptions": {"minTotalDepth": 10,"minAltAlleleCount": 2,"hyperMutator":8000,"consequences":["missense_variant","frameshift_variant"]}, "cnvOptions":{"lossThreshold":-0.4, "gainThreshold": 0.3, "segLength":2000000, "hyperMutator":500}, "chromosomes":["chr1","chr2","chr3"], "max_record": 100000}' | ./target/release/gdcGRIN2 --from-file
+
 */
 
 use flate2::read::GzDecoder;
@@ -21,11 +36,12 @@ use futures::StreamExt;
 use memchr::memchr;
 use serde::{Deserialize, Serialize};
 use serde_json;
-use std::collections::HashMap;
+use std::collections::{HashMap, HashSet};
+use std::env;
+use std::fs;
 use std::io::{self, Read};
 use std::sync::Arc;
 use std::sync::atomic::{AtomicUsize, Ordering};
-use std::thread::sleep;
 use std::time::Duration;
 use tokio::io::{AsyncReadExt, BufReader};
 use tokio::sync::Mutex;
@@ -73,16 +89,6 @@ struct CnvOptions {
     hyper_mutator: i32,
 }
 
-// Individual successful file output (JSONL format)
-#[derive(serde::Serialize)]
-struct SuccessfulFileOutput {
-    #[serde(rename = "type")]
-    output_type: String, // Always "data"
-    case_id: String,
-    data_type: String,
-    data: Vec<Vec<String>>,
-}
-
 // struct for MAF filter details
 #[derive(Clone, Serialize, Default)]
 struct FilteredMafDetails {
@@ -96,6 +102,7 @@ struct FilteredMafDetails {
     excluded_by_consequence_type: usize,
     total_processed: usize,
     total_included: usize,
+    skipped_chromosomes: HashMap<String, usize>,
 }
 
 // struct for CNV filter details
@@ -109,6 +116,7 @@ struct FilteredCnvDetails {
     excluded_by_segment_length: usize,
     total_processed: usize,
     total_included: usize,
+    skipped_chromosomes: HashMap<String, usize>,
 }
 
 // struct for per-case filter details
@@ -121,8 +129,6 @@ struct FilteredCaseDetails {
 // Final summary output (JSONL format)
 #[derive(serde::Serialize)]
 struct FinalSummary {
-    #[serde(rename = "type")]
-    output_type: String, // Always "summary"
     total_files: usize,
     successful_files: usize,
     failed_files: usize,
@@ -134,6 +140,14 @@ struct FinalSummary {
     included_cnv_records: usize,
     filtered_records_by_case: HashMap<String, FilteredCaseDetails>,
     hyper_mutator_records: HashMap<String, Vec<String>>,
+    excluded_by_max_record: HashMap<String, Vec<String>>,
+}
+
+// Enum to hold both SuccessfulFileoutput and FinalSummary
+#[derive(Serialize)]
+struct Output {
+    grin2lesion: String,
+    summary: FinalSummary,
 }
 
 // Define the top-level input structure
@@ -145,6 +159,8 @@ struct InputData {
     maf_options: Option<MafOptions>,
     #[serde(rename = "cnvOptions")]
     cnv_options: Option<CnvOptions>,
+    chromosomes: Vec<String>,
+    max_record: usize,
 }
 
 // Configuration for different data types
@@ -167,6 +183,7 @@ async fn parse_content(
     loss_threshold: f32,
     seg_length: i32,
     cnv_hyper_mutator: i32,
+    chromosomes: &HashSet<String>,
     filtered_records: &Arc<Mutex<HashMap<String, FilteredCaseDetails>>>,
     filtered_maf_records: &AtomicUsize,
     filtered_cnv_records: &AtomicUsize,
@@ -219,9 +236,6 @@ async fn parse_content(
     let mut parsed_data = Vec::new();
     let mut columns_indices: Vec<usize> = Vec::new();
     let mut variant_classification_index: Option<usize> = None;
-    //let mut header_mk: &str = "";
-    //let mut columns = Vec::new();
-
     let mut header: Vec<String> = Vec::new();
 
     for line in lines {
@@ -243,26 +257,49 @@ async fn parse_content(
             continue;
         };
 
-        let row = process_row(
-            line,
-            case_id,
-            data_type,
-            &header,
-            &columns_indices,
-            variant_classification_index,
-            consequences,
-            min_total_depth,
-            min_alt_allele_count,
-            gain_threshold,
-            loss_threshold,
-            seg_length,
-            filtered_records,
-            filtered_maf_records,
-            filtered_cnv_records,
-            included_maf_records,
-            included_cnv_records,
-        )
-        .await?;
+        let row = match data_type {
+            "maf" => {
+                process_mafline(
+                    line,
+                    case_id,
+                    data_type,
+                    &columns_indices,
+                    variant_classification_index,
+                    consequences,
+                    min_total_depth,
+                    min_alt_allele_count,
+                    chromosomes,
+                    filtered_records,
+                    filtered_maf_records,
+                    included_maf_records,
+                )
+                .await
+            }
+            "cnv" => {
+                process_cnvline(
+                    line,
+                    case_id,
+                    data_type,
+                    &header,
+                    &columns_indices,
+                    gain_threshold,
+                    loss_threshold,
+                    seg_length,
+                    chromosomes,
+                    filtered_records,
+                    filtered_cnv_records,
+                    included_cnv_records,
+                )
+                .await
+            }
+            _ => {
+                return Err((
+                    case_id.to_string(),
+                    data_type.to_string(),
+                    "Invalid data type".to_string(),
+                ));
+            }
+        }?;
 
         if let Some(out_lst) = row {
             parsed_data.push(out_lst);
@@ -316,25 +353,20 @@ fn setup_columns(
     Ok(())
 }
 
-// Process a single row of data
-async fn process_row(
+// Process a single row of MAF file
+async fn process_mafline(
     line: &str,
     case_id: &str,
     data_type: &str,
-    header: &[String],
     columns_indices: &[usize],
     variant_classification_index: Option<usize>,
     consequences: &Option<Vec<String>>,
     min_total_depth: i32,
     min_alt_allele_count: i32,
-    gain_threshold: f32,
-    loss_threshold: f32,
-    seg_length: i32,
+    chromosomes: &HashSet<String>,
     filtered_records: &Arc<Mutex<HashMap<String, FilteredCaseDetails>>>,
     filtered_maf_records: &AtomicUsize,
-    filtered_cnv_records: &AtomicUsize,
     included_maf_records: &AtomicUsize,
-    included_cnv_records: &AtomicUsize,
 ) -> Result<Option<Vec<String>>, (String, String, String)> {
     let cont_lst: Vec<String> = line.split("\t").map(|s| s.to_string()).collect();
     let mut out_lst = vec![case_id.to_string()];
@@ -347,51 +379,38 @@ async fn process_row(
             maf: FilteredMafDetails::default(),
             cnv: FilteredCnvDetails::default(),
         });
-
     let case_details = filtered_map.get_mut(case_id).unwrap();
 
     // Track total processed records
-    if data_type == "maf" {
-        case_details.maf.total_processed += 1;
-    } else if data_type == "cnv" {
-        case_details.cnv.total_processed += 1;
-    }
+    case_details.maf.total_processed += 1;
 
     // Handle consequence filtering and counting for MAF files
-    if data_type == "maf" {
-        if let Some(var_class_idx) = variant_classification_index {
-            if var_class_idx < cont_lst.len() {
-                let variant_classification = &cont_lst[var_class_idx];
-                if let Some(consequence_filter) = consequences {
-                    if !consequence_filter.is_empty() {
-                        if consequence_filter.contains(variant_classification) {
-                            // Matched consequence
-                            *case_details
-                                .maf
-                                .matched_consequences
-                                .entry(variant_classification.to_string())
-                                .or_insert(0) += 1;
-                        } else {
-                            // Unmatched consequence
-                            *case_details
-                                .maf
-                                .rejected_consequences
-                                .entry(variant_classification.to_string())
-                                .or_insert(0) += 1;
-                            case_details.maf.excluded_by_consequence_type += 1;
-                            filtered_maf_records.fetch_add(1, Ordering::Relaxed);
-                            return Ok(None);
-                        }
-                    } else {
-                        // Empty filter, count as matched
+
+    if let Some(var_class_idx) = variant_classification_index {
+        if var_class_idx < cont_lst.len() {
+            let variant_classification = &cont_lst[var_class_idx];
+            if let Some(consequence_filter) = consequences {
+                if !consequence_filter.is_empty() {
+                    if consequence_filter.contains(variant_classification) {
+                        // Matched consequence
                         *case_details
                             .maf
                             .matched_consequences
                             .entry(variant_classification.to_string())
                             .or_insert(0) += 1;
+                    } else {
+                        // Unmatched consequence
+                        *case_details
+                            .maf
+                            .rejected_consequences
+                            .entry(variant_classification.to_string())
+                            .or_insert(0) += 1;
+                        case_details.maf.excluded_by_consequence_type += 1;
+                        filtered_maf_records.fetch_add(1, Ordering::Relaxed);
+                        return Ok(None);
                     }
                 } else {
-                    // No filter, count as matched
+                    // Empty filter, count as matched
                     *case_details
                         .maf
                         .matched_consequences
@@ -399,32 +418,142 @@ async fn process_row(
                         .or_insert(0) += 1;
                 }
             } else {
-                case_details.maf.invalid_rows += 1;
-                filtered_maf_records.fetch_add(1, Ordering::Relaxed);
-                return Ok(None);
+                // No filter, count as matched
+                *case_details
+                    .maf
+                    .matched_consequences
+                    .entry(variant_classification.to_string())
+                    .or_insert(0) += 1;
             }
         } else {
             case_details.maf.invalid_rows += 1;
             filtered_maf_records.fetch_add(1, Ordering::Relaxed);
             return Ok(None);
         }
+    } else {
+        case_details.maf.invalid_rows += 1;
+        filtered_maf_records.fetch_add(1, Ordering::Relaxed);
+        return Ok(None);
     }
 
     // Extract relevant columns
     for &x in columns_indices {
         if x >= cont_lst.len() {
-            if data_type == "maf" {
-                case_details.maf.invalid_rows += 1;
-                filtered_maf_records.fetch_add(1, Ordering::Relaxed);
-            } else if data_type == "cnv" {
-                case_details.cnv.invalid_rows += 1;
-                filtered_cnv_records.fetch_add(1, Ordering::Relaxed);
-            }
+            case_details.maf.invalid_rows += 1;
+            filtered_maf_records.fetch_add(1, Ordering::Relaxed);
             return Ok(None); // Invalid row
         }
+        let element = cont_lst[x].to_string();
+        out_lst.push(element);
+    }
+
+    // Additional MAF-specific processing
+    if out_lst.len() < 6 {
+        case_details.maf.invalid_rows += 1;
+        filtered_maf_records.fetch_add(1, Ordering::Relaxed);
+        return Ok(None); // Not enough columns
+    }
+
+    let alle_depth = out_lst[4].parse::<i32>().map_err(|_| {
+        case_details.maf.invalid_rows += 1;
+        filtered_maf_records.fetch_add(1, Ordering::Relaxed);
+        (
+            case_id.to_string(),
+            data_type.to_string(),
+            "Failed to convert t_depth to integer.".to_string(),
+        )
+    })?;
+
+    let alt_count = out_lst[5].parse::<i32>().map_err(|_| {
+        case_details.maf.invalid_rows += 1;
+        filtered_maf_records.fetch_add(1, Ordering::Relaxed);
+        (
+            case_id.to_string(),
+            data_type.to_string(),
+            "Failed to convert t_alt_count to integer.".to_string(),
+        )
+    })?;
+
+    if alle_depth < min_total_depth {
+        case_details.maf.t_depth += 1;
+        case_details.maf.excluded_by_min_depth += 1;
+        filtered_maf_records.fetch_add(1, Ordering::Relaxed);
+        return Ok(None);
+    }
+    if alt_count < min_alt_allele_count {
+        case_details.maf.t_alt_count += 1;
+        case_details.maf.excluded_by_min_alt_count += 1;
+        filtered_maf_records.fetch_add(1, Ordering::Relaxed);
+        return Ok(None);
+    }
+
+    // Keep case_id, chr, start, end, and add "mutation"
+    out_lst = out_lst[0..4].to_vec();
+    out_lst.push("mutation".to_string());
+
+    // adding 'chr' to chromosome if it is not start with 'chr'
+    if !out_lst[1].starts_with("chr") {
+        out_lst[1] = format!("chr{}", out_lst[1]);
+    }
+
+    // Chromosome filtering
+    if !chromosomes.is_empty() && !chromosomes.contains(&out_lst[1]) {
+        *case_details
+            .maf
+            .skipped_chromosomes
+            .entry(out_lst[1].clone())
+            .or_insert(0) += 1;
+        filtered_maf_records.fetch_add(1, Ordering::Relaxed);
+        return Ok(None);
+    }
+
+    // Update counters for included MAF records
+    case_details.maf.total_included += 1;
+    included_maf_records.fetch_add(1, Ordering::Relaxed);
 
+    Ok(Some(out_lst))
+}
+
+// Process a single row of CNV file
+async fn process_cnvline(
+    line: &str,
+    case_id: &str,
+    data_type: &str,
+    header: &[String],
+    columns_indices: &[usize],
+    gain_threshold: f32,
+    loss_threshold: f32,
+    seg_length: i32,
+    chromosomes: &HashSet<String>,
+    filtered_records: &Arc<Mutex<HashMap<String, FilteredCaseDetails>>>,
+    filtered_cnv_records: &AtomicUsize,
+    included_cnv_records: &AtomicUsize,
+) -> Result<Option<Vec<String>>, (String, String, String)> {
+    let cont_lst: Vec<String> = line.split("\t").map(|s| s.to_string()).collect();
+    let mut out_lst = vec![case_id.to_string()];
+
+    // Initialize or update case details
+    let mut filtered_map = filtered_records.lock().await;
+    filtered_map
+        .entry(case_id.to_string())
+        .or_insert_with(|| FilteredCaseDetails {
+            maf: FilteredMafDetails::default(),
+            cnv: FilteredCnvDetails::default(),
+        });
+    let case_details = filtered_map.get_mut(case_id).unwrap();
+
+    // Track total processed records
+    case_details.cnv.total_processed += 1;
+
+    // Extract relevant columns
+    for &x in columns_indices {
+        if x >= cont_lst.len() {
+            case_details.cnv.invalid_rows += 1;
+            filtered_cnv_records.fetch_add(1, Ordering::Relaxed);
+            return Ok(None); // Invalid row
+        }
         let mut element = cont_lst[x].to_string();
-        if data_type == "cnv" && header[x] == "Segment_Mean" {
+        if header[x] == "Segment_Mean" {
             element = process_segment_mean(&element, case_id, data_type, gain_threshold, loss_threshold)?;
             if element.is_empty() {
                 case_details.cnv.segment_mean += 1;
@@ -444,89 +573,55 @@ async fn process_row(
         out_lst.push(element);
     }
 
-    // Additional MAF-specific processing
-    if data_type == "maf" {
-        if out_lst.len() < 6 {
-            case_details.maf.invalid_rows += 1;
-            filtered_maf_records.fetch_add(1, Ordering::Relaxed);
-            return Ok(None); // Not enough columns
-        }
-
-        let alle_depth = out_lst[4].parse::<i32>().map_err(|_| {
-            case_details.maf.invalid_rows += 1;
-            filtered_maf_records.fetch_add(1, Ordering::Relaxed);
-            (
-                case_id.to_string(),
-                data_type.to_string(),
-                "Failed to convert t_depth to integer.".to_string(),
-            )
-        })?;
-
-        let alt_count = out_lst[5].parse::<i32>().map_err(|_| {
-            case_details.maf.invalid_rows += 1;
-            filtered_maf_records.fetch_add(1, Ordering::Relaxed);
-            (
-                case_id.to_string(),
-                data_type.to_string(),
-                "Failed to convert t_alt_count to integer.".to_string(),
-            )
-        })?;
-
-        if alle_depth < min_total_depth {
-            case_details.maf.t_depth += 1;
-            case_details.maf.excluded_by_min_depth += 1;
-            filtered_maf_records.fetch_add(1, Ordering::Relaxed);
-            return Ok(None);
-        }
-        if alt_count < min_alt_allele_count {
-            case_details.maf.t_alt_count += 1;
-            case_details.maf.excluded_by_min_alt_count += 1;
-            filtered_maf_records.fetch_add(1, Ordering::Relaxed);
-            return Ok(None);
-        }
-
-        // Keep case_id, chr, start, end, and add "mutation"
-        out_lst = out_lst[0..4].to_vec();
-        out_lst.push("mutation".to_string());
+    // filter cnvs based on segment length. Default: 0 (no filtering)
+    // calculate segment length (End_Position - Start_Position)
+    let end_position = out_lst[3].parse::<i32>().map_err(|_| {
+        case_details.cnv.invalid_rows += 1;
+        filtered_cnv_records.fetch_add(1, Ordering::Relaxed);
+        (
+            case_id.to_string(),
+            data_type.to_string(),
+            "Failed to convert End Position of cnv to integer.".to_string(),
+        )
+    })?;
 
-        // Update counters for included MAF records
-        case_details.maf.total_included += 1;
-        included_maf_records.fetch_add(1, Ordering::Relaxed);
+    let start_position = out_lst[2].parse::<i32>().map_err(|_| {
+        case_details.cnv.invalid_rows += 1;
+        filtered_cnv_records.fetch_add(1, Ordering::Relaxed);
+        (
+            case_id.to_string(),
+            data_type.to_string(),
+            "Failed to convert Start Position of cnv to integer.".to_string(),
+        )
+    })?;
+    let cnv_length = end_position - start_position;
+    if seg_length > 0 && cnv_length > seg_length {
+        case_details.cnv.seg_length += 1;
+        case_details.cnv.excluded_by_segment_length += 1;
+        filtered_cnv_records.fetch_add(1, Ordering::Relaxed);
+        return Ok(None);
     }
 
-    // filter cnvs based on segment length. Default: 0 (no filtering)
-    if data_type == "cnv" {
-        // calculate segment length (End_Position - Start_Position)
-        let end_position = out_lst[3].parse::<i32>().map_err(|_| {
-            case_details.cnv.invalid_rows += 1;
-            filtered_cnv_records.fetch_add(1, Ordering::Relaxed);
-            (
-                case_id.to_string(),
-                data_type.to_string(),
-                "Failed to convert End Position of cnv to integer.".to_string(),
-            )
-        })?;
+    // adding 'chr' to chromosome if it is not start with 'chr'
+    if !out_lst[1].starts_with("chr") {
+        out_lst[1] = format!("chr{}", out_lst[1]);
+    }
 
-        let start_position = out_lst[2].parse::<i32>().map_err(|_| {
-            case_details.cnv.invalid_rows += 1;
-            filtered_cnv_records.fetch_add(1, Ordering::Relaxed);
-            (
-                case_id.to_string(),
-                data_type.to_string(),
-                "Failed to convert Start Position of cnv to integer.".to_string(),
-            )
-        })?;
-        let cnv_length = end_position - start_position;
-        if seg_length > 0 && cnv_length > seg_length {
-            case_details.cnv.seg_length += 1;
-            case_details.cnv.excluded_by_segment_length += 1;
-            filtered_cnv_records.fetch_add(1, Ordering::Relaxed);
-            return Ok(None);
-        }
-        case_details.cnv.total_included += 1;
-        included_cnv_records.fetch_add(1, Ordering::Relaxed);
+    // Chromosome filtering
+    if !chromosomes.is_empty() && !chromosomes.contains(&out_lst[1]) {
+        *case_details
+            .cnv
+            .skipped_chromosomes
+            .entry(out_lst[1].clone())
+            .or_insert(0) += 1;
+        filtered_cnv_records.fetch_add(1, Ordering::Relaxed);
+        return Ok(None);
     }
 
+    // Update counters for included MAF records
+    case_details.cnv.total_included += 1;
+    included_cnv_records.fetch_add(1, Ordering::Relaxed);
+
     Ok(Some(out_lst))
 }
 
@@ -658,10 +753,9 @@ async fn download_single_file(
     ))
 }
 
-/// Streaming download function
+/// Downloading from GDC
 /// Outputs JSONL format: one JSON object per line
-/// Node.js will read this line-by-line but still wait for completion
-async fn download_data_streaming(
+async fn download_data(
     data4dl: HashMap<String, DataType>,
     host: &str,
     min_total_depth: i32,
@@ -672,6 +766,8 @@ async fn download_data_streaming(
     loss_threshold: f32,
     seg_length: i32,
     cnv_hyper_mutator: i32,
+    chromosomes: &HashSet<String>,
+    max_record: usize,
 ) {
     let data_urls: Vec<(String, String, String)> = data4dl
         .into_iter()
@@ -696,8 +792,11 @@ async fn download_data_streaming(
     let filtered_cnv_records = Arc::new(AtomicUsize::new(0));
     let filtered_records = Arc::new(Mutex::new(HashMap::<String, FilteredCaseDetails>::new()));
     let hyper_mutator_records = Arc::new(Mutex::new(HashMap::<String, Vec<String>>::new()));
+    let excluded_by_max_record = Arc::new(Mutex::new(HashMap::<String, Vec<String>>::new()));
     let included_maf_records = Arc::new(AtomicUsize::new(0));
     let included_cnv_records = Arc::new(AtomicUsize::new(0));
+    let all_records = Arc::new(Mutex::new(Vec::<Vec<String>>::new()));
+    let data_count = Arc::new(AtomicUsize::new(0));
 
     // Only collect errors (successful data is output immediately)
     let errors = Arc::new(Mutex::new(Vec::<ErrorEntry>::new()));
@@ -720,9 +819,25 @@ async fn download_data_streaming(
             let included_maf_records = Arc::clone(&included_maf_records);
             let included_cnv_records = Arc::clone(&included_cnv_records);
             let hyper_mutator_records = Arc::clone(&hyper_mutator_records);
+            let excluded_by_max_record = Arc::clone(&excluded_by_max_record);
             let errors = Arc::clone(&errors);
+            let all_records = Arc::clone(&all_records);
+            let data_count = Arc::clone(&data_count);
 
             async move {
+                let current_count = data_count.load(Ordering::Relaxed);
+                if current_count >= max_record {
+                    // Skip processing and mark as excluded by max_record
+                    if let Ok((case_id, data_type, _)) = download_result {
+                        let mut exclud_max_record = excluded_by_max_record.lock().await;
+                        exclud_max_record
+                            .entry(data_type.to_string())
+                            .or_insert_with(Vec::new)
+                            .push(case_id.to_string());
+                        successful_downloads.fetch_add(1, Ordering::Relaxed);
+                    }
+                    return;
+                }
                 match download_result {
                     Ok((case_id, data_type, content)) => {
                         // Try to parse the content
@@ -738,6 +853,7 @@ async fn download_data_streaming(
                             loss_threshold,
                             seg_length,
                             cnv_hyper_mutator,
+                            &chromosomes,
                             &filtered_records,
                             &filtered_maf_records,
                             &filtered_cnv_records,
@@ -748,24 +864,18 @@ async fn download_data_streaming(
                         .await
                         {
                             Ok(parsed_data) => {
-                                // SUCCESS: Output immediately as JSONL
-                                let success_output = SuccessfulFileOutput {
-                                    output_type: "data".to_string(),
-                                    case_id: case_id.clone(),
-                                    data_type: data_type.clone(),
-                                    data: parsed_data,
-                                };
-
-                                // Output this successful result immediately - Node.js will see this in real-time
-                                if let Ok(json) = serde_json::to_string(&success_output) {
-                                    println!("{}", json); // IMMEDIATE output to stdout
-                                    // Force flush to ensure Node.js sees it immediately
-                                    use std::io::Write;
-                                    let _ = std::io::stdout().flush();
-                                    // Optional: Add small delay to separate lines
-                                    sleep(Duration::from_millis(10));
+                                let remaining = max_record - current_count;
+                                if parsed_data.len() <= remaining {
+                                    data_count.fetch_add(parsed_data.len(), Ordering::Relaxed);
+                                    all_records.lock().await.extend(parsed_data);
+                                } else {
+                                    // Skip file if it would exceed max_record
+                                    let mut exclud_max_record = excluded_by_max_record.lock().await;
+                                    exclud_max_record
+                                        .entry(data_type.to_string())
+                                        .or_insert_with(Vec::new)
+                                        .push(case_id.to_string());
                                 }
-
                                 successful_downloads.fetch_add(1, Ordering::Relaxed);
                             }
                             Err((cid, dtp, error)) => {
@@ -816,7 +926,6 @@ async fn download_data_streaming(
     let included_cnv_count = included_cnv_records.load(Ordering::Relaxed);
 
     let summary = FinalSummary {
-        output_type: "summary".to_string(),
         total_files,
         successful_files: success_count,
         failed_files: failed_count,
@@ -828,10 +937,214 @@ async fn download_data_streaming(
         included_maf_records: included_maf_count,
         included_cnv_records: included_cnv_count,
         hyper_mutator_records: hyper_mutator_records.lock().await.clone(),
+        excluded_by_max_record: excluded_by_max_record.lock().await.clone(),
     };
 
+    let grin2lesion = serde_json::to_string(&all_records.lock().await.drain(..).collect::<Vec<Vec<String>>>())
+        .unwrap_or_else(|_| "[]".to_string());
+    let output = Output { grin2lesion, summary };
+
     // Output final summary - Node.js will know processing is complete when it sees this
-    if let Ok(json) = serde_json::to_string(&summary) {
+    // if let Ok(json) = serde_json::to_string(&summary) {
+    if let Ok(json) = serde_json::to_string(&output) {
+        println!("{}", json);
+        use std::io::Write;
+        let _ = std::io::stdout().flush();
+    }
+}
+
+/// Read data from local file
+async fn localread_data(
+    case_files: HashMap<String, DataType>,
+    min_total_depth: i32,
+    min_alt_allele_count: i32,
+    maf_hyper_mutator: i32,
+    consequences: &Option<Vec<String>>,
+    gain_threshold: f32,
+    loss_threshold: f32,
+    seg_length: i32,
+    cnv_hyper_mutator: i32,
+    chromosomes: &HashSet<String>,
+    max_record: usize,
+) {
+    let data_files: Vec<(String, String, String)> = case_files
+        .into_iter()
+        .flat_map(|(case_id, data_types)| {
+            let mut files = Vec::new();
+            if let Some(cnv_file) = &data_types.cnv {
+                files.push((case_id.clone(), "cnv".to_string(), cnv_file.clone()));
+            }
+            if let Some(maf_file) = &data_types.maf {
+                files.push((case_id.clone(), "maf".to_string(), maf_file.clone()));
+            }
+            files
+        })
+        .collect();
+    let total_files = data_files.len();
+
+    // Counters for final summary
+    let successful_reads = Arc::new(AtomicUsize::new(0));
+    let failed_reads = Arc::new(AtomicUsize::new(0));
+    let filtered_maf_records = Arc::new(AtomicUsize::new(0));
+    let filtered_cnv_records = Arc::new(AtomicUsize::new(0));
+    let filtered_records = Arc::new(Mutex::new(HashMap::<String, FilteredCaseDetails>::new()));
+    let hyper_mutator_records = Arc::new(Mutex::new(HashMap::<String, Vec<String>>::new()));
+    let excluded_by_max_record = Arc::new(Mutex::new(HashMap::<String, Vec<String>>::new()));
+    let included_maf_records = Arc::new(AtomicUsize::new(0));
+    let included_cnv_records = Arc::new(AtomicUsize::new(0));
+    let errors = Arc::new(Mutex::new(Vec::<ErrorEntry>::new()));
+    let all_records = Arc::new(Mutex::new(Vec::<Vec<String>>::new()));
+    let data_count = Arc::new(AtomicUsize::new(0));
+
+    // Process files concurrently
+    let read_futures = futures::stream::iter(data_files.into_iter().map(
+        |(case_id, data_type, file_path)| async move {
+            // read the local file
+            match fs::read_to_string(&file_path) {
+                Ok(content) => Ok((case_id, data_type, content)),
+                Err(e) => Err((
+                    case_id,
+                    data_type,
+                    format!("file_read_error: {}", e),
+                    1, // Single attempt for local file readng
+                )),
+            }
+        },
+    ));
+
+    // Process files and output results
+    read_futures
+        .buffer_unordered(3)
+        .for_each(|read_result| {
+            let successful_reads = Arc::clone(&successful_reads);
+            let failed_reads = Arc::clone(&failed_reads);
+            let filtered_maf_records = Arc::clone(&filtered_maf_records);
+            let filtered_cnv_records = Arc::clone(&filtered_cnv_records);
+            let filtered_records = Arc::clone(&filtered_records);
+            let included_maf_records = Arc::clone(&included_maf_records);
+            let included_cnv_records = Arc::clone(&included_cnv_records);
+            let hyper_mutator_records = Arc::clone(&hyper_mutator_records);
+            let excluded_by_max_record = Arc::clone(&excluded_by_max_record);
+            let errors = Arc::clone(&errors);
+            let all_records = Arc::clone(&all_records);
+            let data_count = Arc::clone(&data_count);
+
+            async move {
+                let current_count = data_count.load(Ordering::Relaxed);
+                if current_count >= max_record {
+                    // Skip processing and mark as excluded by max_record
+                    if let Ok((case_id, data_type, _)) = read_result {
+                        let mut exclud_max_record = excluded_by_max_record.lock().await;
+                        exclud_max_record
+                            .entry(data_type.to_string())
+                            .or_insert_with(Vec::new)
+                            .push(case_id.to_string());
+                        successful_reads.fetch_add(1, Ordering::Relaxed);
+                    }
+                    return;
+                }
+                match read_result {
+                    Ok((case_id, data_type, content)) => {
+                        match parse_content(
+                            &content,
+                            &case_id,
+                            &data_type,
+                            min_total_depth,
+                            min_alt_allele_count,
+                            maf_hyper_mutator,
+                            consequences,
+                            gain_threshold,
+                            loss_threshold,
+                            seg_length,
+                            cnv_hyper_mutator,
+                            chromosomes,
+                            &filtered_records,
+                            &filtered_maf_records,
+                            &filtered_cnv_records,
+                            &included_maf_records,
+                            &included_cnv_records,
+                            &hyper_mutator_records,
+                        )
+                        .await
+                        {
+                            Ok(parsed_data) => {
+                                let remaining = max_record - current_count;
+                                if parsed_data.len() <= remaining {
+                                    data_count.fetch_add(parsed_data.len(), Ordering::Relaxed);
+                                    all_records.lock().await.extend(parsed_data);
+                                } else {
+                                    // Skip file if it would exceed max_record
+                                    let mut exclud_max_record = excluded_by_max_record.lock().await;
+                                    exclud_max_record
+                                        .entry(data_type.to_string())
+                                        .or_insert_with(Vec::new)
+                                        .push(case_id.to_string());
+                                }
+                                successful_reads.fetch_add(1, Ordering::Relaxed);
+                            }
+                            Err((cid, dtp, error)) => {
+                                failed_reads.fetch_add(1, Ordering::Relaxed);
+                                let error = ErrorEntry {
+                                    case_id: cid,
+                                    data_type: dtp,
+                                    error_type: "parsing_error".to_string(),
+                                    error_details: error,
+                                    attempts_made: 1,
+                                };
+                                errors.lock().await.push(error);
+                            }
+                        }
+                    }
+                    Err((case_id, data_type, error_details, attempts)) => {
+                        failed_reads.fetch_add(1, Ordering::Relaxed);
+                        let (error_type, clean_details) = if error_details.contains(":") {
+                            let parts: Vec<&str> = error_details.splitn(2, ": ").collect();
+                            (parts[0].to_string(), parts[1].to_string())
+                        } else {
+                            ("unknown_error".to_string(), error_details)
+                        };
+                        let error = ErrorEntry {
+                            case_id,
+                            data_type,
+                            error_type,
+                            error_details: clean_details,
+                            attempts_made: attempts,
+                        };
+                        errors.lock().await.push(error);
+                    }
+                }
+            }
+        })
+        .await;
+    // Output final summary as the last line
+    let success_count = successful_reads.load(Ordering::Relaxed);
+    let failed_count = failed_reads.load(Ordering::Relaxed);
+    let filtered_maf_count = filtered_maf_records.load(Ordering::Relaxed);
+    let filtered_cnv_count = filtered_cnv_records.load(Ordering::Relaxed);
+    let included_maf_count = included_maf_records.load(Ordering::Relaxed);
+    let included_cnv_count = included_cnv_records.load(Ordering::Relaxed);
+
+    let summary = FinalSummary {
+        total_files,
+        successful_files: success_count,
+        failed_files: failed_count,
+        errors: errors.lock().await.clone(),
+        filtered_records: filtered_maf_count + filtered_cnv_count,
+        filtered_maf_records: filtered_maf_count,
+        filtered_cnv_records: filtered_cnv_count,
+        filtered_records_by_case: filtered_records.lock().await.clone(),
+        included_maf_records: included_maf_count,
+        included_cnv_records: included_cnv_count,
+        hyper_mutator_records: hyper_mutator_records.lock().await.clone(),
+        excluded_by_max_record: excluded_by_max_record.lock().await.clone(),
+    };
+
+    let grin2lesion = serde_json::to_string(&all_records.lock().await.drain(..).collect::<Vec<Vec<String>>>())
+        .unwrap_or_else(|_| "[]".to_string());
+    let output = Output { grin2lesion, summary };
+
+    // Output final JSON array
+    if let Ok(json) = serde_json::to_string(&output) {
         println!("{}", json);
         use std::io::Write;
         let _ = std::io::stdout().flush();
@@ -840,6 +1153,9 @@ async fn download_data_streaming(
 
 #[tokio::main]
 async fn main() -> Result<(), Box<dyn std::error::Error>> {
+    let args: Vec<String> = env::args().collect();
+    let from_file = args.contains(&"--from-file".to_string());
+
     const HOST: &str = "https://api.gdc.cancer.gov/data/";
 
     // Read input with timeout
@@ -879,6 +1195,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
     }
 
     let case_files = input_js.case_files;
+    let max_record: usize = input_js.max_record;
 
     // Set default maf_options
     let (min_total_depth, min_alt_allele_count, maf_hyper_mutator, consequences) = match input_js.maf_options {
@@ -902,20 +1219,42 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
         None => (0.3, -0.4, 0, 500), // Default values
     };
 
-    // Download data - this will now handle errors gracefully
-    download_data_streaming(
-        case_files,
-        HOST,
-        min_total_depth,
-        min_alt_allele_count,
-        maf_hyper_mutator,
-        &consequences,
-        gain_threshold,
-        loss_threshold,
-        seg_length,
-        cnv_hyper_mutator,
-    )
-    .await;
+    // Convert Vec<String> to HashSet<String> for faster lookup
+    let chromosomes = input_js.chromosomes.into_iter().collect::<HashSet<String>>();
+
+    if from_file {
+        localread_data(
+            case_files,
+            min_total_depth,
+            min_alt_allele_count,
+            maf_hyper_mutator,
+            &consequences,
+            gain_threshold,
+            loss_threshold,
+            seg_length,
+            cnv_hyper_mutator,
+            &chromosomes,
+            max_record,
+        )
+        .await;
+    } else {
+        // Download data from GDC- this will now handle errors gracefully
+        download_data(
+            case_files,
+            HOST,
+            min_total_depth,
+            min_alt_allele_count,
+            maf_hyper_mutator,
+            &consequences,
+            gain_threshold,
+            loss_threshold,
+            seg_length,
+            cnv_hyper_mutator,
+            &chromosomes,
+            max_record,
+        )
+        .await;
+    }
 
     // Always exit successfully - individual file failures are logged but don't stop the process
     Ok(())