@sjcrh/proteinpaint-rust 2.30.2 → 2.31.0

package/package.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.30.2",
+  "version": "2.31.0",
   "name": "@sjcrh/proteinpaint-rust",
   "description": "Rust-based utilities for proteinpaint",
   "main": "index.js",

package/src/gdcmaf.rs

@@ -1,35 +1,56 @@
 use flate2::read::GzDecoder;
 use flate2::write::GzEncoder;
+use flate2::Compression;
 use serde_json::Value;
-use std::fs::File;
 use std::path::Path;
 use futures::StreamExt;
 use std::io;
 use std::io::{Read,Write};
 use std::sync::mpsc;
-use std::collections::HashMap;
 
 
-fn gen_map(d:String) -> HashMap<String,Vec<String>> {
-    let mut map: HashMap<String, Vec<String>> = HashMap::new();
-    let mut header: Vec<String> = Vec::new();
+
+fn gen_vec(d:String) -> (Vec<String>,Vec<Vec<u8>>) {
+    let mut maf_bit: Vec<Vec<u8>> = Vec::new();
+    let mut lst_chrom_pos: Vec<String> = Vec::new();
+    let mut header_indices: Vec<usize> = Vec::new();
+    let mut chrom_index: usize = 9999;
+    let mut pos_index: usize = 9999;
     let lines = d.trim_end().split("\n");
     for line in lines {
         if line.starts_with("#") {
             continue
         } else if line.contains("Hugo_Symbol") {
-            header = line.split("\t").map(|s| s.to_string()).collect();
-            for k in &header {
-                map.insert(k.to_string(),Vec::new());
+            let header: Vec<String> = line.split("\t").map(|s| s.to_string()).collect();
+            for col in MAF_COL {
+                let col_index: usize = header.iter().position(|x| x == col).unwrap();
+                header_indices.push(col_index);
+                if col == "Chromosome" {
+                    chrom_index = col_index;
+                } else if col == "Start_Position" {
+                    pos_index = col_index;
+                }
             }
         } else {
             let maf_cont_lst: Vec<String> = line.split("\t").map(|s| s.to_string()).collect();
-            for (i,x) in maf_cont_lst.iter().enumerate() {
-                map.get_mut(&header[i]).map(|val| val.push(x.to_string()));
-            }
+            let mut maf_out_lst: Vec<String> = Vec::new();
+            let mut chrom = String::new();
+            let mut pos = String::new();
+            for (i,x) in header_indices.iter().enumerate() {
+                maf_out_lst.push(maf_cont_lst[*x].to_string());
+                if chrom_index != 9999 && i == chrom_index {
+                    chrom = maf_cont_lst[*x].to_string();
+                } else if pos_index != 9999 && i == pos_index {
+                    pos = maf_cont_lst[*x].to_string();
+                }
+            };
+            maf_out_lst.push("\n".to_string());
+            let maf_compress_data = gen_gzip_vec(maf_out_lst);
+            maf_bit.push(maf_compress_data);
+            lst_chrom_pos.push(chrom+"\t"+&pos);
         }
-    }
-    map
+    };
+    (lst_chrom_pos,maf_bit)
 }
 
 fn get_sorted_indices(lst: &Vec<String>) -> Vec<usize>{
@@ -41,6 +62,14 @@ fn get_sorted_indices(lst: &Vec<String>) -> Vec<usize>{
     indices
 }
 
+// convert vector (maf row) to GZIP encoded format
+fn gen_gzip_vec(s:Vec<String>) -> Vec<u8> {
+    let mut encoder = GzEncoder::new(Vec::new(), Compression::default());
+    let _ = encoder.write_all(s.join("\t").as_bytes());
+    let compress_data = encoder.finish().unwrap();
+    compress_data
+}
+
 // GDC MAF columns (96)
 const MAF_COL: [&str;96] = ["Hugo_Symbol", "Entrez_Gene_Id", "Center", "NCBI_Build", "Chromosome", 
                             "Start_Position", "End_Position", "Strand", "Variant_Classification", 
@@ -66,13 +95,12 @@ const MAF_COL: [&str;96] = ["Hugo_Symbol", "Entrez_Gene_Id", "Center", "NCBI_Bui
 async fn main() -> Result<(),Box<dyn std::error::Error>> {
     // Accepting the piped input json from jodejs and assign to the variable
     // host: GDC host
-    // out_file: save maf to out_file under cachedir
+    // save output into json string
     // url: urls to download single maf files
     let mut buffer = String::new();
     io::stdin().read_line(&mut buffer)?;
     let file_id_lst_js = serde_json::from_str::<Value>(&buffer).expect("Error reading input and serializing to JSON");
     let host = &file_id_lst_js["host"].as_str().unwrap();
-    let out_file = &file_id_lst_js["outFile"].as_str().unwrap();
     let mut url: Vec<String> = Vec::new();
     for v in file_id_lst_js["fileIdLst"].as_array().unwrap() {
         url.push(Path::new(&host).join(&v.as_str().unwrap()).display().to_string());
@@ -89,9 +117,11 @@ async fn main() -> Result<(),Box<dyn std::error::Error>> {
                         let content = resp.bytes().await.unwrap();
                         let mut decoder = GzDecoder::new(&content[..]);
                         let mut decompressed_content = Vec::new();
-                        decoder.read_to_end(&mut decompressed_content).unwrap();
-                        let text = String::from_utf8_lossy(&decompressed_content);
-                        txt.send(text.to_string()).unwrap();
+                        if let Ok(_) = decoder.read_to_end(&mut decompressed_content) {
+                            let text = String::from_utf8_lossy(&decompressed_content);
+                            let (lst_chrom_pos,maf_bit) = gen_vec(text.to_string());
+                            txt.send((lst_chrom_pos,maf_bit)).unwrap();
+                        }
                     }
                     Err(_) => println!("ERROR downloading {}", url),
                 }
@@ -101,61 +131,31 @@ async fn main() -> Result<(),Box<dyn std::error::Error>> {
     fetches.await;
     drop(tx);
 
-    // write downloaded maf into variable received_values
-    let mut received_values: Vec<String> = Vec::new();
-    for value in rx {
-        received_values.push(value);
-    }
-
-    // store downloaed mafs into one HashMap data sturcture based on the common column names
-    let mut maf = HashMap::new();
-    for maf_data in received_values {
-        if maf.is_empty() {
-            maf = gen_map(maf_data);
-            // remove columns if column name is found from MAF_COL
-            let mut keys_to_remove_in_maf: Vec<String> = Vec::new();
-            for key in maf.keys() {
-                if !(MAF_COL.contains(&key.as_str())) {
-                    keys_to_remove_in_maf.push(key.to_string());
-                }
-            };
-            for key in keys_to_remove_in_maf {
-                maf.remove(&key);
-            }
-        } else {
-            let maf1 = gen_map(maf_data);
-            let keys_in_maf1: Vec<String> = maf1.keys().cloned().collect();
-            for key in keys_in_maf1 {
-                if maf.contains_key(&key) {
-                    let key_value = maf1[&key].clone();
-                    maf.get_mut(&key).map(|val| val.extend(key_value));
-                }
-            }
-        }
-    };
-
-
-    // generate a Vec with "chrom\tpos" for sorting
-    // generated indices after sorting 
+    // write downloaded maf (GZIP format) into a Vector
+    // lst_chrom_pos: a vector including chromsome&position info for sorting maf
+    // idx_sorted: indices after sorting basedon chromsome&position
+    let mut maf_bit: Vec<Vec<u8>> = Vec::new();
     let mut lst_chrom_pos: Vec<String> = Vec::new();
-    for (i,v) in maf["Chromosome"].iter().enumerate() {
-         lst_chrom_pos.push(v.to_owned()+"\t"+&maf["Start_Position"][i]);
+    for (chr_pos_lst,maf_bit_lst) in rx {
+        maf_bit.extend_from_slice(&maf_bit_lst);
+        lst_chrom_pos.extend_from_slice(&chr_pos_lst);
     };
     let idx_sorted = get_sorted_indices(&lst_chrom_pos);
 
-    // write to file
-    let file = File::create(out_file).expect("could not create file");
-    let mut encoder = GzEncoder::new(file, Default::default());
-    encoder.write_all(MAF_COL.join("\t").as_bytes())?;
-    encoder.write_all("\n".as_bytes())?;
+    // output
+    // maf_out_bit: A vector of GZIPPED maf
+    // compress_header: output header
+    let mut maf_out_bit: Vec<u8> = Vec::new();
+    let compress_header = gen_gzip_vec(MAF_COL.iter().map(|s| s.to_string()).collect());
+    maf_out_bit.extend(compress_header);
+    let compress_header_line_break = gen_gzip_vec(["\n".to_string()].to_vec());
+    maf_out_bit.extend(compress_header_line_break);
     for i in idx_sorted.iter() {
-        let mut val_lst: Vec<String> = Vec::new();
-        for k in MAF_COL {
-            val_lst.push(maf[k][*i].to_owned());
-        };
-        let val_out = val_lst.join("\t")+"\n";
-        encoder.write_all(val_out.as_bytes())?;
+        maf_out_bit.extend(&maf_bit[*i]);
     };
-    encoder.finish()?;
+
+    // standard output
+    println!("{:?}",maf_out_bit);
+    std::io::stdout().flush().expect("Failed to flush stdout");
     Ok(())
 }