@sjcrh/proteinpaint-rust 2.30.2 → 2.33.0

package/index.js

@@ -1,8 +1,9 @@
 const path = require('path'),
 	spawn = require('child_process').spawn,
-	Readable = require('stream').Readable
+	Readable = require('stream').Readable,
+	Transform = require('stream').Transform
 
-exports.run_rust = function(binfile, input_data) {
+exports.run_rust = function (binfile, input_data) {
 	return new Promise((resolve, reject) => {
 		const binpath = path.join(__dirname, '/target/release/', binfile)
 		const ps = spawn(binpath)
@@ -41,3 +42,31 @@ exports.run_rust = function(binfile, input_data) {
 		})
 	})
 }
+
+exports.run_rust_stream = function (binfile, input_data) {
+	const binpath = path.join(__dirname, '/target/release/', binfile)
+	const ps = spawn(binpath)
+	try {
+		Readable.from(input_data).pipe(ps.stdin)
+	} catch (error) {
+		ps.kill()
+		let errmsg = error
+		if (stderr.length) errmsg += `killed run_rust('${binfile}'), stderr: ${stderr.join('').trim()}`
+		reject(errmsg)
+	}
+
+	const childStream = new Transform({
+		transform(chunk, encoding, callback) {
+			this.push(chunk)
+			callback()
+		}
+	})
+	ps.stdout.pipe(childStream)
+	childStream.on('error', err => {
+		reject(err)
+	})
+	childStream.on('close', code => {
+		childStream.end()
+	})
+	return childStream
+}

package/package.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.30.2",
+  "version": "2.33.0",
   "name": "@sjcrh/proteinpaint-rust",
   "description": "Rust-based utilities for proteinpaint",
   "main": "index.js",

package/src/gdcmaf.rs

@@ -1,35 +1,54 @@
 use flate2::read::GzDecoder;
 use flate2::write::GzEncoder;
+use flate2::Compression;
 use serde_json::Value;
-use std::fs::File;
 use std::path::Path;
 use futures::StreamExt;
-use std::io;
-use std::io::{Read,Write};
+use std::io::{self,Read,Write};
 use std::sync::mpsc;
-use std::collections::HashMap;
 
 
-fn gen_map(d:String) -> HashMap<String,Vec<String>> {
-    let mut map: HashMap<String, Vec<String>> = HashMap::new();
-    let mut header: Vec<String> = Vec::new();
+
+fn gen_vec(d:String) -> (Vec<String>,Vec<Vec<u8>>) {
+    let mut maf_bit: Vec<Vec<u8>> = Vec::new();
+    let mut lst_chrom_pos: Vec<String> = Vec::new();
+    let mut header_indices: Vec<usize> = Vec::new();
+    let mut chrom_index: usize = 9999;
+    let mut pos_index: usize = 9999;
     let lines = d.trim_end().split("\n");
     for line in lines {
         if line.starts_with("#") {
             continue
         } else if line.contains("Hugo_Symbol") {
-            header = line.split("\t").map(|s| s.to_string()).collect();
-            for k in &header {
-                map.insert(k.to_string(),Vec::new());
+            let header: Vec<String> = line.split("\t").map(|s| s.to_string()).collect();
+            for col in MAF_COL {
+                let col_index: usize = header.iter().position(|x| x == col).unwrap();
+                header_indices.push(col_index);
+                if col == "Chromosome" {
+                    chrom_index = col_index;
+                } else if col == "Start_Position" {
+                    pos_index = col_index;
+                }
             }
         } else {
             let maf_cont_lst: Vec<String> = line.split("\t").map(|s| s.to_string()).collect();
-            for (i,x) in maf_cont_lst.iter().enumerate() {
-                map.get_mut(&header[i]).map(|val| val.push(x.to_string()));
-            }
+            let mut maf_out_lst: Vec<String> = Vec::new();
+            let mut chrom = String::new();
+            let mut pos = String::new();
+            for (i,x) in header_indices.iter().enumerate() {
+                maf_out_lst.push(maf_cont_lst[*x].to_string());
+                if chrom_index != 9999 && i == chrom_index {
+                    chrom = maf_cont_lst[*x].to_string();
+                } else if pos_index != 9999 && i == pos_index {
+                    pos = maf_cont_lst[*x].to_string();
+                }
+            };
+            maf_out_lst.push("\n".to_string());
+            maf_bit.push(maf_out_lst.join("\t").as_bytes().to_vec());
+            lst_chrom_pos.push(chrom+"\t"+&pos);
         }
-    }
-    map
+    };
+    (lst_chrom_pos,maf_bit)
 }
 
 fn get_sorted_indices(lst: &Vec<String>) -> Vec<usize>{
@@ -66,13 +85,12 @@ const MAF_COL: [&str;96] = ["Hugo_Symbol", "Entrez_Gene_Id", "Center", "NCBI_Bui
 async fn main() -> Result<(),Box<dyn std::error::Error>> {
     // Accepting the piped input json from jodejs and assign to the variable
     // host: GDC host
-    // out_file: save maf to out_file under cachedir
+    // save output into json string
     // url: urls to download single maf files
     let mut buffer = String::new();
     io::stdin().read_line(&mut buffer)?;
     let file_id_lst_js = serde_json::from_str::<Value>(&buffer).expect("Error reading input and serializing to JSON");
     let host = &file_id_lst_js["host"].as_str().unwrap();
-    let out_file = &file_id_lst_js["outFile"].as_str().unwrap();
     let mut url: Vec<String> = Vec::new();
     for v in file_id_lst_js["fileIdLst"].as_array().unwrap() {
         url.push(Path::new(&host).join(&v.as_str().unwrap()).display().to_string());
@@ -84,16 +102,15 @@ async fn main() -> Result<(),Box<dyn std::error::Error>> {
         url.into_iter().map(|url|{
             let txt = tx.clone();
             async move {
-                match reqwest::get(&url).await{
-                    Ok(resp) => {
-                        let content = resp.bytes().await.unwrap();
-                        let mut decoder = GzDecoder::new(&content[..]);
-                        let mut decompressed_content = Vec::new();
-                        decoder.read_to_end(&mut decompressed_content).unwrap();
+                if let Ok(resp) = reqwest::get(&url).await {
+                    let content = resp.bytes().await.unwrap();
+                    let mut decoder = GzDecoder::new(&content[..]);
+                    let mut decompressed_content = Vec::new();
+                    if let Ok(_) = decoder.read_to_end(&mut decompressed_content) {
                         let text = String::from_utf8_lossy(&decompressed_content);
-                        txt.send(text.to_string()).unwrap();
+                        let (lst_chrom_pos,maf_bit) = gen_vec(text.to_string());
+                        txt.send((lst_chrom_pos,maf_bit)).unwrap();
                     }
-                    Err(_) => println!("ERROR downloading {}", url),
                 }
             }
         })
@@ -101,61 +118,25 @@ async fn main() -> Result<(),Box<dyn std::error::Error>> {
     fetches.await;
     drop(tx);
 
-    // write downloaded maf into variable received_values
-    let mut received_values: Vec<String> = Vec::new();
-    for value in rx {
-        received_values.push(value);
-    }
-
-    // store downloaed mafs into one HashMap data sturcture based on the common column names
-    let mut maf = HashMap::new();
-    for maf_data in received_values {
-        if maf.is_empty() {
-            maf = gen_map(maf_data);
-            // remove columns if column name is found from MAF_COL
-            let mut keys_to_remove_in_maf: Vec<String> = Vec::new();
-            for key in maf.keys() {
-                if !(MAF_COL.contains(&key.as_str())) {
-                    keys_to_remove_in_maf.push(key.to_string());
-                }
-            };
-            for key in keys_to_remove_in_maf {
-                maf.remove(&key);
-            }
-        } else {
-            let maf1 = gen_map(maf_data);
-            let keys_in_maf1: Vec<String> = maf1.keys().cloned().collect();
-            for key in keys_in_maf1 {
-                if maf.contains_key(&key) {
-                    let key_value = maf1[&key].clone();
-                    maf.get_mut(&key).map(|val| val.extend(key_value));
-                }
-            }
-        }
-    };
-
-
-    // generate a Vec with "chrom\tpos" for sorting
-    // generated indices after sorting 
+    // write downloaded maf (GZIP format) into a Vector
+    // lst_chrom_pos: a vector including chromsome&position info for sorting maf
+    // idx_sorted: indices after sorting basedon chromsome&position
+    let mut maf_bit: Vec<Vec<u8>> = Vec::new();
     let mut lst_chrom_pos: Vec<String> = Vec::new();
-    for (i,v) in maf["Chromosome"].iter().enumerate() {
-         lst_chrom_pos.push(v.to_owned()+"\t"+&maf["Start_Position"][i]);
+    for (chr_pos_lst,maf_bit_lst) in rx {
+        maf_bit.extend_from_slice(&maf_bit_lst);
+        lst_chrom_pos.extend_from_slice(&chr_pos_lst);
     };
     let idx_sorted = get_sorted_indices(&lst_chrom_pos);
 
-    // write to file
-    let file = File::create(out_file).expect("could not create file");
-    let mut encoder = GzEncoder::new(file, Default::default());
-    encoder.write_all(MAF_COL.join("\t").as_bytes())?;
-    encoder.write_all("\n".as_bytes())?;
+    // output
+    // maf_out_bit: A vector of GZIPPED maf
+    // compress_header: output header
+    let mut encoder = GzEncoder::new(io::stdout(), Compression::default());
+    let _ = encoder.write_all(&MAF_COL.join("\t").as_bytes().to_vec()).expect("Failed to write header");
+    let _ = encoder.write_all(b"\n").expect("Failed to write newline");
     for i in idx_sorted.iter() {
-        let mut val_lst: Vec<String> = Vec::new();
-        for k in MAF_COL {
-            val_lst.push(maf[k][*i].to_owned());
-        };
-        let val_out = val_lst.join("\t")+"\n";
-        encoder.write_all(val_out.as_bytes())?;
+        let _ = encoder.write_all(&maf_bit[*i]).expect("Failed to write file");
     };
-    encoder.finish()?;
     Ok(())
 }