@sjcrh/proteinpaint-rust 2.99.0 → 2.110.0

package/index.js

@@ -43,16 +43,22 @@ exports.run_rust = function (binfile, input_data) {
 	})
 }
 
-exports.run_rust_stream = function (binfile, input_data) {
+exports.stream_rust = function (binfile, input_data, emitJson) {
 	const binpath = path.join(__dirname, '/target/release/', binfile)
 	const ps = spawn(binpath)
+	const stderr = []
 	try {
+		// from GDC API -> ps.stdin -> ps.stdout -> transformed stream
 		Readable.from(input_data).pipe(ps.stdin)
+		//reader.on('data', ps.stdout.pipe)
+		//reader.on('error', ps.stderr.pipe)
+		//return reader
 	} catch (error) {
 		ps.kill()
 		let errmsg = error
-		if (stderr.length) errmsg += `killed run_rust('${binfile}'), stderr: ${stderr.join('').trim()}`
-		reject(errmsg)
+		//if (stderr.length) errmsg += `killed run_rust('${binfile}'), stderr: ${stderr.join('').trim()}`
+		//reject(errmsg)
+		console.log(59, error)
 	}
 
 	const childStream = new Transform({
@@ -62,11 +68,23 @@ exports.run_rust_stream = function (binfile, input_data) {
 		}
 	})
 	ps.stdout.pipe(childStream)
-	childStream.on('error', err => {
-		reject(err)
+	ps.stderr.on('data', data => stderr.push(data))
+	ps.on('close', code => { //console.log(72, stderr.length)
+		if (stderr.length) {
+			// handle rust stderr
+			const err = stderr.join('').trim()
+			const errmsg = `!!! stream_rust('${binfile}') stderr: !!!\n${err}`
+			console.log(errmsg)
+			emitJson(err)
+		} else {
+			emitJson({ ok: true, status: 'ok', message: 'Processing complete' })
+		}
 	})
-	childStream.on('close', code => {
-		childStream.end()
+	ps.on('error', err => {
+		console.log(74, `stream_rust().on('error')`, err)
+		emitJson(stderr.join('').trim())
 	})
+	// below will duplicate ps.on('close') event above
+	// childStream.on('end', () => console.log(`-- childStream done --`))
 	return childStream
 }

package/package.json

@@ -1,5 +1,5 @@
 {
-	"version": "2.99.0",
+	"version": "2.110.0",
 	"name": "@sjcrh/proteinpaint-rust",
 	"description": "Rust-based utilities for proteinpaint",
 	"main": "index.js",
@@ -38,5 +38,5 @@
 	"devDependencies": {
 		"tape": "^5.2.2"
 	},
-	"pp_release_tag": "v2.99.0"
+	"pp_release_tag": "v2.110.0"
 }

package/src/gdcmaf.rs

@@ -13,17 +13,25 @@
 use flate2::read::GzDecoder;
 use flate2::write::GzEncoder;
 use flate2::Compression;
-use serde_json::Value;
+use serde_json::{Value,json};
 use std::path::Path;
 use futures::StreamExt;
 use std::io::{self,Read,Write};
+use std::sync::Mutex;
 
 
+// Struct to hold error information
+#[derive(serde::Serialize)]
+struct ErrorEntry {
+    url: String,
+    error: String,
+}
 
-fn select_maf_col(d:String,columns:&Vec<String>) -> Vec<u8> {
+fn select_maf_col(d:String,columns:&Vec<String>,url:&str,errors: &Mutex<Vec<ErrorEntry>>) -> (Vec<u8>,i32) {
     let mut maf_str: String = String::new();
     let mut header_indices: Vec<usize> = Vec::new();
     let lines = d.trim_end().split("\n");
+    let mut mafrows = 0;
     for line in lines {
         if line.starts_with("#") {
             continue
@@ -33,6 +41,11 @@ fn select_maf_col(d:String,columns:&Vec<String>) -> Vec<u8> {
                 if let Some(index) = header.iter().position(|x| x == col) {
                     header_indices.push(index);
                 } else {
+                    let error_msg = format!("Column {} was not found", col);
+                    errors.lock().unwrap().push(ErrorEntry {
+                        url: url.to_string().clone(),
+                        error: error_msg.clone(),
+                    });
                     panic!("{} was not found!",col);
                 }
             }
@@ -44,14 +57,17 @@ fn select_maf_col(d:String,columns:&Vec<String>) -> Vec<u8> {
             };
             maf_str.push_str(maf_out_lst.join("\t").as_str());
             maf_str.push_str("\n");
+            mafrows += 1;
         }
     };
-    maf_str.as_bytes().to_vec()
+    (maf_str.as_bytes().to_vec(),mafrows)
 }
 
 
 #[tokio::main]
 async fn main() -> Result<(),Box<dyn std::error::Error>> {
+    // Create a thread-container for errors
+    let errors = Mutex::new(Vec::<ErrorEntry>::new());
     // Accepting the piped input json from jodejs and assign to the variable
     // host: GDC host
     // url: urls to download single maf files
@@ -75,9 +91,17 @@ async fn main() -> Result<(),Box<dyn std::error::Error>> {
                 .map(|v| v.to_string().replace("\"",""))
                 .collect::<Vec<String>>();
         } else {
+            errors.lock().unwrap().push(ErrorEntry {
+                url: String::new(),
+                error: "The columns of arg is not an array".to_string(),
+            });
             panic!("Columns is not an array");
         }
     } else {
+        errors.lock().unwrap().push(ErrorEntry {
+            url: String::new(),
+            error: "The key columns is missed from arg".to_string(),
+        });
         panic!("Columns was not selected");
     };
     
@@ -85,39 +109,89 @@ async fn main() -> Result<(),Box<dyn std::error::Error>> {
     let download_futures = futures::stream::iter(
         url.into_iter().map(|url|{
             async move {
-                let result = reqwest::get(&url).await;
-                if let Ok(resp) = result {
-                    let content = resp.bytes().await.unwrap();
-                    let mut decoder = GzDecoder::new(&content[..]);
-                    let mut decompressed_content = Vec::new();
-                    let read_content = decoder.read_to_end(&mut decompressed_content);
-                    if let Ok(_) = read_content {
-                        let text = String::from_utf8_lossy(&decompressed_content).to_string();
-                        text
-                    } else {
-                        let error_msg = "Failed to read content downloaded from: ".to_string() + &url;
-                        error_msg
+                match reqwest::get(&url).await {
+                    Ok(resp) if resp.status().is_success() => {
+                        match resp.bytes().await {
+                            Ok(content) => {
+                                let mut decoder = GzDecoder::new(&content[..]);
+                                let mut decompressed_content = Vec::new();
+                                match decoder.read_to_end(&mut decompressed_content) {
+                                    Ok(_) => {
+                                        let text = String::from_utf8_lossy(&decompressed_content).to_string();
+                                        return Ok((url.clone(),text))
+                                    }
+                                    Err(e) => {
+                                        let error_msg = format!("Decompression failed: {}", e);
+                                        Err((url.clone(), error_msg))
+                                    }
+                                }
+                            }
+                            Err(e) => {
+                                let error_msg = format!("Decompression failed: {}", e);
+                                Err((url.clone(), error_msg))
+                            }
+                        }
+                    }
+                    Ok(resp) => {
+                        let error_msg = format!("HTTP error: {}", resp.status());
+                        Err((url.clone(), error_msg))
+                    }
+                    Err(e) => {
+                        let error_msg = format!("Server request failed: {}", e);
+                        Err((url.clone(), error_msg))
                     }
-                } else {
-                    let error_msg = "Failed to download: ".to_string() + &url;
-                    error_msg
                 }
             }
         })
     );
 
-    // output
+    // binary output
     let mut encoder = GzEncoder::new(io::stdout(), Compression::default());
     let _ = encoder.write_all(&maf_col.join("\t").as_bytes().to_vec()).expect("Failed to write header");
     let _ = encoder.write_all(b"\n").expect("Failed to write newline");
-    download_futures.buffer_unordered(20).for_each(|item| {
-        if item.starts_with("Failed") {
-            eprintln!("{}",item);
-        } else {
-            let maf_bit = select_maf_col(item,&maf_col);
-            let _ = encoder.write_all(&maf_bit).expect("Failed to write file");
-        };
-        async {}
-    }).await;
+
+    // Collect all results before processing
+    let results = download_futures.buffer_unordered(50).collect::<Vec<_>>().await;
+
+    // Process results after all downloads are complete
+    for result in results {
+        match result {
+            Ok((url, content)) => {
+                let (maf_bit,mafrows) = select_maf_col(content, &maf_col, &url, &errors);
+                if mafrows > 0 {
+                    let _  = encoder.write_all(&maf_bit).expect("Failed to write file");
+                } else {
+                    errors.lock().unwrap().push(ErrorEntry {
+                        url: url.clone(),
+                        error: "Empty maf file".to_string(),
+                    });
+                }
+            }
+            Err((url, error)) => {
+                errors.lock().unwrap().push(ErrorEntry {
+                    url,
+                    error,
+                })
+            }
+        }
+    };
+
+    // Finalize output and printing errors
+    encoder.finish().expect("Maf file output error!");
+
+    // Manually flush stdout
+    io::stdout().flush().expect("Failed to flush stdout");
+    
+    // After processing all downloads, output the errors as JSON to stderr
+    let errors = errors.lock().unwrap();
+    if !errors.is_empty() {
+        let error_json = json!({
+            "errors": errors.iter().collect::<Vec<&ErrorEntry>>()
+        });
+        let mut stderr = io::stderr();
+        writeln!(stderr, "{}", error_json).expect("Failed to output stderr!");
+        io::stderr().flush().expect("Failed to flush stderr");
+    };
+
     Ok(())
 }