@sjcrh/proteinpaint-rust 2.191.6 → 2.191.7

package/index.js

@@ -88,7 +88,12 @@ export function run_rust(binfile, input_data, args = [], { signal } = {}) {
 // will already trigger an error. `stream_rust()` was heavily tested manually
 // while troubleshooting and fixing the idle rust processes the led to memory leaks
 // as part of `/gdc/mafBuild` handler code, leave this code as-is for now.
-export function stream_rust(binfile, input_data, emitJson) {
+// opts.maxElapsed (ms): how long a tracked 'gdcmaf' process may run before the
+// watchdog kills it. Defaults to 300000 (5 min) when omitted. This is a safety
+// BACKSTOP against leaked/hung processes (e.g. a download that stalls against a
+// slow GDC environment) - it is not itself the cause of a hang; tightening the
+// rust-side per-request timeouts is what makes a stuck download fail fast.
+export function stream_rust(binfile, input_data, emitJson, opts = {}) {
 	const binpath = path.join(__dirname, '/target/release/', binfile)
 
 	const ps = spawn(binpath)
@@ -99,7 +104,7 @@ export function stream_rust(binfile, input_data, emitJson) {
 		}
 	})
 	// we only want to run this interval loop inside a container, not in dev/test CI
-	if (binfile == 'gdcmaf') trackByPid(ps.pid, binfile)
+	if (binfile == 'gdcmaf') trackByPid(ps.pid, binfile, opts.maxElapsed)
 	const stderr = []
 	try {
 		// from route handler -> input_data -> ps.stdin -> ps.stdout -> transformed stream -> express response.pipe()
@@ -193,8 +198,9 @@ const killedPids = new Set() // will be used to detect killed processes, to help
 const PSKILL_INTERVAL_MS = 30000 // every 30 seconds
 let psKillInterval
 
-// default maxElapsed = 5 * 60 * 1000 millisecond = 300000 or 5 minutes, change to 0 to test
-// may allow configuration of maxElapsed by dataset/argument
+// maxElapsed: the watchdog kill threshold in ms; defaults to 5 minutes (300000).
+// Configurable per call via stream_rust(..., { maxElapsed }) (wired from
+// serverconfig.features.gdcMafMaxElapsed by the /gdc/mafBuild handler).
 function trackByPid(pid, name, maxElapsed = 300000) {
 	if (!pid) return
 	// only track by value (integer, string), not reference object

package/package.json

@@ -1,5 +1,5 @@
 {
-	"version": "2.191.6",
+	"version": "2.191.7",
 	"name": "@sjcrh/proteinpaint-rust",
 	"type": "module",
 	"description": "Rust-based utilities for proteinpaint",

package/src/gdcmaf.rs

@@ -30,6 +30,30 @@ struct ErrorEntry {
     error: String,
 }
 
+// Flatten a std::error::Error (e.g. reqwest::Error) and its `source()` chain into
+// a single string, so the real transport-level cause (e.g. "connection closed
+// before message completed", connection reset, HTTP/2 stream error) is visible in
+// the per-file error rather than only the top-level wrapper message.
+fn format_error_chain(e: &dyn std::error::Error) -> String {
+    let mut msg = e.to_string();
+    let mut src = e.source();
+    while let Some(s) = src {
+        msg.push_str(" -> ");
+        msg.push_str(&s.to_string());
+        src = s.source();
+    }
+    msg
+}
+
+// Render the first bytes of a response body for diagnostics: helps tell whether the
+// server returned gzip (magic 1f 8b), plain text, or an HTML/JSON error page.
+fn preview_bytes(content: &[u8]) -> String {
+    let n = content.len().min(64);
+    let hex: Vec<String> = content[..n].iter().map(|b| format!("{:02x}", b)).collect();
+    let ascii = String::from_utf8_lossy(&content[..n]);
+    format!("hex=[{}] ascii=\"{}\"", hex.join(" "), ascii.escape_default())
+}
+
 fn select_maf_col(d: String, columns: &Vec<String>, url: &str) -> Result<(Vec<u8>, i32), (String, String)> {
     let mut maf_str: String = String::new();
     let mut header_indices: Vec<usize> = Vec::new();
@@ -204,7 +228,11 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
     let client = reqwest::Client::builder()
         .timeout(Duration::from_secs(30)) // 30-second timeout per request
         .connect_timeout(Duration::from_secs(15))
-        .pool_max_idle_per_host(0) // avoid "connection closed before message completed" race
+        // Allow keep-alive connection reuse. Previously this was pool_max_idle_per_host(0)
+        // to dodge a "connection closed before message completed" race, but disabling reuse
+        // forces a brand-new connection per file, which is far harsher on a server that caps
+        // concurrent connections (e.g. qa-int). Mid-body drops are now handled by retrying the
+        // whole download (see the retry loop below) rather than by refusing to reuse connections.
         .build()
         .map_err(|e| {
             let client_error = ErrorEntry {
@@ -217,58 +245,123 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
             e
         })?;
 
+    // Number of files downloaded concurrently. Driven by the input JSON "concurrency" field,
+    // which the /gdc/mafBuild handler sets from serverconfig.features.gdcMafConcurrency, so it
+    // can be dialed down per environment (e.g. qa-int, which appears to cap simultaneous
+    // connections) without a rebuild; falls back to 20 when absent.
+    let concurrency = file_id_lst_js
+        .get("concurrency")
+        .and_then(|v| v.as_u64())
+        .filter(|n| *n >= 1)
+        .unwrap_or(20) as usize;
+
     //downloading maf files parallelly and merge them into single maf file
     let download_futures = futures::stream::iter(url.into_iter().map(|url| {
         let client = client.clone();
         let req_headers = req_headers.clone();
         async move {
-            // bounded retry for transient transport failures
-            // (IncompleteMessage / TimedOut / connect), up to 3 attempts with backoff
+            // Bounded retry for transient transport failures, up to MAX_ATTEMPTS with backoff.
+            // The whole download (send + body read) is inside the loop so a mid-body drop
+            // ("connection closed before message completed") is retried, not just connect/send
+            // failures. Non-2xx responses and decompression failures are NOT transient and break
+            // immediately. The metadata captured before consuming the body feeds the error text.
+            const MAX_ATTEMPTS: u32 = 3;
             let mut attempt: u32 = 0;
-            let send_result = loop {
+            loop {
                 attempt += 1;
+
                 let mut request = client.get(&url);
                 for (name, value) in req_headers.iter() {
-                    request = request.header(name.clone(), value.clone()); // == .header("X-Forwarded-Agent", …) etc.
+                    request = request.header(name.clone(), value.clone());
                 }
-                match request.send().await {
-                    Ok(resp) => break Ok(resp),
-                    Err(e) if attempt < 3 && (e.is_request() || e.is_timeout() || e.is_connect()) => {
+
+                // --- send ---
+                let resp = match request.send().await {
+                    Ok(resp) => resp,
+                    Err(e) if attempt < MAX_ATTEMPTS && (e.is_request() || e.is_timeout() || e.is_connect()) => {
                         tokio::time::sleep(Duration::from_millis(500 * attempt as u64)).await;
                         continue;
                     }
-                    Err(e) => break Err(e),
+                    Err(e) => {
+                        break Err((url.clone(), format!("Server request failed: {}", format_error_chain(&e))));
+                    }
+                };
+
+                // Capture response metadata before the body is consumed, for diagnostics.
+                let status = resp.status();
+                let version = format!("{:?}", resp.version());
+                let content_type = resp
+                    .headers()
+                    .get(reqwest::header::CONTENT_TYPE)
+                    .and_then(|v| v.to_str().ok())
+                    .unwrap_or("")
+                    .to_string();
+                let content_encoding = resp
+                    .headers()
+                    .get(reqwest::header::CONTENT_ENCODING)
+                    .and_then(|v| v.to_str().ok())
+                    .unwrap_or("")
+                    .to_string();
+
+                if !status.is_success() {
+                    // non-2xx: a server-side decision, not transient - capture a snippet of the
+                    // error body (often an HTML/JSON page from a proxy/WAF) and stop.
+                    let body_snippet = match resp.text().await {
+                        Ok(t) => t.chars().take(200).collect::<String>().escape_default().to_string(),
+                        Err(_) => String::new(),
+                    };
+                    break Err((
+                        url.clone(),
+                        format!(
+                            "HTTP error {} ({}, content-type '{}', content-encoding '{}'), body: {}",
+                            status, version, content_type, content_encoding, body_snippet
+                        ),
+                    ));
                 }
-            };
-            match send_result {
-                Ok(resp) if resp.status().is_success() => match resp.bytes().await {
-                    Ok(content) => {
-                        let mut decoder = GzDecoder::new(&content[..]);
-                        let mut decompressed_content = Vec::new();
-                        match decoder.read_to_end(&mut decompressed_content) {
-                            Ok(_) => {
-                                let text = String::from_utf8_lossy(&decompressed_content).to_string();
-                                return Ok((url.clone(), text));
-                            }
-                            Err(e) => {
-                                let error_msg = format!("Failed to decompress downloaded MAF file: {}", e);
-                                Err((url.clone(), error_msg))
-                            }
-                        }
+
+                // --- read body ---
+                let content = match resp.bytes().await {
+                    Ok(content) => content,
+                    Err(_e) if attempt < MAX_ATTEMPTS => {
+                        // transport/read failure AFTER headers (e.g. "connection closed before
+                        // message completed") - typically transient, so retry the whole download
+                        tokio::time::sleep(Duration::from_millis(500 * attempt as u64)).await;
+                        continue;
                     }
                     Err(e) => {
-                        let error_msg = format!("Failed to decompress downloaded MAF file: {}", e);
-                        Err((url.clone(), error_msg))
+                        break Err((
+                            url.clone(),
+                            format!(
+                                "Failed to read response body (HTTP {}, {}, content-type '{}', content-encoding '{}'): {}",
+                                status,
+                                version,
+                                content_type,
+                                content_encoding,
+                                format_error_chain(&e)
+                            ),
+                        ));
                     }
-                },
-                Ok(resp) => {
-                    let error_msg = format!("HTTP error: {:?}", resp.status());
-                    Err((url.clone(), error_msg))
-                }
-                Err(e) => {
-                    let error_msg = format!("Server request failed: {:?}", e);
-                    Err((url.clone(), error_msg))
-                }
+                };
+
+                // --- decompress (not transient: do not retry) ---
+                let mut decoder = GzDecoder::new(&content[..]);
+                let mut decompressed_content = Vec::new();
+                break match decoder.read_to_end(&mut decompressed_content) {
+                    Ok(_) => Ok((url.clone(), String::from_utf8_lossy(&decompressed_content).to_string())),
+                    Err(e) => Err((
+                        url.clone(),
+                        format!(
+                            "Failed to decompress MAF file (HTTP {}, {}, content-type '{}', content-encoding '{}', body {} bytes, first bytes {}): {}",
+                            status,
+                            version,
+                            content_type,
+                            content_encoding,
+                            content.len(),
+                            preview_bytes(&content),
+                            e
+                        ),
+                    )),
+                };
             }
         }
     }));
@@ -286,7 +379,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
     }
 
     download_futures
-        .buffer_unordered(10)
+        .buffer_unordered(concurrency)
         .for_each(|result| {
             let encoder = Arc::clone(&encoder); // Clone the Arc for each task
             let maf_col_cp = maf_col.clone();
@@ -338,3 +431,135 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
     io::stderr().flush().expect("Failed to flush stderr");
     Ok(())
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    // ---- select_maf_col ----
+
+    // Minimal MAF-like content: a comment line, a header line (detected by the
+    // "Hugo_Symbol" substring), then data rows.
+    const MAF: &str = "# version 2.4\n\
+        Hugo_Symbol\tEntrez_Gene_Id\tChromosome\tStart_Position\n\
+        TP53\t7157\tchr17\t7577120\n\
+        KRAS\t3845\tchr12\t25398284\n";
+
+    fn cols(list: &[&str]) -> Vec<String> {
+        list.iter().map(|s| s.to_string()).collect()
+    }
+
+    #[test]
+    fn select_maf_col_selects_and_orders_requested_columns() {
+        // request a subset, in an order different from the file's column order
+        let (bytes, rows) = select_maf_col(MAF.to_string(), &cols(&["Chromosome", "Hugo_Symbol"]), "u").unwrap();
+        assert_eq!(rows, 2, "two data rows should be emitted");
+        assert_eq!(String::from_utf8(bytes).unwrap(), "chr17\tTP53\nchr12\tKRAS\n");
+    }
+
+    #[test]
+    fn select_maf_col_skips_comment_lines_and_header() {
+        // comment (#) and header lines must not be counted or emitted as data
+        let (bytes, rows) = select_maf_col(MAF.to_string(), &cols(&["Hugo_Symbol"]), "u").unwrap();
+        assert_eq!(rows, 2);
+        assert_eq!(String::from_utf8(bytes).unwrap(), "TP53\nKRAS\n");
+    }
+
+    #[test]
+    fn select_maf_col_errors_on_missing_column() {
+        let err = select_maf_col(MAF.to_string(), &cols(&["No_Such_Column"]), "the-url").unwrap_err();
+        assert_eq!(err.0, "the-url", "error should carry the url");
+        assert!(
+            err.1.contains("No_Such_Column"),
+            "error should name the missing column: {}",
+            err.1
+        );
+    }
+
+    #[test]
+    fn select_maf_col_header_only_yields_no_rows() {
+        let header_only = "Hugo_Symbol\tChromosome\n";
+        let (bytes, rows) = select_maf_col(header_only.to_string(), &cols(&["Hugo_Symbol"]), "u").unwrap();
+        assert_eq!(rows, 0);
+        assert!(bytes.is_empty());
+    }
+
+    // NOTE: a data row with fewer fields than the header currently panics
+    // (maf_cont_lst[*x] index out of bounds); that ragged-row hardening is
+    // deferred to Step 5, where a test for graceful handling should be added.
+
+    // ---- format_error_chain ----
+
+    #[derive(Debug)]
+    struct TestErr {
+        msg: String,
+        src: Option<Box<dyn std::error::Error + 'static>>,
+    }
+    impl std::fmt::Display for TestErr {
+        fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+            write!(f, "{}", self.msg)
+        }
+    }
+    impl std::error::Error for TestErr {
+        fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
+            self.src.as_deref()
+        }
+    }
+
+    #[test]
+    fn format_error_chain_single_error() {
+        let e = TestErr {
+            msg: "top".into(),
+            src: None,
+        };
+        assert_eq!(format_error_chain(&e), "top");
+    }
+
+    #[test]
+    fn format_error_chain_walks_source_chain() {
+        let root = TestErr {
+            msg: "root".into(),
+            src: None,
+        };
+        let middle = TestErr {
+            msg: "middle".into(),
+            src: Some(Box::new(root)),
+        };
+        let top = TestErr {
+            msg: "top".into(),
+            src: Some(Box::new(middle)),
+        };
+        assert_eq!(format_error_chain(&top), "top -> middle -> root");
+    }
+
+    // ---- preview_bytes ----
+
+    #[test]
+    fn preview_bytes_shows_gzip_magic() {
+        // gzip magic 1f 8b should be visible so we can tell gzip from plaintext
+        let out = preview_bytes(&[0x1f, 0x8b, 0x08, 0x00]);
+        assert!(out.contains("hex=[1f 8b 08 00]"), "got: {}", out);
+    }
+
+    #[test]
+    fn preview_bytes_renders_plaintext_in_ascii() {
+        let out = preview_bytes(b"Hugo_Symbol");
+        assert!(
+            out.contains("Hugo_Symbol"),
+            "ascii preview should show plaintext: {}",
+            out
+        );
+    }
+
+    #[test]
+    fn preview_bytes_truncates_to_64_bytes() {
+        // 100 'A' (0x41) bytes -> only 64 should be rendered in the hex section
+        let out = preview_bytes(&[0x41u8; 100]);
+        assert_eq!(
+            out.matches("41").count(),
+            64,
+            "should cap the hex preview at 64 bytes: {}",
+            out
+        );
+    }
+}