@sjcrh/proteinpaint-rust 2.193.0 → 2.195.0

package/Cargo.toml

@@ -75,10 +75,6 @@ path="src/sv.rs"
 name="cluster"
 path="src/cluster.rs"
 
-[[bin]]
-name="gdcmaf"
-path="src/gdcmaf.rs"
-
 [[bin]]
 name="wilcoxon"
 path="src/wilcoxon.rs"

package/index.js

@@ -13,14 +13,11 @@ Standard output of the rust binary is returned.
 // Import necessary modules
 import fs from 'fs'
 import path from 'path'
-import { spawn, exec } from 'child_process'
-import { Readable, Transform } from 'stream'
-import { promisify } from 'util'
+import { spawn } from 'child_process'
+import { Readable } from 'stream'
 
 const __dirname = import.meta.dirname // set __dirname for consistency with cjs code
 
-const execPromise = promisify(exec)
-
 // Check if rust binary directory exists and is not empty
 const binaryDir = path.join(__dirname, '/target/release/')
 if (!fs.existsSync(binaryDir)) throw `missing rust binary directory='${binaryDir}'`
@@ -82,162 +79,3 @@ export function run_rust(binfile, input_data, args = [], { signal } = {}) {
 		})
 	})
 }
-
-// May need to add an abort signal as argument like in run_rust above.
-// Or it's likely not needed since a closed stream connection from a web browser
-// will already trigger an error. `stream_rust()` was heavily tested manually
-// while troubleshooting and fixing the idle rust processes the led to memory leaks
-// as part of `/gdc/mafBuild` handler code, leave this code as-is for now.
-export function stream_rust(binfile, input_data, emitJson) {
-	const binpath = path.join(__dirname, '/target/release/', binfile)
-
-	const ps = spawn(binpath)
-	const childStream = new Transform({
-		transform(chunk, encoding, callback) {
-			this.push(chunk)
-			callback()
-		}
-	})
-	// we only want to run this interval loop inside a container, not in dev/test CI
-	if (binfile == 'gdcmaf') trackByPid(ps.pid, binfile)
-	const stderr = []
-	try {
-		// from route handler -> input_data -> ps.stdin -> ps.stdout -> transformed stream -> express response.pipe()
-		Readable.from(input_data)
-			.pipe(ps.stdin)
-			.on('error', err => {
-				emitErrors({ error: `error piping input data to spawned ${binfile} process` })
-			})
-	} catch (error) {
-		console.log(`Error piping input_data into ${binfile}`, error)
-		return
-	}
-
-	// uncomment to trigger childStream.destroy()
-	// setTimeout(() => { console.log(74, 'childStream.destroy()'); childStream.destroy();}, 1000)
-	// childStream.destroy() does not seem to trigger ps.stdout.pipe('...').on('error') callback,
-	// which is okay as long as the server doesn't crash and ps get's killed eventually
-	ps.stdout.pipe(childStream).on('error', err => console.log('ps.stdout.pipe(childStream) error', err))
-
-	ps.stderr.on('data', data => stderr.push(data))
-
-	ps.on('close', code => {
-		if (trackedPids.has(ps.pid)) trackedPids.delete(ps.pid)
-		if (stderr.length || killedPids.has(ps.pid) || code !== 0) {
-			emitErrors(null, ps.pid, code)
-		} else {
-			emitJson()
-		}
-	})
-	ps.on('error', err => {
-		if (trackedPids.has(ps.pid)) trackedPids.delete(ps.pid)
-		// console.log(74, `stream_rust().on('error')`, err)
-		emitErrors(null, ps.pid)
-	})
-	ps.on('SIGTERM', err => {
-		console.log(err)
-	})
-
-	function emitErrors(error, pid, code = 0) {
-		// concatenate stderr uint8arr into a string
-		let errors = stderr.join('').trim()
-		if (error) errors += `\n` + error
-		if (pid && killedPids.has(ps.pid) && !trackedPids.has(ps.pid)) {
-			errors += '\n' + JSON.stringify({ error: `server error: MAF file processing terminated (expired process)` })
-			killedPids.delete(pid)
-		} else if (pid && code !== 0) {
-			// may result from errors in spawned process code, or external signal (like `kill -9` in terminal)
-			errors += '\n' + JSON.stringify({ error: `server error: MAF file processing terminated (code=${code})` })
-		}
-		emitJson(errors)
-	}
-
-	// on('end') will duplicate ps.on('close') event above
-	// childStream.on('end', () => console.log(`childStream.on(end)`))
-
-	// this may duplicate ps.on('error'), unless the error happened within the transform
-	childStream.on('error', err => {
-		console.log('stream_rust childStream.on(error)', err)
-		try {
-			childStream.destroy(err)
-		} catch (e) {
-			console.log(e)
-		}
-	})
-
-	function endStream() {
-		try {
-			if (!childStream.writableEnded) {
-				console.log('trigger childStream.destroy() in endStream()')
-				childStream.destroy()
-			}
-		} catch (e) {
-			console.log('error triggering childStream.destroy()', e)
-		}
-		try {
-			if (!ps.killed) {
-				console.log('trigger ps.kill() in endStream()')
-				ps.kill()
-			}
-			if (trackedPids.has(ps.pid)) trackedPids.delete(ps.pid)
-		} catch (e) {
-			console.log('error triggering ps.kill()', e)
-		}
-	}
-
-	return { rustStream: childStream, endStream }
-}
-
-const trackedPids = new Map() // will be used to monitor expired processes
-const killedPids = new Set() // will be used to detect killed processes, to help with error detection
-const PSKILL_INTERVAL_MS = 30000 // every 30 seconds
-let psKillInterval
-
-// default maxElapsed = 5 * 60 * 1000 millisecond = 300000 or 5 minutes, change to 0 to test
-// may allow configuration of maxElapsed by dataset/argument
-function trackByPid(pid, name, maxElapsed = 300000) {
-	if (!pid) return
-	// only track by value (integer, string), not reference object
-	// NOTE: a reused/reassigned process.pid will be replaced by the most recent process
-	trackedPids.set(pid, { name, expires: Date.now() + maxElapsed })
-	if (!psKillInterval) psKillInterval = setInterval(killExpiredProcesses, PSKILL_INTERVAL_MS)
-	// uncomment below to test
-	// console.log([...trackedPids.entries()])
-	// if (maxElapsed < 10000) setTimeout(killExpiredProcesses, 1000) // uncomment for testing only
-}
-
-//
-// Use one setInterval() to monitor >= 1 process,
-// instead of a separate setTimeout() for each process.
-// This is more reliable as setTimeout would use spawned ps.kill(),
-// which may not exist when the timeout callback is executed and
-// thus would require clearTimeout(closured_variable). Tracking by
-// pid does not rely on a usable 'ps' variable to kill itself.
-//
-function killExpiredProcesses() {
-	//console.log(149, 'killExpiredProcesses()')
-	killedPids.clear()
-	const time = Date.now()
-	for (const [pid, info] of trackedPids.entries()) {
-		if (info.expires > time) continue
-		try {
-			// true if process exists
-			process.kill(pid, 0)
-		} catch (_) {
-			// no need to kill, but remove from tracking
-			trackedPids.delete(pid)
-			// prevent misleading logs of 'unable to kill ...'
-			continue
-		}
-		const label = `rust process ${info.name} (pid=${pid})`
-		try {
-			// detect if process exists before killing it
-			process.kill(pid, 'SIGTERM')
-			trackedPids.delete(pid)
-			killedPids.add(pid)
-			console.log(`killed ${label}`)
-		} catch (err) {
-			console.log(`unable to kill ${label}`, err)
-		}
-	}
-}

package/package.json

@@ -1,5 +1,5 @@
 {
-	"version": "2.193.0",
+	"version": "2.195.0",
 	"name": "@sjcrh/proteinpaint-rust",
 	"type": "module",
 	"description": "Rust-based utilities for proteinpaint",

package/src/gdcmaf.rs

@@ -1,318 +0,0 @@
-/*
-  This script download cohort maf files from GDC, concatenate them into a single file that includes user specified columns.
-
-  Input JSON:
-    host: GDC host
-    fileIdLst: An array of uuid
-    headers: required headers for GDC API
-  Output gzip compressed maf file to stdout.
-
-  Example of usage:
-      headers='{"X-Forwarded-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.…ML, like Gecko) Chrome/142.0.0.0 Safari/537.36 Edg/142.0.0.0", "X-Forwarded-For": "127.0.0.1"}'
-    echo '{"host": "https://api.gdc.cancer.gov/data/","columns": ["Hugo_Symbol", "Entrez_Gene_Id", "Center", "NCBI_Build", "Chromosome", "Start_Position"], "fileIdLst": ["8b31d6d1-56f7-4aa8-b026-c64bafd531e7", "b429fcc1-2b59-4b4c-a472-fb27758f6249"], "headers": '$headers'}'|./target/release/gdcmaf
-*/
-
-use flate2::Compression;
-use flate2::read::GzDecoder;
-use flate2::write::GzEncoder;
-use futures::StreamExt;
-use serde_json::Value;
-use std::io::{self, Read, Write};
-use std::sync::{Arc, Mutex};
-use std::time::Duration;
-use tokio::io::{AsyncReadExt, BufReader};
-use tokio::time::timeout;
-
-// Struct to hold error information
-#[derive(serde::Serialize)]
-struct ErrorEntry {
-    url: String,
-    error: String,
-}
-
-fn select_maf_col(d: String, columns: &Vec<String>, url: &str) -> Result<(Vec<u8>, i32), (String, String)> {
-    let mut maf_str: String = String::new();
-    let mut header_indices: Vec<usize> = Vec::new();
-    let lines = d.trim_end().split("\n");
-    let mut mafrows = 0;
-    for line in lines {
-        if line.starts_with("#") {
-            continue;
-        } else if line.contains("Hugo_Symbol") {
-            let header: Vec<String> = line.split("\t").map(|s| s.to_string()).collect();
-            for col in columns {
-                match header.iter().position(|x| x == col) {
-                    Some(index) => {
-                        header_indices.push(index);
-                    }
-                    None => {
-                        let error_msg = format!("Column {} was not found", col);
-                        return Err((url.to_string(), error_msg));
-                    }
-                }
-            }
-            if header_indices.is_empty() {
-                return Err((url.to_string(), "No matching columns found".to_string()));
-            }
-        } else {
-            let maf_cont_lst: Vec<String> = line.split("\t").map(|s| s.to_string()).collect();
-            let mut maf_out_lst: Vec<String> = Vec::new();
-            for x in header_indices.iter() {
-                maf_out_lst.push(maf_cont_lst[*x].to_string());
-            }
-            maf_str.push_str(maf_out_lst.join("\t").as_str());
-            maf_str.push_str("\n");
-            mafrows += 1;
-        }
-    }
-    Ok((maf_str.as_bytes().to_vec(), mafrows))
-}
-
-#[tokio::main]
-async fn main() -> Result<(), Box<dyn std::error::Error>> {
-    // Accepting the piped input json from jodejs and assign to the variable
-    // host: GDC host
-    // url: urls to download single maf files
-    let timeout_duration = Duration::from_secs(5); // Set a 10-second timeout
-
-    // Wrap the read operation in a timeout
-    let result = timeout(timeout_duration, async {
-        let mut buffer = String::new(); // Initialize an empty string to store input
-        let mut reader = BufReader::new(tokio::io::stdin()); // Create a buffered reader for stdin 
-        reader.read_to_string(&mut buffer).await?; // Read a line asynchronously
-        Ok::<String, io::Error>(buffer) // Return the input as a Result
-    })
-    .await;
-    // Handle the result of the timeout operation
-    let file_id_lst_js: Value = match result {
-        Ok(Ok(buffer)) => match serde_json::from_str(&buffer) {
-            Ok(js) => js,
-            Err(e) => {
-                let stdin_error = ErrorEntry {
-                    url: String::new(),
-                    error: format!("JSON parsing error: {}", e),
-                };
-                writeln!(io::stderr(), "{}", serde_json::to_string(&stdin_error).unwrap()).unwrap();
-                return Err(Box::new(std::io::Error::new(
-                    std::io::ErrorKind::InvalidInput,
-                    "JSON parsing error!",
-                )) as Box<dyn std::error::Error>);
-            }
-        },
-        Ok(Err(_e)) => {
-            let stdin_error = ErrorEntry {
-                url: String::new(),
-                error: "Error reading from stdin.".to_string(),
-            };
-            let stdin_error_js = serde_json::to_string(&stdin_error).unwrap();
-            writeln!(io::stderr(), "{}", stdin_error_js).expect("Failed to output stderr!");
-            return Err(Box::new(std::io::Error::new(
-                std::io::ErrorKind::InvalidInput,
-                "Failed to output stderr!",
-            )) as Box<dyn std::error::Error>);
-        }
-        Err(_) => {
-            let stdin_error = ErrorEntry {
-                url: String::new(),
-                error: "Timeout while reading from stdin.".to_string(),
-            };
-            let stdin_error_js = serde_json::to_string(&stdin_error).unwrap();
-            writeln!(io::stderr(), "{}", stdin_error_js).expect("Failed to output stderr!");
-            return Err(Box::new(std::io::Error::new(
-                std::io::ErrorKind::InvalidInput,
-                "The columns in arg is not an array",
-            )) as Box<dyn std::error::Error>);
-        }
-    };
-
-    // reading the input from PP
-    let host = file_id_lst_js
-        .get("host")
-        .expect("Host was not provided")
-        .as_str()
-        .expect("Host is not a string");
-    let mut url: Vec<String> = Vec::new();
-    let file_id_lst = file_id_lst_js
-        .get("fileIdLst")
-        .expect("File ID list is missed!")
-        .as_array()
-        .expect("File ID list is not an array");
-    for v in file_id_lst {
-        //url.push(Path::new(&host).join(&v.as_str().unwrap()).display().to_string());
-        url.push(format!("{}/{}", host.trim_end_matches('/'), v.as_str().unwrap()));
-    }
-    let mut req_headers = reqwest::header::HeaderMap::new();
-    if let Some(headers_val) = file_id_lst_js.get("headers") {
-        let headers_obj = match headers_val.as_object() {
-            Some(obj) => obj,
-            None => {
-                let header_error = ErrorEntry {
-                    url: String::new(),
-                    error: "headers is not an object".to_string(),
-                };
-                let header_error_js = serde_json::to_string(&header_error).unwrap();
-                writeln!(io::stderr(), "{}", header_error_js).expect("Failed to output stderr!");
-                return Err(Box::new(std::io::Error::new(
-                    std::io::ErrorKind::InvalidInput,
-                    "headers is not an object",
-                )) as Box<dyn std::error::Error>);
-            }
-        };
-        for (key, value) in headers_obj {
-            req_headers.insert(
-                reqwest::header::HeaderName::from_bytes(key.as_bytes()).expect("Invalid header key"),
-                reqwest::header::HeaderValue::from_str(value.as_str().expect("Invalid string value"))
-                    .expect("Invalid header value"),
-            );
-        }
-    }
-
-    // read columns as array from input json and convert data type from Vec<Value> to Vec<String>
-    let maf_col: Vec<String>;
-    if let Some(maf_col_value) = file_id_lst_js.get("columns") {
-        //convert Vec<Value> to Vec<String>
-        if let Some(maf_col_array) = maf_col_value.as_array() {
-            maf_col = maf_col_array
-                .iter()
-                .map(|v| v.to_string().replace("\"", ""))
-                .collect::<Vec<String>>();
-        } else {
-            let column_error = ErrorEntry {
-                url: String::new(),
-                error: "The columns in arg is not an array".to_string(),
-            };
-            let column_error_js = serde_json::to_string(&column_error).unwrap();
-            writeln!(io::stderr(), "{}", column_error_js).expect("Failed to output stderr!");
-            return Err(Box::new(std::io::Error::new(
-                std::io::ErrorKind::InvalidInput,
-                "The columns in arg is not an array",
-            )) as Box<dyn std::error::Error>);
-        }
-    } else {
-        let column_error = ErrorEntry {
-            url: String::new(),
-            error: "Columns was not selected".to_string(),
-        };
-        let column_error_js = serde_json::to_string(&column_error).unwrap();
-        writeln!(io::stderr(), "{}", column_error_js).expect("Failed to output stderr!");
-        return Err(Box::new(std::io::Error::new(
-            std::io::ErrorKind::InvalidInput,
-            "Columns was not selected",
-        )) as Box<dyn std::error::Error>);
-    };
-
-    //downloading maf files parallelly and merge them into single maf file
-    let download_futures = futures::stream::iter(url.into_iter().map(|url| {
-        let req_headers_clone = req_headers.clone();
-        async move {
-            let client = reqwest::Client::builder()
-                .timeout(Duration::from_secs(60)) // 60-second timeout per request
-                .connect_timeout(Duration::from_secs(15))
-                .default_headers(req_headers_clone.clone())
-                .build()
-                .map_err(|_e| {
-                    let client_error = ErrorEntry {
-                        url: url.clone(),
-                        error: "Client build error".to_string(),
-                    };
-                    let client_error_js = serde_json::to_string(&client_error).unwrap();
-                    writeln!(io::stderr(), "{}", client_error_js).expect("Failed to build reqwest client!");
-                });
-            match client.unwrap().get(&url).send().await {
-                Ok(resp) if resp.status().is_success() => match resp.bytes().await {
-                    Ok(content) => {
-                        let mut decoder = GzDecoder::new(&content[..]);
-                        let mut decompressed_content = Vec::new();
-                        match decoder.read_to_end(&mut decompressed_content) {
-                            Ok(_) => {
-                                let text = String::from_utf8_lossy(&decompressed_content).to_string();
-                                return Ok((url.clone(), text));
-                            }
-                            Err(e) => {
-                                let error_msg = format!("Failed to decompress downloaded MAF file: {}", e);
-                                Err((url.clone(), error_msg))
-                            }
-                        }
-                    }
-                    Err(e) => {
-                        let error_msg = format!("Failed to decompress downloaded MAF file: {}", e);
-                        Err((url.clone(), error_msg))
-                    }
-                },
-                Ok(resp) => {
-                    let error_msg = format!("HTTP error: {}", resp.status());
-                    Err((url.clone(), error_msg))
-                }
-                Err(e) => {
-                    let error_msg = format!("Server request failed: {}", e);
-                    Err((url.clone(), error_msg))
-                }
-            }
-        }
-    }));
-
-    // binary output
-    let encoder = Arc::new(Mutex::new(GzEncoder::new(io::stdout(), Compression::default())));
-
-    // Write the header
-    {
-        let mut encoder_guard = encoder.lock().unwrap(); // Lock the Mutex to get access to the inner GzEncoder
-        encoder_guard
-            .write_all(&maf_col.join("\t").as_bytes().to_vec())
-            .expect("Failed to write header");
-        encoder_guard.write_all(b"\n").expect("Failed to write newline");
-    }
-
-    download_futures
-        .buffer_unordered(20)
-        .for_each(|result| {
-            let encoder = Arc::clone(&encoder); // Clone the Arc for each task
-            let maf_col_cp = maf_col.clone();
-            async move {
-                match result {
-                    Ok((url, content)) => match select_maf_col(content, &maf_col_cp, &url) {
-                        Ok((maf_bit, mafrows)) => {
-                            if mafrows > 0 {
-                                let mut encoder_guard = encoder.lock().unwrap();
-                                encoder_guard.write_all(&maf_bit).expect("Failed to write file");
-                            } else {
-                                let error = ErrorEntry {
-                                    url: url.clone(),
-                                    error: "Empty MAF file".to_string(),
-                                };
-                                let error_js = serde_json::to_string(&error).unwrap();
-                                writeln!(io::stderr(), "{}", error_js).expect("Failed to output stderr!");
-                            }
-                        }
-                        Err((url, error)) => {
-                            let error = ErrorEntry { url, error };
-                            let error_js = serde_json::to_string(&error).unwrap();
-                            writeln!(io::stderr(), "{}", error_js).expect("Failed to output stderr!");
-                        }
-                    },
-                    Err((url, error)) => {
-                        let error = ErrorEntry { url, error };
-                        let error_js = serde_json::to_string(&error).unwrap();
-                        writeln!(io::stderr(), "{}", error_js).expect("Failed to output stderr!");
-                    }
-                };
-            }
-        })
-        .await;
-
-    // Finalize output
-
-    // Replace the value inside the Mutex with a dummy value (e.g., None)
-    let mut encoder_guard = encoder.lock().unwrap();
-    let encoder = std::mem::replace(
-        &mut *encoder_guard,
-        GzEncoder::new(io::stdout(), Compression::default()),
-    );
-    // Finalize the encoder
-    encoder.finish().expect("Maf file output error!");
-
-    // Manually flush stdout and stderr
-    io::stdout().flush().expect("Failed to flush stdout");
-    io::stderr().flush().expect("Failed to flush stderr");
-    Ok(())
-}