@sjcrh/proteinpaint-rust 2.167.0 → 2.170.0

package/src/manhattan_plot.rs

@@ -22,6 +22,7 @@ struct Input {
     plot_height: u64,
     device_pixel_ratio: f64,
     png_dot_radius: u64,
+    log_cutoff: f64,
 }
 
 // chromosome info
@@ -120,6 +121,7 @@ fn cumulative_chrom(
 fn grin2_file_read(
     grin2_file: &str,
     chrom_data: &HashMap<String, ChromInfo>,
+    log_cutoff: f64,
 ) -> Result<(Vec<u64>, Vec<f64>, Vec<String>, Vec<PointDetail>, Vec<usize>), Box<dyn Error>> {
     // Default colours
     let mut colors: HashMap<String, String> = HashMap::new();
@@ -217,11 +219,18 @@ fn grin2_file_read(
                 Some(q) => q,
                 None => continue,
             };
-            let q_val: f64 = match q_val_str.parse() {
-                Ok(v) if v > 0.0 => v,
+            let original_q_val: f64 = match q_val_str.parse() {
+                Ok(v) if v >= 0.0 => v,
                 _ => continue,
             };
-            let neg_log10_q = -q_val.log10();
+
+            // Use log_cutoff for zero q-values to avoid -inf. These will be capped later in plotting at log_cutoff
+            let neg_log10_q = if original_q_val == 0.0 {
+                log_cutoff
+            } else {
+                -original_q_val.log10()
+            };
+
             let n_subj_count: Option<i64> = n_idx_opt
                 .and_then(|i| fields.get(i))
                 .and_then(|s| s.parse::<i64>().ok());
@@ -232,7 +241,8 @@ fn grin2_file_read(
             colors_vec.push(color.clone());
 
             // only add significant points for interactivity
-            if q_val <= 0.05 {
+            // We check against the original q-value here so we send back the correct values instead of the 1e-300 used for log transform
+            if original_q_val <= 0.05 {
                 point_details.push(PointDetail {
                     x: x_pos,
                     y: neg_log10_q,
@@ -243,7 +253,7 @@ fn grin2_file_read(
                     start: gene_start,
                     end: gene_end,
                     pos: gene_start,
-                    q_value: q_val,
+                    q_value: original_q_val,
                     nsubj: n_subj_count,
                     pixel_x: 0.0,
                     pixel_y: 0.0,
@@ -265,6 +275,7 @@ fn plot_grin2_manhattan(
     plot_height: u64,
     device_pixel_ratio: f64,
     png_dot_radius: u64,
+    log_cutoff: f64,
 ) -> Result<(String, InteractiveData), Box<dyn Error>> {
     // ------------------------------------------------
     // 1. Build cumulative chromosome map
@@ -293,7 +304,7 @@ fn plot_grin2_manhattan(
     let mut point_details = Vec::new();
     let mut sig_indices = Vec::new();
 
-    if let Ok((x, y, c, pd, si)) = grin2_file_read(&grin2_result_file, &chrom_data) {
+    if let Ok((x, y, c, pd, si)) = grin2_file_read(&grin2_result_file, &chrom_data, log_cutoff) {
         xs = x;
         ys = y;
         colors_vec = c;
@@ -302,24 +313,26 @@ fn plot_grin2_manhattan(
     }
 
     // ------------------------------------------------
-    // 3. Y-axis scaling
+    // 3. Y-axis scaling (cap at 40)
     // ------------------------------------------------
     let y_padding = png_dot_radius as f64;
     let y_min = 0.0 - y_padding;
+    let y_cap = log_cutoff; // typically 40.0. Use the passed log_cutoff value that user will be able to modify in the future
     let y_max = if !ys.is_empty() {
         let max_y = ys.iter().cloned().fold(f64::MIN, f64::max);
-        if max_y > 40.0 {
-            let target = 40.0;
-            let scale_factor_y = target / max_y;
-
+        if max_y > y_cap {
+            // Clamp values above the cap
             for y in ys.iter_mut() {
-                *y *= scale_factor_y;
+                if *y > y_cap {
+                    *y = y_cap;
+                }
             }
             for p in point_details.iter_mut() {
-                p.y *= scale_factor_y;
+                if p.y > y_cap {
+                    p.y = y_cap;
+                }
             }
-            let scaled_max = ys.iter().cloned().fold(f64::MIN, f64::max);
-            scaled_max + 0.35 + y_padding
+            y_cap + 0.35 + y_padding
         } else {
             max_y + 0.35 + y_padding
         }
@@ -384,7 +397,7 @@ fn plot_grin2_manhattan(
         }
 
         // ------------------------------------------------
-        // 7. capture high-DPR pixel mapping for the points
+        // 7. Capture high-DPR pixel mapping for the points
         //    we do not draw the points with plotters (will use tiny-skia for AA)
         //    but use charts.backend_coord to map data->pixel in the high-DPR backend
         // ------------------------------------------------
@@ -493,6 +506,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
                 let plot_height = &input_json.plot_height;
                 let device_pixel_ratio = &input_json.device_pixel_ratio;
                 let png_dot_radius = &input_json.png_dot_radius;
+                let log_cutoff = &input_json.log_cutoff;
                 if let Ok((base64_string, plot_data)) = plot_grin2_manhattan(
                     grin2_file.clone(),
                     chrom_size.clone(),
@@ -500,6 +514,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
                     plot_height.clone(),
                     device_pixel_ratio.clone(),
                     png_dot_radius.clone(),
+                    log_cutoff.clone(),
                 ) {
                     let output = Output {
                         png: base64_string,

package/src/query_classification.rs

@@ -0,0 +1,152 @@
+// Syntax: cd .. && cargo build --release && time cat ~/sjpp/test.txt | target/release/aichatbot
+#![allow(non_snake_case)]
+use anyhow::Result;
+use json::JsonValue;
+use schemars::JsonSchema;
+use std::io;
+mod aichatbot; // Importing classification agent from aichatbot.rs
+mod ollama; // Importing custom rig module for invoking ollama server
+mod sjprovider; // Importing custom rig module for invoking SJ GPU server
+mod test_ai; // Test examples for AI chatbot
+
+#[tokio::main]
+async fn main() -> Result<()> {
+    let mut input = String::new();
+    match io::stdin().read_line(&mut input) {
+        // Accepting the piped input from nodejs (or command line from testing)
+        Ok(_n) => {
+            let input_json = json::parse(&input);
+            match input_json {
+                Ok(json_string) => {
+                    //println!("json_string:{}", json_string);
+                    let user_input_json: &JsonValue = &json_string["user_input"];
+                    let user_input: &str;
+                    match user_input_json.as_str() {
+                        Some(inp) => user_input = inp,
+                        None => panic!("user_input field is missing in input json"),
+                    }
+
+                    if user_input.len() == 0 {
+                        panic!("The user input is empty");
+                    }
+
+                    let binpath_json: &JsonValue = &json_string["binpath"];
+                    let binpath: &str;
+                    match binpath_json.as_str() {
+                        Some(inp) => binpath = inp,
+                        None => panic!("binpath not found"),
+                    }
+
+                    let aiRoute_json: &JsonValue = &json_string["aiRoute"];
+                    let aiRoute_str: &str;
+                    match aiRoute_json.as_str() {
+                        Some(inp) => aiRoute_str = inp,
+                        None => panic!("aiRoute field is missing in input json"),
+                    }
+                    let airoute = String::from(binpath) + &"/../../" + &aiRoute_str;
+
+                    let apilink_json: &JsonValue = &json_string["apilink"];
+                    let apilink: &str;
+                    match apilink_json.as_str() {
+                        Some(inp) => apilink = inp,
+                        None => panic!("apilink field is missing in input json"),
+                    }
+
+                    let comp_model_name_json: &JsonValue = &json_string["comp_model_name"];
+                    let comp_model_name: &str;
+                    match comp_model_name_json.as_str() {
+                        Some(inp) => comp_model_name = inp,
+                        None => panic!("comp_model_name field is missing in input json"),
+                    }
+
+                    let embedding_model_name_json: &JsonValue = &json_string["embedding_model_name"];
+                    let embedding_model_name: &str;
+                    match embedding_model_name_json.as_str() {
+                        Some(inp) => embedding_model_name = inp,
+                        None => panic!("embedding_model_name field is missing in input json"),
+                    }
+
+                    let llm_backend_name_json: &JsonValue = &json_string["llm_backend_name"];
+                    let llm_backend_name: &str;
+                    match llm_backend_name_json.as_str() {
+                        Some(inp) => llm_backend_name = inp,
+                        None => panic!("llm_backend_name field is missing in input json"),
+                    }
+
+                    let llm_backend_type: aichatbot::llm_backend;
+                    let mut final_output: Option<String> = None;
+                    let temperature: f64 = 0.01;
+                    let max_new_tokens: usize = 512;
+                    let top_p: f32 = 0.95;
+                    if llm_backend_name != "ollama" && llm_backend_name != "SJ" {
+                        panic!(
+                            "This code currently supports only Ollama and SJ provider. llm_backend_name must be \"ollama\" or \"SJ\""
+                        );
+                    } else if llm_backend_name == "ollama".to_string() {
+                        llm_backend_type = aichatbot::llm_backend::Ollama();
+                        // Initialize Ollama client
+                        let ollama_client = ollama::Client::builder()
+                            .base_url(apilink)
+                            .build()
+                            .expect("Ollama server not found");
+                        let embedding_model = ollama_client.embedding_model(embedding_model_name);
+                        let comp_model = ollama_client.completion_model(comp_model_name);
+                        final_output = Some(
+                            aichatbot::classify_query_by_dataset_type(
+                                user_input,
+                                comp_model.clone(),
+                                embedding_model.clone(),
+                                &llm_backend_type,
+                                temperature,
+                                max_new_tokens,
+                                top_p,
+                                &airoute,
+                            )
+                            .await,
+                        );
+                    } else if llm_backend_name == "SJ".to_string() {
+                        llm_backend_type = aichatbot::llm_backend::Sj();
+                        // Initialize Sj provider client
+                        let sj_client = sjprovider::Client::builder()
+                            .base_url(apilink)
+                            .build()
+                            .expect("SJ server not found");
+                        let embedding_model = sj_client.embedding_model(embedding_model_name);
+                        let comp_model = sj_client.completion_model(comp_model_name);
+                        final_output = Some(
+                            aichatbot::classify_query_by_dataset_type(
+                                user_input,
+                                comp_model.clone(),
+                                embedding_model.clone(),
+                                &llm_backend_type,
+                                temperature,
+                                max_new_tokens,
+                                top_p,
+                                &airoute,
+                            )
+                            .await,
+                        );
+                    }
+
+                    match final_output {
+                        Some(fin_out) => {
+                            println!("{{\"{}\":{}}}", "route", fin_out);
+                        }
+                        None => {
+                            println!("{{\"{}\":\"{}\"}}", "route", "unknown");
+                        }
+                    }
+                }
+                Err(error) => println!("Incorrect json:{}", error),
+            }
+        }
+        Err(error) => println!("Piping error: {}", error),
+    }
+    Ok(())
+}
+
+#[derive(Debug, JsonSchema)]
+#[allow(dead_code)]
+struct OutputJson {
+    pub answer: String,
+}

package/src/summary_agent.rs

@@ -0,0 +1,201 @@
+// Syntax: cd .. && cargo build --release && time cat ~/sjpp/test.txt | target/release/aichatbot
+#![allow(non_snake_case)]
+use crate::aichatbot::AiJsonFormat;
+use anyhow::Result;
+use json::JsonValue;
+use std::fs;
+use std::io;
+use std::path::Path;
+mod aichatbot; // Get summary agent
+
+mod ollama; // Importing custom rig module for invoking ollama server
+mod sjprovider; // Importing custom rig module for invoking SJ GPU server
+mod test_ai; // Test examples for AI chatbot
+
+#[tokio::main]
+async fn main() -> Result<()> {
+    let mut input = String::new();
+    match io::stdin().read_line(&mut input) {
+        // Accepting the piped input from nodejs (or command line from testing)
+        Ok(_n) => {
+            let input_json = json::parse(&input);
+            match input_json {
+                Ok(json_string) => {
+                    //println!("json_string:{}", json_string);
+                    let user_input_json: &JsonValue = &json_string["user_input"];
+                    let user_input: &str;
+                    match user_input_json.as_str() {
+                        Some(inp) => user_input = inp,
+                        None => panic!("user_input field is missing in input json"),
+                    }
+
+                    let dataset_db_json: &JsonValue = &json_string["dataset_db"];
+                    let dataset_db_str: &str;
+                    match dataset_db_json.as_str() {
+                        Some(inp) => dataset_db_str = inp,
+                        None => panic!("dataset_db field is missing in input json"),
+                    }
+
+                    let genedb_json: &JsonValue = &json_string["genedb"];
+                    let genedb_str: &str;
+                    match genedb_json.as_str() {
+                        Some(inp) => genedb_str = inp,
+                        None => panic!("genedb field is missing in input json"),
+                    }
+
+                    if user_input.len() == 0 {
+                        panic!("The user input is empty");
+                    }
+
+                    let tpmasterdir_json: &JsonValue = &json_string["tpmasterdir"];
+                    let tpmasterdir: &str;
+                    match tpmasterdir_json.as_str() {
+                        Some(inp) => tpmasterdir = inp,
+                        None => panic!("tpmasterdir not found"),
+                    }
+
+                    let binpath_json: &JsonValue = &json_string["binpath"];
+                    let binpath: &str;
+                    match binpath_json.as_str() {
+                        Some(inp) => binpath = inp,
+                        None => panic!("binpath not found"),
+                    }
+
+                    let ai_json_file_json: &JsonValue = &json_string["aifiles"];
+                    let ai_json_file: String;
+                    match ai_json_file_json.as_str() {
+                        Some(inp) => ai_json_file = String::from(binpath) + &"/../../" + &inp,
+                        None => {
+                            panic!("ai json file not found")
+                        }
+                    }
+
+                    let ai_json_file = Path::new(&ai_json_file);
+                    let ai_json_file_path;
+                    let current_dir = std::env::current_dir().unwrap();
+                    match ai_json_file.canonicalize() {
+                        Ok(p) => ai_json_file_path = p,
+                        Err(_) => {
+                            panic!(
+                                "AI JSON file path not found:{:?}, current directory:{:?}",
+                                ai_json_file, current_dir
+                            )
+                        }
+                    }
+
+                    // Read the file
+                    let ai_data = fs::read_to_string(ai_json_file_path).unwrap();
+
+                    // Parse the JSON data
+                    let ai_json: AiJsonFormat =
+                        serde_json::from_str(&ai_data).expect("AI JSON file does not have the correct format");
+
+                    let genedb = String::from(tpmasterdir) + &"/" + &genedb_str;
+                    let dataset_db = String::from(tpmasterdir) + &"/" + &dataset_db_str;
+
+                    let apilink_json: &JsonValue = &json_string["apilink"];
+                    let apilink: &str;
+                    match apilink_json.as_str() {
+                        Some(inp) => apilink = inp,
+                        None => panic!("apilink field is missing in input json"),
+                    }
+
+                    let comp_model_name_json: &JsonValue = &json_string["comp_model_name"];
+                    let comp_model_name: &str;
+                    match comp_model_name_json.as_str() {
+                        Some(inp) => comp_model_name = inp,
+                        None => panic!("comp_model_name field is missing in input json"),
+                    }
+
+                    let embedding_model_name_json: &JsonValue = &json_string["embedding_model_name"];
+                    let embedding_model_name: &str;
+                    match embedding_model_name_json.as_str() {
+                        Some(inp) => embedding_model_name = inp,
+                        None => panic!("embedding_model_name field is missing in input json"),
+                    }
+
+                    let llm_backend_name_json: &JsonValue = &json_string["llm_backend_name"];
+                    let llm_backend_name: &str;
+                    match llm_backend_name_json.as_str() {
+                        Some(inp) => llm_backend_name = inp,
+                        None => panic!("llm_backend_name field is missing in input json"),
+                    }
+
+                    let llm_backend_type: aichatbot::llm_backend;
+                    let mut final_output: Option<String> = None;
+                    let temperature: f64 = 0.01;
+                    let max_new_tokens: usize = 512;
+                    let top_p: f32 = 0.95;
+                    let testing = false; // This variable is always false in production, this is true in test_ai.rs for testing code
+                    if llm_backend_name != "ollama" && llm_backend_name != "SJ" {
+                        panic!(
+                            "This code currently supports only Ollama and SJ provider. llm_backend_name must be \"ollama\" or \"SJ\""
+                        );
+                    } else if llm_backend_name == "ollama".to_string() {
+                        llm_backend_type = aichatbot::llm_backend::Ollama();
+                        // Initialize Ollama client
+                        let ollama_client = ollama::Client::builder()
+                            .base_url(apilink)
+                            .build()
+                            .expect("Ollama server not found");
+                        let embedding_model = ollama_client.embedding_model(embedding_model_name);
+                        let comp_model = ollama_client.completion_model(comp_model_name);
+                        final_output = Some(
+                            aichatbot::extract_summary_information(
+                                user_input,
+                                comp_model,
+                                embedding_model,
+                                &llm_backend_type,
+                                temperature,
+                                max_new_tokens,
+                                top_p,
+                                &dataset_db,
+                                &genedb,
+                                &ai_json,
+                                testing,
+                            )
+                            .await,
+                        );
+                    } else if llm_backend_name == "SJ".to_string() {
+                        llm_backend_type = aichatbot::llm_backend::Sj();
+                        // Initialize Sj provider client
+                        let sj_client = sjprovider::Client::builder()
+                            .base_url(apilink)
+                            .build()
+                            .expect("SJ server not found");
+                        let embedding_model = sj_client.embedding_model(embedding_model_name);
+                        let comp_model = sj_client.completion_model(comp_model_name);
+                        final_output = Some(
+                            aichatbot::extract_summary_information(
+                                user_input,
+                                comp_model,
+                                embedding_model,
+                                &llm_backend_type,
+                                temperature,
+                                max_new_tokens,
+                                top_p,
+                                &dataset_db,
+                                &genedb,
+                                &ai_json,
+                                testing,
+                            )
+                            .await,
+                        );
+                    }
+
+                    match final_output {
+                        Some(fin_out) => {
+                            println!("final_output:{:?}", fin_out.replace("\\", ""));
+                        }
+                        None => {
+                            println!("final_output:{{\"{}\":\"{}\"}}", "action", "unknown");
+                        }
+                    }
+                }
+                Err(error) => println!("Incorrect json:{}", error),
+            }
+        }
+        Err(error) => println!("Piping error: {}", error),
+    }
+    Ok(())
+}