@sjcrh/proteinpaint-rust 2.74.0 → 2.78.0

package/Cargo.toml

@@ -14,7 +14,8 @@ rayon = "1.7.0"
 bgzip = "0.3.1"
 petgraph = "0.6.3"
 rusqlite="0.31.0"
-ndarray = "0.15.6"
+ndarray = "0.16.1"
+hdf5 = { package = "hdf5-metno", version = "0.9.0" }
 nalgebra = {version = "0.32.2", features = ["serde-serialize"]}
 plotters = "0.3.4"
 colorgrad = "0.6.2"
@@ -90,3 +91,7 @@ path="src/genesetORA.rs"
 [[bin]]
 name="computeTopTerms"
 path="src/computeTopTerms.rs"
+
+[[bin]]
+name="readHDF5"
+path="src/readHDF5.rs"

package/package.json

@@ -1,5 +1,5 @@
 {
-	"version": "2.74.0",
+	"version": "2.78.0",
 	"name": "@sjcrh/proteinpaint-rust",
 	"description": "Rust-based utilities for proteinpaint",
 	"main": "index.js",
@@ -38,5 +38,5 @@
 	"devDependencies": {
 		"tape": "^5.2.2"
 	},
-	"pp_release_tag": "v2.74.0"
+	"pp_release_tag": "v2.78.0"
 }

package/src/genesetORA.rs

@@ -34,23 +34,37 @@ struct pathway_p_value {
     pathway_name: String,
     p_value_original: f64,
     p_value_adjusted: Option<f64>,
+    gene_set_hits: String,
+    gene_set_size: usize,
 }
 
 fn calculate_hypergeometric_p_value(
     sample_genes: &Vec<&str>,
     num_background_genes: usize,
     genes_in_pathway: Vec<pathway_genes>,
-) -> f64 {
-    let matching_sample_genes_counts: f64 = sample_genes
-        .iter()
-        .zip(&genes_in_pathway)
-        .filter(|&(a, b)| *a.to_string() == b.symbol)
-        .count() as f64;
+) -> (f64, f64, String) {
+    let mut matching_sample_genes_counts = 0.0;
+    let mut gene_set_hits: String = "".to_string();
+    for gene in sample_genes {
+        for pathway in &genes_in_pathway {
+            if pathway.symbol == gene.to_string() {
+                matching_sample_genes_counts += 1.0;
+                gene_set_hits += &(gene.to_string() + &",");
+            }
+        }
+    }
+
+    if matching_sample_genes_counts > 0.0 {
+        gene_set_hits.pop();
+    }
+
+    //println!("sample_genes:{:?}", sample_genes);
+    //println!("genes_in_pathway:{:?}", genes_in_pathway);
     //println!("k-1:{}", matching_sample_genes_counts - 1.0);
     //println!("M:{}", genes_in_pathway.len() as f64);
     //println!(
     //    "N-M:{}",
-    //    background_genes.len() as f64 - genes_in_pathway.len() as f64
+    //    num_background_genes as f64 - genes_in_pathway.len() as f64
     //);
     //println!("n:{}", sample_genes.len() as f64);
     let p_value = r_mathlib::hypergeometric_cdf(
@@ -62,7 +76,7 @@ fn calculate_hypergeometric_p_value(
         false,
     );
     //println!("p_value:{}", p_value);
-    p_value
+    (p_value, matching_sample_genes_counts, gene_set_hits)
 }
 
 fn main() -> Result<()> {
@@ -136,7 +150,6 @@ fn main() -> Result<()> {
                             + &genesetgroup
                             + "'"),
                     );
-                    let mut iter = 0;
                     match stmt_result {
                         Ok(mut stmt) => {
                             #[allow(non_snake_case)]
@@ -144,7 +157,6 @@ fn main() -> Result<()> {
                                 stmt.query_map([], |row| Ok(GO_pathway { GO_id: row.get(0)? }))?;
                             #[allow(non_snake_case)]
                             for GO_term in GO_iter {
-                                iter += 1;
                                 match GO_term {
                                     Ok(n) => {
                                         //println!("GO term {:?}", n);
@@ -184,16 +196,20 @@ fn main() -> Result<()> {
                                                 }
                                             }
                                         }
-                                        let p_value = calculate_hypergeometric_p_value(
-                                            &sample_genes,
-                                            num_background_genes,
-                                            names,
-                                        );
-                                        if p_value.is_nan() == false {
+                                        let gene_set_size = names.len();
+                                        let (p_value, matches, gene_set_hits) =
+                                            calculate_hypergeometric_p_value(
+                                                &sample_genes,
+                                                num_background_genes,
+                                                names,
+                                            );
+                                        if matches >= 1.0 && p_value.is_nan() == false {
                                             pathway_p_values.push(pathway_p_value {
                                                 pathway_name: n.GO_id,
                                                 p_value_original: p_value,
                                                 p_value_adjusted: None,
+                                                gene_set_hits: gene_set_hits,
+                                                gene_set_size: gene_set_size,
                                             })
                                         }
                                     }
@@ -206,7 +222,7 @@ fn main() -> Result<()> {
                         Err(_) => panic!("sqlite database file not found"),
                     }
                     let output_string = "{\"num_pathways\":".to_string()
-                        + &iter.to_string()
+                        + &pathway_p_values.len().to_string()
                         + &",\"pathways\":"
                         + &adjust_p_values(pathway_p_values, num_items_output)
                         + &"}";
@@ -263,6 +279,8 @@ fn adjust_p_values(
             pathway_name: original_p_values[i].pathway_name.clone(),
             p_value_original: original_p_values[i].p_value_original,
             p_value_adjusted: Some(adjusted_p_val),
+            gene_set_hits: original_p_values[i].gene_set_hits.clone(),
+            gene_set_size: original_p_values[i].gene_set_size,
         });
     }
     adjusted_p_values.as_mut_slice().sort_by(|a, b| {
@@ -277,8 +295,7 @@ fn adjust_p_values(
 
     let mut output_string = "[".to_string();
     for i in 0..num_items_output {
-        let j = adjusted_p_values.len() - i - 1;
-        output_string += &serde_json::to_string(&adjusted_p_values[j]).unwrap();
+        output_string += &serde_json::to_string(&adjusted_p_values[i]).unwrap();
         if i < num_items_output - 1 {
             output_string += &",".to_string();
         }

package/src/readHDF5.rs

@@ -0,0 +1,227 @@
+// Need to set HDF5_DIR and LD_LIBRARY_PATH in ~/.bash_profile
+// Syntax: HDF5_DIR=/usr/local/Homebrew/Cellar/hdf5/1.14.3_1 && echo $HDF5_DIR && cd .. && cargo build --release && json='{"gene":"TP53","hdf5_file":"matrix_with_na_comp_9.h5"}' && time echo $json | target/release/rust_hdf5
+
+use hdf5::types::FixedAscii;
+use hdf5::{File, Result};
+use json;
+use ndarray::Array1;
+use ndarray::Dim;
+use std::io;
+use std::time::Instant;
+
+fn read_hdf5(hdf5_filename: String, gene_name: String) -> Result<()> {
+    let file = File::open(&hdf5_filename)?; // open for reading
+    let ds_dim = file.dataset("data/dim")?; // open the dataset
+
+    // Check the data type and read the dataset accordingly
+    let data_dim: Array1<_> = ds_dim.read::<usize, Dim<[usize; 1]>>()?;
+    let num_samples = data_dim[0]; // Number of total columns in the dataset
+    let num_genes = data_dim[1];
+    println!("num_samples:{}", num_samples);
+    println!("num_genes:{}", num_genes);
+
+    //let now_partial_i = Instant::now();
+    //let data_partial_i: Array1<usize> = ds_i.read_slice_1d(0..20)?;
+    //println!("Data_partial_i: {:?}", data_partial_i);
+    //println!("Time for partial_i dataset:{:?}", now_partial_i.elapsed());
+    //
+    //let now_x = Instant::now();
+    //let ds_x = file.dataset("data/x")?; // open the dataset
+    //let data_x: Array1<_> = ds_x.read::<f64, Dim<[usize; 1]>>()?;
+    //println!("Data_x: {:?}", data_x);
+    //println!("Time for x dataset:{:?}", now_x.elapsed());
+
+    let now_genes = Instant::now();
+    let ds_genes = file.dataset("gene_names")?;
+    let genes = ds_genes.read_1d::<FixedAscii<104>>()?;
+    //println!("\tgenes = {:?}", genes);
+    //println!("\tgenes.shape() = {:?}", genes.shape());
+    //println!("\tgenes.strides() = {:?}", genes.strides());
+    //println!("\tgenes.ndim() = {:?}", genes.ndim());
+    println!("Time for parsing genes:{:?}", now_genes.elapsed());
+
+    let now_samples = Instant::now();
+    let ds_samples = file.dataset("sample_names")?;
+    let samples = ds_samples.read_1d::<FixedAscii<104>>()?;
+    //println!("\tsamples = {:?}", samples);
+    //println!("\tsamples.shape() = {:?}", samples.shape());
+    //println!("\tsamples.strides() = {:?}", samples.strides());
+    //println!("\tsamples.ndim() = {:?}", samples.ndim());
+    println!("Time for parsing samples:{:?}", now_samples.elapsed());
+
+    let gene_index;
+    match genes.iter().position(|&x| x == gene_name) {
+        Some(index) => {
+            println!(
+                "The index of '{}' is {} in 0-based format (add 1 to compare with R output)",
+                gene_name, index
+            );
+            gene_index = index;
+        }
+        None => panic!(
+            "Gene '{}' not found in the HDF5 file '{}'",
+            gene_name, &hdf5_filename
+        ),
+    }
+
+    // Find the number of columns that are populated for that gene
+    let now_p = Instant::now();
+    let ds_p = file.dataset("data/p")?; // open the dataset
+
+    //let data_p: Array1<_> = ds_p.read::<usize, Dim<[usize; 1]>>()?;
+    let data_partial_p: Array1<usize> = ds_p.read_slice_1d(gene_index..gene_index + 2)?;
+    //println!("Data_p: {:?}", data_p);
+    println!("Data_partial_p: {:?}", data_partial_p);
+    println!("Time for p dataset:{:?}", now_p.elapsed());
+
+    let array_start_point = data_partial_p[0];
+    let array_stop_point = data_partial_p[1];
+    let num_populated_cells = data_partial_p[1] - array_start_point;
+    println!("Number of populated cells:{}", num_populated_cells);
+
+    //Find all columns indices that are populated for the given gene
+    let now_i = Instant::now();
+    let ds_i = file.dataset("data/i")?; // open the dataset
+
+    //let data_i: Array1<_> = ds_i.read::<f64, Dim<[usize; 1]>>()?;
+    //println!("Data_i: {:?}", data_i);
+    let populated_column_ids: Array1<usize> =
+        ds_i.read_slice_1d(array_start_point..array_stop_point - 1)?;
+    println!(
+        "Length of populated_column_ids:{}",
+        populated_column_ids.len()
+    );
+
+    // Do a sanity check (for testing)
+    //let mut min = 0;
+    //for i in 0..populated_column_ids.len() {
+    //    if populated_column_ids[i] < min {
+    //        println!("Value is decreasing {},{}", populated_column_ids[i], min);
+    //    } else {
+    //        min = populated_column_ids[i];
+    //    }
+    //}
+    println!("Populated cells:{:?}", populated_column_ids);
+    println!("Time for i dataset:{:?}", now_i.elapsed());
+
+    //Find all columns values that are populated for the given gene
+    let now_x = Instant::now();
+    let ds_x = file.dataset("data/x")?; // open the dataset
+
+    //let data_x: Array1<_> = ds_x.read::<f64, Dim<[usize; 1]>>()?;
+    //println!("Data_x: {:?}", data_x);
+    let populated_column_values: Array1<f64> =
+        ds_x.read_slice_1d(array_start_point..array_stop_point - 1)?;
+    println!(
+        "Length of populated_column_ids:{}",
+        populated_column_values.len()
+    );
+    println!("Time for x dataset:{:?}", now_x.elapsed());
+
+    // Generate the complete array from the sparse array
+
+    let mut gene_array: Array1<f64> = Array1::zeros(num_samples);
+    let time_generating_full_array = Instant::now();
+    //let mut gene_array: Vec<f64> = Vec::with_capacity(num_samples);
+    for index in 0..num_samples {
+        match populated_column_ids.iter().any(|&x| x == index) {
+            true => match populated_column_ids.iter().position(|&x| x == index) {
+                Some(y) => {
+                    gene_array[index] = populated_column_values[y] //gene_array.push(populated_column_values[y]),
+                }
+                None => {} // should not happen because if the index is found, its position in the array should also be found
+            },
+            false => gene_array[index] = 0.0, //gene_array.push(0.0), // If index not found, it means the value is 0 for that sample
+        }
+    }
+
+    let mut output_string = "{".to_string();
+    for i in 0..gene_array.len() {
+        //let item_json = "{\"".to_string()
+        //    + &samples[i].to_string()
+        //    + &"\","
+        //    + &gene_array[i].to_string()
+        //    + &"}";
+
+        //let item_json = format!("{{\"{}\"}}", samples[i].to_string());
+
+        output_string += &format!(
+            "\"{}\":{}",
+            samples[i].to_string(),
+            gene_array[i].to_string()
+        );
+        //println!("item_json:{}", item_json);
+
+        //let item_json = format!(
+        //    r##"{{"{}",{}}}"##,
+        //    samples[i].to_string().replace("\\", ""),
+        //    gene_array[i].to_string()
+        //);
+        if i != gene_array.len() - 1 {
+            output_string += &",";
+        }
+    }
+    output_string += &"}".to_string();
+    output_string = output_string.replace("\\", "");
+    println!(
+        "Time generating full array:{:?}",
+        time_generating_full_array.elapsed()
+    );
+    println!("output_string:{}", output_string);
+
+    // Print individual element in array
+
+    //let arr = v.iter().collect::<Vec<_>>();
+    //for (idx, val) in arr.iter().enumerate() {
+    //    println!("\tarr[{:?}] = {:?} ({:?})", idx, val.to_string(), val.len());
+    //}
+
+    //for item in data_i {
+    //    println!("i:{}", item);
+    //}
+    Ok(())
+}
+
+fn main() -> Result<()> {
+    let mut input = String::new();
+    match io::stdin().read_line(&mut input) {
+        // Accepting the piped input from nodejs (or command line from testing)
+        Ok(_bytes_read) => {
+            //println!("{} bytes read", bytes_read);
+            //println!("{}", input);
+            let input_json = json::parse(&input);
+            match input_json {
+                Ok(json_string) => {
+                    let now = Instant::now();
+                    let hdf5_filename_result = &json_string["hdf5_file"].to_owned();
+                    let hdf5_filename;
+                    match hdf5_filename_result.as_str() {
+                        Some(x) => {
+                            hdf5_filename = x.to_string();
+                        }
+                        None => {
+                            panic!("HDF5 filename not provided");
+                        }
+                    }
+
+                    let gene_result = &json_string["gene"].to_owned();
+                    let gene_name;
+                    match gene_result.as_str() {
+                        Some(x) => {
+                            gene_name = x.to_string();
+                        }
+                        None => {
+                            panic!("Gene name not provided");
+                        }
+                    }
+
+                    read_hdf5(hdf5_filename, gene_name)?;
+                    println!("Time for parsing genes from HDF5:{:?}", now.elapsed());
+                }
+                Err(error) => println!("Incorrect json: {}", error),
+            }
+        }
+        Err(error) => println!("Piping error: {}", error),
+    }
+    Ok(())
+}