@anysphere/file-service 0.0.0-fcf7dd4f → 0.0.0-ffd549b3

package/Cargo.toml

@@ -6,6 +6,11 @@ version = "0.0.0"
 [lib]
 crate-type = ["cdylib"]
 
+[features]
+default = ["windows-subsystem"]
+windows-subsystem = []
+debugfile = []
+
 [dependencies]
 # Default enable napi4 feature, see https://nodejs.org/api/n-api.html#node-api-version-matrix
 napi = { version = "2.12.2", default-features = false, features = ["napi4", "async", "tokio_rt"] }
@@ -22,6 +27,10 @@ tracing-subscriber = "0.3.17"
 tracing-appender = "0.2.2"
 binaryornot = "1.0.0"
 dunce = "1.0.1"
+encoding_rs = "0.8.33"
+
+[target.'cfg(not(target_os = "linux"))'.dependencies]
+tracing-axiom = "0.4"
 
 [build-dependencies]
 napi-build = "2.0.1"
@@ -29,5 +38,6 @@ tonic-build = "0.9.2"
 anyhow = "1.0.75"
 glob = "0.3.0"
 
+
 [profile.release]
 lto = true

package/build.rs

@@ -3,6 +3,8 @@ use std::path::Path;
 extern crate napi_build;
 
 fn main() -> Result<(), anyhow::Error> {
+  #[cfg(target_os = "windows")]
+  println!("cargo:rustc-cdylib-link-arg=/SUBSYSTEM:WINDOWS");
   napi_build::setup();
 
   // print the relative path.

package/index.d.ts

@@ -5,8 +5,9 @@
 
 export class MerkleClient {
   constructor(absoluteRootDirectory: string)
-  init(): Promise<void>
-  computeMerkleTree(): Promise<void>
+  isTooBig(maxFiles: number, gitIgnoredFiles: Array<string>, isGitRepo: boolean): Promise<boolean>
+  init(gitIgnoredFiles: Array<string>, isGitRepo: boolean): Promise<void>
+  computeMerkleTree(gitIgnoredFiles: Array<string>, isGitRepo: boolean): Promise<void>
   updateFile(filePath: string): Promise<void>
   deleteFile(filePath: string): Promise<void>
   getSubtreeHash(relativePath: string): Promise<string>

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@anysphere/file-service",
-  "version": "0.0.0-fcf7dd4f",
+  "version": "0.0.0-ffd549b3",
   "main": "index.js",
   "types": "index.d.ts",
   "napi": {
@@ -36,12 +36,12 @@
     "version": "napi version"
   },
   "optionalDependencies": {
-    "@anysphere/file-service-win32-x64-msvc": "0.0.0-fcf7dd4f",
-    "@anysphere/file-service-darwin-x64": "0.0.0-fcf7dd4f",
-    "@anysphere/file-service-linux-x64-gnu": "0.0.0-fcf7dd4f",
-    "@anysphere/file-service-darwin-arm64": "0.0.0-fcf7dd4f",
-    "@anysphere/file-service-win32-arm64-msvc": "0.0.0-fcf7dd4f",
-    "@anysphere/file-service-darwin-universal": "0.0.0-fcf7dd4f",
-    "@anysphere/file-service-linux-arm64-gnu": "0.0.0-fcf7dd4f"
+    "@anysphere/file-service-win32-x64-msvc": "0.0.0-ffd549b3",
+    "@anysphere/file-service-darwin-x64": "0.0.0-ffd549b3",
+    "@anysphere/file-service-linux-x64-gnu": "0.0.0-ffd549b3",
+    "@anysphere/file-service-darwin-arm64": "0.0.0-ffd549b3",
+    "@anysphere/file-service-win32-arm64-msvc": "0.0.0-ffd549b3",
+    "@anysphere/file-service-darwin-universal": "0.0.0-ffd549b3",
+    "@anysphere/file-service-linux-arm64-gnu": "0.0.0-ffd549b3"
   }
 }

package/src/file_utils.rs

@@ -5,6 +5,7 @@
 // 4. vscode.fs.stat
 
 use anyhow::Error;
+use encoding_rs::UTF_8;
 use std::path::Path;
 use tokio::fs;
 
@@ -43,8 +44,90 @@ pub fn is_good_file(file_path: &Path) -> Result<(), Error> {
     _ => {}
   }
 
-  match extension {
-    "lock" | "bak" | "tmp" | "bin" | "exe" | "dll" | "so" | "lockb" => {
+  let bad_extensions = vec![
+    "lock",
+    "bak",
+    "tmp",
+    "bin",
+    "exe",
+    "dll",
+    "so",
+    "lockb",
+    "qwoff",
+    "isl",
+    "csv",
+    "pdf",
+    // add ms word, excel, powerpoint, etc.
+    "doc",
+    "docx",
+    "xls",
+    "xlsx",
+    "ppt",
+    "pptx",
+    "odt",
+    "ods",
+    "odp",
+    "odg",
+    "odf",
+    "sxw",
+    "sxc",
+    "sxi",
+    "sxd",
+    "sdc",
+    // add images
+    "jpg",
+    "jpeg",
+    "png",
+    "gif",
+    "bmp",
+    "tif",
+    // add audio
+    "mp3",
+    "wav",
+    "wma",
+    "ogg",
+    "flac",
+    "aac",
+    // add video
+    "mp4",
+    "mov",
+    "wmv",
+    "flv",
+    "avi",
+    // add archives
+    "zip",
+    "tar",
+    "gz",
+    "7z",
+    "rar",
+    "tgz",
+    "dmg",
+    "iso",
+    "cue",
+    "mdf",
+    "mds",
+    "vcd",
+    "toast",
+    "img",
+    "apk",
+    "msi",
+    "cab",
+    "tar.gz",
+    "tar.xz",
+    "tar.bz2",
+    "tar.lzma",
+    "tar.Z",
+    "tar.sz",
+    "lzma",
+    // add fonts
+    "ttf",
+    "otf",
+    "woff",
+    "woff2",
+    "eot",
+  ];
+  match bad_extensions.contains(&extension) {
+    true => {
       return Err(anyhow::anyhow!("File is just a lock file"));
     }
     _ => {}
@@ -93,7 +176,7 @@ pub fn is_good_file(file_path: &Path) -> Result<(), Error> {
 // implement the buffer above:
 pub async fn is_good_file_runtime_check(
   file_path: &Path,
-  _buffer: &[u8],
+  // _buffer: &[u8],
 ) -> Result<(), Error> {
   match get_file_size(file_path).await {
     Ok(size) if size > 2 * 1024 * 1024 => {
@@ -103,13 +186,31 @@ pub async fn is_good_file_runtime_check(
     _ => {}
   }
 
-	// if is_binary(file_path).context("Failed to check if file is binary")? {
-	// 	return Err(anyhow::anyhow!("File is binary"));
-	// }
+  // if is_binary(file_path).context("Failed to check if file is binary")? {
+  // 	return Err(anyhow::anyhow!("File is binary"));
+  // }
 
   Ok(())
 }
 
+pub async fn read_string_without_bom(
+  file_path: &Path,
+) -> Result<String, Error> {
+  let file_buffer = match fs::read(file_path).await {
+    Ok(buffer) => buffer,
+    Err(e) => {
+      return Err(anyhow::anyhow!(
+        "Failed to read file buffer: {}",
+        e.to_string()
+      ))
+    }
+  };
+
+  let (cow, _) = UTF_8.decode_with_bom_removal(&file_buffer);
+
+  Ok(cow.to_string())
+}
+
 pub fn as_relative_path(
   base_path: &Path,
   file_path: &Path,
@@ -167,25 +268,40 @@ mod tests {
     temp_file.write_all(b"Hello, world!").await.unwrap();
     let buffer = fs::read(&temp_file_path).await.unwrap();
     assert_eq!(
-      is_good_file_runtime_check(&temp_file_path, &buffer)
-        .await
-        .is_ok(),
+      is_good_file_runtime_check(&temp_file_path).await.is_ok(),
       true
     );
     temp_dir.close().unwrap();
 
+    // let temp_dir = tempfile::tempdir().unwrap();
+    // let temp_file_path = temp_dir.path().join("test_file");
+    // let mut temp_file = fs::File::create(&temp_file_path).await.unwrap();
+    // temp_file.write_all(&[0, 159, 146, 150]).await.unwrap(); // Invalid UTF-8 sequence
+    // let buffer = fs::read(&temp_file_path).await.unwrap();
+    // assert_eq!(
+    //   is_good_file_runtime_check(&temp_file_path).await.is_err(),
+    //   true
+    // );
+    // temp_dir.close().unwrap();
+  }
+
+  #[tokio::test]
+  async fn test_bom_file() {
+    const BOM: [u8; 3] = [0xEF, 0xBB, 0xBF];
+    const CONTENT: &str = "Hello, world!";
+
+    // Write this to a temp file
     let temp_dir = tempfile::tempdir().unwrap();
     let temp_file_path = temp_dir.path().join("test_file");
     let mut temp_file = fs::File::create(&temp_file_path).await.unwrap();
-    temp_file.write_all(&[0, 159, 146, 150]).await.unwrap(); // Invalid UTF-8 sequence
-    let buffer = fs::read(&temp_file_path).await.unwrap();
-    assert_eq!(
-      is_good_file_runtime_check(&temp_file_path, &buffer)
-        .await
-        .is_err(),
-      true
-    );
-    temp_dir.close().unwrap();
+    temp_file.write_all(&BOM).await.unwrap();
+    temp_file.write_all(CONTENT.as_bytes()).await.unwrap();
+
+    // expect that we read the file with tokio as the CONTENT
+    let file_contents = read_string_without_bom(&temp_file_path).await.unwrap();
+
+    // Check string equality of CONTENT (&str) to file_contents (String)
+    assert_eq!(CONTENT, file_contents);
   }
 
   #[test]

package/src/git_utils.rs

@@ -1,4 +1,5 @@
 use std::collections::HashSet;
+use std::path::MAIN_SEPARATOR_STR;
 use std::process::Command;
 
 pub fn list_ignored_files_and_directories(
@@ -19,9 +20,7 @@ pub fn list_ignored_files_and_directories(
     ],
     // FIXME(sualeh): this is super sketchy and might totally break in like a bazillion ways. i dont like it.
     vec![
-      "sh",
-      "-c",
-      "git submodule foreach --quiet 'git ls-files --others --ignored --exclude-standard --directory --no-empty-directory | sed \"s|^|$path/|\"'",
+      "git submodule foreach --quiet 'git -C $toplevel/$path ls-files --others --ignored --exclude-standard --directory --no-empty-directory | (while read line; do echo $path/$line; done)'",
     ],
   ];
 
@@ -36,9 +35,12 @@ pub fn list_ignored_files_and_directories(
         .lines()
         .filter(|line| !line.is_empty())
         .map(|line| {
+          let line = line.replace("/", MAIN_SEPARATOR_STR);
+
           if should_return_absolute_paths {
             let mut path = std::path::PathBuf::from(workspace_root_path);
-            path.push(line);
+
+            path.push(line.clone());
 
             match path.canonicalize() {
               Ok(canonical_path) => {

package/src/lib.rs

@@ -1,16 +1,15 @@
+#![windows_subsystem = "windows"]
 #![deny(clippy::all)]
 #![deny(unsafe_op_in_unsafe_fn)]
 pub mod file_utils;
-pub mod git_utils;
+pub mod logger;
 pub mod merkle_tree;
 
-use std::vec;
+use std::{collections::HashSet, vec};
 
-use merkle_tree::{LocalConstruction, MerkleTree};
-use tracing::{info, Level};
-use tracing_appender::rolling::{RollingFileAppender, Rotation};
-use tracing_subscriber::fmt;
 use anyhow::Context;
+use merkle_tree::{LocalConstruction, MerkleTree};
+use tracing::{debug, info};
 
 #[macro_use]
 extern crate napi_derive;
@@ -19,30 +18,14 @@ extern crate napi_derive;
 pub struct MerkleClient {
   tree: MerkleTree,
   absolute_root_directory: String,
-  _guard: tracing_appender::non_blocking::WorkerGuard,
-}
-
-pub fn init_logger() -> tracing_appender::non_blocking::WorkerGuard {
-  let file_appender =
-    RollingFileAppender::new(Rotation::NEVER, "./", "rust_log.txt");
-  let (non_blocking, _guard) = tracing_appender::non_blocking(file_appender);
-  let subscriber = fmt::Subscriber::builder()
-    .with_max_level(Level::TRACE)
-    .with_writer(non_blocking)
-    .with_ansi(false)
-    .with_line_number(true)
-    .finish();
-
-  let _ = tracing::subscriber::set_global_default(subscriber);
-
-  _guard
+  _guard: Option<logger::GuardType>,
 }
 
 #[napi]
 impl MerkleClient {
   #[napi(constructor)]
   pub fn new(absolute_root_directory: String) -> MerkleClient {
-    let _guard = init_logger();
+    let _guard = logger::init_logger();
 
     // let canonical_root_directory = std::path::Path::new(&absolute_root_directory);
     // use dunce::canonicalize;
@@ -61,14 +44,74 @@ impl MerkleClient {
     }
   }
 
+	#[napi]
+  pub async fn is_too_big(
+    &self,
+    max_files: i32,
+    git_ignored_files: Vec<String>,
+    is_git_repo: bool,
+  ) -> bool {
+    let git_ignored_set =
+      HashSet::<String>::from_iter(git_ignored_files.into_iter());
+    let mut num_files = 0;
+    let mut dirs_to_check = vec![self.absolute_root_directory.clone()];
+
+    while let Some(dir) = dirs_to_check.pop() {
+			info!("dir: {:?}", dir);
+      let mut entries = match tokio::fs::read_dir(&dir).await {
+        Ok(entries) => entries,
+        Err(_) => continue,
+      };
+			if num_files > max_files {
+				return true;
+			}
+
+
+      while let Some(entry) = entries.next_entry().await.unwrap_or(None) {
+        let path = entry.path();
+				info!("entry: {:?}", path);
+        let path_str = match path.to_str() {
+          Some(path_str) => path_str.to_string(),
+          None => continue,
+        };
+
+        if git_ignored_set.contains(&path_str) {
+          continue;
+        }
+
+				match entry.file_type().await {
+					Ok(file_type) => {
+						if file_type.is_dir() {
+							dirs_to_check.push(path_str);
+						}
+
+						if file_type.is_file() {
+							num_files += 1;
+						}
+					}
+					Err(_) => continue,
+				}
+
+      }
+    }
+    num_files > max_files
+  }
+
   #[napi]
-  pub async unsafe fn init(&mut self) -> Result<(), napi::Error> {
+  pub async unsafe fn init(
+    &mut self,
+    git_ignored_files: Vec<String>,
+    is_git_repo: bool,
+  ) -> Result<(), napi::Error> {
     // 1. compute the merkle tree
     // 2. update the backend
     // 3. sync with the remote
     info!("Merkle tree compute started!");
+    info!("Root directory: {:?}", self.absolute_root_directory);
     unsafe {
-      self.compute_merkle_tree().await?;
+      self
+        .compute_merkle_tree(git_ignored_files, is_git_repo)
+        .await?;
     }
 
     Ok(())
@@ -81,9 +124,24 @@ impl MerkleClient {
   #[napi]
   pub async unsafe fn compute_merkle_tree(
     &mut self,
+    git_ignored_files: Vec<String>,
+    is_git_repo: bool,
   ) -> Result<(), napi::Error> {
-    let t =
-      MerkleTree::construct_merkle_tree(self.absolute_root_directory.clone()).await;
+    // make the git ignored files into a hash set
+    let mut git_ignored_set = HashSet::from_iter(git_ignored_files.into_iter());
+
+    // if the hashset itself contains the root directory, then we should remove it.
+    // this is because the root directory is not a file, and we don't want to ignore it.
+    if git_ignored_set.contains(&self.absolute_root_directory) {
+      git_ignored_set.remove(&self.absolute_root_directory);
+    }
+
+    let t = MerkleTree::construct_merkle_tree(
+      self.absolute_root_directory.clone(),
+      git_ignored_set,
+      is_git_repo,
+    )
+    .await;
 
     match t {
       Ok(tree) => {
@@ -112,29 +170,44 @@ impl MerkleClient {
     &self,
     relative_path: String,
   ) -> Result<String, napi::Error> {
+    debug!("get_subtree_hash: relative_path: {:?}", relative_path);
 
-    let relative_path_without_leading_slash = match relative_path.strip_prefix('.') {
+    let relative_path_without_leading_slash = match relative_path
+      .strip_prefix('.')
+    {
       Some(path) => path.strip_prefix(std::path::MAIN_SEPARATOR).unwrap_or(""),
       None => relative_path.as_str(),
     };
+    debug!(
+      "relative_path_without_leading_slash: {:?}",
+      relative_path_without_leading_slash
+    );
 
     let absolute_path = if !relative_path_without_leading_slash.is_empty() {
-      std::path::Path::new(&self.absolute_root_directory).join(relative_path_without_leading_slash)
+      std::path::Path::new(&self.absolute_root_directory)
+        .join(relative_path_without_leading_slash)
     } else {
       std::path::Path::new(&self.absolute_root_directory).to_path_buf()
     };
 
+    debug!("absolute_path: {:?}", absolute_path);
+
     let absolute_path_string = match absolute_path.to_str() {
       Some(path) => path.to_string(),
       None => {
         return Err(napi::Error::new(
           napi::Status::Unknown,
-          format!("some string error")
+          format!("some string error"),
         ))
       }
     };
 
-    let hash = self.tree.get_subtree_hash(absolute_path_string.as_str()).await;
+    debug!("absolute_path_string: {:?}", absolute_path_string);
+
+    let hash = self
+      .tree
+      .get_subtree_hash(absolute_path_string.as_str())
+      .await;
 
     match hash {
       Ok(hash) => Ok(hash),
@@ -153,7 +226,7 @@ impl MerkleClient {
       Ok(num) => Ok(num),
       Err(e) => Err(napi::Error::new(
         napi::Status::Unknown,
-        format!("Error in get_num_embeddable_files: {:?}", e)
+        format!("Error in get_num_embeddable_files: {:?}", e),
       )),
     }
   }
@@ -175,7 +248,7 @@ impl MerkleClient {
       Ok(num) => Ok(num),
       Err(e) => Err(napi::Error::new(
         napi::Status::Unknown,
-        format!("Error in get_num_embeddable_files_in_subtree: {:?}", e)
+        format!("Error in get_num_embeddable_files_in_subtree: {:?}", e),
       )),
     }
   }
@@ -188,7 +261,7 @@ impl MerkleClient {
       Ok(files) => Ok(files),
       Err(e) => Err(napi::Error::new(
         napi::Status::Unknown,
-        format!("Error in get_all_files: {:?}", e)
+        format!("Error in get_all_files: {:?}", e),
       )),
     }
   }

package/src/logger.rs

@@ -0,0 +1,55 @@
+use tracing::{info, subscriber, Level};
+use tracing_appender::non_blocking::WorkerGuard;
+use tracing_appender::rolling::{RollingFileAppender, Rotation};
+use tracing_subscriber::fmt;
+use tracing_subscriber::prelude::*;
+
+pub enum GuardType {
+  #[cfg(all(not(feature = "debugfile"), not(target_os = "linux")))]
+  Guard(tracing_axiom::Guard),
+  WorkerGuard(tracing_appender::non_blocking::WorkerGuard),
+}
+
+pub fn init_logger() -> Option<GuardType> {
+  #[cfg(feature = "debugfile")]
+  let _guard = {
+    let file_appender =
+      RollingFileAppender::new(Rotation::NEVER, "./", "rust_log.txt");
+    let (non_blocking, _guard) = tracing_appender::non_blocking(file_appender);
+    let subscriber = fmt::Subscriber::builder()
+      .with_max_level(Level::TRACE)
+      .with_writer(non_blocking)
+      .with_ansi(false)
+      .with_line_number(true)
+      .finish();
+
+    let _ = tracing::subscriber::set_global_default(subscriber);
+
+    Some(GuardType::WorkerGuard(_guard))
+  };
+
+  #[cfg(all(not(feature = "debugfile"), not(target_os = "linux")))]
+  let _guard = {
+    let (axiom_layer, _guard) = tracing_axiom::builder()
+      .with_token("xaat-a51088e6-7889-41c0-b440-cfd4601acdd7")
+      .with_dataset("local-indexing")
+      .layer()
+      .ok()?;
+    // let fmt_layer = fmt::layer().with_level(true).with_ansi(false).with_line_number(true);
+
+    let _ = tracing_subscriber::registry()
+      .with(axiom_layer)
+      .try_init()
+      .ok()?;
+    // let _ = tracing::subscriber::set_global_default(subscriber);
+
+    info!("Tracing initialized! in rust");
+
+    Some(GuardType::Guard(_guard))
+  };
+
+  #[cfg(all(not(feature = "debugfile"), target_os = "linux"))]
+  let _guard = { None };
+
+  _guard
+}

package/src/merkle_tree/local_construction.rs

@@ -1,4 +1,3 @@
-use crate::git_utils;
 use crate::merkle_tree::{
   File, MerkleNode, MerkleNodePtr, NodeType, PinnedFuture,
 };
@@ -10,11 +9,18 @@ use tonic::async_trait;
 
 #[async_trait]
 impl LocalConstruction for MerkleTree {
+  #[tracing::instrument]
   async fn new(
     root_directory: Option<String>,
   ) -> Result<MerkleTree, anyhow::Error> {
+    let git_ignored_files = HashSet::<String>::new();
     if let Some(root_directory) = root_directory {
-      let n = MerkleTree::construct_merkle_tree(root_directory).await;
+      let n = MerkleTree::construct_merkle_tree(
+        root_directory,
+        git_ignored_files,
+        false,
+      )
+      .await;
       return n;
     }
 
@@ -30,6 +36,8 @@ impl LocalConstruction for MerkleTree {
   /// 4. return merkle tree
   async fn construct_merkle_tree(
     absolute_path_to_root_directory: String,
+    git_ignored_files_and_dirs: HashSet<String>,
+    is_git_repo: bool,
   ) -> Result<MerkleTree, anyhow::Error> {
     let path = PathBuf::from(absolute_path_to_root_directory.clone());
     if !path.exists() {
@@ -38,28 +46,31 @@ impl LocalConstruction for MerkleTree {
     }
 
     // 1. get all the gitignored files
-    let git_ignored_files_and_dirs =
-      match git_utils::list_ignored_files_and_directories(
-        absolute_path_to_root_directory.as_str(),
-        true,
-      ) {
-        Ok(git_ignored) => git_ignored,
-        Err(_e) => HashSet::new(),
-      };
+    // let git_ignored_files_and_dirs =
+    //   match git_utils::list_ignored_files_and_directories(
+    //     absolute_path_to_root_directory.as_str(),
+    //     true,
+    //   ) {
+    //     Ok(git_ignored) => git_ignored,
+    //     Err(_e) => HashSet::new(),
+    //   };
 
     let root_node = MerkleNode::new(
       path,
       None,
       &git_ignored_files_and_dirs,
       absolute_path_to_root_directory.as_str(),
+      is_git_repo,
     )
     .await;
+
     let mut mt = MerkleTree {
       root: root_node,
       files: BTreeMap::new(),
       root_path: absolute_path_to_root_directory,
       cursor: None,
-      git_ignored_files_and_dirs: git_ignored_files_and_dirs,
+      git_ignored_files_and_dirs,
+      is_git_repo,
     };
 
     // we now iterate over all the nodes and add them to the hashmap
@@ -72,7 +83,6 @@ impl LocalConstruction for MerkleTree {
         let node_reader = node.read().await;
         match &node_reader.node_type {
           NodeType::Branch(n) => {
-            tracing::info!("Branch: {:?}", n.0);
             let children = &n.1;
             files.insert(n.0.clone(), File { node: node.clone() });
             for child in children {
@@ -99,8 +109,7 @@ impl LocalConstruction for MerkleTree {
 
     add_nodes_to_hashmap(&mt.root, &mut mt.files).await;
 
-    tracing::info!("Merkle tree compute finished!");
-    tracing::info!("Merkle tree: {}", mt);
+    tracing::info!("number of files in the tree: {}", mt.files.len());
 
     Ok(mt)
   }
@@ -144,6 +153,7 @@ impl LocalConstruction for MerkleTree {
     Ok(())
   }
 
+  #[tracing::instrument]
   async fn delete_file(
     &mut self,
     file_path: String,

package/src/merkle_tree/mod.rs

@@ -1,5 +1,3 @@
-use crate::git_utils;
-
 use super::file_utils;
 use sha2::Digest;
 use std::collections::{BTreeMap, HashSet};
@@ -8,18 +6,21 @@ use std::vec;
 use std::{fs, path::Path, sync::Arc};
 use tokio::sync::RwLock;
 use tonic::async_trait;
-use tracing::info;
+use tracing::{debug, info};
+
 pub mod local_construction;
 pub mod test;
 
 pub type MerkleNodePtr = Arc<RwLock<MerkleNode>>;
 
+#[derive(Debug)]
 pub struct MerkleTree {
   root_path: String,
   root: MerkleNodePtr,
   files: BTreeMap<String, File>,
   cursor: Option<usize>,
   git_ignored_files_and_dirs: HashSet<String>,
+  is_git_repo: bool,
 }
 
 #[derive(Debug)]
@@ -63,6 +64,8 @@ pub trait LocalConstruction {
 
   async fn construct_merkle_tree(
     root_directory: String,
+    git_ignored_files_and_dirs: HashSet<String>,
+    is_git_repo: bool,
   ) -> Result<MerkleTree, anyhow::Error>;
 
   async fn update_file(
@@ -97,6 +100,7 @@ impl MerkleTree {
       root_path: "".to_string(),
       cursor: None,
       git_ignored_files_and_dirs: HashSet::new(),
+      is_git_repo: false,
     }
   }
 
@@ -104,6 +108,8 @@ impl MerkleTree {
     &self,
     absolute_path: &str,
   ) -> Result<String, anyhow::Error> {
+    debug!("get_subtree_hash: absolute_path: {:?}", absolute_path);
+
     let node = match self.files.get(absolute_path) {
       Some(file) => file.node.clone(),
       None => {
@@ -119,10 +125,7 @@ impl MerkleTree {
     let node_reader = node.read().await;
     let node_hash = node_reader.hash.clone();
 
-    info!(
-      "get_subtree_hash for path: {}, node_hash: {}",
-      absolute_path, node_hash
-    );
+    debug!("node_hash: {:?}", node_hash);
 
     Ok(node_hash)
   }
@@ -287,6 +290,12 @@ impl MerkleTree {
   ) -> Result<Vec<String>, anyhow::Error> {
     let mut files = Vec::new();
 
+    // 1. should check that this absolute path is actually a directory.
+    let file_node = self.files.get(absolute_path);
+    if file_node.is_none() {
+      return Err(anyhow::anyhow!("Could not find directory the in tree!"));
+    }
+
     for (file_path, f) in &self.files {
       if !file_path.contains(absolute_path) {
         continue;
@@ -313,16 +322,11 @@ impl MerkleTree {
     &self,
     absolute_path: &str,
   ) -> Result<Vec<String>, anyhow::Error> {
-    info!("get_spline called with absolute_path: {}", absolute_path);
     let mut files = Vec::new();
 
     let current_node = match self.files.get(absolute_path) {
-      Some(node) => {
-        info!("Found node for absolute_path: {}", absolute_path);
-        node.node.clone()
-      }
+      Some(node) => node.node.clone(),
       None => {
-        info!("File not found for absolute_path: {}", absolute_path);
         return Err(anyhow::anyhow!("File not found: {}", absolute_path));
       }
     };
@@ -333,7 +337,6 @@ impl MerkleTree {
     while let Some(node) = stack.pop() {
       let parent = node.read().await.parent.clone();
       if let Some(parent) = parent {
-        info!("Adding parent hash to files vector");
         {
           let parent_node = parent.read().await;
           match &parent_node.node_type {
@@ -352,7 +355,6 @@ impl MerkleTree {
         stack.push(parent);
       }
     }
-    info!("Returning files vector with {} elements", files.len());
     Ok(files)
   }
 
@@ -401,6 +403,7 @@ impl MerkleTree {
           Some(ancestor.clone()),
           &self.git_ignored_files_and_dirs,
           &absolute_root_path.as_str(),
+          self.is_git_repo,
         )
         .await;
         ancestor.write().await.attach_child(new_node.clone()).await;
@@ -418,6 +421,7 @@ impl MerkleTree {
           Some(ancestor.clone()),
           &self.git_ignored_files_and_dirs,
           &absolute_root_path.as_str(),
+          self.is_git_repo,
         )
         .await;
 
@@ -705,13 +709,14 @@ impl MerkleNode {
     parent: ParentPtr,
     ignored_files: &IgnoredFiles,
     absolute_root_path: &str,
+    is_git_repo: bool,
   ) -> MerkleNodePtr {
-    // check if the root is a git directory.
-    let is_git_repo =
-      match git_utils::is_git_directory(absolute_root_path).await {
-        Ok(is_git_repo) => is_git_repo,
-        Err(e) => false,
-      };
+    // // check if the root is a git directory.
+    // let is_git_repo =
+    //   match git_utils::is_git_directory(absolute_root_path).await {
+    //     Ok(is_git_repo) => is_git_repo,
+    //     Err(_e) => false,
+    //   };
     let bypass_git = !is_git_repo;
 
     MerkleNode::construct_node(
@@ -724,25 +729,20 @@ impl MerkleNode {
     .await
   }
 
+  // #[tracing::instrument]
   async fn new(
     absolute_file_or_directory: PathBuf,
     parent: ParentPtr,
     ignored_files: &IgnoredFiles,
     absolute_root_path: &str,
+    is_git_repo: bool,
   ) -> MerkleNodePtr {
-    // check if the root is a git directory.
-    let is_git_repo =
-      match git_utils::is_git_directory(absolute_root_path).await {
-        Ok(is_git_repo) => is_git_repo,
-        Err(_e) => false,
-      };
     let bypass_git = !is_git_repo;
 
     info!(
       "constructing node for absolute_file_or_directory: {:?}",
       absolute_file_or_directory
     );
-    info!("bypass_git: {}, is_git_repo: {}", bypass_git, is_git_repo);
 
     MerkleNode::construct_node(
       Path::new(&absolute_file_or_directory),
@@ -771,6 +771,7 @@ impl MerkleNode {
     Box::pin(async move {
       // check if it is a file
       let path_str = absolute_file_or_directory.to_str().unwrap().to_string();
+
       if absolute_file_or_directory.is_file() {
         return Arc::new(RwLock::new(
           MerkleNode::construct_file_node_or_error_node(
@@ -795,6 +796,7 @@ impl MerkleNode {
       let is_git_ignored_dir = ignored_files.contains(&path_str);
 
       if is_git_ignored_dir && !bypass_git {
+        tracing::info!("skipping directory: {}", path_str);
         return Arc::new(RwLock::new(MerkleNode::empty_node(
           Some(absolute_file_or_directory),
           Some("Directory is git ignored!".to_string()),
@@ -805,6 +807,7 @@ impl MerkleNode {
       match entries {
         Ok(_) => (),
         Err(e) => {
+          tracing::error!("error reading directory: {}", e);
           return Arc::new(RwLock::new(MerkleNode::empty_node(
             Some(absolute_file_or_directory),
             Some(e.to_string()),
@@ -837,6 +840,7 @@ impl MerkleNode {
             );
           }
           Err(e) => {
+            tracing::error!("error reading directory: {}", e);
             children.push(Arc::new(RwLock::new(MerkleNode::empty_node(
               Some(absolute_file_or_directory),
               Some(e.to_string()),
@@ -883,18 +887,10 @@ impl MerkleNode {
       false => {}
     }
 
-    // read the file_content to a buffer
-    let file_content = match tokio::fs::read(absolute_file_path).await {
-      Ok(content) => content,
-      Err(e) => {
-        return Err(format!("Could not read file! {}", e.to_string()));
-      }
-    };
-
     // check if the file passes runtime checks.
     match file_utils::is_good_file_runtime_check(
       absolute_file_path,
-      &file_content,
+      // &file_content,
     )
     .await
     {
@@ -904,15 +900,14 @@ impl MerkleNode {
       }
     }
 
-    let file_content = match std::str::from_utf8(&file_content) {
-      Ok(content) => content,
-      Err(e) => {
-        return Err(format!(
-          "UTF8 Failure. Could not convert file content to string! {}",
-          e.to_string()
-        ));
-      }
-    };
+    // read the file_content to a buffer
+    let file_content =
+      match file_utils::read_string_without_bom(absolute_file_path).await {
+        Ok(content) => content,
+        Err(e) => {
+          return Err(format!("Could not read file! {}", e.to_string()));
+        }
+      };
 
     let file_hash = compute_hash(&file_content);
     let node = MerkleNode {
@@ -939,11 +934,7 @@ impl MerkleNode {
     .await
     {
       Ok(node) => node,
-      Err(e) => {
-        // println!("constructing error node. error: {}", e);
-        // println!("file_path: {:?}", file_path);
-        MerkleNode::empty_node(Some(absolute_file_path), Some(e))
-      }
+      Err(e) => MerkleNode::empty_node(Some(absolute_file_path), Some(e)),
     };
 
     node
@@ -1003,7 +994,6 @@ impl MerkleNode {
       if hash == "" {
         continue;
       }
-      info!("name: {}, hash: {}", name, hash);
       hasher.update(hash);
     }
 

package/src/merkle_tree/test.rs

@@ -43,8 +43,9 @@ mod tests {
     // let path = Path::new(&temp_dir_path);
 
     // Test construct_merkle_tree() function
+    let new_set = std::collections::HashSet::<String>::new();
     let tree =
-      MerkleTree::construct_merkle_tree(temp_dir_path.clone()).await;
+      MerkleTree::construct_merkle_tree(temp_dir_path.clone(), new_set, false).await;
     let mut tree = match tree {
       Ok(tree) => {
         assert_eq!(tree.files.len(), 2);