@anysphere/file-service 0.0.0-e0c70bcd → 0.0.0-e36a46ab

package/Cargo.toml

@@ -6,6 +6,10 @@ version = "0.0.0"
 [lib]
 crate-type = ["cdylib"]
 
+[features]
+default = ["windows-subsystem"]
+windows-subsystem = []
+
 [dependencies]
 # Default enable napi4 feature, see https://nodejs.org/api/n-api.html#node-api-version-matrix
 napi = { version = "2.12.2", default-features = false, features = ["napi4", "async", "tokio_rt"] }
@@ -21,6 +25,8 @@ tracing = "0.1.37"
 tracing-subscriber = "0.3.17"
 tracing-appender = "0.2.2"
 binaryornot = "1.0.0"
+dunce = "1.0.1"
+encoding_rs = "0.8.33"
 
 [build-dependencies]
 napi-build = "2.0.1"
@@ -28,5 +34,6 @@ tonic-build = "0.9.2"
 anyhow = "1.0.75"
 glob = "0.3.0"
 
+
 [profile.release]
 lto = true

package/build.rs

@@ -3,6 +3,8 @@ use std::path::Path;
 extern crate napi_build;
 
 fn main() -> Result<(), anyhow::Error> {
+  #[cfg(target_os = "windows")]
+  println!("cargo:rustc-cdylib-link-arg=/SUBSYSTEM:WINDOWS");
   napi_build::setup();
 
   // print the relative path.

package/index.d.ts

@@ -4,9 +4,9 @@
 /* auto-generated by NAPI-RS */
 
 export class MerkleClient {
-  constructor(rootDirectory: string)
-  init(): Promise<void>
-  computeMerkleTree(): Promise<void>
+  constructor(absoluteRootDirectory: string)
+  init(gitIgnoredFiles: Array<string>, isGitRepo: boolean): Promise<void>
+  computeMerkleTree(gitIgnoredFiles: Array<string>, isGitRepo: boolean): Promise<void>
   updateFile(filePath: string): Promise<void>
   deleteFile(filePath: string): Promise<void>
   getSubtreeHash(relativePath: string): Promise<string>

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@anysphere/file-service",
-  "version": "0.0.0-e0c70bcd",
+  "version": "0.0.0-e36a46ab",
   "main": "index.js",
   "types": "index.d.ts",
   "napi": {
@@ -36,12 +36,12 @@
     "version": "napi version"
   },
   "optionalDependencies": {
-    "@anysphere/file-service-win32-x64-msvc": "0.0.0-e0c70bcd",
-    "@anysphere/file-service-darwin-x64": "0.0.0-e0c70bcd",
-    "@anysphere/file-service-linux-x64-gnu": "0.0.0-e0c70bcd",
-    "@anysphere/file-service-darwin-arm64": "0.0.0-e0c70bcd",
-    "@anysphere/file-service-win32-arm64-msvc": "0.0.0-e0c70bcd",
-    "@anysphere/file-service-darwin-universal": "0.0.0-e0c70bcd",
-    "@anysphere/file-service-linux-arm64-gnu": "0.0.0-e0c70bcd"
+    "@anysphere/file-service-win32-x64-msvc": "0.0.0-e36a46ab",
+    "@anysphere/file-service-darwin-x64": "0.0.0-e36a46ab",
+    "@anysphere/file-service-linux-x64-gnu": "0.0.0-e36a46ab",
+    "@anysphere/file-service-darwin-arm64": "0.0.0-e36a46ab",
+    "@anysphere/file-service-win32-arm64-msvc": "0.0.0-e36a46ab",
+    "@anysphere/file-service-darwin-universal": "0.0.0-e36a46ab",
+    "@anysphere/file-service-linux-arm64-gnu": "0.0.0-e36a46ab"
   }
 }

package/src/file_utils.rs

@@ -5,7 +5,8 @@
 // 4. vscode.fs.stat
 
 use anyhow::Error;
-use std::{path::Path, any};
+use encoding_rs::UTF_8;
+use std::path::Path;
 use tokio::fs;
 
 pub fn is_in_bad_dir(file_path: &Path) -> Result<bool, Error> {
@@ -43,8 +44,90 @@ pub fn is_good_file(file_path: &Path) -> Result<(), Error> {
     _ => {}
   }
 
-  match extension {
-    "lock" | "bak" | "tmp" | "bin" | "exe" | "dll" | "so" | "lockb" => {
+  let bad_extensions = vec![
+    "lock",
+    "bak",
+    "tmp",
+    "bin",
+    "exe",
+    "dll",
+    "so",
+    "lockb",
+    "qwoff",
+    "isl",
+    "csv",
+    "pdf",
+    // add ms word, excel, powerpoint, etc.
+    "doc",
+    "docx",
+    "xls",
+    "xlsx",
+    "ppt",
+    "pptx",
+    "odt",
+    "ods",
+    "odp",
+    "odg",
+    "odf",
+    "sxw",
+    "sxc",
+    "sxi",
+    "sxd",
+    "sdc",
+    // add images
+    "jpg",
+    "jpeg",
+    "png",
+    "gif",
+    "bmp",
+    "tif",
+    // add audio
+    "mp3",
+    "wav",
+    "wma",
+    "ogg",
+    "flac",
+    "aac",
+    // add video
+    "mp4",
+    "mov",
+    "wmv",
+    "flv",
+    "avi",
+    // add archives
+    "zip",
+    "tar",
+    "gz",
+    "7z",
+    "rar",
+    "tgz",
+    "dmg",
+    "iso",
+    "cue",
+    "mdf",
+    "mds",
+    "vcd",
+    "toast",
+    "img",
+    "apk",
+    "msi",
+    "cab",
+    "tar.gz",
+    "tar.xz",
+    "tar.bz2",
+    "tar.lzma",
+    "tar.Z",
+    "tar.sz",
+    "lzma",
+    // add fonts
+    "ttf",
+    "otf",
+    "woff",
+    "woff2",
+    "eot",
+  ];
+  match bad_extensions.contains(&extension) {
+    true => {
       return Err(anyhow::anyhow!("File is just a lock file"));
     }
     _ => {}
@@ -93,7 +176,7 @@ pub fn is_good_file(file_path: &Path) -> Result<(), Error> {
 // implement the buffer above:
 pub async fn is_good_file_runtime_check(
   file_path: &Path,
-  _buffer: &[u8],
+  // _buffer: &[u8],
 ) -> Result<(), Error> {
   match get_file_size(file_path).await {
     Ok(size) if size > 2 * 1024 * 1024 => {
@@ -103,13 +186,31 @@ pub async fn is_good_file_runtime_check(
     _ => {}
   }
 
-	// if is_binary(file_path).context("Failed to check if file is binary")? {
-	// 	return Err(anyhow::anyhow!("File is binary"));
-	// }
+  // if is_binary(file_path).context("Failed to check if file is binary")? {
+  // 	return Err(anyhow::anyhow!("File is binary"));
+  // }
 
   Ok(())
 }
 
+pub async fn read_string_without_bom(
+  file_path: &Path,
+) -> Result<String, Error> {
+  let file_buffer = match fs::read(file_path).await {
+    Ok(buffer) => buffer,
+    Err(e) => {
+      return Err(anyhow::anyhow!(
+        "Failed to read file buffer: {}",
+        e.to_string()
+      ))
+    }
+  };
+
+  let (cow, _) = UTF_8.decode_with_bom_removal(&file_buffer);
+
+  Ok(cow.to_string())
+}
+
 pub fn as_relative_path(
   base_path: &Path,
   file_path: &Path,
@@ -167,25 +268,40 @@ mod tests {
     temp_file.write_all(b"Hello, world!").await.unwrap();
     let buffer = fs::read(&temp_file_path).await.unwrap();
     assert_eq!(
-      is_good_file_runtime_check(&temp_file_path, &buffer)
-        .await
-        .is_ok(),
+      is_good_file_runtime_check(&temp_file_path).await.is_ok(),
       true
     );
     temp_dir.close().unwrap();
 
+    // let temp_dir = tempfile::tempdir().unwrap();
+    // let temp_file_path = temp_dir.path().join("test_file");
+    // let mut temp_file = fs::File::create(&temp_file_path).await.unwrap();
+    // temp_file.write_all(&[0, 159, 146, 150]).await.unwrap(); // Invalid UTF-8 sequence
+    // let buffer = fs::read(&temp_file_path).await.unwrap();
+    // assert_eq!(
+    //   is_good_file_runtime_check(&temp_file_path).await.is_err(),
+    //   true
+    // );
+    // temp_dir.close().unwrap();
+  }
+
+  #[tokio::test]
+  async fn test_bom_file() {
+    const BOM: [u8; 3] = [0xEF, 0xBB, 0xBF];
+    const CONTENT: &str = "Hello, world!";
+
+    // Write this to a temp file
     let temp_dir = tempfile::tempdir().unwrap();
     let temp_file_path = temp_dir.path().join("test_file");
     let mut temp_file = fs::File::create(&temp_file_path).await.unwrap();
-    temp_file.write_all(&[0, 159, 146, 150]).await.unwrap(); // Invalid UTF-8 sequence
-    let buffer = fs::read(&temp_file_path).await.unwrap();
-    assert_eq!(
-      is_good_file_runtime_check(&temp_file_path, &buffer)
-        .await
-        .is_err(),
-      true
-    );
-    temp_dir.close().unwrap();
+    temp_file.write_all(&BOM).await.unwrap();
+    temp_file.write_all(CONTENT.as_bytes()).await.unwrap();
+
+    // expect that we read the file with tokio as the CONTENT
+    let file_contents = read_string_without_bom(&temp_file_path).await.unwrap();
+
+    // Check string equality of CONTENT (&str) to file_contents (String)
+    assert_eq!(CONTENT, file_contents);
   }
 
   #[test]

package/src/git_utils.rs

@@ -1,7 +1,8 @@
 use std::collections::HashSet;
+use std::path::MAIN_SEPARATOR_STR;
 use std::process::Command;
 
-pub fn list_ignored_files(
+pub fn list_ignored_files_and_directories(
   workspace_root_path: &str,
   should_return_absolute_paths: bool,
 ) -> Result<HashSet<String>, Box<dyn std::error::Error>> {
@@ -14,12 +15,12 @@ pub fn list_ignored_files(
       "--others",
       "--ignored",
       "--exclude-standard",
+      "--directory",
+      "--no-empty-directory"
     ],
     // FIXME(sualeh): this is super sketchy and might totally break in like a bazillion ways. i dont like it.
     vec![
-      "sh",
-      "-c",
-      "git submodule foreach --quiet 'git ls-files --others --ignored --exclude-standard | sed \"s|^|$path/|\"'",
+      "git submodule foreach --quiet 'git -C $toplevel/$path ls-files --others --ignored --exclude-standard --directory --no-empty-directory | (while read line; do echo $path/$line; done)'",
     ],
   ];
 
@@ -34,9 +35,12 @@ pub fn list_ignored_files(
         .lines()
         .filter(|line| !line.is_empty())
         .map(|line| {
+          let line = line.replace("/", MAIN_SEPARATOR_STR);
+
           if should_return_absolute_paths {
             let mut path = std::path::PathBuf::from(workspace_root_path);
-            path.push(line);
+
+            path.push(line.clone());
 
             match path.canonicalize() {
               Ok(canonical_path) => {
@@ -133,7 +137,8 @@ mod tests {
   fn test_no_ignored_files() {
     let dir = tempfile::tempdir().unwrap();
     let gitignored_files =
-      list_ignored_files(dir.path().to_str().unwrap(), false).unwrap();
+      list_ignored_files_and_directories(dir.path().to_str().unwrap(), false)
+        .unwrap();
     Command::new("git")
       .args(&["init"])
       .current_dir(dir.path())
@@ -160,7 +165,8 @@ mod tests {
       .output()
       .unwrap();
     let gitignored_files =
-      list_ignored_files(dir.path().to_str().unwrap(), false).unwrap();
+      list_ignored_files_and_directories(dir.path().to_str().unwrap(), false)
+        .unwrap();
     println!(
       "ignored files for test_one_ignored_file: {:?}",
       gitignored_files
@@ -190,7 +196,8 @@ mod tests {
       .output()
       .unwrap();
     let gitignored_files =
-      list_ignored_files(dir.path().to_str().unwrap(), false).unwrap();
+      list_ignored_files_and_directories(dir.path().to_str().unwrap(), false)
+        .unwrap();
     println!(
       "ignored files for test_multiple_ignored_files: {:?}",
       gitignored_files
@@ -254,7 +261,8 @@ mod tests {
     println!("git submodule add output: {:?}", o);
 
     let gitignored_files =
-      list_ignored_files(dir.path().to_str().unwrap(), false).unwrap();
+      list_ignored_files_and_directories(dir.path().to_str().unwrap(), false)
+        .unwrap();
     println!(
       "ignored files for test_git_submodule_ignored_files: {:?}",
       gitignored_files
@@ -265,7 +273,8 @@ mod tests {
 
   #[test]
   fn test_multiple_ignored_files_in_current_dir() {
-    let gitignored_files = list_ignored_files(".", false).unwrap();
+    let gitignored_files =
+      list_ignored_files_and_directories(".", false).unwrap();
     assert!(gitignored_files.len() > 1);
 
     // print a sample of the ignored files

package/src/lib.rs

@@ -1,11 +1,12 @@
+#![windows_subsystem = "windows"]
 #![deny(clippy::all)]
 #![deny(unsafe_op_in_unsafe_fn)]
 pub mod file_utils;
-pub mod git_utils;
 pub mod merkle_tree;
 
-use std::vec;
+use std::{vec, collections::HashSet};
 
+use anyhow::Context;
 use merkle_tree::{LocalConstruction, MerkleTree};
 use tracing::{info, Level};
 use tracing_appender::rolling::{RollingFileAppender, Rotation};
@@ -17,7 +18,7 @@ extern crate napi_derive;
 #[napi]
 pub struct MerkleClient {
   tree: MerkleTree,
-  root_directory: String,
+  absolute_root_directory: String,
   _guard: tracing_appender::non_blocking::WorkerGuard,
 }
 
@@ -40,24 +41,34 @@ pub fn init_logger() -> tracing_appender::non_blocking::WorkerGuard {
 #[napi]
 impl MerkleClient {
   #[napi(constructor)]
-  pub fn new(root_directory: String) -> MerkleClient {
+  pub fn new(absolute_root_directory: String) -> MerkleClient {
     let _guard = init_logger();
 
+    // let canonical_root_directory = std::path::Path::new(&absolute_root_directory);
+    // use dunce::canonicalize;
+    // let canonical_root_directory = match dunce::canonicalize(&canonical_root_directory) {
+    //   Ok(path) => path.to_str().unwrap_or(&absolute_root_directory).to_string().to_lowercase(),
+    //   Err(e) => {
+    //     info!("Error in canonicalizing path: path: {:?}, error {:?}", canonical_root_directory, e);
+    //     absolute_root_directory
+    //   }
+    // };
+
     MerkleClient {
       tree: MerkleTree::empty_tree(),
-      root_directory,
+      absolute_root_directory,
       _guard,
     }
   }
 
   #[napi]
-  pub async unsafe fn init(&mut self) -> Result<(), napi::Error> {
+  pub async unsafe fn init(&mut self, git_ignored_files: Vec<String>, is_git_repo: bool) -> Result<(), napi::Error> {
     // 1. compute the merkle tree
     // 2. update the backend
     // 3. sync with the remote
     info!("Merkle tree compute started!");
     unsafe {
-      self.compute_merkle_tree().await?;
+      self.compute_merkle_tree(git_ignored_files, is_git_repo).await?;
     }
 
     Ok(())
@@ -70,9 +81,15 @@ impl MerkleClient {
   #[napi]
   pub async unsafe fn compute_merkle_tree(
     &mut self,
+    git_ignored_files: Vec<String>,
+    is_git_repo: bool
   ) -> Result<(), napi::Error> {
+    // make the git ignored files into a hash set
+    let git_ignored_set = HashSet::from_iter(git_ignored_files.into_iter());
+
     let t =
-      MerkleTree::construct_merkle_tree(self.root_directory.clone()).await;
+      MerkleTree::construct_merkle_tree(self.absolute_root_directory.clone(), git_ignored_set, is_git_repo)
+        .await;
 
     match t {
       Ok(tree) => {
@@ -101,24 +118,40 @@ impl MerkleClient {
     &self,
     relative_path: String,
   ) -> Result<String, napi::Error> {
-    let absolute_path =
-      std::path::Path::new(&self.root_directory).join(&relative_path);
+    let relative_path_without_leading_slash = match relative_path
+      .strip_prefix('.')
+    {
+      Some(path) => path.strip_prefix(std::path::MAIN_SEPARATOR).unwrap_or(""),
+      None => relative_path.as_str(),
+    };
 
-    let canonical_path = match absolute_path.canonicalize() {
-      Ok(path) => path,
-      Err(e) => return Err(napi::Error::new(
-        napi::Status::Unknown,
-        format!("Error in canonicalizing path: {:?}", e),
-      )),
+    let absolute_path = if !relative_path_without_leading_slash.is_empty() {
+      std::path::Path::new(&self.absolute_root_directory)
+        .join(relative_path_without_leading_slash)
+    } else {
+      std::path::Path::new(&self.absolute_root_directory).to_path_buf()
     };
 
-    let hash = self.tree.get_subtree_hash(canonical_path.clone()).await;
+    let absolute_path_string = match absolute_path.to_str() {
+      Some(path) => path.to_string(),
+      None => {
+        return Err(napi::Error::new(
+          napi::Status::Unknown,
+          format!("some string error"),
+        ))
+      }
+    };
+
+    let hash = self
+      .tree
+      .get_subtree_hash(absolute_path_string.as_str())
+      .await;
 
     match hash {
       Ok(hash) => Ok(hash),
       Err(e) => Err(napi::Error::new(
         napi::Status::Unknown,
-        format!("Error in get_subtree_hash. \nRelative path: {:?}, \nAbsolute path: {:?}, \nCanonical path: {:?}, \nRoot directory: {:?}\nError: {:?}", &relative_path, absolute_path, canonical_path, self.root_directory, e),
+        format!("Error in get_subtree_hash. \nRelative path: {:?}, \nAbsolute path: {:?}, \nRoot directory: {:?}\nError: {:?}", &relative_path, absolute_path, self.absolute_root_directory, e)
       )),
     }
   }
@@ -140,7 +173,7 @@ impl MerkleClient {
     &self,
     relative_path: String,
   ) -> Result<i32, napi::Error> {
-    let absolute_path = std::path::Path::new(&self.root_directory)
+    let absolute_path = std::path::Path::new(&self.absolute_root_directory)
       .join(relative_path)
       .canonicalize()?;
 
@@ -176,10 +209,12 @@ impl MerkleClient {
     &self,
     absolute_file_path: String,
   ) -> Result<Vec<String>, napi::Error> {
-    let absolute_path_str = absolute_file_path.as_str();
+    // let absolute_path = absolute_file_path.to_lowercase();
+    // let absolute_path_str = absolute_path.as_str();
+
     let files = self
       .tree
-      .get_all_dir_files_to_embed(absolute_path_str)
+      .get_all_dir_files_to_embed(absolute_file_path.as_str())
       .await;
 
     match files {
@@ -220,8 +255,9 @@ impl MerkleClient {
     &self,
     absolute_file_path: String,
   ) -> Result<Vec<String>, napi::Error> {
-    let absolute_path_str = absolute_file_path.as_str();
-    let spline = self.tree.get_spline(absolute_path_str).await;
+    // let absolute_path = absolute_file_path.to_lowercase();
+    // let absolute_path_str = absolute_path.as_str();
+    let spline = self.tree.get_spline(absolute_file_path.as_str()).await;
 
     match spline {
       Ok(spline) => Ok(spline),
@@ -250,6 +286,6 @@ impl MerkleClient {
 
   #[napi]
   pub fn update_root_directory(&mut self, root_directory: String) {
-    self.root_directory = root_directory;
+    self.absolute_root_directory = root_directory;
   }
 }

package/src/merkle_tree/local_construction.rs

@@ -1,4 +1,3 @@
-use crate::git_utils;
 use crate::merkle_tree::{
   File, MerkleNode, MerkleNodePtr, NodeType, PinnedFuture,
 };
@@ -13,8 +12,13 @@ impl LocalConstruction for MerkleTree {
   async fn new(
     root_directory: Option<String>,
   ) -> Result<MerkleTree, anyhow::Error> {
+    let git_ignored_files = HashSet::<String>::new();
     if let Some(root_directory) = root_directory {
-      let n = MerkleTree::construct_merkle_tree(root_directory).await;
+      let n = MerkleTree::construct_merkle_tree(
+        root_directory,
+        git_ignored_files,
+        false
+      ).await;
       return n;
     }
 
@@ -30,6 +34,8 @@ impl LocalConstruction for MerkleTree {
   /// 4. return merkle tree
   async fn construct_merkle_tree(
     absolute_path_to_root_directory: String,
+    git_ignored_files_and_dirs: HashSet<String>,
+    is_git_repo: bool
   ) -> Result<MerkleTree, anyhow::Error> {
     let path = PathBuf::from(absolute_path_to_root_directory.clone());
     if !path.exists() {
@@ -38,19 +44,21 @@ impl LocalConstruction for MerkleTree {
     }
 
     // 1. get all the gitignored files
-    let git_ignored_files = match git_utils::list_ignored_files(
-      absolute_path_to_root_directory.as_str(),
-      true,
-    ) {
-      Ok(git_ignored) => git_ignored,
-      Err(_e) => HashSet::new(),
-    };
+    // let git_ignored_files_and_dirs =
+    //   match git_utils::list_ignored_files_and_directories(
+    //     absolute_path_to_root_directory.as_str(),
+    //     true,
+    //   ) {
+    //     Ok(git_ignored) => git_ignored,
+    //     Err(_e) => HashSet::new(),
+    //   };
 
     let root_node = MerkleNode::new(
       path,
       None,
-      &git_ignored_files,
+      &git_ignored_files_and_dirs,
       absolute_path_to_root_directory.as_str(),
+      is_git_repo
     )
     .await;
     let mut mt = MerkleTree {
@@ -58,7 +66,8 @@ impl LocalConstruction for MerkleTree {
       files: BTreeMap::new(),
       root_path: absolute_path_to_root_directory,
       cursor: None,
-      git_ignored_files,
+      git_ignored_files_and_dirs: git_ignored_files_and_dirs,
+      is_git_repo
     };
 
     // we now iterate over all the nodes and add them to the hashmap
@@ -80,7 +89,13 @@ impl LocalConstruction for MerkleTree {
           }
           NodeType::File(file_name) => {
             let f = File { node: node.clone() };
-            tracing::info!("File: {:?}", file_name);
+
+            // i dont reallly  like this :(((
+            // let canonical_file_name = match dunce::canonicalize(file_name) {
+            //   Ok(path) => path.to_str().unwrap_or(file_name).to_string(),
+            //   Err(_) => file_name.clone(),
+            // };
+
             files.insert(file_name.clone(), f);
           }
           NodeType::ErrorNode(_) => {

package/src/merkle_tree/mod.rs

@@ -1,13 +1,13 @@
-use crate::git_utils;
-
 use super::file_utils;
 use sha2::Digest;
 use std::collections::{BTreeMap, HashSet};
 use std::path::PathBuf;
+use std::vec;
 use std::{fs, path::Path, sync::Arc};
 use tokio::sync::RwLock;
 use tonic::async_trait;
 use tracing::info;
+
 pub mod local_construction;
 pub mod test;
 
@@ -18,7 +18,8 @@ pub struct MerkleTree {
   root: MerkleNodePtr,
   files: BTreeMap<String, File>,
   cursor: Option<usize>,
-  git_ignored_files: HashSet<String>,
+  git_ignored_files_and_dirs: HashSet<String>,
+  is_git_repo: bool,
 }
 
 #[derive(Debug)]
@@ -62,6 +63,8 @@ pub trait LocalConstruction {
 
   async fn construct_merkle_tree(
     root_directory: String,
+    git_ignored_files_and_dirs: HashSet<String>,
+    is_git_repo: bool
   ) -> Result<MerkleTree, anyhow::Error>;
 
   async fn update_file(
@@ -95,34 +98,35 @@ impl MerkleTree {
       files: BTreeMap::new(),
       root_path: "".to_string(),
       cursor: None,
-      git_ignored_files: HashSet::new(),
+      git_ignored_files_and_dirs: HashSet::new(),
+      is_git_repo: false
     }
   }
 
   pub async fn get_subtree_hash(
     &self,
-    absolute_path: PathBuf,
+    absolute_path: &str,
   ) -> Result<String, anyhow::Error> {
-    let abs_string = match absolute_path.to_str() {
-      Some(s) => s.to_string(),
-      None => {
-        return Err(anyhow::anyhow!(
-          "get_subtree_hash: Failed to convert path to string"
-        ))
-      }
-    };
-
-    let node = match self.files.get(&abs_string) {
+    let node = match self.files.get(absolute_path) {
       Some(file) => file.node.clone(),
       None => {
         let all_files: Vec<String> = self.files.keys().cloned().collect();
-        return Err(anyhow::anyhow!("Could not find file in tree! Looking for: {}. All files: {:?}", abs_string, all_files));
+        return Err(anyhow::anyhow!(
+          "Could not find file in tree! Looking for: {}. All files: {:?}",
+          absolute_path,
+          all_files
+        ));
       }
     };
 
     let node_reader = node.read().await;
     let node_hash = node_reader.hash.clone();
 
+    info!(
+      "get_subtree_hash for path: {}, node_hash: {}",
+      absolute_path, node_hash
+    );
+
     Ok(node_hash)
   }
 
@@ -286,6 +290,12 @@ impl MerkleTree {
   ) -> Result<Vec<String>, anyhow::Error> {
     let mut files = Vec::new();
 
+    // 1. should check that this absolute path is actually a directory.
+    let file_node = self.files.get(absolute_path);
+    if file_node.is_none() {
+      return Err(anyhow::anyhow!("Could not find directory the in tree!"));
+    }
+
     for (file_path, f) in &self.files {
       if !file_path.contains(absolute_path) {
         continue;
@@ -398,8 +408,9 @@ impl MerkleTree {
         let new_node = MerkleNode::new(
           file_path.clone(),
           Some(ancestor.clone()),
-          &self.git_ignored_files,
+          &self.git_ignored_files_and_dirs,
           &absolute_root_path.as_str(),
+          self.is_git_repo
         )
         .await;
         ancestor.write().await.attach_child(new_node.clone()).await;
@@ -415,8 +426,9 @@ impl MerkleTree {
         let first_child = MerkleNode::new(
           first_child_path.clone(),
           Some(ancestor.clone()),
-          &self.git_ignored_files,
+          &self.git_ignored_files_and_dirs,
           &absolute_root_path.as_str(),
+          self.is_git_repo
         )
         .await;
 
@@ -704,13 +716,14 @@ impl MerkleNode {
     parent: ParentPtr,
     ignored_files: &IgnoredFiles,
     absolute_root_path: &str,
+    is_git_repo: bool,
   ) -> MerkleNodePtr {
-    // check if the root is a git directory.
-    let is_git_repo =
-      match git_utils::is_git_directory(absolute_root_path).await {
-        Ok(is_git_repo) => is_git_repo,
-        Err(e) => false,
-      };
+    // // check if the root is a git directory.
+    // let is_git_repo =
+    //   match git_utils::is_git_directory(absolute_root_path).await {
+    //     Ok(is_git_repo) => is_git_repo,
+    //     Err(_e) => false,
+    //   };
     let bypass_git = !is_git_repo;
 
     MerkleNode::construct_node(
@@ -728,13 +741,8 @@ impl MerkleNode {
     parent: ParentPtr,
     ignored_files: &IgnoredFiles,
     absolute_root_path: &str,
+    is_git_repo: bool,
   ) -> MerkleNodePtr {
-    // check if the root is a git directory.
-    let is_git_repo =
-      match git_utils::is_git_directory(absolute_root_path).await {
-        Ok(is_git_repo) => is_git_repo,
-        Err(_e) => false,
-      };
     let bypass_git = !is_git_repo;
 
     info!(
@@ -791,26 +799,9 @@ impl MerkleNode {
         )));
       }
 
-      // check if the directory is git ignored
-      let is_git_ignored =
-        match git_utils::is_git_ignored(absolute_root_path, path_str.as_str())
-          .await
-        {
-          Ok(is_git_ignored) => is_git_ignored,
-          Err(e) => {
-            return Arc::new(RwLock::new(MerkleNode::empty_node(
-              Some(absolute_file_or_directory),
-              Some(e.to_string()),
-            )));
-          }
-        };
+      let is_git_ignored_dir = ignored_files.contains(&path_str);
 
-      if is_git_ignored && !bypass_git {
-        // println!("skipping directory: {}", path_str);
-        tracing::info!(
-          "skipping directory because its git ignored: {}",
-          path_str
-        );
+      if is_git_ignored_dir && !bypass_git {
         return Arc::new(RwLock::new(MerkleNode::empty_node(
           Some(absolute_file_or_directory),
           Some("Directory is git ignored!".to_string()),
@@ -899,18 +890,10 @@ impl MerkleNode {
       false => {}
     }
 
-    // read the file_content to a buffer
-    let file_content = match tokio::fs::read(absolute_file_path).await {
-      Ok(content) => content,
-      Err(e) => {
-        return Err(format!("Could not read file! {}", e.to_string()));
-      }
-    };
-
     // check if the file passes runtime checks.
     match file_utils::is_good_file_runtime_check(
       absolute_file_path,
-      &file_content,
+      // &file_content,
     )
     .await
     {
@@ -920,15 +903,14 @@ impl MerkleNode {
       }
     }
 
-    let file_content = match std::str::from_utf8(&file_content) {
-      Ok(content) => content,
-      Err(e) => {
-        return Err(format!(
-          "UTF8 Failure. Could not convert file content to string! {}",
-          e.to_string()
-        ));
-      }
-    };
+    // read the file_content to a buffer
+    let file_content =
+      match file_utils::read_string_without_bom(absolute_file_path).await {
+        Ok(content) => content,
+        Err(e) => {
+          return Err(format!("Could not read file! {}", e.to_string()));
+        }
+      };
 
     let file_hash = compute_hash(&file_content);
     let node = MerkleNode {
@@ -984,15 +966,51 @@ impl MerkleNode {
 
   async fn compute_branch_hash(children: &[MerkleNodePtr]) -> String {
     let mut hasher = sha2::Sha256::new();
+    let mut names_and_hashes = vec![];
+    let mut non_zero_children = 0;
+
     for child in children {
       // check if it is an error node
       let child_reader = child.read().await;
-      if let NodeType::ErrorNode(_) = &child_reader.node_type {
+
+      match &child_reader.node_type {
+        NodeType::File(file_name) => {
+          non_zero_children += 1;
+          names_and_hashes.push((file_name.clone(), child_reader.hash.clone()));
+        }
+        NodeType::Branch((file_name, _)) => {
+          let hash = child_reader.hash.clone();
+          if hash == "" {
+            continue;
+          }
+
+          non_zero_children += 1;
+          names_and_hashes.push((file_name.clone(), hash));
+        }
+        NodeType::ErrorNode(_) => {
+          continue;
+        }
+      }
+    }
+
+    // sort the list of names and hashes by the hashes!!
+    names_and_hashes
+      .sort_by(|a, b| a.1.to_lowercase().cmp(&b.1.to_lowercase()));
+
+    for (name, hash) in names_and_hashes {
+      if hash == "" {
         continue;
       }
+      info!("name: {}, hash: {}", name, hash);
+      hasher.update(hash);
+    }
 
-      hasher.update(child_reader.hash.as_bytes());
+    if non_zero_children == 0 {
+      // this means that the branch is empty.
+      // we should return an empty string.
+      return "".to_string();
     }
+
     let result = hasher.finalize();
     format!("{:x}", result)
   }

package/src/merkle_tree/test.rs

@@ -43,8 +43,9 @@ mod tests {
     // let path = Path::new(&temp_dir_path);
 
     // Test construct_merkle_tree() function
+    let new_set = std::collections::HashSet::<String>::new();
     let tree =
-      MerkleTree::construct_merkle_tree(temp_dir_path.clone()).await;
+      MerkleTree::construct_merkle_tree(temp_dir_path.clone(), new_set, false).await;
     let mut tree = match tree {
       Ok(tree) => {
         assert_eq!(tree.files.len(), 2);