@anysphere/file-service 0.0.0-dbd43428 → 0.0.0-e15bb6ec

package/Cargo.toml

@@ -6,6 +6,11 @@ version = "0.0.0"
 [lib]
 crate-type = ["cdylib"]
 
+[features]
+default = ["windows-subsystem"]
+windows-subsystem = []
+debugfile = []
+
 [dependencies]
 # Default enable napi4 feature, see https://nodejs.org/api/n-api.html#node-api-version-matrix
 napi = { version = "2.12.2", default-features = false, features = ["napi4", "async", "tokio_rt"] }
@@ -20,6 +25,12 @@ prost = "0.11.9"
 tracing = "0.1.37"
 tracing-subscriber = "0.3.17"
 tracing-appender = "0.2.2"
+binaryornot = "1.0.0"
+dunce = "1.0.1"
+encoding_rs = "0.8.33"
+
+[target.'cfg(not(target_os = "linux"))'.dependencies]
+tracing-axiom = "0.4"
 
 [build-dependencies]
 napi-build = "2.0.1"
@@ -27,5 +38,6 @@ tonic-build = "0.9.2"
 anyhow = "1.0.75"
 glob = "0.3.0"
 
+
 [profile.release]
 lto = true

package/build.rs

@@ -3,6 +3,8 @@ use std::path::Path;
 extern crate napi_build;
 
 fn main() -> Result<(), anyhow::Error> {
+  #[cfg(target_os = "windows")]
+  println!("cargo:rustc-cdylib-link-arg=/SUBSYSTEM:WINDOWS");
   napi_build::setup();
 
   // print the relative path.

package/index.d.ts

@@ -4,9 +4,10 @@
 /* auto-generated by NAPI-RS */
 
 export class MerkleClient {
-  constructor(rootDirectory: string)
-  init(): Promise<void>
-  computeMerkleTree(): Promise<void>
+  constructor(absoluteRootDirectory: string)
+  isTooBig(maxFiles: number, gitIgnoredFiles: Array<string>, isGitRepo: boolean): Promise<boolean>
+  init(gitIgnoredFiles: Array<string>, isGitRepo: boolean): Promise<void>
+  computeMerkleTree(gitIgnoredFiles: Array<string>, isGitRepo: boolean): Promise<void>
   updateFile(filePath: string): Promise<void>
   deleteFile(filePath: string): Promise<void>
   getSubtreeHash(relativePath: string): Promise<string>

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@anysphere/file-service",
-  "version": "0.0.0-dbd43428",
+  "version": "0.0.0-e15bb6ec",
   "main": "index.js",
   "types": "index.d.ts",
   "napi": {
@@ -9,7 +9,8 @@
       "additional": [
         "aarch64-apple-darwin",
         "aarch64-pc-windows-msvc",
-        "universal-apple-darwin"
+        "universal-apple-darwin",
+        "aarch64-unknown-linux-gnu"
       ]
     }
   },
@@ -35,11 +36,12 @@
     "version": "napi version"
   },
   "optionalDependencies": {
-    "@anysphere/file-service-win32-x64-msvc": "0.0.0-dbd43428",
-    "@anysphere/file-service-darwin-x64": "0.0.0-dbd43428",
-    "@anysphere/file-service-linux-x64-gnu": "0.0.0-dbd43428",
-    "@anysphere/file-service-darwin-arm64": "0.0.0-dbd43428",
-    "@anysphere/file-service-win32-arm64-msvc": "0.0.0-dbd43428",
-    "@anysphere/file-service-darwin-universal": "0.0.0-dbd43428"
+    "@anysphere/file-service-win32-x64-msvc": "0.0.0-e15bb6ec",
+    "@anysphere/file-service-darwin-x64": "0.0.0-e15bb6ec",
+    "@anysphere/file-service-linux-x64-gnu": "0.0.0-e15bb6ec",
+    "@anysphere/file-service-darwin-arm64": "0.0.0-e15bb6ec",
+    "@anysphere/file-service-win32-arm64-msvc": "0.0.0-e15bb6ec",
+    "@anysphere/file-service-darwin-universal": "0.0.0-e15bb6ec",
+    "@anysphere/file-service-linux-arm64-gnu": "0.0.0-e15bb6ec"
   }
 }

package/src/file_utils.rs

@@ -5,6 +5,7 @@
 // 4. vscode.fs.stat
 
 use anyhow::Error;
+use encoding_rs::UTF_8;
 use std::path::Path;
 use tokio::fs;
 
@@ -43,8 +44,21 @@ pub fn is_good_file(file_path: &Path) -> Result<(), Error> {
     _ => {}
   }
 
-  match extension {
-    "lock" | "bak" | "tmp" | "bin" | "exe" | "dll" | "so" | "lockb" => {
+  let bad_extensions = vec![
+    "lock", "bak", "tmp", "bin", "exe", "dll", "so", "lockb", "qwoff", "isl",
+    "csv", "pdf", // add ms word, excel, powerpoint, etc.
+    "doc", "docx", "xls", "xlsx", "ppt", "pptx", "odt", "ods", "odp", "odg",
+    "odf", "sxw", "sxc", "sxi", "sxd", "sdc", // add images
+    "jpg", "jpeg", "png", "gif", "bmp", "tif", // add audio
+    "mp3", "wav", "wma", "ogg", "flac", "aac", // add video
+    "mp4", "mov", "wmv", "flv", "avi", // add archives
+    "zip", "tar", "gz", "7z", "rar", "tgz", "dmg", "iso", "cue", "mdf", "mds",
+    "vcd", "toast", "img", "apk", "msi", "cab", "tar.gz", "tar.xz", "tar.bz2",
+    "tar.lzma", "tar.Z", "tar.sz", "lzma", // add fonts
+    "ttf", "otf", "woff", "woff2", "eot", "webp", "vsix",
+  ];
+  match bad_extensions.contains(&extension) {
+    true => {
       return Err(anyhow::anyhow!("File is just a lock file"));
     }
     _ => {}
@@ -62,7 +76,7 @@ pub fn is_good_file(file_path: &Path) -> Result<(), Error> {
     Some(extension) => match extension.to_str() {
       Some(ext_str) => {
         if bad_extensions.contains(&ext_str) {
-          return Err(anyhow::anyhow!("File is not a valid UTF-8 string"));
+          return Err(anyhow::anyhow!("Binary file excluded from indexing."));
         }
       }
       None => {
@@ -88,10 +102,12 @@ pub fn is_good_file(file_path: &Path) -> Result<(), Error> {
   Ok(())
 }
 
+// use binaryornot::is_binary;
+// use anyhow::Context;
 // implement the buffer above:
 pub async fn is_good_file_runtime_check(
   file_path: &Path,
-  buffer: &[u8],
+  // _buffer: &[u8],
 ) -> Result<(), Error> {
   match get_file_size(file_path).await {
     Ok(size) if size > 2 * 1024 * 1024 => {
@@ -101,16 +117,31 @@ pub async fn is_good_file_runtime_check(
     _ => {}
   }
 
-  for &byte in buffer.iter().take(2048) {
-    if byte.is_ascii() {
-      continue;
-    } else {
-      return Err(anyhow::anyhow!("File is not a valid UTF-8 string"));
-    }
-  }
+  // if is_binary(file_path).context("Failed to check if file is binary")? {
+  // 	return Err(anyhow::anyhow!("File is binary"));
+  // }
+
   Ok(())
 }
 
+pub async fn read_string_without_bom(
+  file_path: &Path,
+) -> Result<String, Error> {
+  let file_buffer = match fs::read(file_path).await {
+    Ok(buffer) => buffer,
+    Err(e) => {
+      return Err(anyhow::anyhow!(
+        "Failed to read file buffer: {}",
+        e.to_string()
+      ))
+    }
+  };
+
+  let (cow, _) = UTF_8.decode_with_bom_removal(&file_buffer);
+
+  Ok(cow.to_string())
+}
+
 pub fn as_relative_path(
   base_path: &Path,
   file_path: &Path,
@@ -168,25 +199,40 @@ mod tests {
     temp_file.write_all(b"Hello, world!").await.unwrap();
     let buffer = fs::read(&temp_file_path).await.unwrap();
     assert_eq!(
-      is_good_file_runtime_check(&temp_file_path, &buffer)
-        .await
-        .is_ok(),
+      is_good_file_runtime_check(&temp_file_path).await.is_ok(),
       true
     );
     temp_dir.close().unwrap();
 
+    // let temp_dir = tempfile::tempdir().unwrap();
+    // let temp_file_path = temp_dir.path().join("test_file");
+    // let mut temp_file = fs::File::create(&temp_file_path).await.unwrap();
+    // temp_file.write_all(&[0, 159, 146, 150]).await.unwrap(); // Invalid UTF-8 sequence
+    // let buffer = fs::read(&temp_file_path).await.unwrap();
+    // assert_eq!(
+    //   is_good_file_runtime_check(&temp_file_path).await.is_err(),
+    //   true
+    // );
+    // temp_dir.close().unwrap();
+  }
+
+  #[tokio::test]
+  async fn test_bom_file() {
+    const BOM: [u8; 3] = [0xEF, 0xBB, 0xBF];
+    const CONTENT: &str = "Hello, world!";
+
+    // Write this to a temp file
     let temp_dir = tempfile::tempdir().unwrap();
     let temp_file_path = temp_dir.path().join("test_file");
     let mut temp_file = fs::File::create(&temp_file_path).await.unwrap();
-    temp_file.write_all(&[0, 159, 146, 150]).await.unwrap(); // Invalid UTF-8 sequence
-    let buffer = fs::read(&temp_file_path).await.unwrap();
-    assert_eq!(
-      is_good_file_runtime_check(&temp_file_path, &buffer)
-        .await
-        .is_err(),
-      true
-    );
-    temp_dir.close().unwrap();
+    temp_file.write_all(&BOM).await.unwrap();
+    temp_file.write_all(CONTENT.as_bytes()).await.unwrap();
+
+    // expect that we read the file with tokio as the CONTENT
+    let file_contents = read_string_without_bom(&temp_file_path).await.unwrap();
+
+    // Check string equality of CONTENT (&str) to file_contents (String)
+    assert_eq!(CONTENT, file_contents);
   }
 
   #[test]

package/src/git_utils.rs

@@ -1,7 +1,8 @@
 use std::collections::HashSet;
+use std::path::MAIN_SEPARATOR_STR;
 use std::process::Command;
 
-pub fn list_ignored_files(
+pub fn list_ignored_files_and_directories(
   workspace_root_path: &str,
   should_return_absolute_paths: bool,
 ) -> Result<HashSet<String>, Box<dyn std::error::Error>> {
@@ -14,12 +15,12 @@ pub fn list_ignored_files(
       "--others",
       "--ignored",
       "--exclude-standard",
+      "--directory",
+      "--no-empty-directory"
     ],
     // FIXME(sualeh): this is super sketchy and might totally break in like a bazillion ways. i dont like it.
     vec![
-      "sh",
-      "-c",
-      "git submodule foreach --quiet 'git ls-files --others --ignored --exclude-standard | sed \"s|^|$path/|\"'",
+      "git submodule foreach --quiet 'git -C $toplevel/$path ls-files --others --ignored --exclude-standard --directory --no-empty-directory | (while read line; do echo $path/$line; done)'",
     ],
   ];
 
@@ -34,9 +35,12 @@ pub fn list_ignored_files(
         .lines()
         .filter(|line| !line.is_empty())
         .map(|line| {
+          let line = line.replace("/", MAIN_SEPARATOR_STR);
+
           if should_return_absolute_paths {
             let mut path = std::path::PathBuf::from(workspace_root_path);
-            path.push(line);
+
+            path.push(line.clone());
 
             match path.canonicalize() {
               Ok(canonical_path) => {
@@ -133,7 +137,8 @@ mod tests {
   fn test_no_ignored_files() {
     let dir = tempfile::tempdir().unwrap();
     let gitignored_files =
-      list_ignored_files(dir.path().to_str().unwrap(), false).unwrap();
+      list_ignored_files_and_directories(dir.path().to_str().unwrap(), false)
+        .unwrap();
     Command::new("git")
       .args(&["init"])
       .current_dir(dir.path())
@@ -160,7 +165,8 @@ mod tests {
       .output()
       .unwrap();
     let gitignored_files =
-      list_ignored_files(dir.path().to_str().unwrap(), false).unwrap();
+      list_ignored_files_and_directories(dir.path().to_str().unwrap(), false)
+        .unwrap();
     println!(
       "ignored files for test_one_ignored_file: {:?}",
       gitignored_files
@@ -190,7 +196,8 @@ mod tests {
       .output()
       .unwrap();
     let gitignored_files =
-      list_ignored_files(dir.path().to_str().unwrap(), false).unwrap();
+      list_ignored_files_and_directories(dir.path().to_str().unwrap(), false)
+        .unwrap();
     println!(
       "ignored files for test_multiple_ignored_files: {:?}",
       gitignored_files
@@ -254,7 +261,8 @@ mod tests {
     println!("git submodule add output: {:?}", o);
 
     let gitignored_files =
-      list_ignored_files(dir.path().to_str().unwrap(), false).unwrap();
+      list_ignored_files_and_directories(dir.path().to_str().unwrap(), false)
+        .unwrap();
     println!(
       "ignored files for test_git_submodule_ignored_files: {:?}",
       gitignored_files
@@ -265,7 +273,8 @@ mod tests {
 
   #[test]
   fn test_multiple_ignored_files_in_current_dir() {
-    let gitignored_files = list_ignored_files(".", false).unwrap();
+    let gitignored_files =
+      list_ignored_files_and_directories(".", false).unwrap();
     assert!(gitignored_files.len() > 1);
 
     // print a sample of the ignored files

package/src/lib.rs

@@ -1,15 +1,15 @@
+#![windows_subsystem = "windows"]
 #![deny(clippy::all)]
 #![deny(unsafe_op_in_unsafe_fn)]
 pub mod file_utils;
-pub mod git_utils;
+pub mod logger;
 pub mod merkle_tree;
 
-use std::vec;
+use std::{collections::HashSet, vec};
 
+use anyhow::Context;
 use merkle_tree::{LocalConstruction, MerkleTree};
-use tracing::{info, Level};
-use tracing_appender::rolling::{RollingFileAppender, Rotation};
-use tracing_subscriber::fmt;
+use tracing::{debug, info};
 
 #[macro_use]
 extern crate napi_derive;
@@ -17,46 +17,101 @@ extern crate napi_derive;
 #[napi]
 pub struct MerkleClient {
   tree: MerkleTree,
-  root_directory: String,
-  _guard: tracing_appender::non_blocking::WorkerGuard,
-}
-
-pub fn init_logger() -> tracing_appender::non_blocking::WorkerGuard {
-  let file_appender =
-    RollingFileAppender::new(Rotation::NEVER, "./", "rust_log.txt");
-  let (non_blocking, _guard) = tracing_appender::non_blocking(file_appender);
-  let subscriber = fmt::Subscriber::builder()
-    .with_max_level(Level::TRACE)
-    .with_writer(non_blocking)
-    .with_ansi(false)
-    .with_line_number(true)
-    .finish();
-
-  let _ = tracing::subscriber::set_global_default(subscriber);
-
-  _guard
+  absolute_root_directory: String,
+  _guard: Option<logger::GuardType>,
 }
 
 #[napi]
 impl MerkleClient {
   #[napi(constructor)]
-  pub fn new(root_directory: String) -> MerkleClient {
-    let _guard = init_logger();
+  pub fn new(absolute_root_directory: String) -> MerkleClient {
+    let _guard = logger::init_logger();
+
+    // let canonical_root_directory = std::path::Path::new(&absolute_root_directory);
+    // use dunce::canonicalize;
+    // let canonical_root_directory = match dunce::canonicalize(&canonical_root_directory) {
+    //   Ok(path) => path.to_str().unwrap_or(&absolute_root_directory).to_string().to_lowercase(),
+    //   Err(e) => {
+    //     info!("Error in canonicalizing path: path: {:?}, error {:?}", canonical_root_directory, e);
+    //     absolute_root_directory
+    //   }
+    // };
 
     MerkleClient {
       tree: MerkleTree::empty_tree(),
-      root_directory,
+      absolute_root_directory,
       _guard,
     }
   }
 
+	#[napi]
+  pub async fn is_too_big(
+    &self,
+    max_files: i32,
+    git_ignored_files: Vec<String>,
+    is_git_repo: bool,
+  ) -> bool {
+    let git_ignored_set =
+      HashSet::<String>::from_iter(git_ignored_files.into_iter());
+    let mut num_files = 0;
+    let mut dirs_to_check = vec![self.absolute_root_directory.clone()];
+
+    while let Some(dir) = dirs_to_check.pop() {
+			info!("dir: {:?}", dir);
+      let mut entries = match tokio::fs::read_dir(&dir).await {
+        Ok(entries) => entries,
+        Err(_) => continue,
+      };
+			if num_files > max_files {
+				return true;
+			}
+
+
+      while let Some(entry) = entries.next_entry().await.unwrap_or(None) {
+        let path = entry.path();
+				info!("entry: {:?}", path);
+        let path_str = match path.to_str() {
+          Some(path_str) => path_str.to_string(),
+          None => continue,
+        };
+
+        if git_ignored_set.contains(&path_str) {
+          continue;
+        }
+
+				match entry.file_type().await {
+					Ok(file_type) => {
+						if file_type.is_dir() {
+							dirs_to_check.push(path_str);
+						}
+
+						if file_type.is_file() {
+							num_files += 1;
+						}
+					}
+					Err(_) => continue,
+				}
+
+      }
+    }
+    num_files > max_files
+  }
+
   #[napi]
-  pub async unsafe fn init(&mut self) -> Result<(), napi::Error> {
+  pub async unsafe fn init(
+    &mut self,
+    git_ignored_files: Vec<String>,
+    is_git_repo: bool,
+  ) -> Result<(), napi::Error> {
     // 1. compute the merkle tree
     // 2. update the backend
     // 3. sync with the remote
+    info!("Merkle tree compute started!");
+    info!("Root directory: {:?}", self.absolute_root_directory);
     unsafe {
-      self.compute_merkle_tree().await?;
+      self
+        .compute_merkle_tree(git_ignored_files, is_git_repo)
+        .await?;
     }
 
     Ok(())
@@ -69,21 +124,25 @@ impl MerkleClient {
   #[napi]
   pub async unsafe fn compute_merkle_tree(
     &mut self,
+    git_ignored_files: Vec<String>,
+    is_git_repo: bool,
   ) -> Result<(), napi::Error> {
-    let t =
-      MerkleTree::construct_merkle_tree(self.root_directory.clone()).await;
+    // make the git ignored files into a hash set
+    let mut git_ignored_set = HashSet::from_iter(git_ignored_files.into_iter());
 
-    let files = self.tree.get_all_files().await;
-
-    match files {
-      Ok(files) => {
-        info!("files: {:?}", files);
-      }
-      Err(e) => {
-        info!("Error in get_all_files: {:?}", e);
-      }
+    // if the hashset itself contains the root directory, then we should remove it.
+    // this is because the root directory is not a file, and we don't want to ignore it.
+    if git_ignored_set.contains(&self.absolute_root_directory) {
+      git_ignored_set.remove(&self.absolute_root_directory);
     }
 
+    let t = MerkleTree::construct_merkle_tree(
+      self.absolute_root_directory.clone(),
+      git_ignored_set,
+      is_git_repo,
+    )
+    .await;
+
     match t {
       Ok(tree) => {
         self.tree = tree;
@@ -111,19 +170,50 @@ impl MerkleClient {
     &self,
     relative_path: String,
   ) -> Result<String, napi::Error> {
-    info!("relative_path: {:?}", relative_path);
-    let absolute_path =
-      std::path::Path::new(&self.root_directory).join(relative_path);
-    let canonical_path = absolute_path.canonicalize().unwrap();
+    debug!("get_subtree_hash: relative_path: {:?}", relative_path);
+
+    let relative_path_without_leading_slash = match relative_path
+      .strip_prefix('.')
+    {
+      Some(path) => path.strip_prefix(std::path::MAIN_SEPARATOR).unwrap_or(""),
+      None => relative_path.as_str(),
+    };
+    debug!(
+      "relative_path_without_leading_slash: {:?}",
+      relative_path_without_leading_slash
+    );
+
+    let absolute_path = if !relative_path_without_leading_slash.is_empty() {
+      std::path::Path::new(&self.absolute_root_directory)
+        .join(relative_path_without_leading_slash)
+    } else {
+      std::path::Path::new(&self.absolute_root_directory).to_path_buf()
+    };
+
+    debug!("absolute_path: {:?}", absolute_path);
+
+    let absolute_path_string = match absolute_path.to_str() {
+      Some(path) => path.to_string(),
+      None => {
+        return Err(napi::Error::new(
+          napi::Status::Unknown,
+          format!("some string error"),
+        ))
+      }
+    };
 
-    info!("canonical_path: {:?}", canonical_path);
-    let hash = self.tree.get_subtree_hash(canonical_path).await;
+    debug!("absolute_path_string: {:?}", absolute_path_string);
+
+    let hash = self
+      .tree
+      .get_subtree_hash(absolute_path_string.as_str())
+      .await;
 
     match hash {
       Ok(hash) => Ok(hash),
       Err(e) => Err(napi::Error::new(
         napi::Status::Unknown,
-        format!("Error in get_subtree_hash: {:?}", e),
+        format!("Error in get_subtree_hash. \nRelative path: {:?}, \nAbsolute path: {:?}, \nRoot directory: {:?}\nError: {:?}", &relative_path, absolute_path, self.absolute_root_directory, e)
       )),
     }
   }
@@ -145,7 +235,7 @@ impl MerkleClient {
     &self,
     relative_path: String,
   ) -> Result<i32, napi::Error> {
-    let absolute_path = std::path::Path::new(&self.root_directory)
+    let absolute_path = std::path::Path::new(&self.absolute_root_directory)
       .join(relative_path)
       .canonicalize()?;
 
@@ -181,10 +271,12 @@ impl MerkleClient {
     &self,
     absolute_file_path: String,
   ) -> Result<Vec<String>, napi::Error> {
-    let absolute_path_str = absolute_file_path.as_str();
+    // let absolute_path = absolute_file_path.to_lowercase();
+    // let absolute_path_str = absolute_path.as_str();
+
     let files = self
       .tree
-      .get_all_dir_files_to_embed(absolute_path_str)
+      .get_all_dir_files_to_embed(absolute_file_path.as_str())
       .await;
 
     match files {
@@ -209,11 +301,7 @@ impl MerkleClient {
         // TODO(sualeh): we should assert that the path is ascending up to the path.
 
         let ret = vec![file];
-        info!("file: {:?}", ret);
-
         let ret = ret.into_iter().chain(path.into_iter()).collect::<Vec<_>>();
-        info!("ret to js: {:?}", ret);
-
         Ok(ret)
       }
       Err(e) => Err(napi::Error::new(
@@ -229,8 +317,9 @@ impl MerkleClient {
     &self,
     absolute_file_path: String,
   ) -> Result<Vec<String>, napi::Error> {
-    let absolute_path_str = absolute_file_path.as_str();
-    let spline = self.tree.get_spline(absolute_path_str).await;
+    // let absolute_path = absolute_file_path.to_lowercase();
+    // let absolute_path_str = absolute_path.as_str();
+    let spline = self.tree.get_spline(absolute_file_path.as_str()).await;
 
     match spline {
       Ok(spline) => Ok(spline),
@@ -259,6 +348,6 @@ impl MerkleClient {
 
   #[napi]
   pub fn update_root_directory(&mut self, root_directory: String) {
-    self.root_directory = root_directory;
+    self.absolute_root_directory = root_directory;
   }
 }

package/src/logger.rs

@@ -0,0 +1,55 @@
+use tracing::{info, subscriber, Level};
+use tracing_appender::non_blocking::WorkerGuard;
+use tracing_appender::rolling::{RollingFileAppender, Rotation};
+use tracing_subscriber::fmt;
+use tracing_subscriber::prelude::*;
+
+pub enum GuardType {
+  #[cfg(all(not(feature = "debugfile"), not(target_os = "linux")))]
+  Guard(tracing_axiom::Guard),
+  WorkerGuard(tracing_appender::non_blocking::WorkerGuard),
+}
+
+pub fn init_logger() -> Option<GuardType> {
+  #[cfg(feature = "debugfile")]
+  let _guard = {
+    let file_appender =
+      RollingFileAppender::new(Rotation::NEVER, "./", "rust_log.txt");
+    let (non_blocking, _guard) = tracing_appender::non_blocking(file_appender);
+    let subscriber = fmt::Subscriber::builder()
+      .with_max_level(Level::TRACE)
+      .with_writer(non_blocking)
+      .with_ansi(false)
+      .with_line_number(true)
+      .finish();
+
+    let _ = tracing::subscriber::set_global_default(subscriber);
+
+    Some(GuardType::WorkerGuard(_guard))
+  };
+
+  #[cfg(all(not(feature = "debugfile"), not(target_os = "linux")))]
+  let _guard = {
+    let (axiom_layer, _guard) = tracing_axiom::builder()
+      .with_token("xaat-a51088e6-7889-41c0-b440-cfd4601acdd7")
+      .with_dataset("local-indexing")
+      .layer()
+      .ok()?;
+    // let fmt_layer = fmt::layer().with_level(true).with_ansi(false).with_line_number(true);
+
+    let _ = tracing_subscriber::registry()
+      .with(axiom_layer)
+      .try_init()
+      .ok()?;
+    // let _ = tracing::subscriber::set_global_default(subscriber);
+
+    info!("Tracing initialized! in rust");
+
+    Some(GuardType::Guard(_guard))
+  };
+
+  #[cfg(all(not(feature = "debugfile"), target_os = "linux"))]
+  let _guard = { None };
+
+  _guard
+}

package/src/merkle_tree/local_construction.rs

@@ -1,4 +1,3 @@
-use crate::git_utils;
 use crate::merkle_tree::{
   File, MerkleNode, MerkleNodePtr, NodeType, PinnedFuture,
 };
@@ -10,11 +9,18 @@ use tonic::async_trait;
 
 #[async_trait]
 impl LocalConstruction for MerkleTree {
+  #[tracing::instrument]
   async fn new(
     root_directory: Option<String>,
   ) -> Result<MerkleTree, anyhow::Error> {
+    let git_ignored_files = HashSet::<String>::new();
     if let Some(root_directory) = root_directory {
-      let n = MerkleTree::construct_merkle_tree(root_directory).await;
+      let n = MerkleTree::construct_merkle_tree(
+        root_directory,
+        git_ignored_files,
+        false,
+      )
+      .await;
       return n;
     }
 
@@ -30,6 +36,8 @@ impl LocalConstruction for MerkleTree {
   /// 4. return merkle tree
   async fn construct_merkle_tree(
     absolute_path_to_root_directory: String,
+    git_ignored_files_and_dirs: HashSet<String>,
+    is_git_repo: bool,
   ) -> Result<MerkleTree, anyhow::Error> {
     let path = PathBuf::from(absolute_path_to_root_directory.clone());
     if !path.exists() {
@@ -38,29 +46,31 @@ impl LocalConstruction for MerkleTree {
     }
 
     // 1. get all the gitignored files
-    let git_ignored_files = match git_utils::list_ignored_files(
-      absolute_path_to_root_directory.as_str(),
-      true,
-    ) {
-      Ok(git_ignored) => git_ignored,
-      Err(_e) => HashSet::new(),
-    };
-
-    tracing::info!("git_ignored_files: {:?}", git_ignored_files);
+    // let git_ignored_files_and_dirs =
+    //   match git_utils::list_ignored_files_and_directories(
+    //     absolute_path_to_root_directory.as_str(),
+    //     true,
+    //   ) {
+    //     Ok(git_ignored) => git_ignored,
+    //     Err(_e) => HashSet::new(),
+    //   };
 
     let root_node = MerkleNode::new(
       path,
       None,
-      &git_ignored_files,
+      &git_ignored_files_and_dirs,
       absolute_path_to_root_directory.as_str(),
+      is_git_repo,
     )
     .await;
+
     let mut mt = MerkleTree {
       root: root_node,
       files: BTreeMap::new(),
       root_path: absolute_path_to_root_directory,
       cursor: None,
-      git_ignored_files,
+      git_ignored_files_and_dirs,
+      is_git_repo,
     };
 
     // we now iterate over all the nodes and add them to the hashmap
@@ -81,6 +91,13 @@ impl LocalConstruction for MerkleTree {
           }
           NodeType::File(file_name) => {
             let f = File { node: node.clone() };
+
+            // i dont reallly  like this :(((
+            // let canonical_file_name = match dunce::canonicalize(file_name) {
+            //   Ok(path) => path.to_str().unwrap_or(file_name).to_string(),
+            //   Err(_) => file_name.clone(),
+            // };
+
             files.insert(file_name.clone(), f);
           }
           NodeType::ErrorNode(_) => {
@@ -92,6 +109,8 @@ impl LocalConstruction for MerkleTree {
 
     add_nodes_to_hashmap(&mt.root, &mut mt.files).await;
 
+    tracing::info!("number of files in the tree: {}", mt.files.len());
+
     Ok(mt)
   }
 
@@ -134,6 +153,7 @@ impl LocalConstruction for MerkleTree {
     Ok(())
   }
 
+  #[tracing::instrument]
   async fn delete_file(
     &mut self,
     file_path: String,

package/src/merkle_tree/mod.rs

@@ -1,24 +1,26 @@
-use crate::git_utils;
-
 use super::file_utils;
 use sha2::Digest;
 use std::collections::{BTreeMap, HashSet};
 use std::path::PathBuf;
+use std::vec;
 use std::{fs, path::Path, sync::Arc};
 use tokio::sync::RwLock;
 use tonic::async_trait;
-use tracing::info;
+use tracing::{debug, info};
+
 pub mod local_construction;
 pub mod test;
 
 pub type MerkleNodePtr = Arc<RwLock<MerkleNode>>;
 
+#[derive(Debug)]
 pub struct MerkleTree {
   root_path: String,
   root: MerkleNodePtr,
   files: BTreeMap<String, File>,
   cursor: Option<usize>,
-  git_ignored_files: HashSet<String>,
+  git_ignored_files_and_dirs: HashSet<String>,
+  is_git_repo: bool,
 }
 
 #[derive(Debug)]
@@ -62,6 +64,8 @@ pub trait LocalConstruction {
 
   async fn construct_merkle_tree(
     root_directory: String,
+    git_ignored_files_and_dirs: HashSet<String>,
+    is_git_repo: bool,
   ) -> Result<MerkleTree, anyhow::Error>;
 
   async fn update_file(
@@ -95,33 +99,34 @@ impl MerkleTree {
       files: BTreeMap::new(),
       root_path: "".to_string(),
       cursor: None,
-      git_ignored_files: HashSet::new(),
+      git_ignored_files_and_dirs: HashSet::new(),
+      is_git_repo: false,
     }
   }
 
   pub async fn get_subtree_hash(
     &self,
-    absolute_path: PathBuf,
+    absolute_path: &str,
   ) -> Result<String, anyhow::Error> {
-    let abs_string = match absolute_path.to_str() {
-      Some(s) => s.to_string(),
-      None => {
-        return Err(anyhow::anyhow!(
-          "get_subtree_hash: Failed to convert path to string"
-        ))
-      }
-    };
+    debug!("get_subtree_hash: absolute_path: {:?}", absolute_path);
 
-    let node = match self.files.get(&abs_string) {
+    let node = match self.files.get(absolute_path) {
       Some(file) => file.node.clone(),
       None => {
-        return Err(anyhow::anyhow!("Could not find file in tree!"));
+        let all_files: Vec<String> = self.files.keys().cloned().collect();
+        return Err(anyhow::anyhow!(
+          "Could not find file in tree! Looking for: {}. All files: {:?}",
+          absolute_path,
+          all_files
+        ));
       }
     };
 
     let node_reader = node.read().await;
     let node_hash = node_reader.hash.clone();
 
+    debug!("node_hash: {:?}", node_hash);
+
     Ok(node_hash)
   }
 
@@ -285,6 +290,12 @@ impl MerkleTree {
   ) -> Result<Vec<String>, anyhow::Error> {
     let mut files = Vec::new();
 
+    // 1. should check that this absolute path is actually a directory.
+    let file_node = self.files.get(absolute_path);
+    if file_node.is_none() {
+      return Err(anyhow::anyhow!("Could not find directory the in tree!"));
+    }
+
     for (file_path, f) in &self.files {
       if !file_path.contains(absolute_path) {
         continue;
@@ -311,16 +322,11 @@ impl MerkleTree {
     &self,
     absolute_path: &str,
   ) -> Result<Vec<String>, anyhow::Error> {
-    info!("get_spline called with absolute_path: {}", absolute_path);
     let mut files = Vec::new();
 
     let current_node = match self.files.get(absolute_path) {
-      Some(node) => {
-        info!("Found node for absolute_path: {}", absolute_path);
-        node.node.clone()
-      }
+      Some(node) => node.node.clone(),
       None => {
-        info!("File not found for absolute_path: {}", absolute_path);
         return Err(anyhow::anyhow!("File not found: {}", absolute_path));
       }
     };
@@ -331,7 +337,6 @@ impl MerkleTree {
     while let Some(node) = stack.pop() {
       let parent = node.read().await.parent.clone();
       if let Some(parent) = parent {
-        info!("Adding parent hash to files vector");
         {
           let parent_node = parent.read().await;
           match &parent_node.node_type {
@@ -350,7 +355,6 @@ impl MerkleTree {
         stack.push(parent);
       }
     }
-    info!("Returning files vector with {} elements", files.len());
     Ok(files)
   }
 
@@ -397,8 +401,9 @@ impl MerkleTree {
         let new_node = MerkleNode::new(
           file_path.clone(),
           Some(ancestor.clone()),
-          &self.git_ignored_files,
+          &self.git_ignored_files_and_dirs,
           &absolute_root_path.as_str(),
+          self.is_git_repo,
         )
         .await;
         ancestor.write().await.attach_child(new_node.clone()).await;
@@ -414,8 +419,9 @@ impl MerkleTree {
         let first_child = MerkleNode::new(
           first_child_path.clone(),
           Some(ancestor.clone()),
-          &self.git_ignored_files,
+          &self.git_ignored_files_and_dirs,
           &absolute_root_path.as_str(),
+          self.is_git_repo,
         )
         .await;
 
@@ -703,13 +709,14 @@ impl MerkleNode {
     parent: ParentPtr,
     ignored_files: &IgnoredFiles,
     absolute_root_path: &str,
+    is_git_repo: bool,
   ) -> MerkleNodePtr {
-    // check if the root is a git directory.
-    let is_git_repo =
-      match git_utils::is_git_directory(absolute_root_path).await {
-        Ok(is_git_repo) => is_git_repo,
-        Err(e) => false,
-      };
+    // // check if the root is a git directory.
+    // let is_git_repo =
+    //   match git_utils::is_git_directory(absolute_root_path).await {
+    //     Ok(is_git_repo) => is_git_repo,
+    //     Err(_e) => false,
+    //   };
     let bypass_git = !is_git_repo;
 
     MerkleNode::construct_node(
@@ -722,25 +729,20 @@ impl MerkleNode {
     .await
   }
 
+  // #[tracing::instrument]
   async fn new(
     absolute_file_or_directory: PathBuf,
     parent: ParentPtr,
     ignored_files: &IgnoredFiles,
     absolute_root_path: &str,
+    is_git_repo: bool,
   ) -> MerkleNodePtr {
-    // check if the root is a git directory.
-    let is_git_repo =
-      match git_utils::is_git_directory(absolute_root_path).await {
-        Ok(is_git_repo) => is_git_repo,
-        Err(_e) => false,
-      };
     let bypass_git = !is_git_repo;
 
     info!(
       "constructing node for absolute_file_or_directory: {:?}",
       absolute_file_or_directory
     );
-    info!("bypass_git: {}, is_git_repo: {}", bypass_git, is_git_repo);
 
     MerkleNode::construct_node(
       Path::new(&absolute_file_or_directory),
@@ -769,6 +771,7 @@ impl MerkleNode {
     Box::pin(async move {
       // check if it is a file
       let path_str = absolute_file_or_directory.to_str().unwrap().to_string();
+
       if absolute_file_or_directory.is_file() {
         return Arc::new(RwLock::new(
           MerkleNode::construct_file_node_or_error_node(
@@ -790,22 +793,10 @@ impl MerkleNode {
         )));
       }
 
-      // check if the directory is git ignored
-      let is_git_ignored =
-        match git_utils::is_git_ignored(absolute_root_path, path_str.as_str())
-          .await
-        {
-          Ok(is_git_ignored) => is_git_ignored,
-          Err(e) => {
-            return Arc::new(RwLock::new(MerkleNode::empty_node(
-              Some(absolute_file_or_directory),
-              Some(e.to_string()),
-            )));
-          }
-        };
+      let is_git_ignored_dir = ignored_files.contains(&path_str);
 
-      if is_git_ignored && !bypass_git {
-        // println!("skipping directory: {}", path_str);
+      if is_git_ignored_dir && !bypass_git {
+        tracing::info!("skipping directory: {}", path_str);
         return Arc::new(RwLock::new(MerkleNode::empty_node(
           Some(absolute_file_or_directory),
           Some("Directory is git ignored!".to_string()),
@@ -816,6 +807,7 @@ impl MerkleNode {
       match entries {
         Ok(_) => (),
         Err(e) => {
+          tracing::error!("error reading directory: {}", e);
           return Arc::new(RwLock::new(MerkleNode::empty_node(
             Some(absolute_file_or_directory),
             Some(e.to_string()),
@@ -848,6 +840,7 @@ impl MerkleNode {
             );
           }
           Err(e) => {
+            tracing::error!("error reading directory: {}", e);
             children.push(Arc::new(RwLock::new(MerkleNode::empty_node(
               Some(absolute_file_or_directory),
               Some(e.to_string()),
@@ -894,18 +887,10 @@ impl MerkleNode {
       false => {}
     }
 
-    // read the file_content to a buffer
-    let file_content = match tokio::fs::read(absolute_file_path).await {
-      Ok(content) => content,
-      Err(e) => {
-        return Err(format!("Could not read file! {}", e.to_string()));
-      }
-    };
-
     // check if the file passes runtime checks.
     match file_utils::is_good_file_runtime_check(
       absolute_file_path,
-      &file_content,
+      // &file_content,
     )
     .await
     {
@@ -915,15 +900,14 @@ impl MerkleNode {
       }
     }
 
-    let file_content = match std::str::from_utf8(&file_content) {
-      Ok(content) => content,
-      Err(e) => {
-        return Err(format!(
-          "UTF8 Failure. Could not convert file content to string! {}",
-          e.to_string()
-        ));
-      }
-    };
+    // read the file_content to a buffer
+    let file_content =
+      match file_utils::read_string_without_bom(absolute_file_path).await {
+        Ok(content) => content,
+        Err(e) => {
+          return Err(format!("Could not read file! {}", e.to_string()));
+        }
+      };
 
     let file_hash = compute_hash(&file_content);
     let node = MerkleNode {
@@ -950,11 +934,7 @@ impl MerkleNode {
     .await
     {
       Ok(node) => node,
-      Err(e) => {
-        // println!("constructing error node. error: {}", e);
-        // println!("file_path: {:?}", file_path);
-        MerkleNode::empty_node(Some(absolute_file_path), Some(e))
-      }
+      Err(e) => MerkleNode::empty_node(Some(absolute_file_path), Some(e)),
     };
 
     node
@@ -979,15 +959,50 @@ impl MerkleNode {
 
   async fn compute_branch_hash(children: &[MerkleNodePtr]) -> String {
     let mut hasher = sha2::Sha256::new();
+    let mut names_and_hashes = vec![];
+    let mut non_zero_children = 0;
+
     for child in children {
       // check if it is an error node
       let child_reader = child.read().await;
-      if let NodeType::ErrorNode(_) = &child_reader.node_type {
+
+      match &child_reader.node_type {
+        NodeType::File(file_name) => {
+          non_zero_children += 1;
+          names_and_hashes.push((file_name.clone(), child_reader.hash.clone()));
+        }
+        NodeType::Branch((file_name, _)) => {
+          let hash = child_reader.hash.clone();
+          if hash == "" {
+            continue;
+          }
+
+          non_zero_children += 1;
+          names_and_hashes.push((file_name.clone(), hash));
+        }
+        NodeType::ErrorNode(_) => {
+          continue;
+        }
+      }
+    }
+
+    // sort the list of names and hashes by the hashes!!
+    names_and_hashes
+      .sort_by(|a, b| a.1.to_lowercase().cmp(&b.1.to_lowercase()));
+
+    for (name, hash) in names_and_hashes {
+      if hash == "" {
         continue;
       }
+      hasher.update(hash);
+    }
 
-      hasher.update(child_reader.hash.as_bytes());
+    if non_zero_children == 0 {
+      // this means that the branch is empty.
+      // we should return an empty string.
+      return "".to_string();
     }
+
     let result = hasher.finalize();
     format!("{:x}", result)
   }

package/src/merkle_tree/test.rs

@@ -43,8 +43,9 @@ mod tests {
     // let path = Path::new(&temp_dir_path);
 
     // Test construct_merkle_tree() function
+    let new_set = std::collections::HashSet::<String>::new();
     let tree =
-      MerkleTree::construct_merkle_tree(temp_dir_path.clone()).await;
+      MerkleTree::construct_merkle_tree(temp_dir_path.clone(), new_set, false).await;
     let mut tree = match tree {
       Ok(tree) => {
         assert_eq!(tree.files.len(), 2);