@anysphere/file-service 0.0.0-c0e75c6f → 0.0.0-c8c4f8fc

package/Cargo.toml

@@ -17,6 +17,11 @@ tempfile = "3.8.0"
 anyhow = "1.0.75"
 tonic = "0.9.2"
 prost = "0.11.9"
+tracing = "0.1.37"
+tracing-subscriber = "0.3.17"
+tracing-appender = "0.2.2"
+binaryornot = "1.0.0"
+dunce = "1.0.1"
 
 [build-dependencies]
 napi-build = "2.0.1"

package/index.d.ts

@@ -4,13 +4,17 @@
 /* auto-generated by NAPI-RS */
 
 export class MerkleClient {
-  constructor(rootDirectory: string)
+  constructor(absoluteRootDirectory: string)
   init(): Promise<void>
+  computeMerkleTree(): Promise<void>
   updateFile(filePath: string): Promise<void>
   deleteFile(filePath: string): Promise<void>
-  getSubtreeHash(path: string): Promise<string>
+  getSubtreeHash(relativePath: string): Promise<string>
   getNumEmbeddableFiles(): Promise<number>
   getAllFiles(): Promise<Array<string>>
+  getAllDirFilesToEmbed(absoluteFilePath: string): Promise<Array<string>>
   getNextFileToEmbed(): Promise<Array<string>>
+  getSpline(absoluteFilePath: string): Promise<Array<string>>
   getHashesForFiles(files: Array<string>): Promise<Array<string>>
+  updateRootDirectory(rootDirectory: string): void
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@anysphere/file-service",
-  "version": "0.0.0-c0e75c6f",
+  "version": "0.0.0-c8c4f8fc",
   "main": "index.js",
   "types": "index.d.ts",
   "napi": {
@@ -9,7 +9,8 @@
       "additional": [
         "aarch64-apple-darwin",
         "aarch64-pc-windows-msvc",
-        "universal-apple-darwin"
+        "universal-apple-darwin",
+        "aarch64-unknown-linux-gnu"
       ]
     }
   },
@@ -35,11 +36,12 @@
     "version": "napi version"
   },
   "optionalDependencies": {
-    "@anysphere/file-service-win32-x64-msvc": "0.0.0-c0e75c6f",
-    "@anysphere/file-service-darwin-x64": "0.0.0-c0e75c6f",
-    "@anysphere/file-service-linux-x64-gnu": "0.0.0-c0e75c6f",
-    "@anysphere/file-service-darwin-arm64": "0.0.0-c0e75c6f",
-    "@anysphere/file-service-win32-arm64-msvc": "0.0.0-c0e75c6f",
-    "@anysphere/file-service-darwin-universal": "0.0.0-c0e75c6f"
+    "@anysphere/file-service-win32-x64-msvc": "0.0.0-c8c4f8fc",
+    "@anysphere/file-service-darwin-x64": "0.0.0-c8c4f8fc",
+    "@anysphere/file-service-linux-x64-gnu": "0.0.0-c8c4f8fc",
+    "@anysphere/file-service-darwin-arm64": "0.0.0-c8c4f8fc",
+    "@anysphere/file-service-win32-arm64-msvc": "0.0.0-c8c4f8fc",
+    "@anysphere/file-service-darwin-universal": "0.0.0-c8c4f8fc",
+    "@anysphere/file-service-linux-arm64-gnu": "0.0.0-c8c4f8fc"
   }
 }

package/src/file_utils.rs

@@ -12,9 +12,8 @@ pub fn is_in_bad_dir(file_path: &Path) -> Result<bool, Error> {
   let item_path = file_path
     .to_str()
     .ok_or(anyhow::anyhow!("Failed to convert path to string"))?;
-  let is_bad_dir = (item_path.contains("node_modules")
-    || item_path.contains(".git"))
-    && !(item_path.ends_with(".git") || item_path.ends_with("node_modules"));
+  let is_bad_dir =
+    item_path.contains("node_modules") || item_path.contains(".git");
   Ok(is_bad_dir)
 }
 
@@ -38,14 +37,14 @@ pub fn is_good_file(file_path: &Path) -> Result<(), Error> {
 
   match file_name {
     "package-lock.json" | "pnpm-lock.yaml" | "yarn.lock" | "composer.lock"
-    | "Gemfile.lock" => {
+    | "Gemfile.lock" | "bun.lockb" => {
       return Err(anyhow::anyhow!("File is just a lock file"));
     }
     _ => {}
   }
 
   match extension {
-    "lock" | "bak" | "tmp" | "bin" | "exe" | "dll" | "so" => {
+    "lock" | "bak" | "tmp" | "bin" | "exe" | "dll" | "so" | "lockb" => {
       return Err(anyhow::anyhow!("File is just a lock file"));
     }
     _ => {}
@@ -63,7 +62,7 @@ pub fn is_good_file(file_path: &Path) -> Result<(), Error> {
     Some(extension) => match extension.to_str() {
       Some(ext_str) => {
         if bad_extensions.contains(&ext_str) {
-          return Err(anyhow::anyhow!("File is not a valid UTF-8 string"));
+          return Err(anyhow::anyhow!("Binary file excluded from indexing."));
         }
       }
       None => {
@@ -89,10 +88,12 @@ pub fn is_good_file(file_path: &Path) -> Result<(), Error> {
   Ok(())
 }
 
+// use binaryornot::is_binary;
+// use anyhow::Context;
 // implement the buffer above:
 pub async fn is_good_file_runtime_check(
   file_path: &Path,
-  buffer: &[u8],
+  _buffer: &[u8],
 ) -> Result<(), Error> {
   match get_file_size(file_path).await {
     Ok(size) if size > 2 * 1024 * 1024 => {
@@ -102,13 +103,10 @@ pub async fn is_good_file_runtime_check(
     _ => {}
   }
 
-  for &byte in buffer.iter().take(2048) {
-    if byte.is_ascii() {
-      continue;
-    } else {
-      return Err(anyhow::anyhow!("File is not a valid UTF-8 string"));
-    }
-  }
+	// if is_binary(file_path).context("Failed to check if file is binary")? {
+	// 	return Err(anyhow::anyhow!("File is binary"));
+	// }
+
   Ok(())
 }
 

package/src/git_utils.rs

@@ -1,10 +1,10 @@
 use std::collections::HashSet;
-use std::path::PathBuf;
 use std::process::Command;
 
 pub fn list_ignored_files(
   workspace_root_path: &str,
-) -> Result<HashSet<PathBuf>, Box<dyn std::error::Error>> {
+  should_return_absolute_paths: bool,
+) -> Result<HashSet<String>, Box<dyn std::error::Error>> {
   let mut gitignored_files = HashSet::new();
 
   let commands = vec![
@@ -15,12 +15,66 @@ pub fn list_ignored_files(
       "--ignored",
       "--exclude-standard",
     ],
+    // FIXME(sualeh): this is super sketchy and might totally break in like a bazillion ways. i dont like it.
+    vec![
+      "sh",
+      "-c",
+      "git submodule foreach --quiet 'git ls-files --others --ignored --exclude-standard | sed \"s|^|$path/|\"'",
+    ],
+  ];
+
+  for command in commands {
+    let output = Command::new(&command[0])
+      .args(&command[1..])
+      .current_dir(workspace_root_path)
+      .output()?;
+
+    if output.status.success() {
+      let files = String::from_utf8(output.stdout)?
+        .lines()
+        .filter(|line| !line.is_empty())
+        .map(|line| {
+          if should_return_absolute_paths {
+            let mut path = std::path::PathBuf::from(workspace_root_path);
+            path.push(line);
+
+            match path.canonicalize() {
+              Ok(canonical_path) => {
+                canonical_path.to_string_lossy().into_owned()
+              }
+              Err(_) => String::from(line),
+            }
+          } else {
+            String::from(line)
+          }
+        })
+        .collect::<HashSet<_>>();
+
+      gitignored_files.extend(files);
+    }
+  }
+
+  Ok(gitignored_files)
+}
+
+pub fn list_ignored_files_with_absolute_paths(
+  workspace_root_path: &str,
+) -> Result<HashSet<String>, Box<dyn std::error::Error>> {
+  let mut gitignored_files = HashSet::new();
+
+  let commands = vec![
     vec![
       "git",
-      "submodule",
-      "foreach",
-      "--quiet",
-      "git ls-files --others --ignored --exclude-standard | sed 's|^|$path/|'",
+      "ls-files",
+      "--others",
+      "--ignored",
+      "--exclude-standard",
+    ],
+    // FIXME(sualeh): this is super sketchy and might totally break in like a bazillion ways. i dont like it.
+    vec![
+      "sh",
+      "-c",
+      "git submodule foreach --quiet 'git ls-files --others --ignored --exclude-standard | sed \"s|^|$path/|\"'",
     ],
   ];
 
@@ -34,7 +88,7 @@ pub fn list_ignored_files(
       let files = String::from_utf8(output.stdout)?
         .lines()
         .filter(|line| !line.is_empty())
-        .map(|line| PathBuf::from(line))
+        .map(|line| String::from(line))
         .collect::<HashSet<_>>();
 
       gitignored_files.extend(files);
@@ -57,6 +111,18 @@ pub async fn is_git_ignored(
   Ok(output.status.success())
 }
 
+pub async fn is_git_directory(
+  workspace_root_path: &str,
+) -> Result<bool, anyhow::Error> {
+  let output = tokio::process::Command::new("git")
+    .args(&["rev-parse", "--is-inside-work-tree"])
+    .current_dir(workspace_root_path)
+    .output()
+    .await?;
+
+  Ok(output.status.success())
+}
+
 #[cfg(test)]
 mod tests {
   use super::*;
@@ -66,7 +132,8 @@ mod tests {
   #[test]
   fn test_no_ignored_files() {
     let dir = tempfile::tempdir().unwrap();
-    let gitignored_files = list_ignored_files(dir.path().to_str().unwrap()).unwrap();
+    let gitignored_files =
+      list_ignored_files(dir.path().to_str().unwrap(), false).unwrap();
     Command::new("git")
       .args(&["init"])
       .current_dir(dir.path())
@@ -92,13 +159,14 @@ mod tests {
       .current_dir(dir.path())
       .output()
       .unwrap();
-    let gitignored_files = list_ignored_files(dir.path().to_str().unwrap()).unwrap();
+    let gitignored_files =
+      list_ignored_files(dir.path().to_str().unwrap(), false).unwrap();
     println!(
       "ignored files for test_one_ignored_file: {:?}",
       gitignored_files
     );
     // assert_eq!(gitignored_files.len(), 1);
-    assert!(gitignored_files.contains(&PathBuf::from("ignored.txt")));
+    assert!(gitignored_files.contains(&String::from("ignored.txt")));
   }
 
   #[test]
@@ -121,19 +189,83 @@ mod tests {
       .current_dir(dir.path())
       .output()
       .unwrap();
-    let gitignored_files = list_ignored_files(dir.path().to_str().unwrap()).unwrap();
+    let gitignored_files =
+      list_ignored_files(dir.path().to_str().unwrap(), false).unwrap();
     println!(
       "ignored files for test_multiple_ignored_files: {:?}",
       gitignored_files
     );
     // assert_eq!(gitignored_files.len(), 2);
-    assert!(gitignored_files.contains(&PathBuf::from("ignored1.txt")));
-    assert!(gitignored_files.contains(&PathBuf::from("ignored2.txt")));
+    assert!(gitignored_files.contains(&String::from("ignored1.txt")));
+    assert!(gitignored_files.contains(&String::from("ignored2.txt")));
+  }
+
+  #[test]
+  fn test_git_submodule_ignored_files() {
+    let dir = tempfile::tempdir().unwrap();
+    let submodule_path = dir.path().join("submodule");
+    std::fs::create_dir(&submodule_path).unwrap();
+
+    let o = Command::new("git")
+      .args(&["init"])
+      .current_dir(&submodule_path)
+      .output()
+      .unwrap();
+    println!("git init output: {:?}", o);
+
+    let file_path = submodule_path.join("ignored.txt");
+    let mut file = File::create(&file_path).unwrap();
+    writeln!(file, "This is an ignored file.").unwrap();
+
+    let file2 = submodule_path.join("ignored2.txt");
+    let mut file = File::create(&file2).unwrap();
+    writeln!(file, "This is another ignored file.").unwrap();
+
+    let gitignore_path = submodule_path.join(".gitignore");
+    let mut gitignore = File::create(&gitignore_path).unwrap();
+    writeln!(gitignore, "*.txt").unwrap();
+
+    let o = Command::new("git")
+      .args(&["init"])
+      .current_dir(dir.path())
+      .output()
+      .unwrap();
+    println!("git init output: {:?}", o);
+
+    // make a commit in the submodule
+    let o = Command::new("git")
+      .args(&["add", "."])
+      .current_dir(&submodule_path)
+      .output()
+      .unwrap();
+    println!("git add output: {:?}", o);
+    let o = Command::new("git")
+      .args(&["commit", "-m", "initial commit"])
+      .current_dir(&submodule_path)
+      .output()
+      .unwrap();
+    println!("git commit output: {:?}", o);
+
+    let o = Command::new("git")
+      .args(&["submodule", "add", "./submodule"])
+      .current_dir(dir.path())
+      .output()
+      .unwrap();
+    println!("git submodule add output: {:?}", o);
+
+    let gitignored_files =
+      list_ignored_files(dir.path().to_str().unwrap(), false).unwrap();
+    println!(
+      "ignored files for test_git_submodule_ignored_files: {:?}",
+      gitignored_files
+    );
+    assert!(gitignored_files.contains(&String::from("submodule/ignored.txt")));
+    assert!(gitignored_files.contains(&String::from("submodule/ignored2.txt")));
   }
 
   #[test]
   fn test_multiple_ignored_files_in_current_dir() {
-    let gitignored_files = list_ignored_files(".").unwrap();
+    let gitignored_files = list_ignored_files(".", false).unwrap();
     assert!(gitignored_files.len() > 1);
 
     // print a sample of the ignored files
@@ -147,7 +279,6 @@ mod tests {
     }
   }
 
-
   #[tokio::test]
   async fn test_file_not_ignored() {
     let dir = tempfile::tempdir().unwrap();
@@ -160,7 +291,10 @@ mod tests {
       .current_dir(dir.path())
       .output()
       .unwrap();
-    let is_ignored = is_git_ignored(dir.path().to_str().unwrap(), "not_ignored.txt").await.unwrap();
+    let is_ignored =
+      is_git_ignored(dir.path().to_str().unwrap(), "not_ignored.txt")
+        .await
+        .unwrap();
     assert_eq!(is_ignored, false);
   }
 
@@ -180,7 +314,10 @@ mod tests {
       .current_dir(dir.path())
       .output()
       .unwrap();
-    let is_ignored = is_git_ignored(dir.path().to_str().unwrap(), "ignored.txt").await.unwrap();
+    let is_ignored =
+      is_git_ignored(dir.path().to_str().unwrap(), "ignored.txt")
+        .await
+        .unwrap();
     assert_eq!(is_ignored, true);
   }
 
@@ -200,8 +337,10 @@ mod tests {
       .current_dir(dir.path())
       .output()
       .unwrap();
-    let is_ignored = is_git_ignored(dir.path().to_str().unwrap(), "ignored.txt").await.unwrap();
+    let is_ignored =
+      is_git_ignored(dir.path().to_str().unwrap(), "ignored.txt")
+        .await
+        .unwrap();
     assert_eq!(is_ignored, true);
   }
-
 }

package/src/lib.rs

@@ -1,4 +1,5 @@
 #![deny(clippy::all)]
+#![deny(unsafe_op_in_unsafe_fn)]
 pub mod file_utils;
 pub mod git_utils;
 pub mod merkle_tree;
@@ -6,6 +7,9 @@ pub mod merkle_tree;
 use std::vec;
 
 use merkle_tree::{LocalConstruction, MerkleTree};
+use tracing::{info, Level};
+use tracing_appender::rolling::{RollingFileAppender, Rotation};
+use tracing_subscriber::fmt;
 
 #[macro_use]
 extern crate napi_derive;
@@ -13,16 +17,46 @@ extern crate napi_derive;
 #[napi]
 pub struct MerkleClient {
   tree: MerkleTree,
-  root_directory: String,
+  absolute_root_directory: String,
+  _guard: tracing_appender::non_blocking::WorkerGuard,
+}
+
+pub fn init_logger() -> tracing_appender::non_blocking::WorkerGuard {
+  let file_appender =
+    RollingFileAppender::new(Rotation::NEVER, "./", "rust_log.txt");
+  let (non_blocking, _guard) = tracing_appender::non_blocking(file_appender);
+  let subscriber = fmt::Subscriber::builder()
+    .with_max_level(Level::TRACE)
+    .with_writer(non_blocking)
+    .with_ansi(false)
+    .with_line_number(true)
+    .finish();
+
+  let _ = tracing::subscriber::set_global_default(subscriber);
+
+  _guard
 }
 
 #[napi]
 impl MerkleClient {
   #[napi(constructor)]
-  pub fn new(root_directory: String) -> MerkleClient {
+  pub fn new(absolute_root_directory: String) -> MerkleClient {
+    let _guard = init_logger();
+
+    let canonical_root_directory = std::path::Path::new(&absolute_root_directory);
+    // use dunce::canonicalize;
+    let canonical_root_directory = match dunce::canonicalize(&canonical_root_directory) {
+      Ok(path) => path.to_str().unwrap_or(&absolute_root_directory).to_string(),
+      Err(e) => {
+        info!("Error in canonicalizing path: path: {:?}, error {:?}", canonical_root_directory, e);
+        absolute_root_directory
+      }
+    };
+
     MerkleClient {
       tree: MerkleTree::empty_tree(),
-      root_directory,
+      absolute_root_directory: canonical_root_directory,
+      _guard,
     }
   }
 
@@ -31,7 +65,10 @@ impl MerkleClient {
     // 1. compute the merkle tree
     // 2. update the backend
     // 3. sync with the remote
-    self.compute_merkle_tree().await?;
+    info!("Merkle tree compute started!");
+    unsafe {
+      self.compute_merkle_tree().await?;
+    }
 
     Ok(())
   }
@@ -40,12 +77,12 @@ impl MerkleClient {
     unimplemented!("Interrupt is not implemented yet");
   }
 
-  // #[napi]
+  #[napi]
   pub async unsafe fn compute_merkle_tree(
     &mut self,
   ) -> Result<(), napi::Error> {
     let t =
-      MerkleTree::construct_merkle_tree(self.root_directory.clone()).await;
+      MerkleTree::construct_merkle_tree(self.absolute_root_directory.clone()).await;
 
     match t {
       Ok(tree) => {
@@ -72,15 +109,38 @@ impl MerkleClient {
   #[napi]
   pub async fn get_subtree_hash(
     &self,
-    path: String,
+    relative_path: String,
   ) -> Result<String, napi::Error> {
-    let hash = self.tree.get_subtree_hash(path).await;
+    let absolute_path =
+      std::path::Path::new(&self.absolute_root_directory).join(&relative_path);
+
+    let canonical_path = match dunce::canonicalize(&absolute_path) {
+      Ok(path) => path,
+      Err(e) => {
+        return Err(napi::Error::new(
+          napi::Status::Unknown,
+          format!("Error in canonicalizing path: {:?}", e),
+        ))
+      }
+    };
+
+    let cononical_str = match canonical_path.to_str() {
+      Some(s) => s,
+      None => {
+        return Err(napi::Error::new(
+          napi::Status::Unknown,
+          format!("Error in converting canonical path to string"),
+        ))
+      }
+    };
+
+    let hash = self.tree.get_subtree_hash(cononical_str).await;
 
     match hash {
       Ok(hash) => Ok(hash),
       Err(e) => Err(napi::Error::new(
         napi::Status::Unknown,
-        format!("Error in get_subtree_hash: {:?}", e),
+        format!("Error in get_subtree_hash. \nRelative path: {:?}, \nAbsolute path: {:?}, \nCanonical path: {:?}, \nRoot directory: {:?}\nError: {:?}", &relative_path, absolute_path, canonical_path, self.absolute_root_directory, e),
       )),
     }
   }
@@ -98,6 +158,28 @@ impl MerkleClient {
     }
   }
 
+  pub async fn get_num_embeddable_files_in_subtree(
+    &self,
+    relative_path: String,
+  ) -> Result<i32, napi::Error> {
+    let absolute_path = std::path::Path::new(&self.absolute_root_directory)
+      .join(relative_path)
+      .canonicalize()?;
+
+    let num = self
+      .tree
+      .get_num_embeddable_files_in_subtree(absolute_path)
+      .await;
+
+    match num {
+      Ok(num) => Ok(num),
+      Err(e) => Err(napi::Error::new(
+        napi::Status::Unknown,
+        format!("Error in get_num_embeddable_files_in_subtree: {:?}", e),
+      )),
+    }
+  }
+
   #[napi]
   pub async fn get_all_files(&self) -> Result<Vec<String>, napi::Error> {
     let files = self.tree.get_all_files().await;
@@ -111,6 +193,26 @@ impl MerkleClient {
     }
   }
 
+  #[napi]
+  pub async fn get_all_dir_files_to_embed(
+    &self,
+    absolute_file_path: String,
+  ) -> Result<Vec<String>, napi::Error> {
+    let absolute_path_str = absolute_file_path.as_str();
+    let files = self
+      .tree
+      .get_all_dir_files_to_embed(absolute_path_str)
+      .await;
+
+    match files {
+      Ok(files) => Ok(files),
+      Err(e) => Err(napi::Error::new(
+        napi::Status::Unknown,
+        format!("Error in get_all_dir_files_to_embed: {:?}", e),
+      )),
+    }
+  }
+
   #[napi]
   pub async unsafe fn get_next_file_to_embed(
     &mut self,
@@ -125,7 +227,6 @@ impl MerkleClient {
 
         let ret = vec![file];
         let ret = ret.into_iter().chain(path.into_iter()).collect::<Vec<_>>();
-
         Ok(ret)
       }
       Err(e) => Err(napi::Error::new(
@@ -135,6 +236,24 @@ impl MerkleClient {
     }
   }
 
+  // FIXME(sualeh): get_spline
+  #[napi]
+  pub async fn get_spline(
+    &self,
+    absolute_file_path: String,
+  ) -> Result<Vec<String>, napi::Error> {
+    let absolute_path_str = absolute_file_path.as_str();
+    let spline = self.tree.get_spline(absolute_path_str).await;
+
+    match spline {
+      Ok(spline) => Ok(spline),
+      Err(e) => Err(napi::Error::new(
+        napi::Status::Unknown,
+        format!("Error in get_spline: {:?}", e),
+      )),
+    }
+  }
+
   #[napi]
   pub async fn get_hashes_for_files(
     &self,
@@ -151,8 +270,8 @@ impl MerkleClient {
     }
   }
 
-  // #[napi]
+  #[napi]
   pub fn update_root_directory(&mut self, root_directory: String) {
-    self.root_directory = root_directory;
+    self.absolute_root_directory = root_directory;
   }
 }

package/src/merkle_tree/local_construction.rs

@@ -1,10 +1,11 @@
+use crate::git_utils;
 use crate::merkle_tree::{
   File, MerkleNode, MerkleNodePtr, NodeType, PinnedFuture,
 };
 
 use super::{LocalConstruction, MerkleTree};
-use std::path::PathBuf;
-use std::{collections::HashMap, path::Path, sync::Arc};
+use std::collections::{BTreeMap, HashSet};
+use std::path::{Path, PathBuf};
 use tonic::async_trait;
 
 #[async_trait]
@@ -28,32 +29,49 @@ impl LocalConstruction for MerkleTree {
   /// 3. construct merkle tree
   /// 4. return merkle tree
   async fn construct_merkle_tree(
-    root_directory: String,
+    absolute_path_to_root_directory: String,
   ) -> Result<MerkleTree, anyhow::Error> {
-    let path = PathBuf::from(root_directory.clone());
+    let path = PathBuf::from(absolute_path_to_root_directory.clone());
     if !path.exists() {
       // FIXME: we should report this via a good logger.
       panic!("Root directory does not exist!");
     }
 
-    let root_node = MerkleNode::new(path, None).await;
+    // 1. get all the gitignored files
+    let git_ignored_files = match git_utils::list_ignored_files(
+      absolute_path_to_root_directory.as_str(),
+      true,
+    ) {
+      Ok(git_ignored) => git_ignored,
+      Err(_e) => HashSet::new(),
+    };
+
+    let root_node = MerkleNode::new(
+      path,
+      None,
+      &git_ignored_files,
+      absolute_path_to_root_directory.as_str(),
+    )
+    .await;
     let mut mt = MerkleTree {
       root: root_node,
-      files: HashMap::new(),
-      root_path: root_directory,
+      files: BTreeMap::new(),
+      root_path: absolute_path_to_root_directory,
       cursor: None,
+      git_ignored_files,
     };
 
     // we now iterate over all the nodes and add them to the hashmap
     // TODO(later): i can make this parallel.
     fn add_nodes_to_hashmap<'a>(
       node: &'a MerkleNodePtr,
-      files: &'a mut HashMap<String, File>,
+      files: &'a mut BTreeMap<String, File>,
     ) -> PinnedFuture<'a, ()> {
       Box::pin(async move {
         let node_reader = node.read().await;
         match &node_reader.node_type {
           NodeType::Branch(n) => {
+            tracing::info!("Branch: {:?}", n.0);
             let children = &n.1;
             files.insert(n.0.clone(), File { node: node.clone() });
             for child in children {
@@ -62,7 +80,12 @@ impl LocalConstruction for MerkleTree {
           }
           NodeType::File(file_name) => {
             let f = File { node: node.clone() };
-            files.insert(file_name.clone(), f);
+            let canonical_file_name = match dunce::canonicalize(file_name) {
+              Ok(path) => path.to_str().unwrap_or(file_name).to_string(),
+              Err(_) => file_name.clone(),
+            };
+
+            files.insert(canonical_file_name, f);
           }
           NodeType::ErrorNode(_) => {
             // do nothing
@@ -73,6 +96,9 @@ impl LocalConstruction for MerkleTree {
 
     add_nodes_to_hashmap(&mt.root, &mut mt.files).await;
 
+    tracing::info!("Merkle tree compute finished!");
+    tracing::info!("Merkle tree: {}", mt);
+
     Ok(mt)
   }
 

package/src/merkle_tree/mod.rs

@@ -1,9 +1,13 @@
+use crate::git_utils;
+
 use super::file_utils;
 use sha2::Digest;
+use std::collections::{BTreeMap, HashSet};
 use std::path::PathBuf;
-use std::{collections::HashMap, fs, path::Path, sync::Arc};
+use std::{fs, path::Path, sync::Arc};
 use tokio::sync::RwLock;
 use tonic::async_trait;
+use tracing::info;
 pub mod local_construction;
 pub mod test;
 
@@ -12,8 +16,9 @@ pub type MerkleNodePtr = Arc<RwLock<MerkleNode>>;
 pub struct MerkleTree {
   root_path: String,
   root: MerkleNodePtr,
-  files: HashMap<String, File>,
-  cursor: Option<MerkleNodePtr>,
+  files: BTreeMap<String, File>,
+  cursor: Option<usize>,
+  git_ignored_files: HashSet<String>,
 }
 
 #[derive(Debug)]
@@ -87,21 +92,23 @@ impl MerkleTree {
   pub fn empty_tree() -> MerkleTree {
     MerkleTree {
       root: Arc::new(RwLock::new(MerkleNode::empty_node(None, None))),
-      files: HashMap::new(),
+      files: BTreeMap::new(),
       root_path: "".to_string(),
       cursor: None,
+      git_ignored_files: HashSet::new(),
     }
   }
 
   pub async fn get_subtree_hash(
     &self,
-    path: String,
+    absolute_path: &str,
   ) -> Result<String, anyhow::Error> {
-    let path = PathBuf::from(path);
-    let node = match self.files.get(path.to_str().unwrap()) {
+
+    let node = match self.files.get(absolute_path) {
       Some(file) => file.node.clone(),
       None => {
-        return Err(anyhow::anyhow!("Could not find file in tree!"));
+        let all_files: Vec<String> = self.files.keys().cloned().collect();
+        return Err(anyhow::anyhow!("Could not find file in tree! Looking for: {}. All files: {:?}", absolute_path, all_files));
       }
     };
 
@@ -132,6 +139,43 @@ impl MerkleTree {
     Ok(count)
   }
 
+  pub async fn get_num_embeddable_files_in_subtree(
+    &self,
+    absolute_path: PathBuf,
+  ) -> Result<i32, anyhow::Error> {
+    let mut count = 0;
+
+    let absolute_path = match absolute_path.to_str() {
+      Some(s) => s.to_string(),
+      None => {
+        return Err(anyhow::anyhow!(
+        "get_num_embeddable_files_in_subtree: Failed to convert path to string"
+      ))
+      }
+    };
+
+    // TODO(sualeh): worth keeping this list sorted. its now a btree
+
+    for (_, file) in &self.files {
+      let file_reader = file.node.read().await;
+      match &file_reader.node_type {
+        NodeType::File(file_name) => {
+          if file_name.contains(&absolute_path) {
+            count += 1;
+          }
+        }
+        NodeType::Branch(_) => {
+          continue;
+        }
+        NodeType::ErrorNode(_) => {
+          continue;
+        }
+      }
+    }
+
+    Ok(count)
+  }
+
   pub async fn get_all_files(&self) -> Result<Vec<String>, anyhow::Error> {
     let mut files = Vec::new();
 
@@ -188,83 +232,119 @@ impl MerkleTree {
   pub async fn get_next_file_to_embed(
     &mut self,
   ) -> Result<(String, Vec<String>), anyhow::Error> {
-    // the plan is to do an in-order traversal of the tree.
-
-    // first the edge case to deal with:
-    // cursor == None
-    if self.cursor.is_none() {
-      // If the root is a file, return its name.
-      if let NodeType::File(file_path) = &self.root.read().await.node_type {
-        return Ok((file_path.clone(), vec![]));
-      }
-
-      // if the path is not empty, we can iterate till we find the first child.
-      let mut potential_first_child = self.root.clone();
-      let mut is_branch = true;
-      let mut path = Vec::new();
-
-      while is_branch {
-        let node = {
-          let potential_first_child_reader = potential_first_child.read().await;
-          match &potential_first_child_reader.node_type {
-            NodeType::Branch(branch) => branch.clone(),
-            NodeType::File(_) => {
-              return Err(anyhow::anyhow!(
-                "get_next_file_to_embed: This should not happen! the branch happened to be file."
-              ));
-            }
-            NodeType::ErrorNode(_) => {
-              return Err(anyhow::anyhow!("Cursor is an error node!"));
-            }
-          }
-        };
+    // if the cursor is none, set it to 0
+    let cursor = match self.cursor {
+      Some(cursor) => cursor,
+      None => {
+        self.cursor = Some(0);
+        0
+      }
+    };
 
-        let current_node_name = &node.0;
-        let child_list = &node.1;
+    // get the thing at the cursor. while we dont find a file, we keep incrementing the cursor.
+    let mut cursor = cursor;
+    loop {
+      // O(log n)
+      let file = match self.files.values().nth(cursor) {
+        Some(file) => file,
+        None => {
+          return Err(anyhow::anyhow!("Could not find file to embed!"));
+        }
+      };
 
-        if let Some(c) = child_list.first() {
-          let c_reader = c.read().await;
+      let file_reader = file.node.read().await;
+      match &file_reader.node_type {
+        NodeType::File(f) => {
+          // update the cursor.
+          self.cursor = Some(cursor + 1);
+          let spline = self.get_spline(f).await?;
+          return Ok((f.clone(), spline));
+        }
+        NodeType::Branch(_) => {
+          cursor += 1;
+          continue;
+        }
+        NodeType::ErrorNode(_) => {
+          cursor += 1;
+          continue;
+        }
+      }
+    }
+  }
 
-          match &c_reader.node_type {
-            NodeType::File(file_path) => {
-              // must set the cursor!
-              self.cursor = Some(c.clone());
+  pub async fn get_all_dir_files_to_embed(
+    &self,
+    absolute_path: &str,
+  ) -> Result<Vec<String>, anyhow::Error> {
+    let mut files = Vec::new();
 
-              return Ok((file_path.clone(), path));
-            }
-            NodeType::Branch(_) => {
-              potential_first_child = c.clone();
-              is_branch = true;
+    for (file_path, f) in &self.files {
+      if !file_path.contains(absolute_path) {
+        continue;
+      }
 
-              // add the path to the current node.
-              path.push(current_node_name.clone());
-            }
-            NodeType::ErrorNode(_) => {
-              return Err(anyhow::anyhow!("Cursor is an error node!"));
-            }
-          }
-        } else {
-          // If the root has no children, return an error.
-          return Err(anyhow::anyhow!("Root has no children!"));
+      match f.node.read().await.node_type {
+        NodeType::File(_) => {
+          files.push(file_path.clone());
+        }
+        NodeType::Branch(_) => {
+          continue;
+        }
+        NodeType::ErrorNode(_) => {
+          continue;
         }
       }
     }
 
-    // THE DEFAULT CASE:
-    // we already have a cursor at a file.
-
-    // UNWRAP checked and fine. see the none case above.
-    let cursor_name = self.cursor.as_ref().unwrap();
-    let cursor_reader = cursor_name.read().await;
+    Ok(files)
+  }
 
-    // invariant: you must be a file!!
+  // TODO(sualeh): i need tests for this!!
+  pub async fn get_spline(
+    &self,
+    absolute_path: &str,
+  ) -> Result<Vec<String>, anyhow::Error> {
+    info!("get_spline called with absolute_path: {}", absolute_path);
+    let mut files = Vec::new();
 
-    // everytime we get to a child list, we will add all the children to a fifo, and then pull from it as long as we need it.
+    let current_node = match self.files.get(absolute_path) {
+      Some(node) => {
+        info!("Found node for absolute_path: {}", absolute_path);
+        node.node.clone()
+      }
+      None => {
+        info!("File not found for absolute_path: {}", absolute_path);
+        return Err(anyhow::anyhow!("File not found: {}", absolute_path));
+      }
+    };
 
-    // algorithm:
-    // 1.
+    let mut stack = Vec::new();
+    stack.push(current_node);
+
+    while let Some(node) = stack.pop() {
+      let parent = node.read().await.parent.clone();
+      if let Some(parent) = parent {
+        info!("Adding parent hash to files vector");
+        {
+          let parent_node = parent.read().await;
+          match &parent_node.node_type {
+            NodeType::File(file_name) => {
+              files.push(file_name.clone());
+            }
+            NodeType::Branch((branch_name, _)) => {
+              files.push(branch_name.clone());
+            }
+            _ => {
+              continue;
+            }
+          }
+        }
 
-    Err(anyhow::anyhow!("Could not find file to embed!"))
+        stack.push(parent);
+      }
+    }
+    info!("Returning files vector with {} elements", files.len());
+    Ok(files)
   }
 
   /// creates a new node and attaches it to the current tree.
@@ -302,12 +382,18 @@ impl MerkleTree {
     // 1. the path is empty. this means that the ancestor is the root.
     // 2. the path is non-empty. that means there exist a non-empty element btwn till the root.
 
+    let absolute_root_path = self.root_path.clone();
     let new_node = match path.len() {
       0 => {
         // this means that the ancestor is the root.
         // we need to create a new node and attach it to the ancestor.
-        let new_node =
-          MerkleNode::new(file_path.clone(), Some(ancestor.clone())).await;
+        let new_node = MerkleNode::new(
+          file_path.clone(),
+          Some(ancestor.clone()),
+          &self.git_ignored_files,
+          &absolute_root_path.as_str(),
+        )
+        .await;
         ancestor.write().await.attach_child(new_node.clone()).await;
         new_node
       }
@@ -318,9 +404,13 @@ impl MerkleTree {
         // UNSURE: not sure this is the correct thing to do but it is the fastest.
         // get the last thing that is not in the tree.
         let first_child_path = path.last().unwrap();
-        let first_child =
-          MerkleNode::new(first_child_path.clone(), Some(ancestor.clone()))
-            .await;
+        let first_child = MerkleNode::new(
+          first_child_path.clone(),
+          Some(ancestor.clone()),
+          &self.git_ignored_files,
+          &absolute_root_path.as_str(),
+        )
+        .await;
 
         // TODO(sualeh): we should do an assertion check that the entire vec is contained here.
 
@@ -597,18 +687,62 @@ use std::future::Future;
 use std::pin::Pin;
 
 type PinnedFuture<'a, T> = Pin<Box<dyn Future<Output = T> + Send + 'a>>;
+type IgnoredFiles = HashSet<String>;
 
 impl MerkleNode {
   /// please be careful using this.
   async fn __new_unchecked(
     file_or_directory: String,
     parent: ParentPtr,
+    ignored_files: &IgnoredFiles,
+    absolute_root_path: &str,
   ) -> MerkleNodePtr {
-    MerkleNode::construct_node(Path::new(&file_or_directory), parent).await
+    // check if the root is a git directory.
+    let is_git_repo =
+      match git_utils::is_git_directory(absolute_root_path).await {
+        Ok(is_git_repo) => is_git_repo,
+        Err(e) => false,
+      };
+    let bypass_git = !is_git_repo;
+
+    MerkleNode::construct_node(
+      Path::new(&file_or_directory),
+      parent,
+      ignored_files,
+      absolute_root_path,
+      bypass_git,
+    )
+    .await
   }
 
-  async fn new(file_or_directory: PathBuf, parent: ParentPtr) -> MerkleNodePtr {
-    MerkleNode::construct_node(Path::new(&file_or_directory), parent).await
+  async fn new(
+    absolute_file_or_directory: PathBuf,
+    parent: ParentPtr,
+    ignored_files: &IgnoredFiles,
+    absolute_root_path: &str,
+  ) -> MerkleNodePtr {
+    // check if the root is a git directory.
+    let is_git_repo =
+      match git_utils::is_git_directory(absolute_root_path).await {
+        Ok(is_git_repo) => is_git_repo,
+        Err(_e) => false,
+      };
+    let bypass_git = !is_git_repo;
+
+    info!(
+      "constructing node for absolute_file_or_directory: {:?}",
+      absolute_file_or_directory
+    );
+    info!("bypass_git: {}, is_git_repo: {}", bypass_git, is_git_repo);
+
+    MerkleNode::construct_node(
+      Path::new(&absolute_file_or_directory),
+      parent,
+      ignored_files,
+      absolute_root_path,
+      bypass_git,
+    )
+    .await
   }
 
   /// NOT added to the tree by default.
@@ -619,38 +753,68 @@ impl MerkleNode {
   // 		let file_hash = self.files.get_mut(&file_path).unwrap();
 
   fn construct_node<'a>(
-    file_or_directory: &'a Path,
+    absolute_file_or_directory: &'a Path,
     parent: ParentPtr,
+    ignored_files: &'a IgnoredFiles,
+    absolute_root_path: &'a str,
+    bypass_git: bool,
   ) -> PinnedFuture<'a, MerkleNodePtr> {
     Box::pin(async move {
       // check if it is a file
-      let path_str = file_or_directory.to_str().unwrap().to_string();
-      if file_or_directory.is_file() {
+      let path_str = absolute_file_or_directory.to_str().unwrap().to_string();
+      if absolute_file_or_directory.is_file() {
         return Arc::new(RwLock::new(
           MerkleNode::construct_file_node_or_error_node(
-            file_or_directory,
+            absolute_file_or_directory,
             parent,
+            ignored_files,
           )
           .await,
         ));
       }
 
       // check if the directory fails the bad dir test.
-      let is_bad_dir = file_utils::is_in_bad_dir(file_or_directory);
+      let is_bad_dir = file_utils::is_in_bad_dir(absolute_file_or_directory);
       if is_bad_dir.is_err() || is_bad_dir.unwrap_or(false) {
         // println!("skipping directory: {}", path_str);
         return Arc::new(RwLock::new(MerkleNode::empty_node(
-          Some(file_or_directory),
+          Some(absolute_file_or_directory),
           Some("Directory is in bad dir!".to_string()),
         )));
       }
 
-      let entries = fs::read_dir(file_or_directory);
+      // check if the directory is git ignored
+      let is_git_ignored =
+        match git_utils::is_git_ignored(absolute_root_path, path_str.as_str())
+          .await
+        {
+          Ok(is_git_ignored) => is_git_ignored,
+          Err(e) => {
+            return Arc::new(RwLock::new(MerkleNode::empty_node(
+              Some(absolute_file_or_directory),
+              Some(e.to_string()),
+            )));
+          }
+        };
+
+      if is_git_ignored && !bypass_git {
+        // println!("skipping directory: {}", path_str);
+        tracing::info!(
+          "skipping directory because its git ignored: {}",
+          path_str
+        );
+        return Arc::new(RwLock::new(MerkleNode::empty_node(
+          Some(absolute_file_or_directory),
+          Some("Directory is git ignored!".to_string()),
+        )));
+      }
+
+      let entries = fs::read_dir(absolute_file_or_directory);
       match entries {
         Ok(_) => (),
         Err(e) => {
           return Arc::new(RwLock::new(MerkleNode::empty_node(
-            Some(file_or_directory),
+            Some(absolute_file_or_directory),
             Some(e.to_string()),
           )));
         }
@@ -670,13 +834,19 @@ impl MerkleNode {
         match entry {
           Ok(entry) => {
             children.push(
-              MerkleNode::construct_node(&entry.path(), Some(node.clone()))
-                .await,
+              MerkleNode::construct_node(
+                &entry.path(),
+                Some(node.clone()),
+                ignored_files,
+                absolute_root_path,
+                bypass_git,
+              )
+              .await,
             );
           }
           Err(e) => {
             children.push(Arc::new(RwLock::new(MerkleNode::empty_node(
-              Some(file_or_directory),
+              Some(absolute_file_or_directory),
               Some(e.to_string()),
             ))));
           }
@@ -696,23 +866,33 @@ impl MerkleNode {
   }
 
   async fn construct_file_node(
-    file_path: &Path,
+    absolute_file_path: &Path,
     parent: ParentPtr,
+    ignored_files: &IgnoredFiles,
   ) -> Result<MerkleNode, String> {
-    let file_str = file_path
+    let file_str = absolute_file_path
       .to_str()
       .ok_or("Could not convert file path to string!")?
       .to_string();
     // first see if it passes the
-    match file_utils::is_good_file(file_path) {
+    match file_utils::is_good_file(absolute_file_path) {
       Ok(_) => {}
       Err(e) => {
         return Err(format!("File failed runtime checks! {}", e.to_string()));
       }
     }
 
+    // check if the file is in the git ignore buffer.
+    // this is a bug right because we are not checking absoluteness here.
+    match ignored_files.contains(&file_str) {
+      true => {
+        return Err(format!("File is in git ignore buffer!"));
+      }
+      false => {}
+    }
+
     // read the file_content to a buffer
-    let file_content = match tokio::fs::read(file_path).await {
+    let file_content = match tokio::fs::read(absolute_file_path).await {
       Ok(content) => content,
       Err(e) => {
         return Err(format!("Could not read file! {}", e.to_string()));
@@ -720,7 +900,11 @@ impl MerkleNode {
     };
 
     // check if the file passes runtime checks.
-    match file_utils::is_good_file_runtime_check(file_path, &file_content).await
+    match file_utils::is_good_file_runtime_check(
+      absolute_file_path,
+      &file_content,
+    )
+    .await
     {
       Ok(_) => {}
       Err(e) => {
@@ -751,15 +935,22 @@ impl MerkleNode {
   }
 
   async fn construct_file_node_or_error_node(
-    file_path: &Path,
+    absolute_file_path: &Path,
     parent: ParentPtr,
+    ignored_files: &IgnoredFiles,
   ) -> MerkleNode {
-    let node = match MerkleNode::construct_file_node(file_path, parent).await {
+    let node = match MerkleNode::construct_file_node(
+      absolute_file_path,
+      parent,
+      ignored_files,
+    )
+    .await
+    {
       Ok(node) => node,
       Err(e) => {
         // println!("constructing error node. error: {}", e);
         // println!("file_path: {:?}", file_path);
-        MerkleNode::empty_node(Some(file_path), Some(e))
+        MerkleNode::empty_node(Some(absolute_file_path), Some(e))
       }
     };
 

package/src/test.rs

@@ -0,0 +1,5 @@
+#[cfg(test)]
+mod tests {
+  use super::super::*;
+  use std::path::PathBuf;
+}