@anysphere/file-service 0.0.0-bff0125b → 0.0.0-c2b75b32

package/Cargo.toml

@@ -20,6 +20,8 @@ prost = "0.11.9"
 tracing = "0.1.37"
 tracing-subscriber = "0.3.17"
 tracing-appender = "0.2.2"
+binaryornot = "1.0.0"
+dunce = "1.0.1"
 
 [build-dependencies]
 napi-build = "2.0.1"

package/index.d.ts

@@ -4,7 +4,7 @@
 /* auto-generated by NAPI-RS */
 
 export class MerkleClient {
-  constructor(rootDirectory: string)
+  constructor(absoluteRootDirectory: string)
   init(): Promise<void>
   computeMerkleTree(): Promise<void>
   updateFile(filePath: string): Promise<void>

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@anysphere/file-service",
-  "version": "0.0.0-bff0125b",
+  "version": "0.0.0-c2b75b32",
   "main": "index.js",
   "types": "index.d.ts",
   "napi": {
@@ -9,7 +9,8 @@
       "additional": [
         "aarch64-apple-darwin",
         "aarch64-pc-windows-msvc",
-        "universal-apple-darwin"
+        "universal-apple-darwin",
+        "aarch64-unknown-linux-gnu"
       ]
     }
   },
@@ -35,11 +36,12 @@
     "version": "napi version"
   },
   "optionalDependencies": {
-    "@anysphere/file-service-win32-x64-msvc": "0.0.0-bff0125b",
-    "@anysphere/file-service-darwin-x64": "0.0.0-bff0125b",
-    "@anysphere/file-service-linux-x64-gnu": "0.0.0-bff0125b",
-    "@anysphere/file-service-darwin-arm64": "0.0.0-bff0125b",
-    "@anysphere/file-service-win32-arm64-msvc": "0.0.0-bff0125b",
-    "@anysphere/file-service-darwin-universal": "0.0.0-bff0125b"
+    "@anysphere/file-service-win32-x64-msvc": "0.0.0-c2b75b32",
+    "@anysphere/file-service-darwin-x64": "0.0.0-c2b75b32",
+    "@anysphere/file-service-linux-x64-gnu": "0.0.0-c2b75b32",
+    "@anysphere/file-service-darwin-arm64": "0.0.0-c2b75b32",
+    "@anysphere/file-service-win32-arm64-msvc": "0.0.0-c2b75b32",
+    "@anysphere/file-service-darwin-universal": "0.0.0-c2b75b32",
+    "@anysphere/file-service-linux-arm64-gnu": "0.0.0-c2b75b32"
   }
 }

package/src/file_utils.rs

@@ -62,7 +62,7 @@ pub fn is_good_file(file_path: &Path) -> Result<(), Error> {
     Some(extension) => match extension.to_str() {
       Some(ext_str) => {
         if bad_extensions.contains(&ext_str) {
-          return Err(anyhow::anyhow!("File is not a valid UTF-8 string"));
+          return Err(anyhow::anyhow!("Binary file excluded from indexing."));
         }
       }
       None => {
@@ -88,10 +88,12 @@ pub fn is_good_file(file_path: &Path) -> Result<(), Error> {
   Ok(())
 }
 
+// use binaryornot::is_binary;
+// use anyhow::Context;
 // implement the buffer above:
 pub async fn is_good_file_runtime_check(
   file_path: &Path,
-  buffer: &[u8],
+  _buffer: &[u8],
 ) -> Result<(), Error> {
   match get_file_size(file_path).await {
     Ok(size) if size > 2 * 1024 * 1024 => {
@@ -101,13 +103,10 @@ pub async fn is_good_file_runtime_check(
     _ => {}
   }
 
-  for &byte in buffer.iter().take(2048) {
-    if byte.is_ascii() {
-      continue;
-    } else {
-      return Err(anyhow::anyhow!("File is not a valid UTF-8 string"));
-    }
-  }
+	// if is_binary(file_path).context("Failed to check if file is binary")? {
+	// 	return Err(anyhow::anyhow!("File is binary"));
+	// }
+
   Ok(())
 }
 

package/src/git_utils.rs

@@ -1,7 +1,7 @@
 use std::collections::HashSet;
 use std::process::Command;
 
-pub fn list_ignored_files(
+pub fn list_ignored_files_and_directories(
   workspace_root_path: &str,
   should_return_absolute_paths: bool,
 ) -> Result<HashSet<String>, Box<dyn std::error::Error>> {
@@ -14,12 +14,14 @@ pub fn list_ignored_files(
       "--others",
       "--ignored",
       "--exclude-standard",
+      "--directory",
+      "--no-empty-directory"
     ],
     // FIXME(sualeh): this is super sketchy and might totally break in like a bazillion ways. i dont like it.
     vec![
       "sh",
       "-c",
-      "git submodule foreach --quiet 'git ls-files --others --ignored --exclude-standard | sed \"s|^|$path/|\"'",
+      "git submodule foreach --quiet 'git ls-files --others --ignored --exclude-standard --directory --no-empty-directory | sed \"s|^|$path/|\"'",
     ],
   ];
 
@@ -133,7 +135,8 @@ mod tests {
   fn test_no_ignored_files() {
     let dir = tempfile::tempdir().unwrap();
     let gitignored_files =
-      list_ignored_files(dir.path().to_str().unwrap(), false).unwrap();
+      list_ignored_files_and_directories(dir.path().to_str().unwrap(), false)
+        .unwrap();
     Command::new("git")
       .args(&["init"])
       .current_dir(dir.path())
@@ -160,7 +163,8 @@ mod tests {
       .output()
       .unwrap();
     let gitignored_files =
-      list_ignored_files(dir.path().to_str().unwrap(), false).unwrap();
+      list_ignored_files_and_directories(dir.path().to_str().unwrap(), false)
+        .unwrap();
     println!(
       "ignored files for test_one_ignored_file: {:?}",
       gitignored_files
@@ -190,7 +194,8 @@ mod tests {
       .output()
       .unwrap();
     let gitignored_files =
-      list_ignored_files(dir.path().to_str().unwrap(), false).unwrap();
+      list_ignored_files_and_directories(dir.path().to_str().unwrap(), false)
+        .unwrap();
     println!(
       "ignored files for test_multiple_ignored_files: {:?}",
       gitignored_files
@@ -254,7 +259,8 @@ mod tests {
     println!("git submodule add output: {:?}", o);
 
     let gitignored_files =
-      list_ignored_files(dir.path().to_str().unwrap(), false).unwrap();
+      list_ignored_files_and_directories(dir.path().to_str().unwrap(), false)
+        .unwrap();
     println!(
       "ignored files for test_git_submodule_ignored_files: {:?}",
       gitignored_files
@@ -265,7 +271,8 @@ mod tests {
 
   #[test]
   fn test_multiple_ignored_files_in_current_dir() {
-    let gitignored_files = list_ignored_files(".", false).unwrap();
+    let gitignored_files =
+      list_ignored_files_and_directories(".", false).unwrap();
     assert!(gitignored_files.len() > 1);
 
     // print a sample of the ignored files

package/src/lib.rs

@@ -10,6 +10,7 @@ use merkle_tree::{LocalConstruction, MerkleTree};
 use tracing::{info, Level};
 use tracing_appender::rolling::{RollingFileAppender, Rotation};
 use tracing_subscriber::fmt;
+use anyhow::Context;
 
 #[macro_use]
 extern crate napi_derive;
@@ -17,7 +18,7 @@ extern crate napi_derive;
 #[napi]
 pub struct MerkleClient {
   tree: MerkleTree,
-  root_directory: String,
+  absolute_root_directory: String,
   _guard: tracing_appender::non_blocking::WorkerGuard,
 }
 
@@ -40,12 +41,22 @@ pub fn init_logger() -> tracing_appender::non_blocking::WorkerGuard {
 #[napi]
 impl MerkleClient {
   #[napi(constructor)]
-  pub fn new(root_directory: String) -> MerkleClient {
+  pub fn new(absolute_root_directory: String) -> MerkleClient {
     let _guard = init_logger();
 
+    // let canonical_root_directory = std::path::Path::new(&absolute_root_directory);
+    // use dunce::canonicalize;
+    // let canonical_root_directory = match dunce::canonicalize(&canonical_root_directory) {
+    //   Ok(path) => path.to_str().unwrap_or(&absolute_root_directory).to_string().to_lowercase(),
+    //   Err(e) => {
+    //     info!("Error in canonicalizing path: path: {:?}, error {:?}", canonical_root_directory, e);
+    //     absolute_root_directory
+    //   }
+    // };
+
     MerkleClient {
       tree: MerkleTree::empty_tree(),
-      root_directory,
+      absolute_root_directory,
       _guard,
     }
   }
@@ -55,6 +66,7 @@ impl MerkleClient {
     // 1. compute the merkle tree
     // 2. update the backend
     // 3. sync with the remote
+    info!("Merkle tree compute started!");
     unsafe {
       self.compute_merkle_tree().await?;
     }
@@ -71,18 +83,7 @@ impl MerkleClient {
     &mut self,
   ) -> Result<(), napi::Error> {
     let t =
-      MerkleTree::construct_merkle_tree(self.root_directory.clone()).await;
-
-    let files = self.tree.get_all_files().await;
-
-    match files {
-      Ok(files) => {
-        info!("files: {:?}", files);
-      }
-      Err(e) => {
-        info!("Error in get_all_files: {:?}", e);
-      }
-    }
+      MerkleTree::construct_merkle_tree(self.absolute_root_directory.clone()).await;
 
     match t {
       Ok(tree) => {
@@ -111,19 +112,35 @@ impl MerkleClient {
     &self,
     relative_path: String,
   ) -> Result<String, napi::Error> {
-    info!("relative_path: {:?}", relative_path);
-    let absolute_path =
-      std::path::Path::new(&self.root_directory).join(relative_path);
-    let canonical_path = absolute_path.canonicalize().unwrap();
 
-    info!("canonical_path: {:?}", canonical_path);
-    let hash = self.tree.get_subtree_hash(canonical_path).await;
+    let relative_path_without_leading_slash = match relative_path.strip_prefix('.') {
+      Some(path) => path.strip_prefix(std::path::MAIN_SEPARATOR).unwrap_or(""),
+      None => relative_path.as_str(),
+    };
+
+    let absolute_path = if !relative_path_without_leading_slash.is_empty() {
+      std::path::Path::new(&self.absolute_root_directory).join(relative_path_without_leading_slash)
+    } else {
+      std::path::Path::new(&self.absolute_root_directory).to_path_buf()
+    };
+
+    let absolute_path_string = match absolute_path.to_str() {
+      Some(path) => path.to_string(),
+      None => {
+        return Err(napi::Error::new(
+          napi::Status::Unknown,
+          format!("some string error")
+        ))
+      }
+    };
+
+    let hash = self.tree.get_subtree_hash(absolute_path_string.as_str()).await;
 
     match hash {
       Ok(hash) => Ok(hash),
       Err(e) => Err(napi::Error::new(
         napi::Status::Unknown,
-        format!("Error in get_subtree_hash: {:?}", e),
+        format!("Error in get_subtree_hash. \nRelative path: {:?}, \nAbsolute path: {:?}, \nRoot directory: {:?}\nError: {:?}", &relative_path, absolute_path, self.absolute_root_directory, e)
       )),
     }
   }
@@ -136,7 +153,7 @@ impl MerkleClient {
       Ok(num) => Ok(num),
       Err(e) => Err(napi::Error::new(
         napi::Status::Unknown,
-        format!("Error in get_num_embeddable_files: {:?}", e),
+        format!("Error in get_num_embeddable_files: {:?}", e)
       )),
     }
   }
@@ -145,7 +162,7 @@ impl MerkleClient {
     &self,
     relative_path: String,
   ) -> Result<i32, napi::Error> {
-    let absolute_path = std::path::Path::new(&self.root_directory)
+    let absolute_path = std::path::Path::new(&self.absolute_root_directory)
       .join(relative_path)
       .canonicalize()?;
 
@@ -158,7 +175,7 @@ impl MerkleClient {
       Ok(num) => Ok(num),
       Err(e) => Err(napi::Error::new(
         napi::Status::Unknown,
-        format!("Error in get_num_embeddable_files_in_subtree: {:?}", e),
+        format!("Error in get_num_embeddable_files_in_subtree: {:?}", e)
       )),
     }
   }
@@ -171,7 +188,7 @@ impl MerkleClient {
       Ok(files) => Ok(files),
       Err(e) => Err(napi::Error::new(
         napi::Status::Unknown,
-        format!("Error in get_all_files: {:?}", e),
+        format!("Error in get_all_files: {:?}", e)
       )),
     }
   }
@@ -181,10 +198,12 @@ impl MerkleClient {
     &self,
     absolute_file_path: String,
   ) -> Result<Vec<String>, napi::Error> {
-    let absolute_path_str = absolute_file_path.as_str();
+    // let absolute_path = absolute_file_path.to_lowercase();
+    // let absolute_path_str = absolute_path.as_str();
+
     let files = self
       .tree
-      .get_all_dir_files_to_embed(absolute_path_str)
+      .get_all_dir_files_to_embed(absolute_file_path.as_str())
       .await;
 
     match files {
@@ -209,11 +228,7 @@ impl MerkleClient {
         // TODO(sualeh): we should assert that the path is ascending up to the path.
 
         let ret = vec![file];
-        info!("file: {:?}", ret);
-
         let ret = ret.into_iter().chain(path.into_iter()).collect::<Vec<_>>();
-        info!("ret to js: {:?}", ret);
-
         Ok(ret)
       }
       Err(e) => Err(napi::Error::new(
@@ -229,8 +244,9 @@ impl MerkleClient {
     &self,
     absolute_file_path: String,
   ) -> Result<Vec<String>, napi::Error> {
-    let absolute_path_str = absolute_file_path.as_str();
-    let spline = self.tree.get_spline(absolute_path_str).await;
+    // let absolute_path = absolute_file_path.to_lowercase();
+    // let absolute_path_str = absolute_path.as_str();
+    let spline = self.tree.get_spline(absolute_file_path.as_str()).await;
 
     match spline {
       Ok(spline) => Ok(spline),
@@ -259,6 +275,6 @@ impl MerkleClient {
 
   #[napi]
   pub fn update_root_directory(&mut self, root_directory: String) {
-    self.root_directory = root_directory;
+    self.absolute_root_directory = root_directory;
   }
 }

package/src/merkle_tree/local_construction.rs

@@ -38,20 +38,19 @@ impl LocalConstruction for MerkleTree {
     }
 
     // 1. get all the gitignored files
-    let git_ignored_files = match git_utils::list_ignored_files(
-      absolute_path_to_root_directory.as_str(),
-      true,
-    ) {
-      Ok(git_ignored) => git_ignored,
-      Err(_e) => HashSet::new(),
-    };
-
-    tracing::info!("git_ignored_files: {:?}", git_ignored_files);
+    let git_ignored_files_and_dirs =
+      match git_utils::list_ignored_files_and_directories(
+        absolute_path_to_root_directory.as_str(),
+        true,
+      ) {
+        Ok(git_ignored) => git_ignored,
+        Err(_e) => HashSet::new(),
+      };
 
     let root_node = MerkleNode::new(
       path,
       None,
-      &git_ignored_files,
+      &git_ignored_files_and_dirs,
       absolute_path_to_root_directory.as_str(),
     )
     .await;
@@ -60,7 +59,7 @@ impl LocalConstruction for MerkleTree {
       files: BTreeMap::new(),
       root_path: absolute_path_to_root_directory,
       cursor: None,
-      git_ignored_files,
+      git_ignored_files_and_dirs: git_ignored_files_and_dirs,
     };
 
     // we now iterate over all the nodes and add them to the hashmap
@@ -73,6 +72,7 @@ impl LocalConstruction for MerkleTree {
         let node_reader = node.read().await;
         match &node_reader.node_type {
           NodeType::Branch(n) => {
+            tracing::info!("Branch: {:?}", n.0);
             let children = &n.1;
             files.insert(n.0.clone(), File { node: node.clone() });
             for child in children {
@@ -81,6 +81,13 @@ impl LocalConstruction for MerkleTree {
           }
           NodeType::File(file_name) => {
             let f = File { node: node.clone() };
+
+            // i dont reallly  like this :(((
+            // let canonical_file_name = match dunce::canonicalize(file_name) {
+            //   Ok(path) => path.to_str().unwrap_or(file_name).to_string(),
+            //   Err(_) => file_name.clone(),
+            // };
+
             files.insert(file_name.clone(), f);
           }
           NodeType::ErrorNode(_) => {
@@ -92,6 +99,9 @@ impl LocalConstruction for MerkleTree {
 
     add_nodes_to_hashmap(&mt.root, &mut mt.files).await;
 
+    tracing::info!("Merkle tree compute finished!");
+    tracing::info!("Merkle tree: {}", mt);
+
     Ok(mt)
   }
 

package/src/merkle_tree/mod.rs

@@ -4,6 +4,7 @@ use super::file_utils;
 use sha2::Digest;
 use std::collections::{BTreeMap, HashSet};
 use std::path::PathBuf;
+use std::vec;
 use std::{fs, path::Path, sync::Arc};
 use tokio::sync::RwLock;
 use tonic::async_trait;
@@ -18,7 +19,7 @@ pub struct MerkleTree {
   root: MerkleNodePtr,
   files: BTreeMap<String, File>,
   cursor: Option<usize>,
-  git_ignored_files: HashSet<String>,
+  git_ignored_files_and_dirs: HashSet<String>,
 }
 
 #[derive(Debug)]
@@ -95,33 +96,34 @@ impl MerkleTree {
       files: BTreeMap::new(),
       root_path: "".to_string(),
       cursor: None,
-      git_ignored_files: HashSet::new(),
+      git_ignored_files_and_dirs: HashSet::new(),
     }
   }
 
   pub async fn get_subtree_hash(
     &self,
-    absolute_path: PathBuf,
+    absolute_path: &str,
   ) -> Result<String, anyhow::Error> {
-    let abs_string = match absolute_path.to_str() {
-      Some(s) => s.to_string(),
-      None => {
-        return Err(anyhow::anyhow!(
-          "get_subtree_hash: Failed to convert path to string"
-        ))
-      }
-    };
-
-    let node = match self.files.get(&abs_string) {
+    let node = match self.files.get(absolute_path) {
       Some(file) => file.node.clone(),
       None => {
-        return Err(anyhow::anyhow!("Could not find file in tree!"));
+        let all_files: Vec<String> = self.files.keys().cloned().collect();
+        return Err(anyhow::anyhow!(
+          "Could not find file in tree! Looking for: {}. All files: {:?}",
+          absolute_path,
+          all_files
+        ));
       }
     };
 
     let node_reader = node.read().await;
     let node_hash = node_reader.hash.clone();
 
+    info!(
+      "get_subtree_hash for path: {}, node_hash: {}",
+      absolute_path, node_hash
+    );
+
     Ok(node_hash)
   }
 
@@ -397,7 +399,7 @@ impl MerkleTree {
         let new_node = MerkleNode::new(
           file_path.clone(),
           Some(ancestor.clone()),
-          &self.git_ignored_files,
+          &self.git_ignored_files_and_dirs,
           &absolute_root_path.as_str(),
         )
         .await;
@@ -414,7 +416,7 @@ impl MerkleTree {
         let first_child = MerkleNode::new(
           first_child_path.clone(),
           Some(ancestor.clone()),
-          &self.git_ignored_files,
+          &self.git_ignored_files_and_dirs,
           &absolute_root_path.as_str(),
         )
         .await;
@@ -790,22 +792,9 @@ impl MerkleNode {
         )));
       }
 
-      // check if the directory is git ignored
-      let is_git_ignored =
-        match git_utils::is_git_ignored(absolute_root_path, path_str.as_str())
-          .await
-        {
-          Ok(is_git_ignored) => is_git_ignored,
-          Err(e) => {
-            return Arc::new(RwLock::new(MerkleNode::empty_node(
-              Some(absolute_file_or_directory),
-              Some(e.to_string()),
-            )));
-          }
-        };
+      let is_git_ignored_dir = ignored_files.contains(&path_str);
 
-      if is_git_ignored && !bypass_git {
-        // println!("skipping directory: {}", path_str);
+      if is_git_ignored_dir && !bypass_git {
         return Arc::new(RwLock::new(MerkleNode::empty_node(
           Some(absolute_file_or_directory),
           Some("Directory is git ignored!".to_string()),
@@ -979,15 +968,51 @@ impl MerkleNode {
 
   async fn compute_branch_hash(children: &[MerkleNodePtr]) -> String {
     let mut hasher = sha2::Sha256::new();
+    let mut names_and_hashes = vec![];
+    let mut non_zero_children = 0;
+
     for child in children {
       // check if it is an error node
       let child_reader = child.read().await;
-      if let NodeType::ErrorNode(_) = &child_reader.node_type {
+
+      match &child_reader.node_type {
+        NodeType::File(file_name) => {
+          non_zero_children += 1;
+          names_and_hashes.push((file_name.clone(), child_reader.hash.clone()));
+        }
+        NodeType::Branch((file_name, _)) => {
+          let hash = child_reader.hash.clone();
+          if hash == "" {
+            continue;
+          }
+
+          non_zero_children += 1;
+          names_and_hashes.push((file_name.clone(), hash));
+        }
+        NodeType::ErrorNode(_) => {
+          continue;
+        }
+      }
+    }
+
+    // sort the list of names and hashes by the hashes!!
+    names_and_hashes
+      .sort_by(|a, b| a.1.to_lowercase().cmp(&b.1.to_lowercase()));
+
+    for (name, hash) in names_and_hashes {
+      if hash == "" {
         continue;
       }
+      info!("name: {}, hash: {}", name, hash);
+      hasher.update(hash);
+    }
 
-      hasher.update(child_reader.hash.as_bytes());
+    if non_zero_children == 0 {
+      // this means that the branch is empty.
+      // we should return an empty string.
+      return "".to_string();
     }
+
     let result = hasher.finalize();
     format!("{:x}", result)
   }