@anysphere/file-service 0.0.0-d0bf373f → 0.0.0-dbd43428

package/Cargo.toml

@@ -17,6 +17,9 @@ tempfile = "3.8.0"
 anyhow = "1.0.75"
 tonic = "0.9.2"
 prost = "0.11.9"
+tracing = "0.1.37"
+tracing-subscriber = "0.3.17"
+tracing-appender = "0.2.2"
 
 [build-dependencies]
 napi-build = "2.0.1"

package/index.d.ts

@@ -6,10 +6,15 @@
 export class MerkleClient {
   constructor(rootDirectory: string)
   init(): Promise<void>
+  computeMerkleTree(): Promise<void>
   updateFile(filePath: string): Promise<void>
   deleteFile(filePath: string): Promise<void>
-  getSubtreeHash(path: string): Promise<string>
+  getSubtreeHash(relativePath: string): Promise<string>
   getNumEmbeddableFiles(): Promise<number>
   getAllFiles(): Promise<Array<string>>
+  getAllDirFilesToEmbed(absoluteFilePath: string): Promise<Array<string>>
+  getNextFileToEmbed(): Promise<Array<string>>
+  getSpline(absoluteFilePath: string): Promise<Array<string>>
   getHashesForFiles(files: Array<string>): Promise<Array<string>>
+  updateRootDirectory(rootDirectory: string): void
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@anysphere/file-service",
-  "version": "0.0.0-d0bf373f",
+  "version": "0.0.0-dbd43428",
   "main": "index.js",
   "types": "index.d.ts",
   "napi": {
@@ -35,11 +35,11 @@
     "version": "napi version"
   },
   "optionalDependencies": {
-    "@anysphere/file-service-win32-x64-msvc": "0.0.0-d0bf373f",
-    "@anysphere/file-service-darwin-x64": "0.0.0-d0bf373f",
-    "@anysphere/file-service-linux-x64-gnu": "0.0.0-d0bf373f",
-    "@anysphere/file-service-darwin-arm64": "0.0.0-d0bf373f",
-    "@anysphere/file-service-win32-arm64-msvc": "0.0.0-d0bf373f",
-    "@anysphere/file-service-darwin-universal": "0.0.0-d0bf373f"
+    "@anysphere/file-service-win32-x64-msvc": "0.0.0-dbd43428",
+    "@anysphere/file-service-darwin-x64": "0.0.0-dbd43428",
+    "@anysphere/file-service-linux-x64-gnu": "0.0.0-dbd43428",
+    "@anysphere/file-service-darwin-arm64": "0.0.0-dbd43428",
+    "@anysphere/file-service-win32-arm64-msvc": "0.0.0-dbd43428",
+    "@anysphere/file-service-darwin-universal": "0.0.0-dbd43428"
   }
 }

package/src/file_utils.rs

@@ -12,9 +12,8 @@ pub fn is_in_bad_dir(file_path: &Path) -> Result<bool, Error> {
   let item_path = file_path
     .to_str()
     .ok_or(anyhow::anyhow!("Failed to convert path to string"))?;
-  let is_bad_dir = (item_path.contains("node_modules")
-    || item_path.contains(".git"))
-    && !(item_path.ends_with(".git") || item_path.ends_with("node_modules"));
+  let is_bad_dir =
+    item_path.contains("node_modules") || item_path.contains(".git");
   Ok(is_bad_dir)
 }
 
@@ -38,14 +37,14 @@ pub fn is_good_file(file_path: &Path) -> Result<(), Error> {
 
   match file_name {
     "package-lock.json" | "pnpm-lock.yaml" | "yarn.lock" | "composer.lock"
-    | "Gemfile.lock" => {
+    | "Gemfile.lock" | "bun.lockb" => {
       return Err(anyhow::anyhow!("File is just a lock file"));
     }
     _ => {}
   }
 
   match extension {
-    "lock" | "bak" | "tmp" | "bin" | "exe" | "dll" | "so" => {
+    "lock" | "bak" | "tmp" | "bin" | "exe" | "dll" | "so" | "lockb" => {
       return Err(anyhow::anyhow!("File is just a lock file"));
     }
     _ => {}

package/src/git_utils.rs

@@ -1,10 +1,10 @@
 use std::collections::HashSet;
-use std::path::PathBuf;
 use std::process::Command;
 
 pub fn list_ignored_files(
   workspace_root_path: &str,
-) -> Result<HashSet<PathBuf>, Box<dyn std::error::Error>> {
+  should_return_absolute_paths: bool,
+) -> Result<HashSet<String>, Box<dyn std::error::Error>> {
   let mut gitignored_files = HashSet::new();
 
   let commands = vec![
@@ -15,12 +15,66 @@ pub fn list_ignored_files(
       "--ignored",
       "--exclude-standard",
     ],
+    // FIXME(sualeh): this is super sketchy and might totally break in like a bazillion ways. i dont like it.
+    vec![
+      "sh",
+      "-c",
+      "git submodule foreach --quiet 'git ls-files --others --ignored --exclude-standard | sed \"s|^|$path/|\"'",
+    ],
+  ];
+
+  for command in commands {
+    let output = Command::new(&command[0])
+      .args(&command[1..])
+      .current_dir(workspace_root_path)
+      .output()?;
+
+    if output.status.success() {
+      let files = String::from_utf8(output.stdout)?
+        .lines()
+        .filter(|line| !line.is_empty())
+        .map(|line| {
+          if should_return_absolute_paths {
+            let mut path = std::path::PathBuf::from(workspace_root_path);
+            path.push(line);
+
+            match path.canonicalize() {
+              Ok(canonical_path) => {
+                canonical_path.to_string_lossy().into_owned()
+              }
+              Err(_) => String::from(line),
+            }
+          } else {
+            String::from(line)
+          }
+        })
+        .collect::<HashSet<_>>();
+
+      gitignored_files.extend(files);
+    }
+  }
+
+  Ok(gitignored_files)
+}
+
+pub fn list_ignored_files_with_absolute_paths(
+  workspace_root_path: &str,
+) -> Result<HashSet<String>, Box<dyn std::error::Error>> {
+  let mut gitignored_files = HashSet::new();
+
+  let commands = vec![
     vec![
       "git",
-      "submodule",
-      "foreach",
-      "--quiet",
-      "git ls-files --others --ignored --exclude-standard | sed 's|^|$path/|'",
+      "ls-files",
+      "--others",
+      "--ignored",
+      "--exclude-standard",
+    ],
+    // FIXME(sualeh): this is super sketchy and might totally break in like a bazillion ways. i dont like it.
+    vec![
+      "sh",
+      "-c",
+      "git submodule foreach --quiet 'git ls-files --others --ignored --exclude-standard | sed \"s|^|$path/|\"'",
     ],
   ];
 
@@ -34,7 +88,7 @@ pub fn list_ignored_files(
       let files = String::from_utf8(output.stdout)?
         .lines()
         .filter(|line| !line.is_empty())
-        .map(|line| PathBuf::from(line))
+        .map(|line| String::from(line))
         .collect::<HashSet<_>>();
 
       gitignored_files.extend(files);
@@ -57,6 +111,18 @@ pub async fn is_git_ignored(
   Ok(output.status.success())
 }
 
+pub async fn is_git_directory(
+  workspace_root_path: &str,
+) -> Result<bool, anyhow::Error> {
+  let output = tokio::process::Command::new("git")
+    .args(&["rev-parse", "--is-inside-work-tree"])
+    .current_dir(workspace_root_path)
+    .output()
+    .await?;
+
+  Ok(output.status.success())
+}
+
 #[cfg(test)]
 mod tests {
   use super::*;
@@ -66,7 +132,8 @@ mod tests {
   #[test]
   fn test_no_ignored_files() {
     let dir = tempfile::tempdir().unwrap();
-    let gitignored_files = list_ignored_files(dir.path().to_str().unwrap()).unwrap();
+    let gitignored_files =
+      list_ignored_files(dir.path().to_str().unwrap(), false).unwrap();
     Command::new("git")
       .args(&["init"])
       .current_dir(dir.path())
@@ -92,13 +159,14 @@ mod tests {
       .current_dir(dir.path())
       .output()
       .unwrap();
-    let gitignored_files = list_ignored_files(dir.path().to_str().unwrap()).unwrap();
+    let gitignored_files =
+      list_ignored_files(dir.path().to_str().unwrap(), false).unwrap();
     println!(
       "ignored files for test_one_ignored_file: {:?}",
       gitignored_files
     );
     // assert_eq!(gitignored_files.len(), 1);
-    assert!(gitignored_files.contains(&PathBuf::from("ignored.txt")));
+    assert!(gitignored_files.contains(&String::from("ignored.txt")));
   }
 
   #[test]
@@ -121,19 +189,83 @@ mod tests {
       .current_dir(dir.path())
       .output()
       .unwrap();
-    let gitignored_files = list_ignored_files(dir.path().to_str().unwrap()).unwrap();
+    let gitignored_files =
+      list_ignored_files(dir.path().to_str().unwrap(), false).unwrap();
     println!(
       "ignored files for test_multiple_ignored_files: {:?}",
       gitignored_files
     );
     // assert_eq!(gitignored_files.len(), 2);
-    assert!(gitignored_files.contains(&PathBuf::from("ignored1.txt")));
-    assert!(gitignored_files.contains(&PathBuf::from("ignored2.txt")));
+    assert!(gitignored_files.contains(&String::from("ignored1.txt")));
+    assert!(gitignored_files.contains(&String::from("ignored2.txt")));
+  }
+
+  #[test]
+  fn test_git_submodule_ignored_files() {
+    let dir = tempfile::tempdir().unwrap();
+    let submodule_path = dir.path().join("submodule");
+    std::fs::create_dir(&submodule_path).unwrap();
+
+    let o = Command::new("git")
+      .args(&["init"])
+      .current_dir(&submodule_path)
+      .output()
+      .unwrap();
+    println!("git init output: {:?}", o);
+
+    let file_path = submodule_path.join("ignored.txt");
+    let mut file = File::create(&file_path).unwrap();
+    writeln!(file, "This is an ignored file.").unwrap();
+
+    let file2 = submodule_path.join("ignored2.txt");
+    let mut file = File::create(&file2).unwrap();
+    writeln!(file, "This is another ignored file.").unwrap();
+
+    let gitignore_path = submodule_path.join(".gitignore");
+    let mut gitignore = File::create(&gitignore_path).unwrap();
+    writeln!(gitignore, "*.txt").unwrap();
+
+    let o = Command::new("git")
+      .args(&["init"])
+      .current_dir(dir.path())
+      .output()
+      .unwrap();
+    println!("git init output: {:?}", o);
+
+    // make a commit in the submodule
+    let o = Command::new("git")
+      .args(&["add", "."])
+      .current_dir(&submodule_path)
+      .output()
+      .unwrap();
+    println!("git add output: {:?}", o);
+    let o = Command::new("git")
+      .args(&["commit", "-m", "initial commit"])
+      .current_dir(&submodule_path)
+      .output()
+      .unwrap();
+    println!("git commit output: {:?}", o);
+
+    let o = Command::new("git")
+      .args(&["submodule", "add", "./submodule"])
+      .current_dir(dir.path())
+      .output()
+      .unwrap();
+    println!("git submodule add output: {:?}", o);
+
+    let gitignored_files =
+      list_ignored_files(dir.path().to_str().unwrap(), false).unwrap();
+    println!(
+      "ignored files for test_git_submodule_ignored_files: {:?}",
+      gitignored_files
+    );
+    assert!(gitignored_files.contains(&String::from("submodule/ignored.txt")));
+    assert!(gitignored_files.contains(&String::from("submodule/ignored2.txt")));
   }
 
   #[test]
   fn test_multiple_ignored_files_in_current_dir() {
-    let gitignored_files = list_ignored_files(".").unwrap();
+    let gitignored_files = list_ignored_files(".", false).unwrap();
     assert!(gitignored_files.len() > 1);
 
     // print a sample of the ignored files
@@ -147,7 +279,6 @@ mod tests {
     }
   }
 
-
   #[tokio::test]
   async fn test_file_not_ignored() {
     let dir = tempfile::tempdir().unwrap();
@@ -160,7 +291,10 @@ mod tests {
       .current_dir(dir.path())
       .output()
       .unwrap();
-    let is_ignored = is_git_ignored(dir.path().to_str().unwrap(), "not_ignored.txt").await.unwrap();
+    let is_ignored =
+      is_git_ignored(dir.path().to_str().unwrap(), "not_ignored.txt")
+        .await
+        .unwrap();
     assert_eq!(is_ignored, false);
   }
 
@@ -180,7 +314,10 @@ mod tests {
       .current_dir(dir.path())
       .output()
       .unwrap();
-    let is_ignored = is_git_ignored(dir.path().to_str().unwrap(), "ignored.txt").await.unwrap();
+    let is_ignored =
+      is_git_ignored(dir.path().to_str().unwrap(), "ignored.txt")
+        .await
+        .unwrap();
     assert_eq!(is_ignored, true);
   }
 
@@ -200,8 +337,10 @@ mod tests {
       .current_dir(dir.path())
       .output()
       .unwrap();
-    let is_ignored = is_git_ignored(dir.path().to_str().unwrap(), "ignored.txt").await.unwrap();
+    let is_ignored =
+      is_git_ignored(dir.path().to_str().unwrap(), "ignored.txt")
+        .await
+        .unwrap();
     assert_eq!(is_ignored, true);
   }
-
 }

package/src/lib.rs

@@ -1,9 +1,15 @@
 #![deny(clippy::all)]
+#![deny(unsafe_op_in_unsafe_fn)]
 pub mod file_utils;
 pub mod git_utils;
 pub mod merkle_tree;
 
+use std::vec;
+
 use merkle_tree::{LocalConstruction, MerkleTree};
+use tracing::{info, Level};
+use tracing_appender::rolling::{RollingFileAppender, Rotation};
+use tracing_subscriber::fmt;
 
 #[macro_use]
 extern crate napi_derive;
@@ -12,15 +18,35 @@ extern crate napi_derive;
 pub struct MerkleClient {
   tree: MerkleTree,
   root_directory: String,
+  _guard: tracing_appender::non_blocking::WorkerGuard,
+}
+
+pub fn init_logger() -> tracing_appender::non_blocking::WorkerGuard {
+  let file_appender =
+    RollingFileAppender::new(Rotation::NEVER, "./", "rust_log.txt");
+  let (non_blocking, _guard) = tracing_appender::non_blocking(file_appender);
+  let subscriber = fmt::Subscriber::builder()
+    .with_max_level(Level::TRACE)
+    .with_writer(non_blocking)
+    .with_ansi(false)
+    .with_line_number(true)
+    .finish();
+
+  let _ = tracing::subscriber::set_global_default(subscriber);
+
+  _guard
 }
 
 #[napi]
 impl MerkleClient {
   #[napi(constructor)]
   pub fn new(root_directory: String) -> MerkleClient {
+    let _guard = init_logger();
+
     MerkleClient {
       tree: MerkleTree::empty_tree(),
       root_directory,
+      _guard,
     }
   }
 
@@ -29,22 +55,35 @@ impl MerkleClient {
     // 1. compute the merkle tree
     // 2. update the backend
     // 3. sync with the remote
-    self.compute_merkle_tree().await?;
+    unsafe {
+      self.compute_merkle_tree().await?;
+    }
 
-		Ok(())
+    Ok(())
   }
 
   pub async unsafe fn interrupt(&mut self) -> Result<(), napi::Error> {
     unimplemented!("Interrupt is not implemented yet");
   }
 
-  // #[napi]
+  #[napi]
   pub async unsafe fn compute_merkle_tree(
     &mut self,
   ) -> Result<(), napi::Error> {
     let t =
       MerkleTree::construct_merkle_tree(self.root_directory.clone()).await;
 
+    let files = self.tree.get_all_files().await;
+
+    match files {
+      Ok(files) => {
+        info!("files: {:?}", files);
+      }
+      Err(e) => {
+        info!("Error in get_all_files: {:?}", e);
+      }
+    }
+
     match t {
       Ok(tree) => {
         self.tree = tree;
@@ -67,59 +106,158 @@ impl MerkleClient {
     let _r = self.tree.delete_file(file_path);
   }
 
-	#[napi]
-	pub async fn get_subtree_hash(&self, path: String) -> Result<String, napi::Error> {
-		let hash = self.tree.get_subtree_hash(path).await;
-
-		match hash {
-			Ok(hash) => Ok(hash),
-			Err(e) => Err(napi::Error::new(
-				napi::Status::Unknown,
-				format!("Error in get_subtree_hash: {:?}", e),
-			)),
-		}
-	}
-
-	#[napi]
-	pub async fn get_num_embeddable_files(&self) -> Result<i32, napi::Error> {
-		let num = self.tree.get_num_embeddable_files().await;
-
-		match num {
-			Ok(num) => Ok(num),
-			Err(e) => Err(napi::Error::new(
-				napi::Status::Unknown,
-				format!("Error in get_num_embeddable_files: {:?}", e),
-			)),
-		}
-	}
-
-	#[napi]
-	pub async fn get_all_files(&self) -> Result<Vec<String>, napi::Error> {
-		let files = self.tree.get_all_files().await;
-
-		match files {
-			Ok(files) => Ok(files),
-			Err(e) => Err(napi::Error::new(
-				napi::Status::Unknown,
-				format!("Error in get_all_files: {:?}", e),
-			)),
-		}
-	}
-
-	#[napi]
-	pub async fn get_hashes_for_files(&self, files: Vec<String>) -> Result<Vec<String>, napi::Error> {
-		let hashes = self.tree.get_hashes_for_files(files).await;
-
-		match hashes {
-			Ok(hashes) => Ok(hashes),
-			Err(e) => Err(napi::Error::new(
-				napi::Status::Unknown,
-				format!("Error in get_hashes_for_files: {:?}", e),
-			)),
-		}
-	}
-
-  // #[napi]
+  #[napi]
+  pub async fn get_subtree_hash(
+    &self,
+    relative_path: String,
+  ) -> Result<String, napi::Error> {
+    info!("relative_path: {:?}", relative_path);
+    let absolute_path =
+      std::path::Path::new(&self.root_directory).join(relative_path);
+    let canonical_path = absolute_path.canonicalize().unwrap();
+
+    info!("canonical_path: {:?}", canonical_path);
+    let hash = self.tree.get_subtree_hash(canonical_path).await;
+
+    match hash {
+      Ok(hash) => Ok(hash),
+      Err(e) => Err(napi::Error::new(
+        napi::Status::Unknown,
+        format!("Error in get_subtree_hash: {:?}", e),
+      )),
+    }
+  }
+
+  #[napi]
+  pub async fn get_num_embeddable_files(&self) -> Result<i32, napi::Error> {
+    let num = self.tree.get_num_embeddable_files().await;
+
+    match num {
+      Ok(num) => Ok(num),
+      Err(e) => Err(napi::Error::new(
+        napi::Status::Unknown,
+        format!("Error in get_num_embeddable_files: {:?}", e),
+      )),
+    }
+  }
+
+  pub async fn get_num_embeddable_files_in_subtree(
+    &self,
+    relative_path: String,
+  ) -> Result<i32, napi::Error> {
+    let absolute_path = std::path::Path::new(&self.root_directory)
+      .join(relative_path)
+      .canonicalize()?;
+
+    let num = self
+      .tree
+      .get_num_embeddable_files_in_subtree(absolute_path)
+      .await;
+
+    match num {
+      Ok(num) => Ok(num),
+      Err(e) => Err(napi::Error::new(
+        napi::Status::Unknown,
+        format!("Error in get_num_embeddable_files_in_subtree: {:?}", e),
+      )),
+    }
+  }
+
+  #[napi]
+  pub async fn get_all_files(&self) -> Result<Vec<String>, napi::Error> {
+    let files = self.tree.get_all_files().await;
+
+    match files {
+      Ok(files) => Ok(files),
+      Err(e) => Err(napi::Error::new(
+        napi::Status::Unknown,
+        format!("Error in get_all_files: {:?}", e),
+      )),
+    }
+  }
+
+  #[napi]
+  pub async fn get_all_dir_files_to_embed(
+    &self,
+    absolute_file_path: String,
+  ) -> Result<Vec<String>, napi::Error> {
+    let absolute_path_str = absolute_file_path.as_str();
+    let files = self
+      .tree
+      .get_all_dir_files_to_embed(absolute_path_str)
+      .await;
+
+    match files {
+      Ok(files) => Ok(files),
+      Err(e) => Err(napi::Error::new(
+        napi::Status::Unknown,
+        format!("Error in get_all_dir_files_to_embed: {:?}", e),
+      )),
+    }
+  }
+
+  #[napi]
+  pub async unsafe fn get_next_file_to_embed(
+    &mut self,
+  ) -> Result<Vec<String>, napi::Error> {
+    let n = self.tree.get_next_file_to_embed().await;
+
+    match n {
+      Ok((file, path)) => {
+        // now our job is to put the filename as the first element of the path.
+
+        // TODO(sualeh): we should assert that the path is ascending up to the path.
+
+        let ret = vec![file];
+        info!("file: {:?}", ret);
+
+        let ret = ret.into_iter().chain(path.into_iter()).collect::<Vec<_>>();
+        info!("ret to js: {:?}", ret);
+
+        Ok(ret)
+      }
+      Err(e) => Err(napi::Error::new(
+        napi::Status::Unknown,
+        format!("Error in get_next_file_to_embed: {:?}", e),
+      )),
+    }
+  }
+
+  // FIXME(sualeh): get_spline
+  #[napi]
+  pub async fn get_spline(
+    &self,
+    absolute_file_path: String,
+  ) -> Result<Vec<String>, napi::Error> {
+    let absolute_path_str = absolute_file_path.as_str();
+    let spline = self.tree.get_spline(absolute_path_str).await;
+
+    match spline {
+      Ok(spline) => Ok(spline),
+      Err(e) => Err(napi::Error::new(
+        napi::Status::Unknown,
+        format!("Error in get_spline: {:?}", e),
+      )),
+    }
+  }
+
+  #[napi]
+  pub async fn get_hashes_for_files(
+    &self,
+    files: Vec<String>,
+  ) -> Result<Vec<String>, napi::Error> {
+    let hashes = self.tree.get_hashes_for_files(files).await;
+
+    match hashes {
+      Ok(hashes) => Ok(hashes),
+      Err(e) => Err(napi::Error::new(
+        napi::Status::Unknown,
+        format!("Error in get_hashes_for_files: {:?}", e),
+      )),
+    }
+  }
+
+  #[napi]
   pub fn update_root_directory(&mut self, root_directory: String) {
     self.root_directory = root_directory;
   }