@anysphere/file-service 0.0.0-e3fdf62d → 0.0.0-e9ef07ca

package/Cargo.toml

@@ -6,6 +6,11 @@ version = "0.0.0"
 [lib]
 crate-type = ["cdylib"]
 
+[features]
+default = ["windows-subsystem"]
+windows-subsystem = []
+debugfile = []
+
 [dependencies]
 # Default enable napi4 feature, see https://nodejs.org/api/n-api.html#node-api-version-matrix
 napi = { version = "2.12.2", default-features = false, features = ["napi4", "async", "tokio_rt"] }
@@ -17,6 +22,12 @@ tempfile = "3.8.0"
 anyhow = "1.0.75"
 tonic = "0.9.2"
 prost = "0.11.9"
+tracing = "0.1.37"
+tracing-subscriber = "0.3.17"
+tracing-appender = "0.2.2"
+binaryornot = "1.0.0"
+dunce = "1.0.1"
+encoding_rs = "0.8.33"
 
 [build-dependencies]
 napi-build = "2.0.1"
@@ -24,5 +35,6 @@ tonic-build = "0.9.2"
 anyhow = "1.0.75"
 glob = "0.3.0"
 
+
 [profile.release]
 lto = true

package/build.rs

@@ -3,6 +3,8 @@ use std::path::Path;
 extern crate napi_build;
 
 fn main() -> Result<(), anyhow::Error> {
+  #[cfg(target_os = "windows")]
+  println!("cargo:rustc-cdylib-link-arg=/SUBSYSTEM:WINDOWS");
   napi_build::setup();
 
   // print the relative path.

package/index.d.ts

@@ -4,13 +4,17 @@
 /* auto-generated by NAPI-RS */
 
 export class MerkleClient {
-  constructor(rootDirectory: string)
-  init(): Promise<void>
+  constructor(absoluteRootDirectory: string)
+  init(gitIgnoredFiles: Array<string>, isGitRepo: boolean): Promise<void>
+  computeMerkleTree(gitIgnoredFiles: Array<string>, isGitRepo: boolean): Promise<void>
   updateFile(filePath: string): Promise<void>
   deleteFile(filePath: string): Promise<void>
-  getSubtreeHash(path: string): Promise<string>
+  getSubtreeHash(relativePath: string): Promise<string>
   getNumEmbeddableFiles(): Promise<number>
   getAllFiles(): Promise<Array<string>>
+  getAllDirFilesToEmbed(absoluteFilePath: string): Promise<Array<string>>
   getNextFileToEmbed(): Promise<Array<string>>
+  getSpline(absoluteFilePath: string): Promise<Array<string>>
   getHashesForFiles(files: Array<string>): Promise<Array<string>>
+  updateRootDirectory(rootDirectory: string): void
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@anysphere/file-service",
-  "version": "0.0.0-e3fdf62d",
+  "version": "0.0.0-e9ef07ca",
   "main": "index.js",
   "types": "index.d.ts",
   "napi": {
@@ -9,7 +9,8 @@
       "additional": [
         "aarch64-apple-darwin",
         "aarch64-pc-windows-msvc",
-        "universal-apple-darwin"
+        "universal-apple-darwin",
+        "aarch64-unknown-linux-gnu"
       ]
     }
   },
@@ -35,11 +36,12 @@
     "version": "napi version"
   },
   "optionalDependencies": {
-    "@anysphere/file-service-win32-x64-msvc": "0.0.0-e3fdf62d",
-    "@anysphere/file-service-darwin-x64": "0.0.0-e3fdf62d",
-    "@anysphere/file-service-linux-x64-gnu": "0.0.0-e3fdf62d",
-    "@anysphere/file-service-darwin-arm64": "0.0.0-e3fdf62d",
-    "@anysphere/file-service-win32-arm64-msvc": "0.0.0-e3fdf62d",
-    "@anysphere/file-service-darwin-universal": "0.0.0-e3fdf62d"
+    "@anysphere/file-service-win32-x64-msvc": "0.0.0-e9ef07ca",
+    "@anysphere/file-service-darwin-x64": "0.0.0-e9ef07ca",
+    "@anysphere/file-service-linux-x64-gnu": "0.0.0-e9ef07ca",
+    "@anysphere/file-service-darwin-arm64": "0.0.0-e9ef07ca",
+    "@anysphere/file-service-win32-arm64-msvc": "0.0.0-e9ef07ca",
+    "@anysphere/file-service-darwin-universal": "0.0.0-e9ef07ca",
+    "@anysphere/file-service-linux-arm64-gnu": "0.0.0-e9ef07ca"
   }
 }

package/src/file_utils.rs

@@ -5,6 +5,7 @@
 // 4. vscode.fs.stat
 
 use anyhow::Error;
+use encoding_rs::UTF_8;
 use std::path::Path;
 use tokio::fs;
 
@@ -12,9 +13,8 @@ pub fn is_in_bad_dir(file_path: &Path) -> Result<bool, Error> {
   let item_path = file_path
     .to_str()
     .ok_or(anyhow::anyhow!("Failed to convert path to string"))?;
-  let is_bad_dir = (item_path.contains("node_modules")
-    || item_path.contains(".git"))
-    && !(item_path.ends_with(".git") || item_path.ends_with("node_modules"));
+  let is_bad_dir =
+    item_path.contains("node_modules") || item_path.contains(".git");
   Ok(is_bad_dir)
 }
 
@@ -38,14 +38,96 @@ pub fn is_good_file(file_path: &Path) -> Result<(), Error> {
 
   match file_name {
     "package-lock.json" | "pnpm-lock.yaml" | "yarn.lock" | "composer.lock"
-    | "Gemfile.lock" => {
+    | "Gemfile.lock" | "bun.lockb" => {
       return Err(anyhow::anyhow!("File is just a lock file"));
     }
     _ => {}
   }
 
-  match extension {
-    "lock" | "bak" | "tmp" | "bin" | "exe" | "dll" | "so" => {
+  let bad_extensions = vec![
+    "lock",
+    "bak",
+    "tmp",
+    "bin",
+    "exe",
+    "dll",
+    "so",
+    "lockb",
+    "qwoff",
+    "isl",
+    "csv",
+    "pdf",
+    // add ms word, excel, powerpoint, etc.
+    "doc",
+    "docx",
+    "xls",
+    "xlsx",
+    "ppt",
+    "pptx",
+    "odt",
+    "ods",
+    "odp",
+    "odg",
+    "odf",
+    "sxw",
+    "sxc",
+    "sxi",
+    "sxd",
+    "sdc",
+    // add images
+    "jpg",
+    "jpeg",
+    "png",
+    "gif",
+    "bmp",
+    "tif",
+    // add audio
+    "mp3",
+    "wav",
+    "wma",
+    "ogg",
+    "flac",
+    "aac",
+    // add video
+    "mp4",
+    "mov",
+    "wmv",
+    "flv",
+    "avi",
+    // add archives
+    "zip",
+    "tar",
+    "gz",
+    "7z",
+    "rar",
+    "tgz",
+    "dmg",
+    "iso",
+    "cue",
+    "mdf",
+    "mds",
+    "vcd",
+    "toast",
+    "img",
+    "apk",
+    "msi",
+    "cab",
+    "tar.gz",
+    "tar.xz",
+    "tar.bz2",
+    "tar.lzma",
+    "tar.Z",
+    "tar.sz",
+    "lzma",
+    // add fonts
+    "ttf",
+    "otf",
+    "woff",
+    "woff2",
+    "eot",
+  ];
+  match bad_extensions.contains(&extension) {
+    true => {
       return Err(anyhow::anyhow!("File is just a lock file"));
     }
     _ => {}
@@ -63,7 +145,7 @@ pub fn is_good_file(file_path: &Path) -> Result<(), Error> {
     Some(extension) => match extension.to_str() {
       Some(ext_str) => {
         if bad_extensions.contains(&ext_str) {
-          return Err(anyhow::anyhow!("File is not a valid UTF-8 string"));
+          return Err(anyhow::anyhow!("Binary file excluded from indexing."));
         }
       }
       None => {
@@ -89,10 +171,12 @@ pub fn is_good_file(file_path: &Path) -> Result<(), Error> {
   Ok(())
 }
 
+// use binaryornot::is_binary;
+// use anyhow::Context;
 // implement the buffer above:
 pub async fn is_good_file_runtime_check(
   file_path: &Path,
-  buffer: &[u8],
+  // _buffer: &[u8],
 ) -> Result<(), Error> {
   match get_file_size(file_path).await {
     Ok(size) if size > 2 * 1024 * 1024 => {
@@ -102,16 +186,31 @@ pub async fn is_good_file_runtime_check(
     _ => {}
   }
 
-  for &byte in buffer.iter().take(2048) {
-    if byte.is_ascii() {
-      continue;
-    } else {
-      return Err(anyhow::anyhow!("File is not a valid UTF-8 string"));
-    }
-  }
+  // if is_binary(file_path).context("Failed to check if file is binary")? {
+  // 	return Err(anyhow::anyhow!("File is binary"));
+  // }
+
   Ok(())
 }
 
+pub async fn read_string_without_bom(
+  file_path: &Path,
+) -> Result<String, Error> {
+  let file_buffer = match fs::read(file_path).await {
+    Ok(buffer) => buffer,
+    Err(e) => {
+      return Err(anyhow::anyhow!(
+        "Failed to read file buffer: {}",
+        e.to_string()
+      ))
+    }
+  };
+
+  let (cow, _) = UTF_8.decode_with_bom_removal(&file_buffer);
+
+  Ok(cow.to_string())
+}
+
 pub fn as_relative_path(
   base_path: &Path,
   file_path: &Path,
@@ -169,25 +268,40 @@ mod tests {
     temp_file.write_all(b"Hello, world!").await.unwrap();
     let buffer = fs::read(&temp_file_path).await.unwrap();
     assert_eq!(
-      is_good_file_runtime_check(&temp_file_path, &buffer)
-        .await
-        .is_ok(),
+      is_good_file_runtime_check(&temp_file_path).await.is_ok(),
       true
     );
     temp_dir.close().unwrap();
 
+    // let temp_dir = tempfile::tempdir().unwrap();
+    // let temp_file_path = temp_dir.path().join("test_file");
+    // let mut temp_file = fs::File::create(&temp_file_path).await.unwrap();
+    // temp_file.write_all(&[0, 159, 146, 150]).await.unwrap(); // Invalid UTF-8 sequence
+    // let buffer = fs::read(&temp_file_path).await.unwrap();
+    // assert_eq!(
+    //   is_good_file_runtime_check(&temp_file_path).await.is_err(),
+    //   true
+    // );
+    // temp_dir.close().unwrap();
+  }
+
+  #[tokio::test]
+  async fn test_bom_file() {
+    const BOM: [u8; 3] = [0xEF, 0xBB, 0xBF];
+    const CONTENT: &str = "Hello, world!";
+
+    // Write this to a temp file
     let temp_dir = tempfile::tempdir().unwrap();
     let temp_file_path = temp_dir.path().join("test_file");
     let mut temp_file = fs::File::create(&temp_file_path).await.unwrap();
-    temp_file.write_all(&[0, 159, 146, 150]).await.unwrap(); // Invalid UTF-8 sequence
-    let buffer = fs::read(&temp_file_path).await.unwrap();
-    assert_eq!(
-      is_good_file_runtime_check(&temp_file_path, &buffer)
-        .await
-        .is_err(),
-      true
-    );
-    temp_dir.close().unwrap();
+    temp_file.write_all(&BOM).await.unwrap();
+    temp_file.write_all(CONTENT.as_bytes()).await.unwrap();
+
+    // expect that we read the file with tokio as the CONTENT
+    let file_contents = read_string_without_bom(&temp_file_path).await.unwrap();
+
+    // Check string equality of CONTENT (&str) to file_contents (String)
+    assert_eq!(CONTENT, file_contents);
   }
 
   #[test]

package/src/git_utils.rs

@@ -1,10 +1,11 @@
 use std::collections::HashSet;
-use std::path::PathBuf;
+use std::path::MAIN_SEPARATOR_STR;
 use std::process::Command;
 
-pub fn list_ignored_files(
+pub fn list_ignored_files_and_directories(
   workspace_root_path: &str,
-) -> Result<HashSet<PathBuf>, Box<dyn std::error::Error>> {
+  should_return_absolute_paths: bool,
+) -> Result<HashSet<String>, Box<dyn std::error::Error>> {
   let mut gitignored_files = HashSet::new();
 
   let commands = vec![
@@ -14,13 +15,70 @@ pub fn list_ignored_files(
       "--others",
       "--ignored",
       "--exclude-standard",
+      "--directory",
+      "--no-empty-directory"
     ],
+    // FIXME(sualeh): this is super sketchy and might totally break in like a bazillion ways. i dont like it.
+    vec![
+      "git submodule foreach --quiet 'git -C $toplevel/$path ls-files --others --ignored --exclude-standard --directory --no-empty-directory | (while read line; do echo $path/$line; done)'",
+    ],
+  ];
+
+  for command in commands {
+    let output = Command::new(&command[0])
+      .args(&command[1..])
+      .current_dir(workspace_root_path)
+      .output()?;
+
+    if output.status.success() {
+      let files = String::from_utf8(output.stdout)?
+        .lines()
+        .filter(|line| !line.is_empty())
+        .map(|line| {
+          let line = line.replace("/", MAIN_SEPARATOR_STR);
+
+          if should_return_absolute_paths {
+            let mut path = std::path::PathBuf::from(workspace_root_path);
+
+            path.push(line.clone());
+
+            match path.canonicalize() {
+              Ok(canonical_path) => {
+                canonical_path.to_string_lossy().into_owned()
+              }
+              Err(_) => String::from(line),
+            }
+          } else {
+            String::from(line)
+          }
+        })
+        .collect::<HashSet<_>>();
+
+      gitignored_files.extend(files);
+    }
+  }
+
+  Ok(gitignored_files)
+}
+
+pub fn list_ignored_files_with_absolute_paths(
+  workspace_root_path: &str,
+) -> Result<HashSet<String>, Box<dyn std::error::Error>> {
+  let mut gitignored_files = HashSet::new();
+
+  let commands = vec![
     vec![
       "git",
-      "submodule",
-      "foreach",
-      "--quiet",
-      "git ls-files --others --ignored --exclude-standard | sed 's|^|$path/|'",
+      "ls-files",
+      "--others",
+      "--ignored",
+      "--exclude-standard",
+    ],
+    // FIXME(sualeh): this is super sketchy and might totally break in like a bazillion ways. i dont like it.
+    vec![
+      "sh",
+      "-c",
+      "git submodule foreach --quiet 'git ls-files --others --ignored --exclude-standard | sed \"s|^|$path/|\"'",
     ],
   ];
 
@@ -34,7 +92,7 @@ pub fn list_ignored_files(
       let files = String::from_utf8(output.stdout)?
         .lines()
         .filter(|line| !line.is_empty())
-        .map(|line| PathBuf::from(line))
+        .map(|line| String::from(line))
         .collect::<HashSet<_>>();
 
       gitignored_files.extend(files);
@@ -57,6 +115,18 @@ pub async fn is_git_ignored(
   Ok(output.status.success())
 }
 
+pub async fn is_git_directory(
+  workspace_root_path: &str,
+) -> Result<bool, anyhow::Error> {
+  let output = tokio::process::Command::new("git")
+    .args(&["rev-parse", "--is-inside-work-tree"])
+    .current_dir(workspace_root_path)
+    .output()
+    .await?;
+
+  Ok(output.status.success())
+}
+
 #[cfg(test)]
 mod tests {
   use super::*;
@@ -66,7 +136,9 @@ mod tests {
   #[test]
   fn test_no_ignored_files() {
     let dir = tempfile::tempdir().unwrap();
-    let gitignored_files = list_ignored_files(dir.path().to_str().unwrap()).unwrap();
+    let gitignored_files =
+      list_ignored_files_and_directories(dir.path().to_str().unwrap(), false)
+        .unwrap();
     Command::new("git")
       .args(&["init"])
       .current_dir(dir.path())
@@ -92,13 +164,15 @@ mod tests {
       .current_dir(dir.path())
       .output()
       .unwrap();
-    let gitignored_files = list_ignored_files(dir.path().to_str().unwrap()).unwrap();
+    let gitignored_files =
+      list_ignored_files_and_directories(dir.path().to_str().unwrap(), false)
+        .unwrap();
     println!(
       "ignored files for test_one_ignored_file: {:?}",
       gitignored_files
     );
     // assert_eq!(gitignored_files.len(), 1);
-    assert!(gitignored_files.contains(&PathBuf::from("ignored.txt")));
+    assert!(gitignored_files.contains(&String::from("ignored.txt")));
   }
 
   #[test]
@@ -121,19 +195,86 @@ mod tests {
       .current_dir(dir.path())
       .output()
       .unwrap();
-    let gitignored_files = list_ignored_files(dir.path().to_str().unwrap()).unwrap();
+    let gitignored_files =
+      list_ignored_files_and_directories(dir.path().to_str().unwrap(), false)
+        .unwrap();
     println!(
       "ignored files for test_multiple_ignored_files: {:?}",
       gitignored_files
     );
     // assert_eq!(gitignored_files.len(), 2);
-    assert!(gitignored_files.contains(&PathBuf::from("ignored1.txt")));
-    assert!(gitignored_files.contains(&PathBuf::from("ignored2.txt")));
+    assert!(gitignored_files.contains(&String::from("ignored1.txt")));
+    assert!(gitignored_files.contains(&String::from("ignored2.txt")));
+  }
+
+  #[test]
+  fn test_git_submodule_ignored_files() {
+    let dir = tempfile::tempdir().unwrap();
+    let submodule_path = dir.path().join("submodule");
+    std::fs::create_dir(&submodule_path).unwrap();
+
+    let o = Command::new("git")
+      .args(&["init"])
+      .current_dir(&submodule_path)
+      .output()
+      .unwrap();
+    println!("git init output: {:?}", o);
+
+    let file_path = submodule_path.join("ignored.txt");
+    let mut file = File::create(&file_path).unwrap();
+    writeln!(file, "This is an ignored file.").unwrap();
+
+    let file2 = submodule_path.join("ignored2.txt");
+    let mut file = File::create(&file2).unwrap();
+    writeln!(file, "This is another ignored file.").unwrap();
+
+    let gitignore_path = submodule_path.join(".gitignore");
+    let mut gitignore = File::create(&gitignore_path).unwrap();
+    writeln!(gitignore, "*.txt").unwrap();
+
+    let o = Command::new("git")
+      .args(&["init"])
+      .current_dir(dir.path())
+      .output()
+      .unwrap();
+    println!("git init output: {:?}", o);
+
+    // make a commit in the submodule
+    let o = Command::new("git")
+      .args(&["add", "."])
+      .current_dir(&submodule_path)
+      .output()
+      .unwrap();
+    println!("git add output: {:?}", o);
+    let o = Command::new("git")
+      .args(&["commit", "-m", "initial commit"])
+      .current_dir(&submodule_path)
+      .output()
+      .unwrap();
+    println!("git commit output: {:?}", o);
+
+    let o = Command::new("git")
+      .args(&["submodule", "add", "./submodule"])
+      .current_dir(dir.path())
+      .output()
+      .unwrap();
+    println!("git submodule add output: {:?}", o);
+
+    let gitignored_files =
+      list_ignored_files_and_directories(dir.path().to_str().unwrap(), false)
+        .unwrap();
+    println!(
+      "ignored files for test_git_submodule_ignored_files: {:?}",
+      gitignored_files
+    );
+    assert!(gitignored_files.contains(&String::from("submodule/ignored.txt")));
+    assert!(gitignored_files.contains(&String::from("submodule/ignored2.txt")));
   }
 
   #[test]
   fn test_multiple_ignored_files_in_current_dir() {
-    let gitignored_files = list_ignored_files(".").unwrap();
+    let gitignored_files =
+      list_ignored_files_and_directories(".", false).unwrap();
     assert!(gitignored_files.len() > 1);
 
     // print a sample of the ignored files
@@ -147,7 +288,6 @@ mod tests {
     }
   }
 
-
   #[tokio::test]
   async fn test_file_not_ignored() {
     let dir = tempfile::tempdir().unwrap();
@@ -160,7 +300,10 @@ mod tests {
       .current_dir(dir.path())
       .output()
       .unwrap();
-    let is_ignored = is_git_ignored(dir.path().to_str().unwrap(), "not_ignored.txt").await.unwrap();
+    let is_ignored =
+      is_git_ignored(dir.path().to_str().unwrap(), "not_ignored.txt")
+        .await
+        .unwrap();
     assert_eq!(is_ignored, false);
   }
 
@@ -180,7 +323,10 @@ mod tests {
       .current_dir(dir.path())
       .output()
       .unwrap();
-    let is_ignored = is_git_ignored(dir.path().to_str().unwrap(), "ignored.txt").await.unwrap();
+    let is_ignored =
+      is_git_ignored(dir.path().to_str().unwrap(), "ignored.txt")
+        .await
+        .unwrap();
     assert_eq!(is_ignored, true);
   }
 
@@ -200,8 +346,10 @@ mod tests {
       .current_dir(dir.path())
       .output()
       .unwrap();
-    let is_ignored = is_git_ignored(dir.path().to_str().unwrap(), "ignored.txt").await.unwrap();
+    let is_ignored =
+      is_git_ignored(dir.path().to_str().unwrap(), "ignored.txt")
+        .await
+        .unwrap();
     assert_eq!(is_ignored, true);
   }
-
 }