@anysphere/file-service 0.0.0-fcf7dd4f → 0.0.0-fd010bd5

package/Cargo.toml

@@ -6,6 +6,10 @@ version = "0.0.0"
 [lib]
 crate-type = ["cdylib"]
 
+[features]
+default = ["windows-subsystem"]
+windows-subsystem = []
+
 [dependencies]
 # Default enable napi4 feature, see https://nodejs.org/api/n-api.html#node-api-version-matrix
 napi = { version = "2.12.2", default-features = false, features = ["napi4", "async", "tokio_rt"] }
@@ -22,6 +26,7 @@ tracing-subscriber = "0.3.17"
 tracing-appender = "0.2.2"
 binaryornot = "1.0.0"
 dunce = "1.0.1"
+encoding_rs = "0.8.33"
 
 [build-dependencies]
 napi-build = "2.0.1"
@@ -29,5 +34,6 @@ tonic-build = "0.9.2"
 anyhow = "1.0.75"
 glob = "0.3.0"
 
+
 [profile.release]
 lto = true

package/build.rs

@@ -3,6 +3,8 @@ use std::path::Path;
 extern crate napi_build;
 
 fn main() -> Result<(), anyhow::Error> {
+  #[cfg(target_os = "windows")]
+  println!("cargo:rustc-cdylib-link-arg=/SUBSYSTEM:WINDOWS");
   napi_build::setup();
 
   // print the relative path.

package/index.d.ts

@@ -5,8 +5,8 @@
 
 export class MerkleClient {
   constructor(absoluteRootDirectory: string)
-  init(): Promise<void>
-  computeMerkleTree(): Promise<void>
+  init(gitIgnoredFiles: Array<string>, isGitRepo: boolean): Promise<void>
+  computeMerkleTree(gitIgnoredFiles: Array<string>, isGitRepo: boolean): Promise<void>
   updateFile(filePath: string): Promise<void>
   deleteFile(filePath: string): Promise<void>
   getSubtreeHash(relativePath: string): Promise<string>

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@anysphere/file-service",
-  "version": "0.0.0-fcf7dd4f",
+  "version": "0.0.0-fd010bd5",
   "main": "index.js",
   "types": "index.d.ts",
   "napi": {
@@ -36,12 +36,12 @@
     "version": "napi version"
   },
   "optionalDependencies": {
-    "@anysphere/file-service-win32-x64-msvc": "0.0.0-fcf7dd4f",
-    "@anysphere/file-service-darwin-x64": "0.0.0-fcf7dd4f",
-    "@anysphere/file-service-linux-x64-gnu": "0.0.0-fcf7dd4f",
-    "@anysphere/file-service-darwin-arm64": "0.0.0-fcf7dd4f",
-    "@anysphere/file-service-win32-arm64-msvc": "0.0.0-fcf7dd4f",
-    "@anysphere/file-service-darwin-universal": "0.0.0-fcf7dd4f",
-    "@anysphere/file-service-linux-arm64-gnu": "0.0.0-fcf7dd4f"
+    "@anysphere/file-service-win32-x64-msvc": "0.0.0-fd010bd5",
+    "@anysphere/file-service-darwin-x64": "0.0.0-fd010bd5",
+    "@anysphere/file-service-linux-x64-gnu": "0.0.0-fd010bd5",
+    "@anysphere/file-service-darwin-arm64": "0.0.0-fd010bd5",
+    "@anysphere/file-service-win32-arm64-msvc": "0.0.0-fd010bd5",
+    "@anysphere/file-service-darwin-universal": "0.0.0-fd010bd5",
+    "@anysphere/file-service-linux-arm64-gnu": "0.0.0-fd010bd5"
   }
 }

package/src/file_utils.rs

@@ -5,6 +5,7 @@
 // 4. vscode.fs.stat
 
 use anyhow::Error;
+use encoding_rs::UTF_8;
 use std::path::Path;
 use tokio::fs;
 
@@ -43,8 +44,90 @@ pub fn is_good_file(file_path: &Path) -> Result<(), Error> {
     _ => {}
   }
 
-  match extension {
-    "lock" | "bak" | "tmp" | "bin" | "exe" | "dll" | "so" | "lockb" => {
+  let bad_extensions = vec![
+    "lock",
+    "bak",
+    "tmp",
+    "bin",
+    "exe",
+    "dll",
+    "so",
+    "lockb",
+    "qwoff",
+    "isl",
+    "csv",
+    "pdf",
+    // add ms word, excel, powerpoint, etc.
+    "doc",
+    "docx",
+    "xls",
+    "xlsx",
+    "ppt",
+    "pptx",
+    "odt",
+    "ods",
+    "odp",
+    "odg",
+    "odf",
+    "sxw",
+    "sxc",
+    "sxi",
+    "sxd",
+    "sdc",
+    // add images
+    "jpg",
+    "jpeg",
+    "png",
+    "gif",
+    "bmp",
+    "tif",
+    // add audio
+    "mp3",
+    "wav",
+    "wma",
+    "ogg",
+    "flac",
+    "aac",
+    // add video
+    "mp4",
+    "mov",
+    "wmv",
+    "flv",
+    "avi",
+    // add archives
+    "zip",
+    "tar",
+    "gz",
+    "7z",
+    "rar",
+    "tgz",
+    "dmg",
+    "iso",
+    "cue",
+    "mdf",
+    "mds",
+    "vcd",
+    "toast",
+    "img",
+    "apk",
+    "msi",
+    "cab",
+    "tar.gz",
+    "tar.xz",
+    "tar.bz2",
+    "tar.lzma",
+    "tar.Z",
+    "tar.sz",
+    "lzma",
+    // add fonts
+    "ttf",
+    "otf",
+    "woff",
+    "woff2",
+    "eot",
+  ];
+  match bad_extensions.contains(&extension) {
+    true => {
       return Err(anyhow::anyhow!("File is just a lock file"));
     }
     _ => {}
@@ -93,7 +176,7 @@ pub fn is_good_file(file_path: &Path) -> Result<(), Error> {
 // implement the buffer above:
 pub async fn is_good_file_runtime_check(
   file_path: &Path,
-  _buffer: &[u8],
+  // _buffer: &[u8],
 ) -> Result<(), Error> {
   match get_file_size(file_path).await {
     Ok(size) if size > 2 * 1024 * 1024 => {
@@ -103,13 +186,31 @@ pub async fn is_good_file_runtime_check(
     _ => {}
   }
 
-	// if is_binary(file_path).context("Failed to check if file is binary")? {
-	// 	return Err(anyhow::anyhow!("File is binary"));
-	// }
+  // if is_binary(file_path).context("Failed to check if file is binary")? {
+  // 	return Err(anyhow::anyhow!("File is binary"));
+  // }
 
   Ok(())
 }
 
+pub async fn read_string_without_bom(
+  file_path: &Path,
+) -> Result<String, Error> {
+  let file_buffer = match fs::read(file_path).await {
+    Ok(buffer) => buffer,
+    Err(e) => {
+      return Err(anyhow::anyhow!(
+        "Failed to read file buffer: {}",
+        e.to_string()
+      ))
+    }
+  };
+
+  let (cow, _) = UTF_8.decode_with_bom_removal(&file_buffer);
+
+  Ok(cow.to_string())
+}
+
 pub fn as_relative_path(
   base_path: &Path,
   file_path: &Path,
@@ -167,25 +268,40 @@ mod tests {
     temp_file.write_all(b"Hello, world!").await.unwrap();
     let buffer = fs::read(&temp_file_path).await.unwrap();
     assert_eq!(
-      is_good_file_runtime_check(&temp_file_path, &buffer)
-        .await
-        .is_ok(),
+      is_good_file_runtime_check(&temp_file_path).await.is_ok(),
       true
     );
     temp_dir.close().unwrap();
 
+    // let temp_dir = tempfile::tempdir().unwrap();
+    // let temp_file_path = temp_dir.path().join("test_file");
+    // let mut temp_file = fs::File::create(&temp_file_path).await.unwrap();
+    // temp_file.write_all(&[0, 159, 146, 150]).await.unwrap(); // Invalid UTF-8 sequence
+    // let buffer = fs::read(&temp_file_path).await.unwrap();
+    // assert_eq!(
+    //   is_good_file_runtime_check(&temp_file_path).await.is_err(),
+    //   true
+    // );
+    // temp_dir.close().unwrap();
+  }
+
+  #[tokio::test]
+  async fn test_bom_file() {
+    const BOM: [u8; 3] = [0xEF, 0xBB, 0xBF];
+    const CONTENT: &str = "Hello, world!";
+
+    // Write this to a temp file
     let temp_dir = tempfile::tempdir().unwrap();
     let temp_file_path = temp_dir.path().join("test_file");
     let mut temp_file = fs::File::create(&temp_file_path).await.unwrap();
-    temp_file.write_all(&[0, 159, 146, 150]).await.unwrap(); // Invalid UTF-8 sequence
-    let buffer = fs::read(&temp_file_path).await.unwrap();
-    assert_eq!(
-      is_good_file_runtime_check(&temp_file_path, &buffer)
-        .await
-        .is_err(),
-      true
-    );
-    temp_dir.close().unwrap();
+    temp_file.write_all(&BOM).await.unwrap();
+    temp_file.write_all(CONTENT.as_bytes()).await.unwrap();
+
+    // expect that we read the file with tokio as the CONTENT
+    let file_contents = read_string_without_bom(&temp_file_path).await.unwrap();
+
+    // Check string equality of CONTENT (&str) to file_contents (String)
+    assert_eq!(CONTENT, file_contents);
   }
 
   #[test]

package/src/git_utils.rs

@@ -1,4 +1,5 @@
 use std::collections::HashSet;
+use std::path::MAIN_SEPARATOR_STR;
 use std::process::Command;
 
 pub fn list_ignored_files_and_directories(
@@ -19,9 +20,7 @@ pub fn list_ignored_files_and_directories(
     ],
     // FIXME(sualeh): this is super sketchy and might totally break in like a bazillion ways. i dont like it.
     vec![
-      "sh",
-      "-c",
-      "git submodule foreach --quiet 'git ls-files --others --ignored --exclude-standard --directory --no-empty-directory | sed \"s|^|$path/|\"'",
+      "git submodule foreach --quiet 'git -C $toplevel/$path ls-files --others --ignored --exclude-standard --directory --no-empty-directory | (while read line; do echo $path/$line; done)'",
     ],
   ];
 
@@ -36,9 +35,12 @@ pub fn list_ignored_files_and_directories(
         .lines()
         .filter(|line| !line.is_empty())
         .map(|line| {
+          let line = line.replace("/", MAIN_SEPARATOR_STR);
+
           if should_return_absolute_paths {
             let mut path = std::path::PathBuf::from(workspace_root_path);
-            path.push(line);
+
+            path.push(line.clone());
 
             match path.canonicalize() {
               Ok(canonical_path) => {

package/src/lib.rs

@@ -1,16 +1,16 @@
+#![windows_subsystem = "windows"]
 #![deny(clippy::all)]
 #![deny(unsafe_op_in_unsafe_fn)]
 pub mod file_utils;
-pub mod git_utils;
 pub mod merkle_tree;
 
-use std::vec;
+use std::{collections::HashSet, vec};
 
+use anyhow::Context;
 use merkle_tree::{LocalConstruction, MerkleTree};
 use tracing::{info, Level};
 use tracing_appender::rolling::{RollingFileAppender, Rotation};
 use tracing_subscriber::fmt;
-use anyhow::Context;
 
 #[macro_use]
 extern crate napi_derive;
@@ -19,22 +19,27 @@ extern crate napi_derive;
 pub struct MerkleClient {
   tree: MerkleTree,
   absolute_root_directory: String,
-  _guard: tracing_appender::non_blocking::WorkerGuard,
+  _guard: Option<tracing_appender::non_blocking::WorkerGuard>,
 }
 
-pub fn init_logger() -> tracing_appender::non_blocking::WorkerGuard {
-  let file_appender =
-    RollingFileAppender::new(Rotation::NEVER, "./", "rust_log.txt");
-  let (non_blocking, _guard) = tracing_appender::non_blocking(file_appender);
-  let subscriber = fmt::Subscriber::builder()
-    .with_max_level(Level::TRACE)
-    .with_writer(non_blocking)
-    .with_ansi(false)
-    .with_line_number(true)
-    .finish();
-
-  let _ = tracing::subscriber::set_global_default(subscriber);
-
+pub fn init_logger() -> Option<tracing_appender::non_blocking::WorkerGuard> {
+  #[cfg(feature = "debugfile")]
+  let _guard = {
+    let file_appender =
+      RollingFileAppender::new(Rotation::NEVER, "./", "rust_log.txt");
+    let (non_blocking, _guard) = tracing_appender::non_blocking(file_appender);
+    let subscriber = fmt::Subscriber::builder()
+      .with_max_level(Level::TRACE)
+      .with_writer(non_blocking)
+      .with_ansi(false)
+      .with_line_number(true)
+      .finish();
+
+    let _ = tracing::subscriber::set_global_default(subscriber);
+    _guard
+  };
+  #[cfg(not(feature = "debugfile"))]
+  let _guard = { None };
   _guard
 }
 
@@ -62,13 +67,19 @@ impl MerkleClient {
   }
 
   #[napi]
-  pub async unsafe fn init(&mut self) -> Result<(), napi::Error> {
+  pub async unsafe fn init(
+    &mut self,
+    git_ignored_files: Vec<String>,
+    is_git_repo: bool,
+  ) -> Result<(), napi::Error> {
     // 1. compute the merkle tree
     // 2. update the backend
     // 3. sync with the remote
     info!("Merkle tree compute started!");
     unsafe {
-      self.compute_merkle_tree().await?;
+      self
+        .compute_merkle_tree(git_ignored_files, is_git_repo)
+        .await?;
     }
 
     Ok(())
@@ -81,9 +92,18 @@ impl MerkleClient {
   #[napi]
   pub async unsafe fn compute_merkle_tree(
     &mut self,
+    git_ignored_files: Vec<String>,
+    is_git_repo: bool,
   ) -> Result<(), napi::Error> {
-    let t =
-      MerkleTree::construct_merkle_tree(self.absolute_root_directory.clone()).await;
+    // make the git ignored files into a hash set
+    let git_ignored_set = HashSet::from_iter(git_ignored_files.into_iter());
+
+    let t = MerkleTree::construct_merkle_tree(
+      self.absolute_root_directory.clone(),
+      git_ignored_set,
+      is_git_repo,
+    )
+    .await;
 
     match t {
       Ok(tree) => {
@@ -112,14 +132,16 @@ impl MerkleClient {
     &self,
     relative_path: String,
   ) -> Result<String, napi::Error> {
-
-    let relative_path_without_leading_slash = match relative_path.strip_prefix('.') {
+    let relative_path_without_leading_slash = match relative_path
+      .strip_prefix('.')
+    {
       Some(path) => path.strip_prefix(std::path::MAIN_SEPARATOR).unwrap_or(""),
       None => relative_path.as_str(),
     };
 
     let absolute_path = if !relative_path_without_leading_slash.is_empty() {
-      std::path::Path::new(&self.absolute_root_directory).join(relative_path_without_leading_slash)
+      std::path::Path::new(&self.absolute_root_directory)
+        .join(relative_path_without_leading_slash)
     } else {
       std::path::Path::new(&self.absolute_root_directory).to_path_buf()
     };
@@ -129,12 +151,15 @@ impl MerkleClient {
       None => {
         return Err(napi::Error::new(
           napi::Status::Unknown,
-          format!("some string error")
+          format!("some string error"),
         ))
       }
     };
 
-    let hash = self.tree.get_subtree_hash(absolute_path_string.as_str()).await;
+    let hash = self
+      .tree
+      .get_subtree_hash(absolute_path_string.as_str())
+      .await;
 
     match hash {
       Ok(hash) => Ok(hash),
@@ -153,7 +178,7 @@ impl MerkleClient {
       Ok(num) => Ok(num),
       Err(e) => Err(napi::Error::new(
         napi::Status::Unknown,
-        format!("Error in get_num_embeddable_files: {:?}", e)
+        format!("Error in get_num_embeddable_files: {:?}", e),
       )),
     }
   }
@@ -175,7 +200,7 @@ impl MerkleClient {
       Ok(num) => Ok(num),
       Err(e) => Err(napi::Error::new(
         napi::Status::Unknown,
-        format!("Error in get_num_embeddable_files_in_subtree: {:?}", e)
+        format!("Error in get_num_embeddable_files_in_subtree: {:?}", e),
       )),
     }
   }
@@ -188,7 +213,7 @@ impl MerkleClient {
       Ok(files) => Ok(files),
       Err(e) => Err(napi::Error::new(
         napi::Status::Unknown,
-        format!("Error in get_all_files: {:?}", e)
+        format!("Error in get_all_files: {:?}", e),
       )),
     }
   }

package/src/merkle_tree/local_construction.rs

@@ -1,4 +1,3 @@
-use crate::git_utils;
 use crate::merkle_tree::{
   File, MerkleNode, MerkleNodePtr, NodeType, PinnedFuture,
 };
@@ -13,8 +12,13 @@ impl LocalConstruction for MerkleTree {
   async fn new(
     root_directory: Option<String>,
   ) -> Result<MerkleTree, anyhow::Error> {
+    let git_ignored_files = HashSet::<String>::new();
     if let Some(root_directory) = root_directory {
-      let n = MerkleTree::construct_merkle_tree(root_directory).await;
+      let n = MerkleTree::construct_merkle_tree(
+        root_directory,
+        git_ignored_files,
+        false
+      ).await;
       return n;
     }
 
@@ -30,6 +34,8 @@ impl LocalConstruction for MerkleTree {
   /// 4. return merkle tree
   async fn construct_merkle_tree(
     absolute_path_to_root_directory: String,
+    git_ignored_files_and_dirs: HashSet<String>,
+    is_git_repo: bool
   ) -> Result<MerkleTree, anyhow::Error> {
     let path = PathBuf::from(absolute_path_to_root_directory.clone());
     if !path.exists() {
@@ -38,20 +44,21 @@ impl LocalConstruction for MerkleTree {
     }
 
     // 1. get all the gitignored files
-    let git_ignored_files_and_dirs =
-      match git_utils::list_ignored_files_and_directories(
-        absolute_path_to_root_directory.as_str(),
-        true,
-      ) {
-        Ok(git_ignored) => git_ignored,
-        Err(_e) => HashSet::new(),
-      };
+    // let git_ignored_files_and_dirs =
+    //   match git_utils::list_ignored_files_and_directories(
+    //     absolute_path_to_root_directory.as_str(),
+    //     true,
+    //   ) {
+    //     Ok(git_ignored) => git_ignored,
+    //     Err(_e) => HashSet::new(),
+    //   };
 
     let root_node = MerkleNode::new(
       path,
       None,
       &git_ignored_files_and_dirs,
       absolute_path_to_root_directory.as_str(),
+      is_git_repo
     )
     .await;
     let mut mt = MerkleTree {
@@ -60,6 +67,7 @@ impl LocalConstruction for MerkleTree {
       root_path: absolute_path_to_root_directory,
       cursor: None,
       git_ignored_files_and_dirs: git_ignored_files_and_dirs,
+      is_git_repo
     };
 
     // we now iterate over all the nodes and add them to the hashmap

package/src/merkle_tree/mod.rs

@@ -1,5 +1,3 @@
-use crate::git_utils;
-
 use super::file_utils;
 use sha2::Digest;
 use std::collections::{BTreeMap, HashSet};
@@ -9,6 +7,7 @@ use std::{fs, path::Path, sync::Arc};
 use tokio::sync::RwLock;
 use tonic::async_trait;
 use tracing::info;
+
 pub mod local_construction;
 pub mod test;
 
@@ -20,6 +19,7 @@ pub struct MerkleTree {
   files: BTreeMap<String, File>,
   cursor: Option<usize>,
   git_ignored_files_and_dirs: HashSet<String>,
+  is_git_repo: bool,
 }
 
 #[derive(Debug)]
@@ -63,6 +63,8 @@ pub trait LocalConstruction {
 
   async fn construct_merkle_tree(
     root_directory: String,
+    git_ignored_files_and_dirs: HashSet<String>,
+    is_git_repo: bool
   ) -> Result<MerkleTree, anyhow::Error>;
 
   async fn update_file(
@@ -97,6 +99,7 @@ impl MerkleTree {
       root_path: "".to_string(),
       cursor: None,
       git_ignored_files_and_dirs: HashSet::new(),
+      is_git_repo: false
     }
   }
 
@@ -287,6 +290,12 @@ impl MerkleTree {
   ) -> Result<Vec<String>, anyhow::Error> {
     let mut files = Vec::new();
 
+    // 1. should check that this absolute path is actually a directory.
+    let file_node = self.files.get(absolute_path);
+    if file_node.is_none() {
+      return Err(anyhow::anyhow!("Could not find directory the in tree!"));
+    }
+
     for (file_path, f) in &self.files {
       if !file_path.contains(absolute_path) {
         continue;
@@ -401,6 +410,7 @@ impl MerkleTree {
           Some(ancestor.clone()),
           &self.git_ignored_files_and_dirs,
           &absolute_root_path.as_str(),
+          self.is_git_repo
         )
         .await;
         ancestor.write().await.attach_child(new_node.clone()).await;
@@ -418,6 +428,7 @@ impl MerkleTree {
           Some(ancestor.clone()),
           &self.git_ignored_files_and_dirs,
           &absolute_root_path.as_str(),
+          self.is_git_repo
         )
         .await;
 
@@ -705,13 +716,14 @@ impl MerkleNode {
     parent: ParentPtr,
     ignored_files: &IgnoredFiles,
     absolute_root_path: &str,
+    is_git_repo: bool,
   ) -> MerkleNodePtr {
-    // check if the root is a git directory.
-    let is_git_repo =
-      match git_utils::is_git_directory(absolute_root_path).await {
-        Ok(is_git_repo) => is_git_repo,
-        Err(e) => false,
-      };
+    // // check if the root is a git directory.
+    // let is_git_repo =
+    //   match git_utils::is_git_directory(absolute_root_path).await {
+    //     Ok(is_git_repo) => is_git_repo,
+    //     Err(_e) => false,
+    //   };
     let bypass_git = !is_git_repo;
 
     MerkleNode::construct_node(
@@ -729,13 +741,8 @@ impl MerkleNode {
     parent: ParentPtr,
     ignored_files: &IgnoredFiles,
     absolute_root_path: &str,
+    is_git_repo: bool,
   ) -> MerkleNodePtr {
-    // check if the root is a git directory.
-    let is_git_repo =
-      match git_utils::is_git_directory(absolute_root_path).await {
-        Ok(is_git_repo) => is_git_repo,
-        Err(_e) => false,
-      };
     let bypass_git = !is_git_repo;
 
     info!(
@@ -883,18 +890,10 @@ impl MerkleNode {
       false => {}
     }
 
-    // read the file_content to a buffer
-    let file_content = match tokio::fs::read(absolute_file_path).await {
-      Ok(content) => content,
-      Err(e) => {
-        return Err(format!("Could not read file! {}", e.to_string()));
-      }
-    };
-
     // check if the file passes runtime checks.
     match file_utils::is_good_file_runtime_check(
       absolute_file_path,
-      &file_content,
+      // &file_content,
     )
     .await
     {
@@ -904,15 +903,14 @@ impl MerkleNode {
       }
     }
 
-    let file_content = match std::str::from_utf8(&file_content) {
-      Ok(content) => content,
-      Err(e) => {
-        return Err(format!(
-          "UTF8 Failure. Could not convert file content to string! {}",
-          e.to_string()
-        ));
-      }
-    };
+    // read the file_content to a buffer
+    let file_content =
+      match file_utils::read_string_without_bom(absolute_file_path).await {
+        Ok(content) => content,
+        Err(e) => {
+          return Err(format!("Could not read file! {}", e.to_string()));
+        }
+      };
 
     let file_hash = compute_hash(&file_content);
     let node = MerkleNode {

package/src/merkle_tree/test.rs

@@ -43,8 +43,9 @@ mod tests {
     // let path = Path::new(&temp_dir_path);
 
     // Test construct_merkle_tree() function
+    let new_set = std::collections::HashSet::<String>::new();
     let tree =
-      MerkleTree::construct_merkle_tree(temp_dir_path.clone()).await;
+      MerkleTree::construct_merkle_tree(temp_dir_path.clone(), new_set, false).await;
     let mut tree = match tree {
       Ok(tree) => {
         assert_eq!(tree.files.len(), 2);