@anysphere/file-service 0.0.0-d50aa568 → 0.0.0-d785e254

package/Cargo.toml

@@ -9,6 +9,7 @@ crate-type = ["cdylib"]
 [features]
 default = ["windows-subsystem"]
 windows-subsystem = []
+debugfile = []
 
 [dependencies]
 # Default enable napi4 feature, see https://nodejs.org/api/n-api.html#node-api-version-matrix
@@ -28,6 +29,9 @@ binaryornot = "1.0.0"
 dunce = "1.0.1"
 encoding_rs = "0.8.33"
 
+[target.'cfg(not(target_os = "linux"))'.dependencies]
+tracing-axiom = "0.4"
+
 [build-dependencies]
 napi-build = "2.0.1"
 tonic-build = "0.9.2"

package/index.d.ts

@@ -5,6 +5,7 @@
 
 export class MerkleClient {
   constructor(absoluteRootDirectory: string)
+  isTooBig(maxFiles: number, gitIgnoredFiles: Array<string>, isGitRepo: boolean): Promise<boolean>
   init(gitIgnoredFiles: Array<string>, isGitRepo: boolean): Promise<void>
   computeMerkleTree(gitIgnoredFiles: Array<string>, isGitRepo: boolean): Promise<void>
   updateFile(filePath: string): Promise<void>

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@anysphere/file-service",
-  "version": "0.0.0-d50aa568",
+  "version": "0.0.0-d785e254",
   "main": "index.js",
   "types": "index.d.ts",
   "napi": {
@@ -36,12 +36,12 @@
     "version": "napi version"
   },
   "optionalDependencies": {
-    "@anysphere/file-service-win32-x64-msvc": "0.0.0-d50aa568",
-    "@anysphere/file-service-darwin-x64": "0.0.0-d50aa568",
-    "@anysphere/file-service-linux-x64-gnu": "0.0.0-d50aa568",
-    "@anysphere/file-service-darwin-arm64": "0.0.0-d50aa568",
-    "@anysphere/file-service-win32-arm64-msvc": "0.0.0-d50aa568",
-    "@anysphere/file-service-darwin-universal": "0.0.0-d50aa568",
-    "@anysphere/file-service-linux-arm64-gnu": "0.0.0-d50aa568"
+    "@anysphere/file-service-win32-x64-msvc": "0.0.0-d785e254",
+    "@anysphere/file-service-darwin-x64": "0.0.0-d785e254",
+    "@anysphere/file-service-linux-x64-gnu": "0.0.0-d785e254",
+    "@anysphere/file-service-darwin-arm64": "0.0.0-d785e254",
+    "@anysphere/file-service-win32-arm64-msvc": "0.0.0-d785e254",
+    "@anysphere/file-service-darwin-universal": "0.0.0-d785e254",
+    "@anysphere/file-service-linux-arm64-gnu": "0.0.0-d785e254"
   }
 }

package/src/lib.rs

@@ -2,15 +2,14 @@
 #![deny(clippy::all)]
 #![deny(unsafe_op_in_unsafe_fn)]
 pub mod file_utils;
+pub mod logger;
 pub mod merkle_tree;
 
-use std::{vec, collections::HashSet};
+use std::{collections::HashSet, vec};
 
 use anyhow::Context;
 use merkle_tree::{LocalConstruction, MerkleTree};
-use tracing::{info, Level};
-use tracing_appender::rolling::{RollingFileAppender, Rotation};
-use tracing_subscriber::fmt;
+use tracing::{debug, info};
 
 #[macro_use]
 extern crate napi_derive;
@@ -19,30 +18,14 @@ extern crate napi_derive;
 pub struct MerkleClient {
   tree: MerkleTree,
   absolute_root_directory: String,
-  _guard: tracing_appender::non_blocking::WorkerGuard,
-}
-
-pub fn init_logger() -> tracing_appender::non_blocking::WorkerGuard {
-  let file_appender =
-    RollingFileAppender::new(Rotation::NEVER, "./", "rust_log.txt");
-  let (non_blocking, _guard) = tracing_appender::non_blocking(file_appender);
-  let subscriber = fmt::Subscriber::builder()
-    .with_max_level(Level::TRACE)
-    .with_writer(non_blocking)
-    .with_ansi(false)
-    .with_line_number(true)
-    .finish();
-
-  let _ = tracing::subscriber::set_global_default(subscriber);
-
-  _guard
+  _guard: Option<logger::GuardType>,
 }
 
 #[napi]
 impl MerkleClient {
   #[napi(constructor)]
   pub fn new(absolute_root_directory: String) -> MerkleClient {
-    let _guard = init_logger();
+    let _guard = logger::init_logger();
 
     // let canonical_root_directory = std::path::Path::new(&absolute_root_directory);
     // use dunce::canonicalize;
@@ -61,14 +44,74 @@ impl MerkleClient {
     }
   }
 
+	#[napi]
+  pub async fn is_too_big(
+    &self,
+    max_files: i32,
+    git_ignored_files: Vec<String>,
+    is_git_repo: bool,
+  ) -> bool {
+    let git_ignored_set =
+      HashSet::<String>::from_iter(git_ignored_files.into_iter());
+    let mut num_files = 0;
+    let mut dirs_to_check = vec![self.absolute_root_directory.clone()];
+
+    while let Some(dir) = dirs_to_check.pop() {
+			info!("dir: {:?}", dir);
+      let mut entries = match tokio::fs::read_dir(&dir).await {
+        Ok(entries) => entries,
+        Err(_) => continue,
+      };
+			if num_files > max_files {
+				return true;
+			}
+
+
+      while let Some(entry) = entries.next_entry().await.unwrap_or(None) {
+        let path = entry.path();
+				info!("entry: {:?}", path);
+        let path_str = match path.to_str() {
+          Some(path_str) => path_str.to_string(),
+          None => continue,
+        };
+
+        if git_ignored_set.contains(&path_str) {
+          continue;
+        }
+
+				match entry.file_type().await {
+					Ok(file_type) => {
+						if file_type.is_dir() {
+							dirs_to_check.push(path_str);
+						}
+
+						if file_type.is_file() {
+							num_files += 1;
+						}
+					}
+					Err(_) => continue,
+				}
+
+      }
+    }
+    num_files > max_files
+  }
+
   #[napi]
-  pub async unsafe fn init(&mut self, git_ignored_files: Vec<String>, is_git_repo: bool) -> Result<(), napi::Error> {
+  pub async unsafe fn init(
+    &mut self,
+    git_ignored_files: Vec<String>,
+    is_git_repo: bool,
+  ) -> Result<(), napi::Error> {
     // 1. compute the merkle tree
     // 2. update the backend
     // 3. sync with the remote
     info!("Merkle tree compute started!");
+    info!("Root directory: {:?}", self.absolute_root_directory);
     unsafe {
-      self.compute_merkle_tree(git_ignored_files, is_git_repo).await?;
+      self
+        .compute_merkle_tree(git_ignored_files, is_git_repo)
+        .await?;
     }
 
     Ok(())
@@ -82,14 +125,23 @@ impl MerkleClient {
   pub async unsafe fn compute_merkle_tree(
     &mut self,
     git_ignored_files: Vec<String>,
-    is_git_repo: bool
+    is_git_repo: bool,
   ) -> Result<(), napi::Error> {
     // make the git ignored files into a hash set
-    let git_ignored_set = HashSet::from_iter(git_ignored_files.into_iter());
+    let mut git_ignored_set = HashSet::from_iter(git_ignored_files.into_iter());
 
-    let t =
-      MerkleTree::construct_merkle_tree(self.absolute_root_directory.clone(), git_ignored_set, is_git_repo)
-        .await;
+    // if the hashset itself contains the root directory, then we should remove it.
+    // this is because the root directory is not a file, and we don't want to ignore it.
+    if git_ignored_set.contains(&self.absolute_root_directory) {
+      git_ignored_set.remove(&self.absolute_root_directory);
+    }
+
+    let t = MerkleTree::construct_merkle_tree(
+      self.absolute_root_directory.clone(),
+      git_ignored_set,
+      is_git_repo,
+    )
+    .await;
 
     match t {
       Ok(tree) => {
@@ -118,12 +170,18 @@ impl MerkleClient {
     &self,
     relative_path: String,
   ) -> Result<String, napi::Error> {
+    debug!("get_subtree_hash: relative_path: {:?}", relative_path);
+
     let relative_path_without_leading_slash = match relative_path
       .strip_prefix('.')
     {
       Some(path) => path.strip_prefix(std::path::MAIN_SEPARATOR).unwrap_or(""),
       None => relative_path.as_str(),
     };
+    debug!(
+      "relative_path_without_leading_slash: {:?}",
+      relative_path_without_leading_slash
+    );
 
     let absolute_path = if !relative_path_without_leading_slash.is_empty() {
       std::path::Path::new(&self.absolute_root_directory)
@@ -132,6 +190,8 @@ impl MerkleClient {
       std::path::Path::new(&self.absolute_root_directory).to_path_buf()
     };
 
+    debug!("absolute_path: {:?}", absolute_path);
+
     let absolute_path_string = match absolute_path.to_str() {
       Some(path) => path.to_string(),
       None => {
@@ -142,6 +202,8 @@ impl MerkleClient {
       }
     };
 
+    debug!("absolute_path_string: {:?}", absolute_path_string);
+
     let hash = self
       .tree
       .get_subtree_hash(absolute_path_string.as_str())

package/src/logger.rs

@@ -0,0 +1,55 @@
+use tracing::{info, subscriber, Level};
+use tracing_appender::non_blocking::WorkerGuard;
+use tracing_appender::rolling::{RollingFileAppender, Rotation};
+use tracing_subscriber::fmt;
+use tracing_subscriber::prelude::*;
+
+pub enum GuardType {
+  #[cfg(all(not(feature = "debugfile"), not(target_os = "linux")))]
+  Guard(tracing_axiom::Guard),
+  WorkerGuard(tracing_appender::non_blocking::WorkerGuard),
+}
+
+pub fn init_logger() -> Option<GuardType> {
+  #[cfg(feature = "debugfile")]
+  let _guard = {
+    let file_appender =
+      RollingFileAppender::new(Rotation::NEVER, "./", "rust_log.txt");
+    let (non_blocking, _guard) = tracing_appender::non_blocking(file_appender);
+    let subscriber = fmt::Subscriber::builder()
+      .with_max_level(Level::TRACE)
+      .with_writer(non_blocking)
+      .with_ansi(false)
+      .with_line_number(true)
+      .finish();
+
+    let _ = tracing::subscriber::set_global_default(subscriber);
+
+    Some(GuardType::WorkerGuard(_guard))
+  };
+
+  #[cfg(all(not(feature = "debugfile"), not(target_os = "linux")))]
+  let _guard = {
+    let (axiom_layer, _guard) = tracing_axiom::builder()
+      .with_token("xaat-a51088e6-7889-41c0-b440-cfd4601acdd7")
+      .with_dataset("local-indexing")
+      .layer()
+      .ok()?;
+    // let fmt_layer = fmt::layer().with_level(true).with_ansi(false).with_line_number(true);
+
+    let _ = tracing_subscriber::registry()
+      .with(axiom_layer)
+      .try_init()
+      .ok()?;
+    // let _ = tracing::subscriber::set_global_default(subscriber);
+
+    info!("Tracing initialized! in rust");
+
+    Some(GuardType::Guard(_guard))
+  };
+
+  #[cfg(all(not(feature = "debugfile"), target_os = "linux"))]
+  let _guard = { None };
+
+  _guard
+}

package/src/merkle_tree/local_construction.rs

@@ -9,6 +9,7 @@ use tonic::async_trait;
 
 #[async_trait]
 impl LocalConstruction for MerkleTree {
+  #[tracing::instrument]
   async fn new(
     root_directory: Option<String>,
   ) -> Result<MerkleTree, anyhow::Error> {
@@ -17,8 +18,9 @@ impl LocalConstruction for MerkleTree {
       let n = MerkleTree::construct_merkle_tree(
         root_directory,
         git_ignored_files,
-        false
-      ).await;
+        false,
+      )
+      .await;
       return n;
     }
 
@@ -35,7 +37,7 @@ impl LocalConstruction for MerkleTree {
   async fn construct_merkle_tree(
     absolute_path_to_root_directory: String,
     git_ignored_files_and_dirs: HashSet<String>,
-    is_git_repo: bool
+    is_git_repo: bool,
   ) -> Result<MerkleTree, anyhow::Error> {
     let path = PathBuf::from(absolute_path_to_root_directory.clone());
     if !path.exists() {
@@ -58,16 +60,17 @@ impl LocalConstruction for MerkleTree {
       None,
       &git_ignored_files_and_dirs,
       absolute_path_to_root_directory.as_str(),
-      is_git_repo
+      is_git_repo,
     )
     .await;
+
     let mut mt = MerkleTree {
       root: root_node,
       files: BTreeMap::new(),
       root_path: absolute_path_to_root_directory,
       cursor: None,
-      git_ignored_files_and_dirs: git_ignored_files_and_dirs,
-      is_git_repo
+      git_ignored_files_and_dirs,
+      is_git_repo,
     };
 
     // we now iterate over all the nodes and add them to the hashmap
@@ -80,7 +83,6 @@ impl LocalConstruction for MerkleTree {
         let node_reader = node.read().await;
         match &node_reader.node_type {
           NodeType::Branch(n) => {
-            tracing::info!("Branch: {:?}", n.0);
             let children = &n.1;
             files.insert(n.0.clone(), File { node: node.clone() });
             for child in children {
@@ -107,8 +109,7 @@ impl LocalConstruction for MerkleTree {
 
     add_nodes_to_hashmap(&mt.root, &mut mt.files).await;
 
-    tracing::info!("Merkle tree compute finished!");
-    tracing::info!("Merkle tree: {}", mt);
+    tracing::info!("number of files in the tree: {}", mt.files.len());
 
     Ok(mt)
   }
@@ -152,6 +153,7 @@ impl LocalConstruction for MerkleTree {
     Ok(())
   }
 
+  #[tracing::instrument]
   async fn delete_file(
     &mut self,
     file_path: String,

package/src/merkle_tree/mod.rs

@@ -6,13 +6,14 @@ use std::vec;
 use std::{fs, path::Path, sync::Arc};
 use tokio::sync::RwLock;
 use tonic::async_trait;
-use tracing::info;
+use tracing::{debug, info};
 
 pub mod local_construction;
 pub mod test;
 
 pub type MerkleNodePtr = Arc<RwLock<MerkleNode>>;
 
+#[derive(Debug)]
 pub struct MerkleTree {
   root_path: String,
   root: MerkleNodePtr,
@@ -64,7 +65,7 @@ pub trait LocalConstruction {
   async fn construct_merkle_tree(
     root_directory: String,
     git_ignored_files_and_dirs: HashSet<String>,
-    is_git_repo: bool
+    is_git_repo: bool,
   ) -> Result<MerkleTree, anyhow::Error>;
 
   async fn update_file(
@@ -99,7 +100,7 @@ impl MerkleTree {
       root_path: "".to_string(),
       cursor: None,
       git_ignored_files_and_dirs: HashSet::new(),
-      is_git_repo: false
+      is_git_repo: false,
     }
   }
 
@@ -107,6 +108,8 @@ impl MerkleTree {
     &self,
     absolute_path: &str,
   ) -> Result<String, anyhow::Error> {
+    debug!("get_subtree_hash: absolute_path: {:?}", absolute_path);
+
     let node = match self.files.get(absolute_path) {
       Some(file) => file.node.clone(),
       None => {
@@ -122,10 +125,7 @@ impl MerkleTree {
     let node_reader = node.read().await;
     let node_hash = node_reader.hash.clone();
 
-    info!(
-      "get_subtree_hash for path: {}, node_hash: {}",
-      absolute_path, node_hash
-    );
+    debug!("node_hash: {:?}", node_hash);
 
     Ok(node_hash)
   }
@@ -322,16 +322,11 @@ impl MerkleTree {
     &self,
     absolute_path: &str,
   ) -> Result<Vec<String>, anyhow::Error> {
-    info!("get_spline called with absolute_path: {}", absolute_path);
     let mut files = Vec::new();
 
     let current_node = match self.files.get(absolute_path) {
-      Some(node) => {
-        info!("Found node for absolute_path: {}", absolute_path);
-        node.node.clone()
-      }
+      Some(node) => node.node.clone(),
       None => {
-        info!("File not found for absolute_path: {}", absolute_path);
         return Err(anyhow::anyhow!("File not found: {}", absolute_path));
       }
     };
@@ -342,7 +337,6 @@ impl MerkleTree {
     while let Some(node) = stack.pop() {
       let parent = node.read().await.parent.clone();
       if let Some(parent) = parent {
-        info!("Adding parent hash to files vector");
         {
           let parent_node = parent.read().await;
           match &parent_node.node_type {
@@ -361,7 +355,6 @@ impl MerkleTree {
         stack.push(parent);
       }
     }
-    info!("Returning files vector with {} elements", files.len());
     Ok(files)
   }
 
@@ -410,7 +403,7 @@ impl MerkleTree {
           Some(ancestor.clone()),
           &self.git_ignored_files_and_dirs,
           &absolute_root_path.as_str(),
-          self.is_git_repo
+          self.is_git_repo,
         )
         .await;
         ancestor.write().await.attach_child(new_node.clone()).await;
@@ -428,7 +421,7 @@ impl MerkleTree {
           Some(ancestor.clone()),
           &self.git_ignored_files_and_dirs,
           &absolute_root_path.as_str(),
-          self.is_git_repo
+          self.is_git_repo,
         )
         .await;
 
@@ -736,6 +729,7 @@ impl MerkleNode {
     .await
   }
 
+  // #[tracing::instrument]
   async fn new(
     absolute_file_or_directory: PathBuf,
     parent: ParentPtr,
@@ -749,7 +743,6 @@ impl MerkleNode {
       "constructing node for absolute_file_or_directory: {:?}",
       absolute_file_or_directory
     );
-    info!("bypass_git: {}, is_git_repo: {}", bypass_git, is_git_repo);
 
     MerkleNode::construct_node(
       Path::new(&absolute_file_or_directory),
@@ -778,6 +771,7 @@ impl MerkleNode {
     Box::pin(async move {
       // check if it is a file
       let path_str = absolute_file_or_directory.to_str().unwrap().to_string();
+
       if absolute_file_or_directory.is_file() {
         return Arc::new(RwLock::new(
           MerkleNode::construct_file_node_or_error_node(
@@ -802,6 +796,7 @@ impl MerkleNode {
       let is_git_ignored_dir = ignored_files.contains(&path_str);
 
       if is_git_ignored_dir && !bypass_git {
+        tracing::info!("skipping directory: {}", path_str);
         return Arc::new(RwLock::new(MerkleNode::empty_node(
           Some(absolute_file_or_directory),
           Some("Directory is git ignored!".to_string()),
@@ -812,6 +807,7 @@ impl MerkleNode {
       match entries {
         Ok(_) => (),
         Err(e) => {
+          tracing::error!("error reading directory: {}", e);
           return Arc::new(RwLock::new(MerkleNode::empty_node(
             Some(absolute_file_or_directory),
             Some(e.to_string()),
@@ -844,6 +840,7 @@ impl MerkleNode {
             );
           }
           Err(e) => {
+            tracing::error!("error reading directory: {}", e);
             children.push(Arc::new(RwLock::new(MerkleNode::empty_node(
               Some(absolute_file_or_directory),
               Some(e.to_string()),
@@ -937,11 +934,7 @@ impl MerkleNode {
     .await
     {
       Ok(node) => node,
-      Err(e) => {
-        // println!("constructing error node. error: {}", e);
-        // println!("file_path: {:?}", file_path);
-        MerkleNode::empty_node(Some(absolute_file_path), Some(e))
-      }
+      Err(e) => MerkleNode::empty_node(Some(absolute_file_path), Some(e)),
     };
 
     node
@@ -1001,7 +994,6 @@ impl MerkleNode {
       if hash == "" {
         continue;
       }
-      info!("name: {}, hash: {}", name, hash);
       hasher.update(hash);
     }