@anysphere/file-service 0.0.0-e3fdf62d → 0.0.0-e7e53a0a

package/src/lib.rs

@@ -1,37 +1,118 @@
+#![windows_subsystem = "windows"]
 #![deny(clippy::all)]
+#![deny(unsafe_op_in_unsafe_fn)]
 pub mod file_utils;
-pub mod git_utils;
 pub mod merkle_tree;
 
-use std::vec;
+use std::{collections::HashSet, vec};
 
+use anyhow::Context;
 use merkle_tree::{LocalConstruction, MerkleTree};
+use tracing::{info, subscriber, Level};
+use tracing_appender::non_blocking::WorkerGuard;
+use tracing_appender::rolling::{RollingFileAppender, Rotation};
+use tracing_subscriber::fmt;
+use tracing_subscriber::prelude::*;
 
 #[macro_use]
 extern crate napi_derive;
 
+pub enum GuardType {
+  #[cfg(all(not(feature = "debugfile"), not(target_os = "linux")))]
+  Guard(tracing_axiom::Guard),
+  WorkerGuard(tracing_appender::non_blocking::WorkerGuard),
+}
+
 #[napi]
 pub struct MerkleClient {
   tree: MerkleTree,
-  root_directory: String,
+  absolute_root_directory: String,
+  _guard: Option<GuardType>,
+}
+
+pub fn init_logger() -> Option<GuardType> {
+  #[cfg(feature = "debugfile")]
+  let _guard = {
+    let file_appender =
+      RollingFileAppender::new(Rotation::NEVER, "./", "rust_log.txt");
+    let (non_blocking, _guard) = tracing_appender::non_blocking(file_appender);
+    let subscriber = fmt::Subscriber::builder()
+      .with_max_level(Level::TRACE)
+      .with_writer(non_blocking)
+      .with_ansi(false)
+      .with_line_number(true)
+      .finish();
+
+    let _ = tracing::subscriber::set_global_default(subscriber);
+
+    Some(GuardType::WorkerGuard(_guard))
+  };
+
+  #[cfg(all(not(feature = "debugfile"), not(target_os = "linux")))]
+  let _guard = {
+    let (axiom_layer, _guard) = tracing_axiom::builder()
+      .with_token("xaat-a51088e6-7889-41c0-b440-cfd4601acdd7")
+      .with_dataset("local-indexing")
+      .layer()
+      .ok()?;
+    // let fmt_layer = fmt::layer().with_level(true).with_ansi(false).with_line_number(true);
+
+    let _ = tracing_subscriber::registry()
+      .with(axiom_layer)
+      .try_init()
+      .ok()?;
+    // let _ = tracing::subscriber::set_global_default(subscriber);
+
+    info!("Tracing initialized! in rust");
+
+    Some(GuardType::Guard(_guard))
+  };
+
+  #[cfg(all(not(feature = "debugfile"), target_os = "linux"))]
+  let _guard = { None };
+
+  _guard
 }
 
 #[napi]
 impl MerkleClient {
   #[napi(constructor)]
-  pub fn new(root_directory: String) -> MerkleClient {
+  pub fn new(absolute_root_directory: String) -> MerkleClient {
+    let _guard = init_logger();
+
+    // let canonical_root_directory = std::path::Path::new(&absolute_root_directory);
+    // use dunce::canonicalize;
+    // let canonical_root_directory = match dunce::canonicalize(&canonical_root_directory) {
+    //   Ok(path) => path.to_str().unwrap_or(&absolute_root_directory).to_string().to_lowercase(),
+    //   Err(e) => {
+    //     info!("Error in canonicalizing path: path: {:?}, error {:?}", canonical_root_directory, e);
+    //     absolute_root_directory
+    //   }
+    // };
+
     MerkleClient {
       tree: MerkleTree::empty_tree(),
-      root_directory,
+      absolute_root_directory,
+      _guard,
     }
   }
 
   #[napi]
-  pub async unsafe fn init(&mut self) -> Result<(), napi::Error> {
+  pub async unsafe fn init(
+    &mut self,
+    git_ignored_files: Vec<String>,
+    is_git_repo: bool,
+  ) -> Result<(), napi::Error> {
     // 1. compute the merkle tree
     // 2. update the backend
     // 3. sync with the remote
-    self.compute_merkle_tree().await?;
+    info!("Merkle tree compute started!");
+    info!("Root directory: {:?}", self.absolute_root_directory);
+    unsafe {
+      self
+        .compute_merkle_tree(git_ignored_files, is_git_repo)
+        .await?;
+    }
 
     Ok(())
   }
@@ -40,12 +121,27 @@ impl MerkleClient {
     unimplemented!("Interrupt is not implemented yet");
   }
 
-  // #[napi]
+  #[napi]
   pub async unsafe fn compute_merkle_tree(
     &mut self,
+    git_ignored_files: Vec<String>,
+    is_git_repo: bool,
   ) -> Result<(), napi::Error> {
-    let t =
-      MerkleTree::construct_merkle_tree(self.root_directory.clone()).await;
+    // make the git ignored files into a hash set
+    let mut git_ignored_set = HashSet::from_iter(git_ignored_files.into_iter());
+
+    // if the hashset itself contains the root directory, then we should remove it.
+    // this is because the root directory is not a file, and we don't want to ignore it.
+    if git_ignored_set.contains(&self.absolute_root_directory) {
+      git_ignored_set.remove(&self.absolute_root_directory);
+    }
+
+    let t = MerkleTree::construct_merkle_tree(
+      self.absolute_root_directory.clone(),
+      git_ignored_set,
+      is_git_repo,
+    )
+    .await;
 
     match t {
       Ok(tree) => {
@@ -72,15 +168,42 @@ impl MerkleClient {
   #[napi]
   pub async fn get_subtree_hash(
     &self,
-    path: String,
+    relative_path: String,
   ) -> Result<String, napi::Error> {
-    let hash = self.tree.get_subtree_hash(path).await;
+    let relative_path_without_leading_slash = match relative_path
+      .strip_prefix('.')
+    {
+      Some(path) => path.strip_prefix(std::path::MAIN_SEPARATOR).unwrap_or(""),
+      None => relative_path.as_str(),
+    };
+
+    let absolute_path = if !relative_path_without_leading_slash.is_empty() {
+      std::path::Path::new(&self.absolute_root_directory)
+        .join(relative_path_without_leading_slash)
+    } else {
+      std::path::Path::new(&self.absolute_root_directory).to_path_buf()
+    };
+
+    let absolute_path_string = match absolute_path.to_str() {
+      Some(path) => path.to_string(),
+      None => {
+        return Err(napi::Error::new(
+          napi::Status::Unknown,
+          format!("some string error"),
+        ))
+      }
+    };
+
+    let hash = self
+      .tree
+      .get_subtree_hash(absolute_path_string.as_str())
+      .await;
 
     match hash {
       Ok(hash) => Ok(hash),
       Err(e) => Err(napi::Error::new(
         napi::Status::Unknown,
-        format!("Error in get_subtree_hash: {:?}", e),
+        format!("Error in get_subtree_hash. \nRelative path: {:?}, \nAbsolute path: {:?}, \nRoot directory: {:?}\nError: {:?}", &relative_path, absolute_path, self.absolute_root_directory, e)
       )),
     }
   }
@@ -98,6 +221,28 @@ impl MerkleClient {
     }
   }
 
+  pub async fn get_num_embeddable_files_in_subtree(
+    &self,
+    relative_path: String,
+  ) -> Result<i32, napi::Error> {
+    let absolute_path = std::path::Path::new(&self.absolute_root_directory)
+      .join(relative_path)
+      .canonicalize()?;
+
+    let num = self
+      .tree
+      .get_num_embeddable_files_in_subtree(absolute_path)
+      .await;
+
+    match num {
+      Ok(num) => Ok(num),
+      Err(e) => Err(napi::Error::new(
+        napi::Status::Unknown,
+        format!("Error in get_num_embeddable_files_in_subtree: {:?}", e),
+      )),
+    }
+  }
+
   #[napi]
   pub async fn get_all_files(&self) -> Result<Vec<String>, napi::Error> {
     let files = self.tree.get_all_files().await;
@@ -111,6 +256,28 @@ impl MerkleClient {
     }
   }
 
+  #[napi]
+  pub async fn get_all_dir_files_to_embed(
+    &self,
+    absolute_file_path: String,
+  ) -> Result<Vec<String>, napi::Error> {
+    // let absolute_path = absolute_file_path.to_lowercase();
+    // let absolute_path_str = absolute_path.as_str();
+
+    let files = self
+      .tree
+      .get_all_dir_files_to_embed(absolute_file_path.as_str())
+      .await;
+
+    match files {
+      Ok(files) => Ok(files),
+      Err(e) => Err(napi::Error::new(
+        napi::Status::Unknown,
+        format!("Error in get_all_dir_files_to_embed: {:?}", e),
+      )),
+    }
+  }
+
   #[napi]
   pub async unsafe fn get_next_file_to_embed(
     &mut self,
@@ -125,7 +292,6 @@ impl MerkleClient {
 
         let ret = vec![file];
         let ret = ret.into_iter().chain(path.into_iter()).collect::<Vec<_>>();
-
         Ok(ret)
       }
       Err(e) => Err(napi::Error::new(
@@ -135,6 +301,25 @@ impl MerkleClient {
     }
   }
 
+  // FIXME(sualeh): get_spline
+  #[napi]
+  pub async fn get_spline(
+    &self,
+    absolute_file_path: String,
+  ) -> Result<Vec<String>, napi::Error> {
+    // let absolute_path = absolute_file_path.to_lowercase();
+    // let absolute_path_str = absolute_path.as_str();
+    let spline = self.tree.get_spline(absolute_file_path.as_str()).await;
+
+    match spline {
+      Ok(spline) => Ok(spline),
+      Err(e) => Err(napi::Error::new(
+        napi::Status::Unknown,
+        format!("Error in get_spline: {:?}", e),
+      )),
+    }
+  }
+
   #[napi]
   pub async fn get_hashes_for_files(
     &self,
@@ -151,8 +336,8 @@ impl MerkleClient {
     }
   }
 
-  // #[napi]
+  #[napi]
   pub fn update_root_directory(&mut self, root_directory: String) {
-    self.root_directory = root_directory;
+    self.absolute_root_directory = root_directory;
   }
 }

package/src/merkle_tree/local_construction.rs

@@ -3,17 +3,23 @@ use crate::merkle_tree::{
 };
 
 use super::{LocalConstruction, MerkleTree};
-use std::path::PathBuf;
-use std::{collections::HashMap, path::Path, sync::Arc};
+use std::collections::{BTreeMap, HashSet};
+use std::path::{Path, PathBuf};
 use tonic::async_trait;
 
 #[async_trait]
 impl LocalConstruction for MerkleTree {
+	#[tracing::instrument]
   async fn new(
     root_directory: Option<String>,
   ) -> Result<MerkleTree, anyhow::Error> {
+    let git_ignored_files = HashSet::<String>::new();
     if let Some(root_directory) = root_directory {
-      let n = MerkleTree::construct_merkle_tree(root_directory).await;
+      let n = MerkleTree::construct_merkle_tree(
+        root_directory,
+        git_ignored_files,
+        false
+      ).await;
       return n;
     }
 
@@ -27,28 +33,50 @@ impl LocalConstruction for MerkleTree {
   /// 2. compute hash for each file
   /// 3. construct merkle tree
   /// 4. return merkle tree
+	#[tracing::instrument(skip(git_ignored_files_and_dirs))]
   async fn construct_merkle_tree(
-    root_directory: String,
+    absolute_path_to_root_directory: String,
+    git_ignored_files_and_dirs: HashSet<String>,
+    is_git_repo: bool
   ) -> Result<MerkleTree, anyhow::Error> {
-    let path = PathBuf::from(root_directory.clone());
+    let path = PathBuf::from(absolute_path_to_root_directory.clone());
     if !path.exists() {
       // FIXME: we should report this via a good logger.
       panic!("Root directory does not exist!");
     }
 
-    let root_node = MerkleNode::new(path, None).await;
+    // 1. get all the gitignored files
+    // let git_ignored_files_and_dirs =
+    //   match git_utils::list_ignored_files_and_directories(
+    //     absolute_path_to_root_directory.as_str(),
+    //     true,
+    //   ) {
+    //     Ok(git_ignored) => git_ignored,
+    //     Err(_e) => HashSet::new(),
+    //   };
+
+    let root_node = MerkleNode::new(
+      path,
+      None,
+      &git_ignored_files_and_dirs,
+      absolute_path_to_root_directory.as_str(),
+      is_git_repo
+    ).await;
+
     let mut mt = MerkleTree {
       root: root_node,
-      files: HashMap::new(),
-      root_path: root_directory,
+      files: BTreeMap::new(),
+      root_path: absolute_path_to_root_directory,
       cursor: None,
+      git_ignored_files_and_dirs,
+      is_git_repo
     };
 
     // we now iterate over all the nodes and add them to the hashmap
     // TODO(later): i can make this parallel.
     fn add_nodes_to_hashmap<'a>(
       node: &'a MerkleNodePtr,
-      files: &'a mut HashMap<String, File>,
+      files: &'a mut BTreeMap<String, File>,
     ) -> PinnedFuture<'a, ()> {
       Box::pin(async move {
         let node_reader = node.read().await;
@@ -62,6 +90,13 @@ impl LocalConstruction for MerkleTree {
           }
           NodeType::File(file_name) => {
             let f = File { node: node.clone() };
+
+            // i dont reallly  like this :(((
+            // let canonical_file_name = match dunce::canonicalize(file_name) {
+            //   Ok(path) => path.to_str().unwrap_or(file_name).to_string(),
+            //   Err(_) => file_name.clone(),
+            // };
+
             files.insert(file_name.clone(), f);
           }
           NodeType::ErrorNode(_) => {
@@ -73,9 +108,12 @@ impl LocalConstruction for MerkleTree {
 
     add_nodes_to_hashmap(&mt.root, &mut mt.files).await;
 
+		tracing::info!("number of files in the tree: {}", mt.files.len());
+
     Ok(mt)
   }
 
+	#[tracing::instrument]
   async fn update_file(
     &mut self,
     file_path: String,
@@ -115,6 +153,7 @@ impl LocalConstruction for MerkleTree {
     Ok(())
   }
 
+	#[tracing::instrument]
   async fn delete_file(
     &mut self,
     file_path: String,