@anysphere/file-service 0.0.0-dacf38fa → 0.0.0-e0c70bcd

package/src/lib.rs

@@ -1,9 +1,15 @@
 #![deny(clippy::all)]
+#![deny(unsafe_op_in_unsafe_fn)]
 pub mod file_utils;
 pub mod git_utils;
 pub mod merkle_tree;
 
+use std::vec;
+
 use merkle_tree::{LocalConstruction, MerkleTree};
+use tracing::{info, Level};
+use tracing_appender::rolling::{RollingFileAppender, Rotation};
+use tracing_subscriber::fmt;
 
 #[macro_use]
 extern crate napi_derive;
@@ -12,15 +18,35 @@ extern crate napi_derive;
 pub struct MerkleClient {
   tree: MerkleTree,
   root_directory: String,
+  _guard: tracing_appender::non_blocking::WorkerGuard,
+}
+
+pub fn init_logger() -> tracing_appender::non_blocking::WorkerGuard {
+  let file_appender =
+    RollingFileAppender::new(Rotation::NEVER, "./", "rust_log.txt");
+  let (non_blocking, _guard) = tracing_appender::non_blocking(file_appender);
+  let subscriber = fmt::Subscriber::builder()
+    .with_max_level(Level::TRACE)
+    .with_writer(non_blocking)
+    .with_ansi(false)
+    .with_line_number(true)
+    .finish();
+
+  let _ = tracing::subscriber::set_global_default(subscriber);
+
+  _guard
 }
 
 #[napi]
 impl MerkleClient {
   #[napi(constructor)]
   pub fn new(root_directory: String) -> MerkleClient {
+    let _guard = init_logger();
+
     MerkleClient {
       tree: MerkleTree::empty_tree(),
       root_directory,
+      _guard,
     }
   }
 
@@ -29,16 +55,19 @@ impl MerkleClient {
     // 1. compute the merkle tree
     // 2. update the backend
     // 3. sync with the remote
-    self.compute_merkle_tree().await?;
+    info!("Merkle tree compute started!");
+    unsafe {
+      self.compute_merkle_tree().await?;
+    }
 
-		Ok(())
+    Ok(())
   }
 
   pub async unsafe fn interrupt(&mut self) -> Result<(), napi::Error> {
     unimplemented!("Interrupt is not implemented yet");
   }
 
-  // #[napi]
+  #[napi]
   pub async unsafe fn compute_merkle_tree(
     &mut self,
   ) -> Result<(), napi::Error> {
@@ -67,59 +96,159 @@ impl MerkleClient {
     let _r = self.tree.delete_file(file_path);
   }
 
-	#[napi]
-	pub async fn get_subtree_hash(&self, path: String) -> Result<String, napi::Error> {
-		let hash = self.tree.get_subtree_hash(path).await;
-
-		match hash {
-			Ok(hash) => Ok(hash),
-			Err(e) => Err(napi::Error::new(
-				napi::Status::Unknown,
-				format!("Error in get_subtree_hash: {:?}", e),
-			)),
-		}
-	}
-
-	#[napi]
-	pub async fn get_num_embeddable_files(&self) -> Result<i32, napi::Error> {
-		let num = self.tree.get_num_embeddable_files().await;
-
-		match num {
-			Ok(num) => Ok(num),
-			Err(e) => Err(napi::Error::new(
-				napi::Status::Unknown,
-				format!("Error in get_num_embeddable_files: {:?}", e),
-			)),
-		}
-	}
-
-	#[napi]
-	pub async fn get_all_files(&self) -> Result<Vec<String>, napi::Error> {
-		let files = self.tree.get_all_files().await;
-
-		match files {
-			Ok(files) => Ok(files),
-			Err(e) => Err(napi::Error::new(
-				napi::Status::Unknown,
-				format!("Error in get_all_files: {:?}", e),
-			)),
-		}
-	}
-
-	#[napi]
-	pub async fn get_hashes_for_files(&self, files: Vec<String>) -> Result<Vec<String>, napi::Error> {
-		let hashes = self.tree.get_hashes_for_files(files).await;
-
-		match hashes {
-			Ok(hashes) => Ok(hashes),
-			Err(e) => Err(napi::Error::new(
-				napi::Status::Unknown,
-				format!("Error in get_hashes_for_files: {:?}", e),
-			)),
-		}
-	}
-
-  // #[napi]
+  #[napi]
+  pub async fn get_subtree_hash(
+    &self,
+    relative_path: String,
+  ) -> Result<String, napi::Error> {
+    let absolute_path =
+      std::path::Path::new(&self.root_directory).join(&relative_path);
+
+    let canonical_path = match absolute_path.canonicalize() {
+      Ok(path) => path,
+      Err(e) => return Err(napi::Error::new(
+        napi::Status::Unknown,
+        format!("Error in canonicalizing path: {:?}", e),
+      )),
+    };
+
+    let hash = self.tree.get_subtree_hash(canonical_path.clone()).await;
+
+    match hash {
+      Ok(hash) => Ok(hash),
+      Err(e) => Err(napi::Error::new(
+        napi::Status::Unknown,
+        format!("Error in get_subtree_hash. \nRelative path: {:?}, \nAbsolute path: {:?}, \nCanonical path: {:?}, \nRoot directory: {:?}\nError: {:?}", &relative_path, absolute_path, canonical_path, self.root_directory, e),
+      )),
+    }
+  }
+
+  #[napi]
+  pub async fn get_num_embeddable_files(&self) -> Result<i32, napi::Error> {
+    let num = self.tree.get_num_embeddable_files().await;
+
+    match num {
+      Ok(num) => Ok(num),
+      Err(e) => Err(napi::Error::new(
+        napi::Status::Unknown,
+        format!("Error in get_num_embeddable_files: {:?}", e),
+      )),
+    }
+  }
+
+  pub async fn get_num_embeddable_files_in_subtree(
+    &self,
+    relative_path: String,
+  ) -> Result<i32, napi::Error> {
+    let absolute_path = std::path::Path::new(&self.root_directory)
+      .join(relative_path)
+      .canonicalize()?;
+
+    let num = self
+      .tree
+      .get_num_embeddable_files_in_subtree(absolute_path)
+      .await;
+
+    match num {
+      Ok(num) => Ok(num),
+      Err(e) => Err(napi::Error::new(
+        napi::Status::Unknown,
+        format!("Error in get_num_embeddable_files_in_subtree: {:?}", e),
+      )),
+    }
+  }
+
+  #[napi]
+  pub async fn get_all_files(&self) -> Result<Vec<String>, napi::Error> {
+    let files = self.tree.get_all_files().await;
+
+    match files {
+      Ok(files) => Ok(files),
+      Err(e) => Err(napi::Error::new(
+        napi::Status::Unknown,
+        format!("Error in get_all_files: {:?}", e),
+      )),
+    }
+  }
+
+  #[napi]
+  pub async fn get_all_dir_files_to_embed(
+    &self,
+    absolute_file_path: String,
+  ) -> Result<Vec<String>, napi::Error> {
+    let absolute_path_str = absolute_file_path.as_str();
+    let files = self
+      .tree
+      .get_all_dir_files_to_embed(absolute_path_str)
+      .await;
+
+    match files {
+      Ok(files) => Ok(files),
+      Err(e) => Err(napi::Error::new(
+        napi::Status::Unknown,
+        format!("Error in get_all_dir_files_to_embed: {:?}", e),
+      )),
+    }
+  }
+
+  #[napi]
+  pub async unsafe fn get_next_file_to_embed(
+    &mut self,
+  ) -> Result<Vec<String>, napi::Error> {
+    let n = self.tree.get_next_file_to_embed().await;
+
+    match n {
+      Ok((file, path)) => {
+        // now our job is to put the filename as the first element of the path.
+
+        // TODO(sualeh): we should assert that the path is ascending up to the path.
+
+        let ret = vec![file];
+        let ret = ret.into_iter().chain(path.into_iter()).collect::<Vec<_>>();
+        Ok(ret)
+      }
+      Err(e) => Err(napi::Error::new(
+        napi::Status::Unknown,
+        format!("Error in get_next_file_to_embed: {:?}", e),
+      )),
+    }
+  }
+
+  // FIXME(sualeh): get_spline
+  #[napi]
+  pub async fn get_spline(
+    &self,
+    absolute_file_path: String,
+  ) -> Result<Vec<String>, napi::Error> {
+    let absolute_path_str = absolute_file_path.as_str();
+    let spline = self.tree.get_spline(absolute_path_str).await;
+
+    match spline {
+      Ok(spline) => Ok(spline),
+      Err(e) => Err(napi::Error::new(
+        napi::Status::Unknown,
+        format!("Error in get_spline: {:?}", e),
+      )),
+    }
+  }
+
+  #[napi]
+  pub async fn get_hashes_for_files(
+    &self,
+    files: Vec<String>,
+  ) -> Result<Vec<String>, napi::Error> {
+    let hashes = self.tree.get_hashes_for_files(files).await;
+
+    match hashes {
+      Ok(hashes) => Ok(hashes),
+      Err(e) => Err(napi::Error::new(
+        napi::Status::Unknown,
+        format!("Error in get_hashes_for_files: {:?}", e),
+      )),
+    }
+  }
+
+  #[napi]
   pub fn update_root_directory(&mut self, root_directory: String) {
     self.root_directory = root_directory;
   }

package/src/merkle_tree/local_construction.rs

@@ -1,15 +1,18 @@
+use crate::git_utils;
 use crate::merkle_tree::{
   File, MerkleNode, MerkleNodePtr, NodeType, PinnedFuture,
 };
 
 use super::{LocalConstruction, MerkleTree};
-use std::path::PathBuf;
-use std::{collections::HashMap, path::Path, sync::Arc};
+use std::collections::{BTreeMap, HashSet};
+use std::path::{Path, PathBuf};
 use tonic::async_trait;
 
 #[async_trait]
 impl LocalConstruction for MerkleTree {
-  async fn new(root_directory: Option<String>) -> Result<MerkleTree, anyhow::Error> {
+  async fn new(
+    root_directory: Option<String>,
+  ) -> Result<MerkleTree, anyhow::Error> {
     if let Some(root_directory) = root_directory {
       let n = MerkleTree::construct_merkle_tree(root_directory).await;
       return n;
@@ -25,38 +28,59 @@ impl LocalConstruction for MerkleTree {
   /// 2. compute hash for each file
   /// 3. construct merkle tree
   /// 4. return merkle tree
-  async fn construct_merkle_tree(root_directory: String) -> Result<MerkleTree, anyhow::Error> {
-    let path = PathBuf::from(root_directory.clone());
+  async fn construct_merkle_tree(
+    absolute_path_to_root_directory: String,
+  ) -> Result<MerkleTree, anyhow::Error> {
+    let path = PathBuf::from(absolute_path_to_root_directory.clone());
     if !path.exists() {
       // FIXME: we should report this via a good logger.
       panic!("Root directory does not exist!");
     }
 
-    let root_node = MerkleNode::new(path, None).await;
+    // 1. get all the gitignored files
+    let git_ignored_files = match git_utils::list_ignored_files(
+      absolute_path_to_root_directory.as_str(),
+      true,
+    ) {
+      Ok(git_ignored) => git_ignored,
+      Err(_e) => HashSet::new(),
+    };
+
+    let root_node = MerkleNode::new(
+      path,
+      None,
+      &git_ignored_files,
+      absolute_path_to_root_directory.as_str(),
+    )
+    .await;
     let mut mt = MerkleTree {
       root: root_node,
-      files: HashMap::new(),
-      root_path: root_directory,
+      files: BTreeMap::new(),
+      root_path: absolute_path_to_root_directory,
+      cursor: None,
+      git_ignored_files,
     };
 
     // we now iterate over all the nodes and add them to the hashmap
     // TODO(later): i can make this parallel.
     fn add_nodes_to_hashmap<'a>(
       node: &'a MerkleNodePtr,
-      files: &'a mut HashMap<String, File>,
+      files: &'a mut BTreeMap<String, File>,
     ) -> PinnedFuture<'a, ()> {
       Box::pin(async move {
         let node_reader = node.read().await;
         match &node_reader.node_type {
           NodeType::Branch(n) => {
-						let children = &n.1;
-						files.insert(n.0.clone(), File { node: node.clone() });
+            tracing::info!("Branch: {:?}", n.0);
+            let children = &n.1;
+            files.insert(n.0.clone(), File { node: node.clone() });
             for child in children {
               add_nodes_to_hashmap(child, files).await;
             }
           }
           NodeType::File(file_name) => {
             let f = File { node: node.clone() };
+            tracing::info!("File: {:?}", file_name);
             files.insert(file_name.clone(), f);
           }
           NodeType::ErrorNode(_) => {
@@ -68,6 +92,9 @@ impl LocalConstruction for MerkleTree {
 
     add_nodes_to_hashmap(&mt.root, &mut mt.files).await;
 
+    tracing::info!("Merkle tree compute finished!");
+    tracing::info!("Merkle tree: {}", mt);
+
     Ok(mt)
   }
 
@@ -102,9 +129,9 @@ impl LocalConstruction for MerkleTree {
       // File does not exist in the tree, let's add it.
       let e = self.attach_new_node_to_tree(file_path.clone()).await;
 
-			if e.is_err() {
-				return Err(anyhow::anyhow!("Could not attach new node to tree!"));
-			}
+      if e.is_err() {
+        return Err(anyhow::anyhow!("Could not attach new node to tree!"));
+      }
     }
 
     Ok(())
@@ -139,18 +166,17 @@ impl LocalConstruction for MerkleTree {
     let parent_node = parent_node.unwrap();
     let mut mut_parent = parent_node.write().await;
 
+    // BUG(sualeh): need to actually drop everything that is a child here.
+    // idea: enumerate all nodes that are children of this through a starts with query on the hashtable.
+    // then drop all of them.
+    // in opposite order of length.
 
-		// BUG(sualeh): need to actually drop everything that is a child here.
-		// idea: enumerate all nodes that are children of this through a starts with query on the hashtable.
-		// then drop all of them.
-		// in opposite order of length.
-
-		// then remove the node from the parent and you are done
+    // then remove the node from the parent and you are done
 
     // Remove the child from the parent node
     match mut_parent.node_type {
       NodeType::Branch(ref mut node) => {
-				let children = &mut node.1;
+        let children = &mut node.1;
         let mut found = false;
         let mut index = 0;