@anysphere/file-service 0.0.0-dacf38fa → 0.0.0-e0c70bcd

package/src/merkle_tree/mod.rs

@@ -1,9 +1,13 @@
+use crate::git_utils;
+
 use super::file_utils;
 use sha2::Digest;
+use std::collections::{BTreeMap, HashSet};
 use std::path::PathBuf;
-use std::{collections::HashMap, fs, path::Path, sync::Arc};
+use std::{fs, path::Path, sync::Arc};
 use tokio::sync::RwLock;
 use tonic::async_trait;
+use tracing::info;
 pub mod local_construction;
 pub mod test;
 
@@ -12,7 +16,9 @@ pub type MerkleNodePtr = Arc<RwLock<MerkleNode>>;
 pub struct MerkleTree {
   root_path: String,
   root: MerkleNodePtr,
-  files: HashMap<String, File>,
+  files: BTreeMap<String, File>,
+  cursor: Option<usize>,
+  git_ignored_files: HashSet<String>,
 }
 
 #[derive(Debug)]
@@ -50,12 +56,19 @@ fn get_id() -> i32 {
 
 #[async_trait]
 pub trait LocalConstruction {
-  async fn new(root_directory: Option<String>) -> Result<MerkleTree, anyhow::Error>;
-  async fn construct_merkle_tree(root_directory: String) -> Result<MerkleTree, anyhow::Error>;
+  async fn new(
+    root_directory: Option<String>,
+  ) -> Result<MerkleTree, anyhow::Error>;
+
+  async fn construct_merkle_tree(
+    root_directory: String,
+  ) -> Result<MerkleTree, anyhow::Error>;
+
   async fn update_file(
     &mut self,
     file_path: String,
   ) -> Result<(), anyhow::Error>;
+
   async fn delete_file(
     &mut self,
     file_path: String,
@@ -79,96 +92,268 @@ impl MerkleTree {
   pub fn empty_tree() -> MerkleTree {
     MerkleTree {
       root: Arc::new(RwLock::new(MerkleNode::empty_node(None, None))),
-      files: HashMap::new(),
+      files: BTreeMap::new(),
       root_path: "".to_string(),
+      cursor: None,
+      git_ignored_files: HashSet::new(),
+    }
+  }
+
+  pub async fn get_subtree_hash(
+    &self,
+    absolute_path: PathBuf,
+  ) -> Result<String, anyhow::Error> {
+    let abs_string = match absolute_path.to_str() {
+      Some(s) => s.to_string(),
+      None => {
+        return Err(anyhow::anyhow!(
+          "get_subtree_hash: Failed to convert path to string"
+        ))
+      }
+    };
+
+    let node = match self.files.get(&abs_string) {
+      Some(file) => file.node.clone(),
+      None => {
+        let all_files: Vec<String> = self.files.keys().cloned().collect();
+        return Err(anyhow::anyhow!("Could not find file in tree! Looking for: {}. All files: {:?}", abs_string, all_files));
+      }
+    };
+
+    let node_reader = node.read().await;
+    let node_hash = node_reader.hash.clone();
+
+    Ok(node_hash)
+  }
+
+  pub async fn get_num_embeddable_files(&self) -> Result<i32, anyhow::Error> {
+    let mut count = 0;
+
+    for (_, file) in &self.files {
+      let file_reader = file.node.read().await;
+      match &file_reader.node_type {
+        NodeType::File(_) => {
+          count += 1;
+        }
+        NodeType::Branch(_) => {
+          continue;
+        }
+        NodeType::ErrorNode(_) => {
+          continue;
+        }
+      }
+    }
+
+    Ok(count)
+  }
+
+  pub async fn get_num_embeddable_files_in_subtree(
+    &self,
+    absolute_path: PathBuf,
+  ) -> Result<i32, anyhow::Error> {
+    let mut count = 0;
+
+    let absolute_path = match absolute_path.to_str() {
+      Some(s) => s.to_string(),
+      None => {
+        return Err(anyhow::anyhow!(
+        "get_num_embeddable_files_in_subtree: Failed to convert path to string"
+      ))
+      }
+    };
+
+    // TODO(sualeh): worth keeping this list sorted. its now a btree
+
+    for (_, file) in &self.files {
+      let file_reader = file.node.read().await;
+      match &file_reader.node_type {
+        NodeType::File(file_name) => {
+          if file_name.contains(&absolute_path) {
+            count += 1;
+          }
+        }
+        NodeType::Branch(_) => {
+          continue;
+        }
+        NodeType::ErrorNode(_) => {
+          continue;
+        }
+      }
+    }
+
+    Ok(count)
+  }
+
+  pub async fn get_all_files(&self) -> Result<Vec<String>, anyhow::Error> {
+    let mut files = Vec::new();
+
+    for (file_name, file) in &self.files {
+      let file_reader = file.node.read().await;
+      match &file_reader.node_type {
+        NodeType::File(_) => {
+          files.push(file_name.clone());
+        }
+        NodeType::Branch(_) => {
+          continue;
+        }
+        NodeType::ErrorNode(_) => {
+          continue;
+        }
+      }
+    }
+
+    Ok(files)
+  }
+
+  pub async fn get_hashes_for_files(
+    &self,
+    files: Vec<String>,
+  ) -> Result<Vec<String>, anyhow::Error> {
+    let mut hashes = Vec::new();
+
+    for file_name in files {
+      let file = match self.files.get(&file_name) {
+        Some(file) => file,
+        None => {
+          return Err(anyhow::anyhow!("Could not find file in tree!"));
+        }
+      };
+
+      let file_reader = file.node.read().await;
+      match &file_reader.node_type {
+        NodeType::File(_) => {
+          hashes.push(file_reader.hash.clone());
+        }
+        NodeType::Branch(_) => {
+          continue;
+        }
+        NodeType::ErrorNode(_) => {
+          continue;
+        }
+      }
+    }
+
+    Ok(hashes)
+  }
+
+  /// Returns a filename, and then a path from its parent to the root (which can possibly be empty.)
+  pub async fn get_next_file_to_embed(
+    &mut self,
+  ) -> Result<(String, Vec<String>), anyhow::Error> {
+    // if the cursor is none, set it to 0
+    let cursor = match self.cursor {
+      Some(cursor) => cursor,
+      None => {
+        self.cursor = Some(0);
+        0
+      }
+    };
+
+    // get the thing at the cursor. while we dont find a file, we keep incrementing the cursor.
+    let mut cursor = cursor;
+    loop {
+      // O(log n)
+      let file = match self.files.values().nth(cursor) {
+        Some(file) => file,
+        None => {
+          return Err(anyhow::anyhow!("Could not find file to embed!"));
+        }
+      };
+
+      let file_reader = file.node.read().await;
+      match &file_reader.node_type {
+        NodeType::File(f) => {
+          // update the cursor.
+          self.cursor = Some(cursor + 1);
+          let spline = self.get_spline(f).await?;
+          return Ok((f.clone(), spline));
+        }
+        NodeType::Branch(_) => {
+          cursor += 1;
+          continue;
+        }
+        NodeType::ErrorNode(_) => {
+          cursor += 1;
+          continue;
+        }
+      }
+    }
+  }
+
+  pub async fn get_all_dir_files_to_embed(
+    &self,
+    absolute_path: &str,
+  ) -> Result<Vec<String>, anyhow::Error> {
+    let mut files = Vec::new();
+
+    for (file_path, f) in &self.files {
+      if !file_path.contains(absolute_path) {
+        continue;
+      }
+
+      match f.node.read().await.node_type {
+        NodeType::File(_) => {
+          files.push(file_path.clone());
+        }
+        NodeType::Branch(_) => {
+          continue;
+        }
+        NodeType::ErrorNode(_) => {
+          continue;
+        }
+      }
     }
+
+    Ok(files)
   }
 
-	pub async fn get_subtree_hash(&self, path: String) -> Result<String, anyhow::Error> {
-		let path = PathBuf::from(path);
-		let node = match self.files.get(path.to_str().unwrap()) {
-			Some(file) => file.node.clone(),
-			None => {
-				return Err(anyhow::anyhow!("Could not find file in tree!"));
-			}
-		};
-
-		let node_reader = node.read().await;
-		let node_hash = node_reader.hash.clone();
-
-		Ok(node_hash)
-	}
-
-	pub async fn get_num_embeddable_files(&self) -> Result<i32, anyhow::Error> {
-		let mut count = 0;
-
-		for (_, file) in &self.files {
-			let file_reader = file.node.read().await;
-			match &file_reader.node_type {
-				NodeType::File(_) => {
-					count += 1;
-				}
-				NodeType::Branch(_) => {
-					continue;
-				}
-				NodeType::ErrorNode(_) => {
-					continue;
-				}
-			}
-		}
-
-		Ok(count)
-	}
-
-	pub async fn get_all_files(&self) -> Result<Vec<String>, anyhow::Error> {
-		let mut files = Vec::new();
-
-		for (file_name, file) in &self.files {
-			let file_reader = file.node.read().await;
-			match &file_reader.node_type {
-				NodeType::File(_) => {
-					files.push(file_name.clone());
-				}
-				NodeType::Branch(_) => {
-					continue;
-				}
-				NodeType::ErrorNode(_) => {
-					continue;
-				}
-			}
-		}
-
-		Ok(files)
-	}
-
-	pub async fn get_hashes_for_files(&self, files: Vec<String>) -> Result<Vec<String>, anyhow::Error> {
-		let mut hashes = Vec::new();
-
-		for file_name in files {
-			let file = match self.files.get(&file_name) {
-				Some(file) => file,
-				None => {
-					return Err(anyhow::anyhow!("Could not find file in tree!"));
-				}
-			};
-
-			let file_reader = file.node.read().await;
-			match &file_reader.node_type {
-				NodeType::File(_) => {
-					hashes.push(file_reader.hash.clone());
-				}
-				NodeType::Branch(_) => {
-					continue;
-				}
-				NodeType::ErrorNode(_) => {
-					continue;
-				}
-			}
-		}
-
-		Ok(hashes)
-	}
+  // TODO(sualeh): i need tests for this!!
+  pub async fn get_spline(
+    &self,
+    absolute_path: &str,
+  ) -> Result<Vec<String>, anyhow::Error> {
+    info!("get_spline called with absolute_path: {}", absolute_path);
+    let mut files = Vec::new();
+
+    let current_node = match self.files.get(absolute_path) {
+      Some(node) => {
+        info!("Found node for absolute_path: {}", absolute_path);
+        node.node.clone()
+      }
+      None => {
+        info!("File not found for absolute_path: {}", absolute_path);
+        return Err(anyhow::anyhow!("File not found: {}", absolute_path));
+      }
+    };
+
+    let mut stack = Vec::new();
+    stack.push(current_node);
+
+    while let Some(node) = stack.pop() {
+      let parent = node.read().await.parent.clone();
+      if let Some(parent) = parent {
+        info!("Adding parent hash to files vector");
+        {
+          let parent_node = parent.read().await;
+          match &parent_node.node_type {
+            NodeType::File(file_name) => {
+              files.push(file_name.clone());
+            }
+            NodeType::Branch((branch_name, _)) => {
+              files.push(branch_name.clone());
+            }
+            _ => {
+              continue;
+            }
+          }
+        }
 
+        stack.push(parent);
+      }
+    }
+    info!("Returning files vector with {} elements", files.len());
+    Ok(files)
+  }
 
   /// creates a new node and attaches it to the current tree.
   /// SPEC:
@@ -205,12 +390,18 @@ impl MerkleTree {
     // 1. the path is empty. this means that the ancestor is the root.
     // 2. the path is non-empty. that means there exist a non-empty element btwn till the root.
 
+    let absolute_root_path = self.root_path.clone();
     let new_node = match path.len() {
       0 => {
         // this means that the ancestor is the root.
         // we need to create a new node and attach it to the ancestor.
-        let new_node =
-          MerkleNode::new(file_path.clone(), Some(ancestor.clone())).await;
+        let new_node = MerkleNode::new(
+          file_path.clone(),
+          Some(ancestor.clone()),
+          &self.git_ignored_files,
+          &absolute_root_path.as_str(),
+        )
+        .await;
         ancestor.write().await.attach_child(new_node.clone()).await;
         new_node
       }
@@ -221,9 +412,13 @@ impl MerkleTree {
         // UNSURE: not sure this is the correct thing to do but it is the fastest.
         // get the last thing that is not in the tree.
         let first_child_path = path.last().unwrap();
-        let first_child =
-          MerkleNode::new(first_child_path.clone(), Some(ancestor.clone()))
-            .await;
+        let first_child = MerkleNode::new(
+          first_child_path.clone(),
+          Some(ancestor.clone()),
+          &self.git_ignored_files,
+          &absolute_root_path.as_str(),
+        )
+        .await;
 
         // TODO(sualeh): we should do an assertion check that the entire vec is contained here.
 
@@ -234,7 +429,7 @@ impl MerkleTree {
       }
     };
 
-		Ok(new_node)
+    Ok(new_node)
   }
 
   /// Spec:
@@ -298,17 +493,21 @@ impl MerkleTree {
   /// - attaches to the ancestor.
   /// - adds to the filemap
   /// - updates hashes of ancestor path.
-  async fn attach_new_node_to_tree(&mut self, file_path: String) -> Result<(), anyhow::Error> {
+  async fn attach_new_node_to_tree(
+    &mut self,
+    file_path: String,
+  ) -> Result<(), anyhow::Error> {
     let path = PathBuf::from(file_path.clone());
     match self.create_new_node_and_attach_to_ancestors(path).await {
-			Ok(node_ptr) => {
-				self.add_subtree_to_filemap(node_ptr).await;
-				Ok(())
-			}
-			Err(e) => {
-				Err(anyhow::anyhow!("Could not create new node and attach to ancestors! {}", e.to_string()))
-			}
-		}
+      Ok(node_ptr) => {
+        self.add_subtree_to_filemap(node_ptr).await;
+        Ok(())
+      }
+      Err(e) => Err(anyhow::anyhow!(
+        "Could not create new node and attach to ancestors! {}",
+        e.to_string()
+      )),
+    }
   }
 
   /// MUTATES MUTATES MUTATES
@@ -389,20 +588,22 @@ impl MerkleTree {
       .create_new_node_and_attach_to_ancestors(file_path)
       .await;
 
-		match node_ptr {
-			Ok(node_ptr) => {
-
-				self.files.insert(
-					file_string,
-					File {
-						node: node_ptr.clone(),
-					},
-				);
-			}
-			Err(e) => {
-				return Err(anyhow::anyhow!("Could not create new node and attach to ancestors! {}", e.to_string()));
-			}
-		}
+    match node_ptr {
+      Ok(node_ptr) => {
+        self.files.insert(
+          file_string,
+          File {
+            node: node_ptr.clone(),
+          },
+        );
+      }
+      Err(e) => {
+        return Err(anyhow::anyhow!(
+          "Could not create new node and attach to ancestors! {}",
+          e.to_string()
+        ));
+      }
+    }
 
     Ok(())
   }
@@ -494,18 +695,62 @@ use std::future::Future;
 use std::pin::Pin;
 
 type PinnedFuture<'a, T> = Pin<Box<dyn Future<Output = T> + Send + 'a>>;
+type IgnoredFiles = HashSet<String>;
 
 impl MerkleNode {
   /// please be careful using this.
   async fn __new_unchecked(
     file_or_directory: String,
     parent: ParentPtr,
+    ignored_files: &IgnoredFiles,
+    absolute_root_path: &str,
   ) -> MerkleNodePtr {
-    MerkleNode::construct_node(Path::new(&file_or_directory), parent).await
+    // check if the root is a git directory.
+    let is_git_repo =
+      match git_utils::is_git_directory(absolute_root_path).await {
+        Ok(is_git_repo) => is_git_repo,
+        Err(e) => false,
+      };
+    let bypass_git = !is_git_repo;
+
+    MerkleNode::construct_node(
+      Path::new(&file_or_directory),
+      parent,
+      ignored_files,
+      absolute_root_path,
+      bypass_git,
+    )
+    .await
   }
 
-  async fn new(file_or_directory: PathBuf, parent: ParentPtr) -> MerkleNodePtr {
-    MerkleNode::construct_node(Path::new(&file_or_directory), parent).await
+  async fn new(
+    absolute_file_or_directory: PathBuf,
+    parent: ParentPtr,
+    ignored_files: &IgnoredFiles,
+    absolute_root_path: &str,
+  ) -> MerkleNodePtr {
+    // check if the root is a git directory.
+    let is_git_repo =
+      match git_utils::is_git_directory(absolute_root_path).await {
+        Ok(is_git_repo) => is_git_repo,
+        Err(_e) => false,
+      };
+    let bypass_git = !is_git_repo;
+
+    info!(
+      "constructing node for absolute_file_or_directory: {:?}",
+      absolute_file_or_directory
+    );
+    info!("bypass_git: {}, is_git_repo: {}", bypass_git, is_git_repo);
+
+    MerkleNode::construct_node(
+      Path::new(&absolute_file_or_directory),
+      parent,
+      ignored_files,
+      absolute_root_path,
+      bypass_git,
+    )
+    .await
   }
 
   /// NOT added to the tree by default.
@@ -516,38 +761,68 @@ impl MerkleNode {
   // 		let file_hash = self.files.get_mut(&file_path).unwrap();
 
   fn construct_node<'a>(
-    file_or_directory: &'a Path,
+    absolute_file_or_directory: &'a Path,
     parent: ParentPtr,
+    ignored_files: &'a IgnoredFiles,
+    absolute_root_path: &'a str,
+    bypass_git: bool,
   ) -> PinnedFuture<'a, MerkleNodePtr> {
     Box::pin(async move {
       // check if it is a file
-      let path_str = file_or_directory.to_str().unwrap().to_string();
-      if file_or_directory.is_file() {
+      let path_str = absolute_file_or_directory.to_str().unwrap().to_string();
+      if absolute_file_or_directory.is_file() {
         return Arc::new(RwLock::new(
           MerkleNode::construct_file_node_or_error_node(
-            file_or_directory,
+            absolute_file_or_directory,
             parent,
+            ignored_files,
           )
           .await,
         ));
       }
 
-			// check if the directory fails the bad dir test.
-			let is_bad_dir = file_utils::is_in_bad_dir(file_or_directory);
-			if is_bad_dir.is_err() || is_bad_dir.unwrap_or(false) {
-				// println!("skipping directory: {}", path_str);
-				return Arc::new(RwLock::new(MerkleNode::empty_node(
-					Some(file_or_directory),
-					Some("Directory is in bad dir!".to_string()),
-				)));
-			}
+      // check if the directory fails the bad dir test.
+      let is_bad_dir = file_utils::is_in_bad_dir(absolute_file_or_directory);
+      if is_bad_dir.is_err() || is_bad_dir.unwrap_or(false) {
+        // println!("skipping directory: {}", path_str);
+        return Arc::new(RwLock::new(MerkleNode::empty_node(
+          Some(absolute_file_or_directory),
+          Some("Directory is in bad dir!".to_string()),
+        )));
+      }
 
-      let entries = fs::read_dir(file_or_directory);
+      // check if the directory is git ignored
+      let is_git_ignored =
+        match git_utils::is_git_ignored(absolute_root_path, path_str.as_str())
+          .await
+        {
+          Ok(is_git_ignored) => is_git_ignored,
+          Err(e) => {
+            return Arc::new(RwLock::new(MerkleNode::empty_node(
+              Some(absolute_file_or_directory),
+              Some(e.to_string()),
+            )));
+          }
+        };
+
+      if is_git_ignored && !bypass_git {
+        // println!("skipping directory: {}", path_str);
+        tracing::info!(
+          "skipping directory because its git ignored: {}",
+          path_str
+        );
+        return Arc::new(RwLock::new(MerkleNode::empty_node(
+          Some(absolute_file_or_directory),
+          Some("Directory is git ignored!".to_string()),
+        )));
+      }
+
+      let entries = fs::read_dir(absolute_file_or_directory);
       match entries {
         Ok(_) => (),
         Err(e) => {
           return Arc::new(RwLock::new(MerkleNode::empty_node(
-            Some(file_or_directory),
+            Some(absolute_file_or_directory),
             Some(e.to_string()),
           )));
         }
@@ -567,13 +842,19 @@ impl MerkleNode {
         match entry {
           Ok(entry) => {
             children.push(
-              MerkleNode::construct_node(&entry.path(), Some(node.clone()))
-                .await,
+              MerkleNode::construct_node(
+                &entry.path(),
+                Some(node.clone()),
+                ignored_files,
+                absolute_root_path,
+                bypass_git,
+              )
+              .await,
             );
           }
           Err(e) => {
             children.push(Arc::new(RwLock::new(MerkleNode::empty_node(
-              Some(file_or_directory),
+              Some(absolute_file_or_directory),
               Some(e.to_string()),
             ))));
           }
@@ -593,23 +874,33 @@ impl MerkleNode {
   }
 
   async fn construct_file_node(
-    file_path: &Path,
+    absolute_file_path: &Path,
     parent: ParentPtr,
+    ignored_files: &IgnoredFiles,
   ) -> Result<MerkleNode, String> {
-    let file_str = file_path
+    let file_str = absolute_file_path
       .to_str()
       .ok_or("Could not convert file path to string!")?
       .to_string();
     // first see if it passes the
-    match file_utils::is_good_file(file_path) {
+    match file_utils::is_good_file(absolute_file_path) {
       Ok(_) => {}
       Err(e) => {
         return Err(format!("File failed runtime checks! {}", e.to_string()));
       }
     }
 
+    // check if the file is in the git ignore buffer.
+    // this is a bug right because we are not checking absoluteness here.
+    match ignored_files.contains(&file_str) {
+      true => {
+        return Err(format!("File is in git ignore buffer!"));
+      }
+      false => {}
+    }
+
     // read the file_content to a buffer
-    let file_content = match tokio::fs::read(file_path).await {
+    let file_content = match tokio::fs::read(absolute_file_path).await {
       Ok(content) => content,
       Err(e) => {
         return Err(format!("Could not read file! {}", e.to_string()));
@@ -617,7 +908,11 @@ impl MerkleNode {
     };
 
     // check if the file passes runtime checks.
-    match file_utils::is_good_file_runtime_check(file_path, &file_content).await
+    match file_utils::is_good_file_runtime_check(
+      absolute_file_path,
+      &file_content,
+    )
+    .await
     {
       Ok(_) => {}
       Err(e) => {
@@ -648,15 +943,22 @@ impl MerkleNode {
   }
 
   async fn construct_file_node_or_error_node(
-    file_path: &Path,
+    absolute_file_path: &Path,
     parent: ParentPtr,
+    ignored_files: &IgnoredFiles,
   ) -> MerkleNode {
-    let node = match MerkleNode::construct_file_node(file_path, parent).await {
+    let node = match MerkleNode::construct_file_node(
+      absolute_file_path,
+      parent,
+      ignored_files,
+    )
+    .await
+    {
       Ok(node) => node,
       Err(e) => {
         // println!("constructing error node. error: {}", e);
-				// println!("file_path: {:?}", file_path);
-        MerkleNode::empty_node(Some(file_path), Some(e))
+        // println!("file_path: {:?}", file_path);
+        MerkleNode::empty_node(Some(absolute_file_path), Some(e))
       }
     };