@anysphere/file-service 0.0.0-dacf38fa → 0.0.0-e1f2f04d

package/index.d.ts

@@ -11,5 +11,6 @@ export class MerkleClient {
   getSubtreeHash(path: string): Promise<string>
   getNumEmbeddableFiles(): Promise<number>
   getAllFiles(): Promise<Array<string>>
+  getNextFileToEmbed(): Promise<Array<string>>
   getHashesForFiles(files: Array<string>): Promise<Array<string>>
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@anysphere/file-service",
-  "version": "0.0.0-dacf38fa",
+  "version": "0.0.0-e1f2f04d",
   "main": "index.js",
   "types": "index.d.ts",
   "napi": {
@@ -35,11 +35,11 @@
     "version": "napi version"
   },
   "optionalDependencies": {
-    "@anysphere/file-service-win32-x64-msvc": "0.0.0-dacf38fa",
-    "@anysphere/file-service-darwin-x64": "0.0.0-dacf38fa",
-    "@anysphere/file-service-linux-x64-gnu": "0.0.0-dacf38fa",
-    "@anysphere/file-service-darwin-arm64": "0.0.0-dacf38fa",
-    "@anysphere/file-service-win32-arm64-msvc": "0.0.0-dacf38fa",
-    "@anysphere/file-service-darwin-universal": "0.0.0-dacf38fa"
+    "@anysphere/file-service-win32-x64-msvc": "0.0.0-e1f2f04d",
+    "@anysphere/file-service-darwin-x64": "0.0.0-e1f2f04d",
+    "@anysphere/file-service-linux-x64-gnu": "0.0.0-e1f2f04d",
+    "@anysphere/file-service-darwin-arm64": "0.0.0-e1f2f04d",
+    "@anysphere/file-service-win32-arm64-msvc": "0.0.0-e1f2f04d",
+    "@anysphere/file-service-darwin-universal": "0.0.0-e1f2f04d"
   }
 }

package/src/lib.rs

@@ -3,6 +3,8 @@ pub mod file_utils;
 pub mod git_utils;
 pub mod merkle_tree;
 
+use std::vec;
+
 use merkle_tree::{LocalConstruction, MerkleTree};
 
 #[macro_use]
@@ -31,7 +33,7 @@ impl MerkleClient {
     // 3. sync with the remote
     self.compute_merkle_tree().await?;
 
-		Ok(())
+    Ok(())
   }
 
   pub async unsafe fn interrupt(&mut self) -> Result<(), napi::Error> {
@@ -67,57 +69,87 @@ impl MerkleClient {
     let _r = self.tree.delete_file(file_path);
   }
 
-	#[napi]
-	pub async fn get_subtree_hash(&self, path: String) -> Result<String, napi::Error> {
-		let hash = self.tree.get_subtree_hash(path).await;
-
-		match hash {
-			Ok(hash) => Ok(hash),
-			Err(e) => Err(napi::Error::new(
-				napi::Status::Unknown,
-				format!("Error in get_subtree_hash: {:?}", e),
-			)),
-		}
-	}
-
-	#[napi]
-	pub async fn get_num_embeddable_files(&self) -> Result<i32, napi::Error> {
-		let num = self.tree.get_num_embeddable_files().await;
-
-		match num {
-			Ok(num) => Ok(num),
-			Err(e) => Err(napi::Error::new(
-				napi::Status::Unknown,
-				format!("Error in get_num_embeddable_files: {:?}", e),
-			)),
-		}
-	}
-
-	#[napi]
-	pub async fn get_all_files(&self) -> Result<Vec<String>, napi::Error> {
-		let files = self.tree.get_all_files().await;
-
-		match files {
-			Ok(files) => Ok(files),
-			Err(e) => Err(napi::Error::new(
-				napi::Status::Unknown,
-				format!("Error in get_all_files: {:?}", e),
-			)),
-		}
-	}
-
-	#[napi]
-	pub async fn get_hashes_for_files(&self, files: Vec<String>) -> Result<Vec<String>, napi::Error> {
-		let hashes = self.tree.get_hashes_for_files(files).await;
-
-		match hashes {
-			Ok(hashes) => Ok(hashes),
-			Err(e) => Err(napi::Error::new(
-				napi::Status::Unknown,
-				format!("Error in get_hashes_for_files: {:?}", e),
-			)),
-		}
-	}
+  #[napi]
+  pub async fn get_subtree_hash(
+    &self,
+    path: String,
+  ) -> Result<String, napi::Error> {
+    let hash = self.tree.get_subtree_hash(path).await;
+
+    match hash {
+      Ok(hash) => Ok(hash),
+      Err(e) => Err(napi::Error::new(
+        napi::Status::Unknown,
+        format!("Error in get_subtree_hash: {:?}", e),
+      )),
+    }
+  }
+
+  #[napi]
+  pub async fn get_num_embeddable_files(&self) -> Result<i32, napi::Error> {
+    let num = self.tree.get_num_embeddable_files().await;
+
+    match num {
+      Ok(num) => Ok(num),
+      Err(e) => Err(napi::Error::new(
+        napi::Status::Unknown,
+        format!("Error in get_num_embeddable_files: {:?}", e),
+      )),
+    }
+  }
+
+  #[napi]
+  pub async fn get_all_files(&self) -> Result<Vec<String>, napi::Error> {
+    let files = self.tree.get_all_files().await;
+
+    match files {
+      Ok(files) => Ok(files),
+      Err(e) => Err(napi::Error::new(
+        napi::Status::Unknown,
+        format!("Error in get_all_files: {:?}", e),
+      )),
+    }
+  }
+
+  #[napi]
+  pub async unsafe fn get_next_file_to_embed(
+    &mut self,
+  ) -> Result<Vec<String>, napi::Error> {
+    let n = self.tree.get_next_file_to_embed().await;
+
+    match n {
+      Ok((file, path)) => {
+        // now our job is to put the filename as the first element of the path.
+
+        // TODO(sualeh): we should assert that the path is ascending up to the path.
+
+        let ret = vec![file];
+        let ret = ret.into_iter().chain(path.into_iter()).collect::<Vec<_>>();
+
+        Ok(ret)
+      }
+      Err(e) => Err(napi::Error::new(
+        napi::Status::Unknown,
+        format!("Error in get_next_file_to_embed: {:?}", e),
+      )),
+    }
+  }
+
+  #[napi]
+  pub async fn get_hashes_for_files(
+    &self,
+    files: Vec<String>,
+  ) -> Result<Vec<String>, napi::Error> {
+    let hashes = self.tree.get_hashes_for_files(files).await;
+
+    match hashes {
+      Ok(hashes) => Ok(hashes),
+      Err(e) => Err(napi::Error::new(
+        napi::Status::Unknown,
+        format!("Error in get_hashes_for_files: {:?}", e),
+      )),
+    }
+  }
 
   // #[napi]
   pub fn update_root_directory(&mut self, root_directory: String) {

package/src/merkle_tree/local_construction.rs

@@ -9,7 +9,9 @@ use tonic::async_trait;
 
 #[async_trait]
 impl LocalConstruction for MerkleTree {
-  async fn new(root_directory: Option<String>) -> Result<MerkleTree, anyhow::Error> {
+  async fn new(
+    root_directory: Option<String>,
+  ) -> Result<MerkleTree, anyhow::Error> {
     if let Some(root_directory) = root_directory {
       let n = MerkleTree::construct_merkle_tree(root_directory).await;
       return n;
@@ -25,7 +27,9 @@ impl LocalConstruction for MerkleTree {
   /// 2. compute hash for each file
   /// 3. construct merkle tree
   /// 4. return merkle tree
-  async fn construct_merkle_tree(root_directory: String) -> Result<MerkleTree, anyhow::Error> {
+  async fn construct_merkle_tree(
+    root_directory: String,
+  ) -> Result<MerkleTree, anyhow::Error> {
     let path = PathBuf::from(root_directory.clone());
     if !path.exists() {
       // FIXME: we should report this via a good logger.
@@ -37,6 +41,7 @@ impl LocalConstruction for MerkleTree {
       root: root_node,
       files: HashMap::new(),
       root_path: root_directory,
+      cursor: None,
     };
 
     // we now iterate over all the nodes and add them to the hashmap
@@ -49,8 +54,8 @@ impl LocalConstruction for MerkleTree {
         let node_reader = node.read().await;
         match &node_reader.node_type {
           NodeType::Branch(n) => {
-						let children = &n.1;
-						files.insert(n.0.clone(), File { node: node.clone() });
+            let children = &n.1;
+            files.insert(n.0.clone(), File { node: node.clone() });
             for child in children {
               add_nodes_to_hashmap(child, files).await;
             }
@@ -102,9 +107,9 @@ impl LocalConstruction for MerkleTree {
       // File does not exist in the tree, let's add it.
       let e = self.attach_new_node_to_tree(file_path.clone()).await;
 
-			if e.is_err() {
-				return Err(anyhow::anyhow!("Could not attach new node to tree!"));
-			}
+      if e.is_err() {
+        return Err(anyhow::anyhow!("Could not attach new node to tree!"));
+      }
     }
 
     Ok(())
@@ -139,18 +144,17 @@ impl LocalConstruction for MerkleTree {
     let parent_node = parent_node.unwrap();
     let mut mut_parent = parent_node.write().await;
 
+    // BUG(sualeh): need to actually drop everything that is a child here.
+    // idea: enumerate all nodes that are children of this through a starts with query on the hashtable.
+    // then drop all of them.
+    // in opposite order of length.
 
-		// BUG(sualeh): need to actually drop everything that is a child here.
-		// idea: enumerate all nodes that are children of this through a starts with query on the hashtable.
-		// then drop all of them.
-		// in opposite order of length.
-
-		// then remove the node from the parent and you are done
+    // then remove the node from the parent and you are done
 
     // Remove the child from the parent node
     match mut_parent.node_type {
       NodeType::Branch(ref mut node) => {
-				let children = &mut node.1;
+        let children = &mut node.1;
         let mut found = false;
         let mut index = 0;
 

package/src/merkle_tree/mod.rs

@@ -13,6 +13,7 @@ pub struct MerkleTree {
   root_path: String,
   root: MerkleNodePtr,
   files: HashMap<String, File>,
+  cursor: Option<MerkleNodePtr>,
 }
 
 #[derive(Debug)]
@@ -50,12 +51,19 @@ fn get_id() -> i32 {
 
 #[async_trait]
 pub trait LocalConstruction {
-  async fn new(root_directory: Option<String>) -> Result<MerkleTree, anyhow::Error>;
-  async fn construct_merkle_tree(root_directory: String) -> Result<MerkleTree, anyhow::Error>;
+  async fn new(
+    root_directory: Option<String>,
+  ) -> Result<MerkleTree, anyhow::Error>;
+
+  async fn construct_merkle_tree(
+    root_directory: String,
+  ) -> Result<MerkleTree, anyhow::Error>;
+
   async fn update_file(
     &mut self,
     file_path: String,
   ) -> Result<(), anyhow::Error>;
+
   async fn delete_file(
     &mut self,
     file_path: String,
@@ -81,94 +89,183 @@ impl MerkleTree {
       root: Arc::new(RwLock::new(MerkleNode::empty_node(None, None))),
       files: HashMap::new(),
       root_path: "".to_string(),
+      cursor: None,
+    }
+  }
+
+  pub async fn get_subtree_hash(
+    &self,
+    path: String,
+  ) -> Result<String, anyhow::Error> {
+    let path = PathBuf::from(path);
+    let node = match self.files.get(path.to_str().unwrap()) {
+      Some(file) => file.node.clone(),
+      None => {
+        return Err(anyhow::anyhow!("Could not find file in tree!"));
+      }
+    };
+
+    let node_reader = node.read().await;
+    let node_hash = node_reader.hash.clone();
+
+    Ok(node_hash)
+  }
+
+  pub async fn get_num_embeddable_files(&self) -> Result<i32, anyhow::Error> {
+    let mut count = 0;
+
+    for (_, file) in &self.files {
+      let file_reader = file.node.read().await;
+      match &file_reader.node_type {
+        NodeType::File(_) => {
+          count += 1;
+        }
+        NodeType::Branch(_) => {
+          continue;
+        }
+        NodeType::ErrorNode(_) => {
+          continue;
+        }
+      }
+    }
+
+    Ok(count)
+  }
+
+  pub async fn get_all_files(&self) -> Result<Vec<String>, anyhow::Error> {
+    let mut files = Vec::new();
+
+    for (file_name, file) in &self.files {
+      let file_reader = file.node.read().await;
+      match &file_reader.node_type {
+        NodeType::File(_) => {
+          files.push(file_name.clone());
+        }
+        NodeType::Branch(_) => {
+          continue;
+        }
+        NodeType::ErrorNode(_) => {
+          continue;
+        }
+      }
+    }
+
+    Ok(files)
+  }
+
+  pub async fn get_hashes_for_files(
+    &self,
+    files: Vec<String>,
+  ) -> Result<Vec<String>, anyhow::Error> {
+    let mut hashes = Vec::new();
+
+    for file_name in files {
+      let file = match self.files.get(&file_name) {
+        Some(file) => file,
+        None => {
+          return Err(anyhow::anyhow!("Could not find file in tree!"));
+        }
+      };
+
+      let file_reader = file.node.read().await;
+      match &file_reader.node_type {
+        NodeType::File(_) => {
+          hashes.push(file_reader.hash.clone());
+        }
+        NodeType::Branch(_) => {
+          continue;
+        }
+        NodeType::ErrorNode(_) => {
+          continue;
+        }
+      }
     }
+
+    Ok(hashes)
   }
 
-	pub async fn get_subtree_hash(&self, path: String) -> Result<String, anyhow::Error> {
-		let path = PathBuf::from(path);
-		let node = match self.files.get(path.to_str().unwrap()) {
-			Some(file) => file.node.clone(),
-			None => {
-				return Err(anyhow::anyhow!("Could not find file in tree!"));
-			}
-		};
-
-		let node_reader = node.read().await;
-		let node_hash = node_reader.hash.clone();
-
-		Ok(node_hash)
-	}
-
-	pub async fn get_num_embeddable_files(&self) -> Result<i32, anyhow::Error> {
-		let mut count = 0;
-
-		for (_, file) in &self.files {
-			let file_reader = file.node.read().await;
-			match &file_reader.node_type {
-				NodeType::File(_) => {
-					count += 1;
-				}
-				NodeType::Branch(_) => {
-					continue;
-				}
-				NodeType::ErrorNode(_) => {
-					continue;
-				}
-			}
-		}
-
-		Ok(count)
-	}
-
-	pub async fn get_all_files(&self) -> Result<Vec<String>, anyhow::Error> {
-		let mut files = Vec::new();
-
-		for (file_name, file) in &self.files {
-			let file_reader = file.node.read().await;
-			match &file_reader.node_type {
-				NodeType::File(_) => {
-					files.push(file_name.clone());
-				}
-				NodeType::Branch(_) => {
-					continue;
-				}
-				NodeType::ErrorNode(_) => {
-					continue;
-				}
-			}
-		}
-
-		Ok(files)
-	}
-
-	pub async fn get_hashes_for_files(&self, files: Vec<String>) -> Result<Vec<String>, anyhow::Error> {
-		let mut hashes = Vec::new();
-
-		for file_name in files {
-			let file = match self.files.get(&file_name) {
-				Some(file) => file,
-				None => {
-					return Err(anyhow::anyhow!("Could not find file in tree!"));
-				}
-			};
-
-			let file_reader = file.node.read().await;
-			match &file_reader.node_type {
-				NodeType::File(_) => {
-					hashes.push(file_reader.hash.clone());
-				}
-				NodeType::Branch(_) => {
-					continue;
-				}
-				NodeType::ErrorNode(_) => {
-					continue;
-				}
-			}
-		}
-
-		Ok(hashes)
-	}
+  /// Returns a filename, and then a path from its parent to the root (which can possibly be empty.)
+  pub async fn get_next_file_to_embed(
+    &mut self,
+  ) -> Result<(String, Vec<String>), anyhow::Error> {
+    // the plan is to do an in-order traversal of the tree.
+
+    // first the edge case to deal with:
+    // cursor == None
+    if self.cursor.is_none() {
+      // If the root is a file, return its name.
+      if let NodeType::File(file_path) = &self.root.read().await.node_type {
+        return Ok((file_path.clone(), vec![]));
+      }
+
+      // if the path is not empty, we can iterate till we find the first child.
+      let mut potential_first_child = self.root.clone();
+      let mut is_branch = true;
+      let mut path = Vec::new();
+
+      while is_branch {
+        let node = {
+          let potential_first_child_reader = potential_first_child.read().await;
+          match &potential_first_child_reader.node_type {
+            NodeType::Branch(branch) => branch.clone(),
+            NodeType::File(_) => {
+              return Err(anyhow::anyhow!(
+                "get_next_file_to_embed: This should not happen! the branch happened to be file."
+              ));
+            }
+            NodeType::ErrorNode(_) => {
+              return Err(anyhow::anyhow!("Cursor is an error node!"));
+            }
+          }
+        };
+
+        let current_node_name = &node.0;
+        let child_list = &node.1;
+
+        if let Some(c) = child_list.first() {
+          let c_reader = c.read().await;
+
+          match &c_reader.node_type {
+            NodeType::File(file_path) => {
+              // must set the cursor!
+              self.cursor = Some(c.clone());
+
+              return Ok((file_path.clone(), path));
+            }
+            NodeType::Branch(_) => {
+              potential_first_child = c.clone();
+              is_branch = true;
+
+              // add the path to the current node.
+              path.push(current_node_name.clone());
+            }
+            NodeType::ErrorNode(_) => {
+              return Err(anyhow::anyhow!("Cursor is an error node!"));
+            }
+          }
+        } else {
+          // If the root has no children, return an error.
+          return Err(anyhow::anyhow!("Root has no children!"));
+        }
+      }
+    }
+
+    // THE DEFAULT CASE:
+    // we already have a cursor at a file.
+
+    // UNWRAP checked and fine. see the none case above.
+    let cursor_name = self.cursor.as_ref().unwrap();
+    let cursor_reader = cursor_name.read().await;
 
+    // invariant: you must be a file!!
+
+    // everytime we get to a child list, we will add all the children to a fifo, and then pull from it as long as we need it.
+
+    // algorithm:
+    // 1.
+
+    Err(anyhow::anyhow!("Could not find file to embed!"))
+  }
 
   /// creates a new node and attaches it to the current tree.
   /// SPEC:
@@ -234,7 +331,7 @@ impl MerkleTree {
       }
     };
 
-		Ok(new_node)
+    Ok(new_node)
   }
 
   /// Spec:
@@ -298,17 +395,21 @@ impl MerkleTree {
   /// - attaches to the ancestor.
   /// - adds to the filemap
   /// - updates hashes of ancestor path.
-  async fn attach_new_node_to_tree(&mut self, file_path: String) -> Result<(), anyhow::Error> {
+  async fn attach_new_node_to_tree(
+    &mut self,
+    file_path: String,
+  ) -> Result<(), anyhow::Error> {
     let path = PathBuf::from(file_path.clone());
     match self.create_new_node_and_attach_to_ancestors(path).await {
-			Ok(node_ptr) => {
-				self.add_subtree_to_filemap(node_ptr).await;
-				Ok(())
-			}
-			Err(e) => {
-				Err(anyhow::anyhow!("Could not create new node and attach to ancestors! {}", e.to_string()))
-			}
-		}
+      Ok(node_ptr) => {
+        self.add_subtree_to_filemap(node_ptr).await;
+        Ok(())
+      }
+      Err(e) => Err(anyhow::anyhow!(
+        "Could not create new node and attach to ancestors! {}",
+        e.to_string()
+      )),
+    }
   }
 
   /// MUTATES MUTATES MUTATES
@@ -389,20 +490,22 @@ impl MerkleTree {
       .create_new_node_and_attach_to_ancestors(file_path)
       .await;
 
-		match node_ptr {
-			Ok(node_ptr) => {
-
-				self.files.insert(
-					file_string,
-					File {
-						node: node_ptr.clone(),
-					},
-				);
-			}
-			Err(e) => {
-				return Err(anyhow::anyhow!("Could not create new node and attach to ancestors! {}", e.to_string()));
-			}
-		}
+    match node_ptr {
+      Ok(node_ptr) => {
+        self.files.insert(
+          file_string,
+          File {
+            node: node_ptr.clone(),
+          },
+        );
+      }
+      Err(e) => {
+        return Err(anyhow::anyhow!(
+          "Could not create new node and attach to ancestors! {}",
+          e.to_string()
+        ));
+      }
+    }
 
     Ok(())
   }
@@ -532,15 +635,15 @@ impl MerkleNode {
         ));
       }
 
-			// check if the directory fails the bad dir test.
-			let is_bad_dir = file_utils::is_in_bad_dir(file_or_directory);
-			if is_bad_dir.is_err() || is_bad_dir.unwrap_or(false) {
-				// println!("skipping directory: {}", path_str);
-				return Arc::new(RwLock::new(MerkleNode::empty_node(
-					Some(file_or_directory),
-					Some("Directory is in bad dir!".to_string()),
-				)));
-			}
+      // check if the directory fails the bad dir test.
+      let is_bad_dir = file_utils::is_in_bad_dir(file_or_directory);
+      if is_bad_dir.is_err() || is_bad_dir.unwrap_or(false) {
+        // println!("skipping directory: {}", path_str);
+        return Arc::new(RwLock::new(MerkleNode::empty_node(
+          Some(file_or_directory),
+          Some("Directory is in bad dir!".to_string()),
+        )));
+      }
 
       let entries = fs::read_dir(file_or_directory);
       match entries {
@@ -655,7 +758,7 @@ impl MerkleNode {
       Ok(node) => node,
       Err(e) => {
         // println!("constructing error node. error: {}", e);
-				// println!("file_path: {:?}", file_path);
+        // println!("file_path: {:?}", file_path);
         MerkleNode::empty_node(Some(file_path), Some(e))
       }
     };