@anysphere/file-service 0.0.0-eb8b99bf → 0.0.0-ed88c2dc
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Cargo.toml +0 -3
- package/package.json +8 -10
- package/src/file_utils.rs +5 -4
- package/src/git_utils.rs +19 -158
- package/src/lib.rs +5 -46
- package/src/merkle_tree/local_construction.rs +7 -28
- package/src/merkle_tree/mod.rs +32 -174
- package/src/test.rs +0 -5
package/Cargo.toml
CHANGED
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@anysphere/file-service",
|
|
3
|
-
"version": "0.0.0-
|
|
3
|
+
"version": "0.0.0-ed88c2dc",
|
|
4
4
|
"main": "index.js",
|
|
5
5
|
"types": "index.d.ts",
|
|
6
6
|
"napi": {
|
|
@@ -9,8 +9,7 @@
|
|
|
9
9
|
"additional": [
|
|
10
10
|
"aarch64-apple-darwin",
|
|
11
11
|
"aarch64-pc-windows-msvc",
|
|
12
|
-
"universal-apple-darwin"
|
|
13
|
-
"aarch64-unknown-linux-gnu"
|
|
12
|
+
"universal-apple-darwin"
|
|
14
13
|
]
|
|
15
14
|
}
|
|
16
15
|
},
|
|
@@ -36,12 +35,11 @@
|
|
|
36
35
|
"version": "napi version"
|
|
37
36
|
},
|
|
38
37
|
"optionalDependencies": {
|
|
39
|
-
"@anysphere/file-service-win32-x64-msvc": "0.0.0-
|
|
40
|
-
"@anysphere/file-service-darwin-x64": "0.0.0-
|
|
41
|
-
"@anysphere/file-service-linux-x64-gnu": "0.0.0-
|
|
42
|
-
"@anysphere/file-service-darwin-arm64": "0.0.0-
|
|
43
|
-
"@anysphere/file-service-win32-arm64-msvc": "0.0.0-
|
|
44
|
-
"@anysphere/file-service-darwin-universal": "0.0.0-
|
|
45
|
-
"@anysphere/file-service-linux-arm64-gnu": "0.0.0-eb8b99bf"
|
|
38
|
+
"@anysphere/file-service-win32-x64-msvc": "0.0.0-ed88c2dc",
|
|
39
|
+
"@anysphere/file-service-darwin-x64": "0.0.0-ed88c2dc",
|
|
40
|
+
"@anysphere/file-service-linux-x64-gnu": "0.0.0-ed88c2dc",
|
|
41
|
+
"@anysphere/file-service-darwin-arm64": "0.0.0-ed88c2dc",
|
|
42
|
+
"@anysphere/file-service-win32-arm64-msvc": "0.0.0-ed88c2dc",
|
|
43
|
+
"@anysphere/file-service-darwin-universal": "0.0.0-ed88c2dc"
|
|
46
44
|
}
|
|
47
45
|
}
|
package/src/file_utils.rs
CHANGED
|
@@ -12,8 +12,9 @@ pub fn is_in_bad_dir(file_path: &Path) -> Result<bool, Error> {
|
|
|
12
12
|
let item_path = file_path
|
|
13
13
|
.to_str()
|
|
14
14
|
.ok_or(anyhow::anyhow!("Failed to convert path to string"))?;
|
|
15
|
-
let is_bad_dir =
|
|
16
|
-
|
|
15
|
+
let is_bad_dir = (item_path.contains("node_modules")
|
|
16
|
+
|| item_path.contains(".git"))
|
|
17
|
+
&& !(item_path.ends_with(".git") || item_path.ends_with("node_modules"));
|
|
17
18
|
Ok(is_bad_dir)
|
|
18
19
|
}
|
|
19
20
|
|
|
@@ -37,14 +38,14 @@ pub fn is_good_file(file_path: &Path) -> Result<(), Error> {
|
|
|
37
38
|
|
|
38
39
|
match file_name {
|
|
39
40
|
"package-lock.json" | "pnpm-lock.yaml" | "yarn.lock" | "composer.lock"
|
|
40
|
-
| "Gemfile.lock"
|
|
41
|
+
| "Gemfile.lock" => {
|
|
41
42
|
return Err(anyhow::anyhow!("File is just a lock file"));
|
|
42
43
|
}
|
|
43
44
|
_ => {}
|
|
44
45
|
}
|
|
45
46
|
|
|
46
47
|
match extension {
|
|
47
|
-
"lock" | "bak" | "tmp" | "bin" | "exe" | "dll" | "so"
|
|
48
|
+
"lock" | "bak" | "tmp" | "bin" | "exe" | "dll" | "so" => {
|
|
48
49
|
return Err(anyhow::anyhow!("File is just a lock file"));
|
|
49
50
|
}
|
|
50
51
|
_ => {}
|
package/src/git_utils.rs
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
use std::collections::HashSet;
|
|
2
|
+
use std::path::PathBuf;
|
|
2
3
|
use std::process::Command;
|
|
3
4
|
|
|
4
5
|
pub fn list_ignored_files(
|
|
5
6
|
workspace_root_path: &str,
|
|
6
|
-
|
|
7
|
-
) -> Result<HashSet<String>, Box<dyn std::error::Error>> {
|
|
7
|
+
) -> Result<HashSet<PathBuf>, Box<dyn std::error::Error>> {
|
|
8
8
|
let mut gitignored_files = HashSet::new();
|
|
9
9
|
|
|
10
10
|
let commands = vec![
|
|
@@ -15,66 +15,12 @@ pub fn list_ignored_files(
|
|
|
15
15
|
"--ignored",
|
|
16
16
|
"--exclude-standard",
|
|
17
17
|
],
|
|
18
|
-
// FIXME(sualeh): this is super sketchy and might totally break in like a bazillion ways. i dont like it.
|
|
19
|
-
vec![
|
|
20
|
-
"sh",
|
|
21
|
-
"-c",
|
|
22
|
-
"git submodule foreach --quiet 'git ls-files --others --ignored --exclude-standard | sed \"s|^|$path/|\"'",
|
|
23
|
-
],
|
|
24
|
-
];
|
|
25
|
-
|
|
26
|
-
for command in commands {
|
|
27
|
-
let output = Command::new(&command[0])
|
|
28
|
-
.args(&command[1..])
|
|
29
|
-
.current_dir(workspace_root_path)
|
|
30
|
-
.output()?;
|
|
31
|
-
|
|
32
|
-
if output.status.success() {
|
|
33
|
-
let files = String::from_utf8(output.stdout)?
|
|
34
|
-
.lines()
|
|
35
|
-
.filter(|line| !line.is_empty())
|
|
36
|
-
.map(|line| {
|
|
37
|
-
if should_return_absolute_paths {
|
|
38
|
-
let mut path = std::path::PathBuf::from(workspace_root_path);
|
|
39
|
-
path.push(line);
|
|
40
|
-
|
|
41
|
-
match path.canonicalize() {
|
|
42
|
-
Ok(canonical_path) => {
|
|
43
|
-
canonical_path.to_string_lossy().into_owned()
|
|
44
|
-
}
|
|
45
|
-
Err(_) => String::from(line),
|
|
46
|
-
}
|
|
47
|
-
} else {
|
|
48
|
-
String::from(line)
|
|
49
|
-
}
|
|
50
|
-
})
|
|
51
|
-
.collect::<HashSet<_>>();
|
|
52
|
-
|
|
53
|
-
gitignored_files.extend(files);
|
|
54
|
-
}
|
|
55
|
-
}
|
|
56
|
-
|
|
57
|
-
Ok(gitignored_files)
|
|
58
|
-
}
|
|
59
|
-
|
|
60
|
-
pub fn list_ignored_files_with_absolute_paths(
|
|
61
|
-
workspace_root_path: &str,
|
|
62
|
-
) -> Result<HashSet<String>, Box<dyn std::error::Error>> {
|
|
63
|
-
let mut gitignored_files = HashSet::new();
|
|
64
|
-
|
|
65
|
-
let commands = vec![
|
|
66
18
|
vec![
|
|
67
19
|
"git",
|
|
68
|
-
"
|
|
69
|
-
"
|
|
70
|
-
"--
|
|
71
|
-
"--exclude-standard",
|
|
72
|
-
],
|
|
73
|
-
// FIXME(sualeh): this is super sketchy and might totally break in like a bazillion ways. i dont like it.
|
|
74
|
-
vec![
|
|
75
|
-
"sh",
|
|
76
|
-
"-c",
|
|
77
|
-
"git submodule foreach --quiet 'git ls-files --others --ignored --exclude-standard | sed \"s|^|$path/|\"'",
|
|
20
|
+
"submodule",
|
|
21
|
+
"foreach",
|
|
22
|
+
"--quiet",
|
|
23
|
+
"git ls-files --others --ignored --exclude-standard | sed 's|^|$path/|'",
|
|
78
24
|
],
|
|
79
25
|
];
|
|
80
26
|
|
|
@@ -88,7 +34,7 @@ pub fn list_ignored_files_with_absolute_paths(
|
|
|
88
34
|
let files = String::from_utf8(output.stdout)?
|
|
89
35
|
.lines()
|
|
90
36
|
.filter(|line| !line.is_empty())
|
|
91
|
-
.map(|line|
|
|
37
|
+
.map(|line| PathBuf::from(line))
|
|
92
38
|
.collect::<HashSet<_>>();
|
|
93
39
|
|
|
94
40
|
gitignored_files.extend(files);
|
|
@@ -111,18 +57,6 @@ pub async fn is_git_ignored(
|
|
|
111
57
|
Ok(output.status.success())
|
|
112
58
|
}
|
|
113
59
|
|
|
114
|
-
pub async fn is_git_directory(
|
|
115
|
-
workspace_root_path: &str,
|
|
116
|
-
) -> Result<bool, anyhow::Error> {
|
|
117
|
-
let output = tokio::process::Command::new("git")
|
|
118
|
-
.args(&["rev-parse", "--is-inside-work-tree"])
|
|
119
|
-
.current_dir(workspace_root_path)
|
|
120
|
-
.output()
|
|
121
|
-
.await?;
|
|
122
|
-
|
|
123
|
-
Ok(output.status.success())
|
|
124
|
-
}
|
|
125
|
-
|
|
126
60
|
#[cfg(test)]
|
|
127
61
|
mod tests {
|
|
128
62
|
use super::*;
|
|
@@ -132,8 +66,7 @@ mod tests {
|
|
|
132
66
|
#[test]
|
|
133
67
|
fn test_no_ignored_files() {
|
|
134
68
|
let dir = tempfile::tempdir().unwrap();
|
|
135
|
-
let gitignored_files =
|
|
136
|
-
list_ignored_files(dir.path().to_str().unwrap(), false).unwrap();
|
|
69
|
+
let gitignored_files = list_ignored_files(dir.path().to_str().unwrap()).unwrap();
|
|
137
70
|
Command::new("git")
|
|
138
71
|
.args(&["init"])
|
|
139
72
|
.current_dir(dir.path())
|
|
@@ -159,14 +92,13 @@ mod tests {
|
|
|
159
92
|
.current_dir(dir.path())
|
|
160
93
|
.output()
|
|
161
94
|
.unwrap();
|
|
162
|
-
let gitignored_files =
|
|
163
|
-
list_ignored_files(dir.path().to_str().unwrap(), false).unwrap();
|
|
95
|
+
let gitignored_files = list_ignored_files(dir.path().to_str().unwrap()).unwrap();
|
|
164
96
|
println!(
|
|
165
97
|
"ignored files for test_one_ignored_file: {:?}",
|
|
166
98
|
gitignored_files
|
|
167
99
|
);
|
|
168
100
|
// assert_eq!(gitignored_files.len(), 1);
|
|
169
|
-
assert!(gitignored_files.contains(&
|
|
101
|
+
assert!(gitignored_files.contains(&PathBuf::from("ignored.txt")));
|
|
170
102
|
}
|
|
171
103
|
|
|
172
104
|
#[test]
|
|
@@ -189,83 +121,19 @@ mod tests {
|
|
|
189
121
|
.current_dir(dir.path())
|
|
190
122
|
.output()
|
|
191
123
|
.unwrap();
|
|
192
|
-
let gitignored_files =
|
|
193
|
-
list_ignored_files(dir.path().to_str().unwrap(), false).unwrap();
|
|
124
|
+
let gitignored_files = list_ignored_files(dir.path().to_str().unwrap()).unwrap();
|
|
194
125
|
println!(
|
|
195
126
|
"ignored files for test_multiple_ignored_files: {:?}",
|
|
196
127
|
gitignored_files
|
|
197
128
|
);
|
|
198
129
|
// assert_eq!(gitignored_files.len(), 2);
|
|
199
|
-
assert!(gitignored_files.contains(&
|
|
200
|
-
assert!(gitignored_files.contains(&
|
|
201
|
-
}
|
|
202
|
-
|
|
203
|
-
#[test]
|
|
204
|
-
fn test_git_submodule_ignored_files() {
|
|
205
|
-
let dir = tempfile::tempdir().unwrap();
|
|
206
|
-
let submodule_path = dir.path().join("submodule");
|
|
207
|
-
std::fs::create_dir(&submodule_path).unwrap();
|
|
208
|
-
|
|
209
|
-
let o = Command::new("git")
|
|
210
|
-
.args(&["init"])
|
|
211
|
-
.current_dir(&submodule_path)
|
|
212
|
-
.output()
|
|
213
|
-
.unwrap();
|
|
214
|
-
println!("git init output: {:?}", o);
|
|
215
|
-
|
|
216
|
-
let file_path = submodule_path.join("ignored.txt");
|
|
217
|
-
let mut file = File::create(&file_path).unwrap();
|
|
218
|
-
writeln!(file, "This is an ignored file.").unwrap();
|
|
219
|
-
|
|
220
|
-
let file2 = submodule_path.join("ignored2.txt");
|
|
221
|
-
let mut file = File::create(&file2).unwrap();
|
|
222
|
-
writeln!(file, "This is another ignored file.").unwrap();
|
|
223
|
-
|
|
224
|
-
let gitignore_path = submodule_path.join(".gitignore");
|
|
225
|
-
let mut gitignore = File::create(&gitignore_path).unwrap();
|
|
226
|
-
writeln!(gitignore, "*.txt").unwrap();
|
|
227
|
-
|
|
228
|
-
let o = Command::new("git")
|
|
229
|
-
.args(&["init"])
|
|
230
|
-
.current_dir(dir.path())
|
|
231
|
-
.output()
|
|
232
|
-
.unwrap();
|
|
233
|
-
println!("git init output: {:?}", o);
|
|
234
|
-
|
|
235
|
-
// make a commit in the submodule
|
|
236
|
-
let o = Command::new("git")
|
|
237
|
-
.args(&["add", "."])
|
|
238
|
-
.current_dir(&submodule_path)
|
|
239
|
-
.output()
|
|
240
|
-
.unwrap();
|
|
241
|
-
println!("git add output: {:?}", o);
|
|
242
|
-
let o = Command::new("git")
|
|
243
|
-
.args(&["commit", "-m", "initial commit"])
|
|
244
|
-
.current_dir(&submodule_path)
|
|
245
|
-
.output()
|
|
246
|
-
.unwrap();
|
|
247
|
-
println!("git commit output: {:?}", o);
|
|
248
|
-
|
|
249
|
-
let o = Command::new("git")
|
|
250
|
-
.args(&["submodule", "add", "./submodule"])
|
|
251
|
-
.current_dir(dir.path())
|
|
252
|
-
.output()
|
|
253
|
-
.unwrap();
|
|
254
|
-
println!("git submodule add output: {:?}", o);
|
|
255
|
-
|
|
256
|
-
let gitignored_files =
|
|
257
|
-
list_ignored_files(dir.path().to_str().unwrap(), false).unwrap();
|
|
258
|
-
println!(
|
|
259
|
-
"ignored files for test_git_submodule_ignored_files: {:?}",
|
|
260
|
-
gitignored_files
|
|
261
|
-
);
|
|
262
|
-
assert!(gitignored_files.contains(&String::from("submodule/ignored.txt")));
|
|
263
|
-
assert!(gitignored_files.contains(&String::from("submodule/ignored2.txt")));
|
|
130
|
+
assert!(gitignored_files.contains(&PathBuf::from("ignored1.txt")));
|
|
131
|
+
assert!(gitignored_files.contains(&PathBuf::from("ignored2.txt")));
|
|
264
132
|
}
|
|
265
133
|
|
|
266
134
|
#[test]
|
|
267
135
|
fn test_multiple_ignored_files_in_current_dir() {
|
|
268
|
-
let gitignored_files = list_ignored_files("."
|
|
136
|
+
let gitignored_files = list_ignored_files(".").unwrap();
|
|
269
137
|
assert!(gitignored_files.len() > 1);
|
|
270
138
|
|
|
271
139
|
// print a sample of the ignored files
|
|
@@ -279,6 +147,7 @@ mod tests {
|
|
|
279
147
|
}
|
|
280
148
|
}
|
|
281
149
|
|
|
150
|
+
|
|
282
151
|
#[tokio::test]
|
|
283
152
|
async fn test_file_not_ignored() {
|
|
284
153
|
let dir = tempfile::tempdir().unwrap();
|
|
@@ -291,10 +160,7 @@ mod tests {
|
|
|
291
160
|
.current_dir(dir.path())
|
|
292
161
|
.output()
|
|
293
162
|
.unwrap();
|
|
294
|
-
let is_ignored =
|
|
295
|
-
is_git_ignored(dir.path().to_str().unwrap(), "not_ignored.txt")
|
|
296
|
-
.await
|
|
297
|
-
.unwrap();
|
|
163
|
+
let is_ignored = is_git_ignored(dir.path().to_str().unwrap(), "not_ignored.txt").await.unwrap();
|
|
298
164
|
assert_eq!(is_ignored, false);
|
|
299
165
|
}
|
|
300
166
|
|
|
@@ -314,10 +180,7 @@ mod tests {
|
|
|
314
180
|
.current_dir(dir.path())
|
|
315
181
|
.output()
|
|
316
182
|
.unwrap();
|
|
317
|
-
let is_ignored =
|
|
318
|
-
is_git_ignored(dir.path().to_str().unwrap(), "ignored.txt")
|
|
319
|
-
.await
|
|
320
|
-
.unwrap();
|
|
183
|
+
let is_ignored = is_git_ignored(dir.path().to_str().unwrap(), "ignored.txt").await.unwrap();
|
|
321
184
|
assert_eq!(is_ignored, true);
|
|
322
185
|
}
|
|
323
186
|
|
|
@@ -337,10 +200,8 @@ mod tests {
|
|
|
337
200
|
.current_dir(dir.path())
|
|
338
201
|
.output()
|
|
339
202
|
.unwrap();
|
|
340
|
-
let is_ignored =
|
|
341
|
-
is_git_ignored(dir.path().to_str().unwrap(), "ignored.txt")
|
|
342
|
-
.await
|
|
343
|
-
.unwrap();
|
|
203
|
+
let is_ignored = is_git_ignored(dir.path().to_str().unwrap(), "ignored.txt").await.unwrap();
|
|
344
204
|
assert_eq!(is_ignored, true);
|
|
345
205
|
}
|
|
206
|
+
|
|
346
207
|
}
|
package/src/lib.rs
CHANGED
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
#![deny(clippy::all)]
|
|
2
|
-
#![deny(unsafe_op_in_unsafe_fn)]
|
|
3
2
|
pub mod file_utils;
|
|
4
3
|
pub mod git_utils;
|
|
5
4
|
pub mod merkle_tree;
|
|
@@ -7,9 +6,6 @@ pub mod merkle_tree;
|
|
|
7
6
|
use std::vec;
|
|
8
7
|
|
|
9
8
|
use merkle_tree::{LocalConstruction, MerkleTree};
|
|
10
|
-
use tracing::{info, Level};
|
|
11
|
-
use tracing_appender::rolling::{RollingFileAppender, Rotation};
|
|
12
|
-
use tracing_subscriber::fmt;
|
|
13
9
|
|
|
14
10
|
#[macro_use]
|
|
15
11
|
extern crate napi_derive;
|
|
@@ -18,35 +14,15 @@ extern crate napi_derive;
|
|
|
18
14
|
pub struct MerkleClient {
|
|
19
15
|
tree: MerkleTree,
|
|
20
16
|
root_directory: String,
|
|
21
|
-
_guard: tracing_appender::non_blocking::WorkerGuard,
|
|
22
|
-
}
|
|
23
|
-
|
|
24
|
-
pub fn init_logger() -> tracing_appender::non_blocking::WorkerGuard {
|
|
25
|
-
let file_appender =
|
|
26
|
-
RollingFileAppender::new(Rotation::NEVER, "./", "rust_log.txt");
|
|
27
|
-
let (non_blocking, _guard) = tracing_appender::non_blocking(file_appender);
|
|
28
|
-
let subscriber = fmt::Subscriber::builder()
|
|
29
|
-
.with_max_level(Level::TRACE)
|
|
30
|
-
.with_writer(non_blocking)
|
|
31
|
-
.with_ansi(false)
|
|
32
|
-
.with_line_number(true)
|
|
33
|
-
.finish();
|
|
34
|
-
|
|
35
|
-
let _ = tracing::subscriber::set_global_default(subscriber);
|
|
36
|
-
|
|
37
|
-
_guard
|
|
38
17
|
}
|
|
39
18
|
|
|
40
19
|
#[napi]
|
|
41
20
|
impl MerkleClient {
|
|
42
21
|
#[napi(constructor)]
|
|
43
22
|
pub fn new(root_directory: String) -> MerkleClient {
|
|
44
|
-
let _guard = init_logger();
|
|
45
|
-
|
|
46
23
|
MerkleClient {
|
|
47
24
|
tree: MerkleTree::empty_tree(),
|
|
48
25
|
root_directory,
|
|
49
|
-
_guard,
|
|
50
26
|
}
|
|
51
27
|
}
|
|
52
28
|
|
|
@@ -55,10 +31,7 @@ impl MerkleClient {
|
|
|
55
31
|
// 1. compute the merkle tree
|
|
56
32
|
// 2. update the backend
|
|
57
33
|
// 3. sync with the remote
|
|
58
|
-
|
|
59
|
-
unsafe {
|
|
60
|
-
self.compute_merkle_tree().await?;
|
|
61
|
-
}
|
|
34
|
+
self.compute_merkle_tree().await?;
|
|
62
35
|
|
|
63
36
|
Ok(())
|
|
64
37
|
}
|
|
@@ -74,17 +47,6 @@ impl MerkleClient {
|
|
|
74
47
|
let t =
|
|
75
48
|
MerkleTree::construct_merkle_tree(self.root_directory.clone()).await;
|
|
76
49
|
|
|
77
|
-
let files = self.tree.get_all_files().await;
|
|
78
|
-
|
|
79
|
-
match files {
|
|
80
|
-
Ok(files) => {
|
|
81
|
-
info!("files: {:?}", files);
|
|
82
|
-
}
|
|
83
|
-
Err(e) => {
|
|
84
|
-
info!("Error in get_all_files: {:?}", e);
|
|
85
|
-
}
|
|
86
|
-
}
|
|
87
|
-
|
|
88
50
|
match t {
|
|
89
51
|
Ok(tree) => {
|
|
90
52
|
self.tree = tree;
|
|
@@ -114,9 +76,7 @@ impl MerkleClient {
|
|
|
114
76
|
) -> Result<String, napi::Error> {
|
|
115
77
|
let absolute_path =
|
|
116
78
|
std::path::Path::new(&self.root_directory).join(relative_path);
|
|
117
|
-
let
|
|
118
|
-
|
|
119
|
-
let hash = self.tree.get_subtree_hash(canonical_path).await;
|
|
79
|
+
let hash = self.tree.get_subtree_hash(absolute_path).await;
|
|
120
80
|
|
|
121
81
|
match hash {
|
|
122
82
|
Ok(hash) => Ok(hash),
|
|
@@ -144,10 +104,8 @@ impl MerkleClient {
|
|
|
144
104
|
&self,
|
|
145
105
|
relative_path: String,
|
|
146
106
|
) -> Result<i32, napi::Error> {
|
|
147
|
-
let absolute_path =
|
|
148
|
-
.join(relative_path)
|
|
149
|
-
.canonicalize()?;
|
|
150
|
-
|
|
107
|
+
let absolute_path =
|
|
108
|
+
std::path::Path::new(&self.root_directory).join(relative_path);
|
|
151
109
|
let num = self
|
|
152
110
|
.tree
|
|
153
111
|
.get_num_embeddable_files_in_subtree(absolute_path)
|
|
@@ -209,6 +167,7 @@ impl MerkleClient {
|
|
|
209
167
|
|
|
210
168
|
let ret = vec![file];
|
|
211
169
|
let ret = ret.into_iter().chain(path.into_iter()).collect::<Vec<_>>();
|
|
170
|
+
|
|
212
171
|
Ok(ret)
|
|
213
172
|
}
|
|
214
173
|
Err(e) => Err(napi::Error::new(
|
|
@@ -1,11 +1,11 @@
|
|
|
1
|
-
use crate::git_utils;
|
|
2
1
|
use crate::merkle_tree::{
|
|
3
2
|
File, MerkleNode, MerkleNodePtr, NodeType, PinnedFuture,
|
|
4
3
|
};
|
|
5
4
|
|
|
6
5
|
use super::{LocalConstruction, MerkleTree};
|
|
7
|
-
use std::collections::
|
|
8
|
-
use std::path::
|
|
6
|
+
use std::collections::BTreeMap;
|
|
7
|
+
use std::path::PathBuf;
|
|
8
|
+
use std::{collections::HashMap, path::Path, sync::Arc};
|
|
9
9
|
use tonic::async_trait;
|
|
10
10
|
|
|
11
11
|
#[async_trait]
|
|
@@ -29,36 +29,20 @@ impl LocalConstruction for MerkleTree {
|
|
|
29
29
|
/// 3. construct merkle tree
|
|
30
30
|
/// 4. return merkle tree
|
|
31
31
|
async fn construct_merkle_tree(
|
|
32
|
-
|
|
32
|
+
root_directory: String,
|
|
33
33
|
) -> Result<MerkleTree, anyhow::Error> {
|
|
34
|
-
let path = PathBuf::from(
|
|
34
|
+
let path = PathBuf::from(root_directory.clone());
|
|
35
35
|
if !path.exists() {
|
|
36
36
|
// FIXME: we should report this via a good logger.
|
|
37
37
|
panic!("Root directory does not exist!");
|
|
38
38
|
}
|
|
39
39
|
|
|
40
|
-
|
|
41
|
-
let git_ignored_files = match git_utils::list_ignored_files(
|
|
42
|
-
absolute_path_to_root_directory.as_str(),
|
|
43
|
-
true,
|
|
44
|
-
) {
|
|
45
|
-
Ok(git_ignored) => git_ignored,
|
|
46
|
-
Err(_e) => HashSet::new(),
|
|
47
|
-
};
|
|
48
|
-
|
|
49
|
-
let root_node = MerkleNode::new(
|
|
50
|
-
path,
|
|
51
|
-
None,
|
|
52
|
-
&git_ignored_files,
|
|
53
|
-
absolute_path_to_root_directory.as_str(),
|
|
54
|
-
)
|
|
55
|
-
.await;
|
|
40
|
+
let root_node = MerkleNode::new(path, None).await;
|
|
56
41
|
let mut mt = MerkleTree {
|
|
57
42
|
root: root_node,
|
|
58
43
|
files: BTreeMap::new(),
|
|
59
|
-
root_path:
|
|
44
|
+
root_path: root_directory,
|
|
60
45
|
cursor: None,
|
|
61
|
-
git_ignored_files,
|
|
62
46
|
};
|
|
63
47
|
|
|
64
48
|
// we now iterate over all the nodes and add them to the hashmap
|
|
@@ -71,7 +55,6 @@ impl LocalConstruction for MerkleTree {
|
|
|
71
55
|
let node_reader = node.read().await;
|
|
72
56
|
match &node_reader.node_type {
|
|
73
57
|
NodeType::Branch(n) => {
|
|
74
|
-
tracing::info!("Branch: {:?}", n.0);
|
|
75
58
|
let children = &n.1;
|
|
76
59
|
files.insert(n.0.clone(), File { node: node.clone() });
|
|
77
60
|
for child in children {
|
|
@@ -80,7 +63,6 @@ impl LocalConstruction for MerkleTree {
|
|
|
80
63
|
}
|
|
81
64
|
NodeType::File(file_name) => {
|
|
82
65
|
let f = File { node: node.clone() };
|
|
83
|
-
tracing::info!("File: {:?}", file_name);
|
|
84
66
|
files.insert(file_name.clone(), f);
|
|
85
67
|
}
|
|
86
68
|
NodeType::ErrorNode(_) => {
|
|
@@ -92,9 +74,6 @@ impl LocalConstruction for MerkleTree {
|
|
|
92
74
|
|
|
93
75
|
add_nodes_to_hashmap(&mt.root, &mut mt.files).await;
|
|
94
76
|
|
|
95
|
-
tracing::info!("Merkle tree compute finished!");
|
|
96
|
-
// tracing::info!("Merkle tree: {:?}", mt.files);
|
|
97
|
-
|
|
98
77
|
Ok(mt)
|
|
99
78
|
}
|
|
100
79
|
|
package/src/merkle_tree/mod.rs
CHANGED
|
@@ -1,13 +1,10 @@
|
|
|
1
|
-
use crate::git_utils;
|
|
2
|
-
|
|
3
1
|
use super::file_utils;
|
|
4
2
|
use sha2::Digest;
|
|
5
|
-
use std::collections::
|
|
3
|
+
use std::collections::BTreeMap;
|
|
6
4
|
use std::path::PathBuf;
|
|
7
5
|
use std::{fs, path::Path, sync::Arc};
|
|
8
6
|
use tokio::sync::RwLock;
|
|
9
7
|
use tonic::async_trait;
|
|
10
|
-
use tracing::info;
|
|
11
8
|
pub mod local_construction;
|
|
12
9
|
pub mod test;
|
|
13
10
|
|
|
@@ -18,7 +15,6 @@ pub struct MerkleTree {
|
|
|
18
15
|
root: MerkleNodePtr,
|
|
19
16
|
files: BTreeMap<String, File>,
|
|
20
17
|
cursor: Option<usize>,
|
|
21
|
-
git_ignored_files: HashSet<String>,
|
|
22
18
|
}
|
|
23
19
|
|
|
24
20
|
#[derive(Debug)]
|
|
@@ -95,7 +91,6 @@ impl MerkleTree {
|
|
|
95
91
|
files: BTreeMap::new(),
|
|
96
92
|
root_path: "".to_string(),
|
|
97
93
|
cursor: None,
|
|
98
|
-
git_ignored_files: HashSet::new(),
|
|
99
94
|
}
|
|
100
95
|
}
|
|
101
96
|
|
|
@@ -115,8 +110,7 @@ impl MerkleTree {
|
|
|
115
110
|
let node = match self.files.get(&abs_string) {
|
|
116
111
|
Some(file) => file.node.clone(),
|
|
117
112
|
None => {
|
|
118
|
-
|
|
119
|
-
return Err(anyhow::anyhow!("Could not find file in tree! Looking for: {}. All files: {:?}", abs_string, all_files));
|
|
113
|
+
return Err(anyhow::anyhow!("Could not find file in tree!"));
|
|
120
114
|
}
|
|
121
115
|
};
|
|
122
116
|
|
|
@@ -307,51 +301,26 @@ impl MerkleTree {
|
|
|
307
301
|
Ok(files)
|
|
308
302
|
}
|
|
309
303
|
|
|
310
|
-
// TODO(sualeh): i need tests for this!!
|
|
311
304
|
pub async fn get_spline(
|
|
312
305
|
&self,
|
|
313
306
|
absolute_path: &str,
|
|
314
307
|
) -> Result<Vec<String>, anyhow::Error> {
|
|
315
|
-
info!("get_spline called with absolute_path: {}", absolute_path);
|
|
316
308
|
let mut files = Vec::new();
|
|
317
309
|
|
|
318
310
|
let current_node = match self.files.get(absolute_path) {
|
|
319
|
-
Some(node) =>
|
|
320
|
-
|
|
321
|
-
node.node.clone()
|
|
322
|
-
}
|
|
323
|
-
None => {
|
|
324
|
-
info!("File not found for absolute_path: {}", absolute_path);
|
|
325
|
-
return Err(anyhow::anyhow!("File not found: {}", absolute_path));
|
|
326
|
-
}
|
|
311
|
+
Some(node) => node.node.clone(),
|
|
312
|
+
None => return Err(anyhow::anyhow!("File not found: {}", absolute_path)),
|
|
327
313
|
};
|
|
328
|
-
|
|
329
314
|
let mut stack = Vec::new();
|
|
330
315
|
stack.push(current_node);
|
|
331
316
|
|
|
332
317
|
while let Some(node) = stack.pop() {
|
|
333
318
|
let parent = node.read().await.parent.clone();
|
|
334
319
|
if let Some(parent) = parent {
|
|
335
|
-
|
|
336
|
-
{
|
|
337
|
-
let parent_node = parent.read().await;
|
|
338
|
-
match &parent_node.node_type {
|
|
339
|
-
NodeType::File(file_name) => {
|
|
340
|
-
files.push(file_name.clone());
|
|
341
|
-
}
|
|
342
|
-
NodeType::Branch((branch_name, _)) => {
|
|
343
|
-
files.push(branch_name.clone());
|
|
344
|
-
}
|
|
345
|
-
_ => {
|
|
346
|
-
continue;
|
|
347
|
-
}
|
|
348
|
-
}
|
|
349
|
-
}
|
|
350
|
-
|
|
320
|
+
files.push(parent.read().await.hash.clone());
|
|
351
321
|
stack.push(parent);
|
|
352
322
|
}
|
|
353
323
|
}
|
|
354
|
-
info!("Returning files vector with {} elements", files.len());
|
|
355
324
|
Ok(files)
|
|
356
325
|
}
|
|
357
326
|
|
|
@@ -390,18 +359,12 @@ impl MerkleTree {
|
|
|
390
359
|
// 1. the path is empty. this means that the ancestor is the root.
|
|
391
360
|
// 2. the path is non-empty. that means there exist a non-empty element btwn till the root.
|
|
392
361
|
|
|
393
|
-
let absolute_root_path = self.root_path.clone();
|
|
394
362
|
let new_node = match path.len() {
|
|
395
363
|
0 => {
|
|
396
364
|
// this means that the ancestor is the root.
|
|
397
365
|
// we need to create a new node and attach it to the ancestor.
|
|
398
|
-
let new_node =
|
|
399
|
-
file_path.clone(),
|
|
400
|
-
Some(ancestor.clone()),
|
|
401
|
-
&self.git_ignored_files,
|
|
402
|
-
&absolute_root_path.as_str(),
|
|
403
|
-
)
|
|
404
|
-
.await;
|
|
366
|
+
let new_node =
|
|
367
|
+
MerkleNode::new(file_path.clone(), Some(ancestor.clone())).await;
|
|
405
368
|
ancestor.write().await.attach_child(new_node.clone()).await;
|
|
406
369
|
new_node
|
|
407
370
|
}
|
|
@@ -412,13 +375,9 @@ impl MerkleTree {
|
|
|
412
375
|
// UNSURE: not sure this is the correct thing to do but it is the fastest.
|
|
413
376
|
// get the last thing that is not in the tree.
|
|
414
377
|
let first_child_path = path.last().unwrap();
|
|
415
|
-
let first_child =
|
|
416
|
-
first_child_path.clone(),
|
|
417
|
-
|
|
418
|
-
&self.git_ignored_files,
|
|
419
|
-
&absolute_root_path.as_str(),
|
|
420
|
-
)
|
|
421
|
-
.await;
|
|
378
|
+
let first_child =
|
|
379
|
+
MerkleNode::new(first_child_path.clone(), Some(ancestor.clone()))
|
|
380
|
+
.await;
|
|
422
381
|
|
|
423
382
|
// TODO(sualeh): we should do an assertion check that the entire vec is contained here.
|
|
424
383
|
|
|
@@ -695,62 +654,18 @@ use std::future::Future;
|
|
|
695
654
|
use std::pin::Pin;
|
|
696
655
|
|
|
697
656
|
type PinnedFuture<'a, T> = Pin<Box<dyn Future<Output = T> + Send + 'a>>;
|
|
698
|
-
type IgnoredFiles = HashSet<String>;
|
|
699
657
|
|
|
700
658
|
impl MerkleNode {
|
|
701
659
|
/// please be careful using this.
|
|
702
660
|
async fn __new_unchecked(
|
|
703
661
|
file_or_directory: String,
|
|
704
662
|
parent: ParentPtr,
|
|
705
|
-
ignored_files: &IgnoredFiles,
|
|
706
|
-
absolute_root_path: &str,
|
|
707
663
|
) -> MerkleNodePtr {
|
|
708
|
-
|
|
709
|
-
let is_git_repo =
|
|
710
|
-
match git_utils::is_git_directory(absolute_root_path).await {
|
|
711
|
-
Ok(is_git_repo) => is_git_repo,
|
|
712
|
-
Err(e) => false,
|
|
713
|
-
};
|
|
714
|
-
let bypass_git = !is_git_repo;
|
|
715
|
-
|
|
716
|
-
MerkleNode::construct_node(
|
|
717
|
-
Path::new(&file_or_directory),
|
|
718
|
-
parent,
|
|
719
|
-
ignored_files,
|
|
720
|
-
absolute_root_path,
|
|
721
|
-
bypass_git,
|
|
722
|
-
)
|
|
723
|
-
.await
|
|
664
|
+
MerkleNode::construct_node(Path::new(&file_or_directory), parent).await
|
|
724
665
|
}
|
|
725
666
|
|
|
726
|
-
async fn new(
|
|
727
|
-
|
|
728
|
-
parent: ParentPtr,
|
|
729
|
-
ignored_files: &IgnoredFiles,
|
|
730
|
-
absolute_root_path: &str,
|
|
731
|
-
) -> MerkleNodePtr {
|
|
732
|
-
// check if the root is a git directory.
|
|
733
|
-
let is_git_repo =
|
|
734
|
-
match git_utils::is_git_directory(absolute_root_path).await {
|
|
735
|
-
Ok(is_git_repo) => is_git_repo,
|
|
736
|
-
Err(_e) => false,
|
|
737
|
-
};
|
|
738
|
-
let bypass_git = !is_git_repo;
|
|
739
|
-
|
|
740
|
-
info!(
|
|
741
|
-
"constructing node for absolute_file_or_directory: {:?}",
|
|
742
|
-
absolute_file_or_directory
|
|
743
|
-
);
|
|
744
|
-
info!("bypass_git: {}, is_git_repo: {}", bypass_git, is_git_repo);
|
|
745
|
-
|
|
746
|
-
MerkleNode::construct_node(
|
|
747
|
-
Path::new(&absolute_file_or_directory),
|
|
748
|
-
parent,
|
|
749
|
-
ignored_files,
|
|
750
|
-
absolute_root_path,
|
|
751
|
-
bypass_git,
|
|
752
|
-
)
|
|
753
|
-
.await
|
|
667
|
+
async fn new(file_or_directory: PathBuf, parent: ParentPtr) -> MerkleNodePtr {
|
|
668
|
+
MerkleNode::construct_node(Path::new(&file_or_directory), parent).await
|
|
754
669
|
}
|
|
755
670
|
|
|
756
671
|
/// NOT added to the tree by default.
|
|
@@ -761,68 +676,38 @@ impl MerkleNode {
|
|
|
761
676
|
// let file_hash = self.files.get_mut(&file_path).unwrap();
|
|
762
677
|
|
|
763
678
|
fn construct_node<'a>(
|
|
764
|
-
|
|
679
|
+
file_or_directory: &'a Path,
|
|
765
680
|
parent: ParentPtr,
|
|
766
|
-
ignored_files: &'a IgnoredFiles,
|
|
767
|
-
absolute_root_path: &'a str,
|
|
768
|
-
bypass_git: bool,
|
|
769
681
|
) -> PinnedFuture<'a, MerkleNodePtr> {
|
|
770
682
|
Box::pin(async move {
|
|
771
683
|
// check if it is a file
|
|
772
|
-
let path_str =
|
|
773
|
-
if
|
|
684
|
+
let path_str = file_or_directory.to_str().unwrap().to_string();
|
|
685
|
+
if file_or_directory.is_file() {
|
|
774
686
|
return Arc::new(RwLock::new(
|
|
775
687
|
MerkleNode::construct_file_node_or_error_node(
|
|
776
|
-
|
|
688
|
+
file_or_directory,
|
|
777
689
|
parent,
|
|
778
|
-
ignored_files,
|
|
779
690
|
)
|
|
780
691
|
.await,
|
|
781
692
|
));
|
|
782
693
|
}
|
|
783
694
|
|
|
784
695
|
// check if the directory fails the bad dir test.
|
|
785
|
-
let is_bad_dir = file_utils::is_in_bad_dir(
|
|
696
|
+
let is_bad_dir = file_utils::is_in_bad_dir(file_or_directory);
|
|
786
697
|
if is_bad_dir.is_err() || is_bad_dir.unwrap_or(false) {
|
|
787
698
|
// println!("skipping directory: {}", path_str);
|
|
788
699
|
return Arc::new(RwLock::new(MerkleNode::empty_node(
|
|
789
|
-
Some(
|
|
700
|
+
Some(file_or_directory),
|
|
790
701
|
Some("Directory is in bad dir!".to_string()),
|
|
791
702
|
)));
|
|
792
703
|
}
|
|
793
704
|
|
|
794
|
-
|
|
795
|
-
let is_git_ignored =
|
|
796
|
-
match git_utils::is_git_ignored(absolute_root_path, path_str.as_str())
|
|
797
|
-
.await
|
|
798
|
-
{
|
|
799
|
-
Ok(is_git_ignored) => is_git_ignored,
|
|
800
|
-
Err(e) => {
|
|
801
|
-
return Arc::new(RwLock::new(MerkleNode::empty_node(
|
|
802
|
-
Some(absolute_file_or_directory),
|
|
803
|
-
Some(e.to_string()),
|
|
804
|
-
)));
|
|
805
|
-
}
|
|
806
|
-
};
|
|
807
|
-
|
|
808
|
-
if is_git_ignored && !bypass_git {
|
|
809
|
-
// println!("skipping directory: {}", path_str);
|
|
810
|
-
tracing::info!(
|
|
811
|
-
"skipping directory because its git ignored: {}",
|
|
812
|
-
path_str
|
|
813
|
-
);
|
|
814
|
-
return Arc::new(RwLock::new(MerkleNode::empty_node(
|
|
815
|
-
Some(absolute_file_or_directory),
|
|
816
|
-
Some("Directory is git ignored!".to_string()),
|
|
817
|
-
)));
|
|
818
|
-
}
|
|
819
|
-
|
|
820
|
-
let entries = fs::read_dir(absolute_file_or_directory);
|
|
705
|
+
let entries = fs::read_dir(file_or_directory);
|
|
821
706
|
match entries {
|
|
822
707
|
Ok(_) => (),
|
|
823
708
|
Err(e) => {
|
|
824
709
|
return Arc::new(RwLock::new(MerkleNode::empty_node(
|
|
825
|
-
Some(
|
|
710
|
+
Some(file_or_directory),
|
|
826
711
|
Some(e.to_string()),
|
|
827
712
|
)));
|
|
828
713
|
}
|
|
@@ -842,19 +727,13 @@ impl MerkleNode {
|
|
|
842
727
|
match entry {
|
|
843
728
|
Ok(entry) => {
|
|
844
729
|
children.push(
|
|
845
|
-
MerkleNode::construct_node(
|
|
846
|
-
|
|
847
|
-
Some(node.clone()),
|
|
848
|
-
ignored_files,
|
|
849
|
-
absolute_root_path,
|
|
850
|
-
bypass_git,
|
|
851
|
-
)
|
|
852
|
-
.await,
|
|
730
|
+
MerkleNode::construct_node(&entry.path(), Some(node.clone()))
|
|
731
|
+
.await,
|
|
853
732
|
);
|
|
854
733
|
}
|
|
855
734
|
Err(e) => {
|
|
856
735
|
children.push(Arc::new(RwLock::new(MerkleNode::empty_node(
|
|
857
|
-
Some(
|
|
736
|
+
Some(file_or_directory),
|
|
858
737
|
Some(e.to_string()),
|
|
859
738
|
))));
|
|
860
739
|
}
|
|
@@ -874,33 +753,23 @@ impl MerkleNode {
|
|
|
874
753
|
}
|
|
875
754
|
|
|
876
755
|
async fn construct_file_node(
|
|
877
|
-
|
|
756
|
+
file_path: &Path,
|
|
878
757
|
parent: ParentPtr,
|
|
879
|
-
ignored_files: &IgnoredFiles,
|
|
880
758
|
) -> Result<MerkleNode, String> {
|
|
881
|
-
let file_str =
|
|
759
|
+
let file_str = file_path
|
|
882
760
|
.to_str()
|
|
883
761
|
.ok_or("Could not convert file path to string!")?
|
|
884
762
|
.to_string();
|
|
885
763
|
// first see if it passes the
|
|
886
|
-
match file_utils::is_good_file(
|
|
764
|
+
match file_utils::is_good_file(file_path) {
|
|
887
765
|
Ok(_) => {}
|
|
888
766
|
Err(e) => {
|
|
889
767
|
return Err(format!("File failed runtime checks! {}", e.to_string()));
|
|
890
768
|
}
|
|
891
769
|
}
|
|
892
770
|
|
|
893
|
-
// check if the file is in the git ignore buffer.
|
|
894
|
-
// this is a bug right because we are not checking absoluteness here.
|
|
895
|
-
match ignored_files.contains(&file_str) {
|
|
896
|
-
true => {
|
|
897
|
-
return Err(format!("File is in git ignore buffer!"));
|
|
898
|
-
}
|
|
899
|
-
false => {}
|
|
900
|
-
}
|
|
901
|
-
|
|
902
771
|
// read the file_content to a buffer
|
|
903
|
-
let file_content = match tokio::fs::read(
|
|
772
|
+
let file_content = match tokio::fs::read(file_path).await {
|
|
904
773
|
Ok(content) => content,
|
|
905
774
|
Err(e) => {
|
|
906
775
|
return Err(format!("Could not read file! {}", e.to_string()));
|
|
@@ -908,11 +777,7 @@ impl MerkleNode {
|
|
|
908
777
|
};
|
|
909
778
|
|
|
910
779
|
// check if the file passes runtime checks.
|
|
911
|
-
match file_utils::is_good_file_runtime_check(
|
|
912
|
-
absolute_file_path,
|
|
913
|
-
&file_content,
|
|
914
|
-
)
|
|
915
|
-
.await
|
|
780
|
+
match file_utils::is_good_file_runtime_check(file_path, &file_content).await
|
|
916
781
|
{
|
|
917
782
|
Ok(_) => {}
|
|
918
783
|
Err(e) => {
|
|
@@ -943,22 +808,15 @@ impl MerkleNode {
|
|
|
943
808
|
}
|
|
944
809
|
|
|
945
810
|
async fn construct_file_node_or_error_node(
|
|
946
|
-
|
|
811
|
+
file_path: &Path,
|
|
947
812
|
parent: ParentPtr,
|
|
948
|
-
ignored_files: &IgnoredFiles,
|
|
949
813
|
) -> MerkleNode {
|
|
950
|
-
let node = match MerkleNode::construct_file_node(
|
|
951
|
-
absolute_file_path,
|
|
952
|
-
parent,
|
|
953
|
-
ignored_files,
|
|
954
|
-
)
|
|
955
|
-
.await
|
|
956
|
-
{
|
|
814
|
+
let node = match MerkleNode::construct_file_node(file_path, parent).await {
|
|
957
815
|
Ok(node) => node,
|
|
958
816
|
Err(e) => {
|
|
959
817
|
// println!("constructing error node. error: {}", e);
|
|
960
818
|
// println!("file_path: {:?}", file_path);
|
|
961
|
-
MerkleNode::empty_node(Some(
|
|
819
|
+
MerkleNode::empty_node(Some(file_path), Some(e))
|
|
962
820
|
}
|
|
963
821
|
};
|
|
964
822
|
|