@filen/sync 0.2.1 → 0.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.node-version +1 -1
- package/dist/ignorer.d.ts +6 -0
- package/dist/ignorer.js +43 -24
- package/dist/ignorer.js.map +1 -1
- package/dist/index.d.ts +4 -1
- package/dist/index.js +3 -1
- package/dist/index.js.map +1 -1
- package/dist/lib/deltas.d.ts +58 -2
- package/dist/lib/deltas.js +693 -108
- package/dist/lib/deltas.js.map +1 -1
- package/dist/lib/environment.d.ts +47 -0
- package/dist/lib/environment.js +71 -0
- package/dist/lib/environment.js.map +1 -0
- package/dist/lib/filesystems/dirTree.d.ts +70 -0
- package/dist/lib/filesystems/dirTree.js +157 -0
- package/dist/lib/filesystems/dirTree.js.map +1 -0
- package/dist/lib/filesystems/local.d.ts +18 -8
- package/dist/lib/filesystems/local.js +166 -160
- package/dist/lib/filesystems/local.js.map +1 -1
- package/dist/lib/filesystems/remote.d.ts +12 -5
- package/dist/lib/filesystems/remote.js +226 -172
- package/dist/lib/filesystems/remote.js.map +1 -1
- package/dist/lib/ipc.js +1 -2
- package/dist/lib/ipc.js.map +1 -1
- package/dist/lib/lock.js +19 -12
- package/dist/lib/lock.js.map +1 -1
- package/dist/lib/logger.js +9 -7
- package/dist/lib/logger.js.map +1 -1
- package/dist/lib/state.js +159 -63
- package/dist/lib/state.js.map +1 -1
- package/dist/lib/sync.d.ts +18 -0
- package/dist/lib/sync.js +165 -96
- package/dist/lib/sync.js.map +1 -1
- package/dist/lib/tasks.d.ts +7 -8
- package/dist/lib/tasks.js +38 -45
- package/dist/lib/tasks.js.map +1 -1
- package/dist/semaphore.d.ts +1 -0
- package/dist/semaphore.js +22 -5
- package/dist/semaphore.js.map +1 -1
- package/dist/utils.js +51 -35
- package/dist/utils.js.map +1 -1
- package/eslint.config.mjs +36 -0
- package/package.json +19 -15
- package/tests/bench/collapse.bench.ts +114 -0
- package/tests/bench/cycle.bench.ts +111 -0
- package/tests/bench/deltas.bench.ts +151 -0
- package/tests/bench/harness/fake-sync.ts +32 -0
- package/tests/bench/harness/measure.ts +276 -0
- package/tests/bench/harness/scale-world.ts +160 -0
- package/tests/bench/harness/trees.ts +275 -0
- package/tests/bench/local-scan.bench.ts +74 -0
- package/tests/bench/longrun.bench.ts +130 -0
- package/tests/bench/profile-incremental.ts +90 -0
- package/tests/bench/remote-build.bench.ts +104 -0
- package/tests/bench/render.ts +14 -0
- package/tests/bench/semaphore.bench.ts +79 -0
- package/tests/bench/state.bench.ts +85 -0
- package/tests/bench/tasks-dispatch.bench.ts +156 -0
- package/tests/conformance/virtual-fs.test.ts +213 -0
- package/tests/e2e/backup.e2e.test.ts +130 -0
- package/tests/e2e/confirm.e2e.test.ts +191 -0
- package/tests/e2e/conflict.e2e.test.ts +261 -0
- package/tests/e2e/edge.e2e.test.ts +339 -0
- package/tests/e2e/harness/account.ts +104 -0
- package/tests/e2e/harness/assert.ts +127 -0
- package/tests/e2e/harness/drive.ts +88 -0
- package/tests/e2e/harness/mutations.ts +249 -0
- package/tests/e2e/harness/world.ts +222 -0
- package/tests/e2e/ignore.e2e.test.ts +123 -0
- package/tests/e2e/lifecycle.e2e.test.ts +290 -0
- package/tests/e2e/modes.e2e.test.ts +215 -0
- package/tests/e2e/platform.e2e.test.ts +157 -0
- package/tests/e2e/property.e2e.test.ts +163 -0
- package/tests/e2e/races.e2e.test.ts +90 -0
- package/tests/e2e/regressions.e2e.test.ts +212 -0
- package/tests/e2e/resilience.e2e.test.ts +231 -0
- package/tests/e2e/special.e2e.test.ts +185 -0
- package/tests/e2e/state.e2e.test.ts +229 -0
- package/tests/e2e/sync.e2e.test.ts +222 -0
- package/tests/fakes/fake-cloud.test.ts +267 -0
- package/tests/fakes/fake-cloud.ts +1094 -0
- package/tests/fakes/virtual-fs.ts +354 -0
- package/tests/harness/known-bug.ts +17 -0
- package/tests/harness/mutations.ts +65 -0
- package/tests/harness/runner.ts +141 -0
- package/tests/harness/snapshot.ts +113 -0
- package/tests/harness/world.ts +187 -0
- package/tests/scenarios/a-baseline.test.ts +107 -0
- package/tests/scenarios/aa-races.test.ts +258 -0
- package/tests/scenarios/ab-mode-property.test.ts +189 -0
- package/tests/scenarios/ac-platform.test.ts +320 -0
- package/tests/scenarios/ad-unicode-normalization.test.ts +67 -0
- package/tests/scenarios/b-additions.test.ts +160 -0
- package/tests/scenarios/c-modifications.test.ts +194 -0
- package/tests/scenarios/d-deletions.test.ts +259 -0
- package/tests/scenarios/e-rename-move.test.ts +288 -0
- package/tests/scenarios/f-ignore-filter.test.ts +346 -0
- package/tests/scenarios/g-large-deletion.test.ts +277 -0
- package/tests/scenarios/h-resilience.test.ts +167 -0
- package/tests/scenarios/i-lifecycle.test.ts +353 -0
- package/tests/scenarios/j-state-cache.test.ts +264 -0
- package/tests/scenarios/k-scale.test.ts +202 -0
- package/tests/scenarios/l-property.test.ts +145 -0
- package/tests/scenarios/m-golden.test.ts +452 -0
- package/tests/scenarios/o-task-errors.test.ts +497 -0
- package/tests/scenarios/p-remote-originated.test.ts +306 -0
- package/tests/scenarios/q-cycle-lifecycle.test.ts +234 -0
- package/tests/scenarios/r-rename-stress.test.ts +208 -0
- package/tests/scenarios/s-upgrade-transition.test.ts +171 -0
- package/tests/scenarios/t-type-change.test.ts +144 -0
- package/tests/scenarios/u-mode-local-to-cloud.test.ts +347 -0
- package/tests/scenarios/v-mode-local-backup.test.ts +201 -0
- package/tests/scenarios/w-mode-cloud-to-local.test.ts +304 -0
- package/tests/scenarios/x-mode-cloud-backup.test.ts +201 -0
- package/tests/scenarios/y-conflict-matrix.test.ts +292 -0
- package/tests/scenarios/z-cross-ops.test.ts +285 -0
- package/tests/scenarios/zb-dir-rename-cross.test.ts +296 -0
- package/tests/scenarios/zc-crash-recovery.test.ts +189 -0
- package/tests/scenarios/zd-inode-reuse.test.ts +118 -0
- package/tests/scenarios/ze-move-into-new-dir.test.ts +130 -0
- package/tests/scenarios/zf-remote-change-unchanged-local.test.ts +81 -0
- package/tests/scenarios/zg-edit-during-scan.test.ts +68 -0
- package/tests/scenarios/zh-dir-delete-vs-child.test.ts +104 -0
- package/tests/scenarios/zi-smoke-test-outage.test.ts +78 -0
- package/tests/scenarios/zj-trash-cleanup.test.ts +133 -0
- package/tests/scenarios/zk-ignore-asymmetry.test.ts +150 -0
- package/tests/scenarios/zl-mode-atomicity.test.ts +104 -0
- package/tests/scenarios/zm-scan-concurrency.test.ts +78 -0
- package/tests/scenarios/zn-delta-ordering.test.ts +130 -0
- package/tests/scenarios/zo-download-temp-cleanup.test.ts +65 -0
- package/tests/unit/collapse-deltas.test.ts +276 -0
- package/tests/unit/dir-tree.test.ts +159 -0
- package/tests/unit/icloud.test.ts +115 -0
- package/tests/unit/ignorer-cache-regression.test.ts +70 -0
- package/tests/unit/ignorer.test.ts +63 -0
- package/tests/unit/ipc-lock.test.ts +438 -0
- package/tests/unit/lock.test.ts +135 -0
- package/tests/unit/n-unit.test.ts +632 -0
- package/tests/unit/remote-tree-unordered-regression.test.ts +101 -0
- package/tests/unit/semaphore-regression.test.ts +140 -0
- package/tests/unit/state-refencode-regression.test.ts +224 -0
- package/tests/unit/state.test.ts +809 -0
- package/tests/unit/tasks-dispatch-order-regression.test.ts +53 -0
- package/tests/unit/worker-api.test.ts +379 -0
- package/tsconfig.json +10 -1
- package/tsconfig.test.json +12 -0
- package/tsconfig.tsbuildinfo +1 -0
- package/vitest.bench.config.ts +32 -0
- package/vitest.config.ts +27 -0
- package/vitest.e2e.config.ts +68 -0
- package/.eslintrc +0 -16
- package/jest.config.js +0 -5
|
@@ -3,17 +3,24 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
3
3
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
4
|
};
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
exports.RemoteFileSystem = exports.DEVICE_ID_VERSION = void 0;
|
|
6
|
+
exports.RemoteFileSystem = exports.REMOTE_BUILD_CONCURRENCY = exports.DEVICE_ID_VERSION = void 0;
|
|
7
7
|
const sdk_1 = require("@filen/sdk");
|
|
8
8
|
const path_1 = __importDefault(require("path"));
|
|
9
9
|
const semaphore_1 = require("../../semaphore");
|
|
10
|
-
const fs_extra_1 = __importDefault(require("fs-extra"));
|
|
11
10
|
const ipc_1 = require("../ipc");
|
|
12
11
|
const utils_1 = require("../../utils");
|
|
13
12
|
const uuid_1 = require("uuid");
|
|
14
13
|
const constants_1 = require("../../constants");
|
|
15
|
-
const write_file_atomic_1 = __importDefault(require("write-file-atomic"));
|
|
16
14
|
exports.DEVICE_ID_VERSION = 1;
|
|
15
|
+
/**
|
|
16
|
+
* How many remote items (folders, then files) are decrypted concurrently while building the tree. The
|
|
17
|
+
* build decrypts entries in fixed-size batches instead of mapping ALL entries to promises up front — on a
|
|
18
|
+
* tree with millions of entries that eager map allocated millions of pending promises + closures at once
|
|
19
|
+
* (an O(n) peak-memory spike). A batch caps peak concurrency — and therefore peak memory — without
|
|
20
|
+
* reducing throughput (the SDK's decrypt/IO services only a handful in parallel regardless), and the batch
|
|
21
|
+
* size IS the parallel-decrypt width (it replaces the old 1024-wide listSemaphore).
|
|
22
|
+
*/
|
|
23
|
+
exports.REMOTE_BUILD_CONCURRENCY = 1024;
|
|
17
24
|
class RemoteFileSystem {
|
|
18
25
|
constructor(sync) {
|
|
19
26
|
this.getDirectoryTreeCache = {
|
|
@@ -26,7 +33,6 @@ class RemoteFileSystem {
|
|
|
26
33
|
this.mutex = new semaphore_1.Semaphore(1);
|
|
27
34
|
this.mkdirMutex = new semaphore_1.Semaphore(1);
|
|
28
35
|
this.itemsMutex = new semaphore_1.Semaphore(1);
|
|
29
|
-
this.listSemaphore = new semaphore_1.Semaphore(1024);
|
|
30
36
|
this.deviceIdCache = "";
|
|
31
37
|
this.ignoredCache = new Map();
|
|
32
38
|
this.sync = sync;
|
|
@@ -36,16 +42,16 @@ class RemoteFileSystem {
|
|
|
36
42
|
return this.deviceIdCache;
|
|
37
43
|
}
|
|
38
44
|
const deviceIdFile = path_1.default.join(this.sync.dbPath, "deviceId", `v${exports.DEVICE_ID_VERSION}`, this.sync.syncPair.uuid);
|
|
39
|
-
await
|
|
40
|
-
let deviceId
|
|
41
|
-
if (!(await
|
|
45
|
+
await this.sync.environment.fs.ensureDir(path_1.default.dirname(deviceIdFile));
|
|
46
|
+
let deviceId;
|
|
47
|
+
if (!(await this.sync.environment.fs.exists(deviceIdFile))) {
|
|
42
48
|
deviceId = (0, uuid_1.v4)();
|
|
43
|
-
await
|
|
49
|
+
await this.sync.environment.writeFileAtomic(deviceIdFile, deviceId, {
|
|
44
50
|
encoding: "utf-8"
|
|
45
51
|
});
|
|
46
52
|
}
|
|
47
53
|
else {
|
|
48
|
-
deviceId = await
|
|
54
|
+
deviceId = await this.sync.environment.fs.readFile(deviceIdFile, {
|
|
49
55
|
encoding: "utf-8"
|
|
50
56
|
});
|
|
51
57
|
}
|
|
@@ -54,8 +60,8 @@ class RemoteFileSystem {
|
|
|
54
60
|
}
|
|
55
61
|
async clearDeviceId() {
|
|
56
62
|
const deviceIdFile = path_1.default.join(this.sync.dbPath, "deviceId", `v${exports.DEVICE_ID_VERSION}`, this.sync.syncPair.uuid);
|
|
57
|
-
await
|
|
58
|
-
await
|
|
63
|
+
await this.sync.environment.fs.ensureDir(path_1.default.dirname(deviceIdFile));
|
|
64
|
+
await this.sync.environment.fs.rm(deviceIdFile, {
|
|
59
65
|
force: true,
|
|
60
66
|
maxRetries: 60 * 10,
|
|
61
67
|
recursive: true,
|
|
@@ -64,8 +70,10 @@ class RemoteFileSystem {
|
|
|
64
70
|
}
|
|
65
71
|
isPathIgnored({ absolutePath, relativePath, name, type }) {
|
|
66
72
|
const key = absolutePath + ":" + type;
|
|
67
|
-
|
|
68
|
-
|
|
73
|
+
// One cache lookup, not two (the second redundant get() ran per item on every tree build).
|
|
74
|
+
const cached = this.ignoredCache.get(key);
|
|
75
|
+
if (cached) {
|
|
76
|
+
return cached;
|
|
69
77
|
}
|
|
70
78
|
if ((0, utils_1.isPathOverMaxLength)(absolutePath)) {
|
|
71
79
|
this.ignoredCache.set(key, {
|
|
@@ -137,11 +145,10 @@ class RemoteFileSystem {
|
|
|
137
145
|
}
|
|
138
146
|
async getDirectoryTree(skipCache = false) {
|
|
139
147
|
const deviceId = await this.getDeviceId();
|
|
140
|
-
const dir = await this.sync.
|
|
148
|
+
const dir = await this.sync.environment.fetchDirTree(this.sync.sdk, {
|
|
141
149
|
uuid: this.sync.syncPair.remoteParentUUID,
|
|
142
150
|
deviceId,
|
|
143
|
-
skipCache
|
|
144
|
-
includeRaw: false
|
|
151
|
+
skipCache
|
|
145
152
|
});
|
|
146
153
|
const now = Date.now();
|
|
147
154
|
// Data did not change, use cache
|
|
@@ -156,152 +163,171 @@ class RemoteFileSystem {
|
|
|
156
163
|
changed: false
|
|
157
164
|
};
|
|
158
165
|
}
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
166
|
+
// The /v3/dir/tree response can arrive UNORDERED — there is no guarantee a parent folder precedes its
|
|
167
|
+
// children, nor that folders[0] is the sync root. So we decrypt every folder's metadata first (in
|
|
168
|
+
// bounded-concurrency batches, capping peak pending promises/memory), then resolve each folder's full
|
|
169
|
+
// path by walking its parent chain to the sync root (memoized). The previous implementation built
|
|
170
|
+
// paths incrementally in array order and required folders[0].parent === "base", so an out-of-order
|
|
171
|
+
// response corrupted child paths or threw. We still REQUIRE a sync-root folder (parent "base") to
|
|
172
|
+
// exist: a response with none is malformed and must NOT be read as an empty tree (that would look like
|
|
173
|
+
// every remote node was deleted). (perf + unordered correctness)
|
|
174
|
+
if (!dir.folders.some(folder => folder[2] === "base")) {
|
|
165
175
|
throw new Error("Invalid base folder parent.");
|
|
166
176
|
}
|
|
167
|
-
const folderNames = { base: "/" };
|
|
168
177
|
const pathsAdded = {};
|
|
169
178
|
let size = 0;
|
|
170
179
|
this.getDirectoryTreeCache.ignored = [];
|
|
171
180
|
this.getDirectoryTreeCache.tree = {};
|
|
172
181
|
this.getDirectoryTreeCache.uuids = {};
|
|
173
182
|
this.getDirectoryTreeCache.size = 0;
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
folderNames[folder[0]] = folderPath;
|
|
181
|
-
if (folderPath.startsWith(constants_1.LOCAL_TRASH_NAME) ||
|
|
182
|
-
decrypted.name.startsWith(constants_1.LOCAL_TRASH_NAME) ||
|
|
183
|
-
(folder[2] !== "base" && decrypted.name.length === 0)) {
|
|
184
|
-
continue;
|
|
183
|
+
const folderMeta = new Map();
|
|
184
|
+
for (let offset = 0; offset < dir.folders.length; offset += exports.REMOTE_BUILD_CONCURRENCY) {
|
|
185
|
+
await Promise.all(dir.folders.slice(offset, offset + exports.REMOTE_BUILD_CONCURRENCY).map(async (folder) => {
|
|
186
|
+
try {
|
|
187
|
+
const decrypted = await this.sync.sdk.crypto().decrypt().folderMetadata({ metadata: folder[1] });
|
|
188
|
+
folderMeta.set(folder[0], { name: decrypted.name, parent: folder[2] });
|
|
185
189
|
}
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
this.
|
|
189
|
-
localPath,
|
|
190
|
-
relativePath: folderPath,
|
|
191
|
-
reason: "duplicate"
|
|
192
|
-
});
|
|
193
|
-
continue;
|
|
194
|
-
}
|
|
195
|
-
pathsAdded[lowercasePath] = true;
|
|
196
|
-
if (folderPath.length === 0) {
|
|
197
|
-
continue;
|
|
190
|
+
catch (e) {
|
|
191
|
+
this.sync.worker.logger.log("error", e, "filesystems.remote.getDirectoryTree");
|
|
192
|
+
this.sync.worker.logger.log("error", e);
|
|
198
193
|
}
|
|
199
|
-
|
|
200
|
-
|
|
194
|
+
}));
|
|
195
|
+
}
|
|
196
|
+
// Resolve a folder uuid to its relative path ("" = the sync root), memoized. Depth-bounded so a
|
|
197
|
+
// malformed cyclic chain can never infinite-loop; returns undefined for an orphan (parent absent).
|
|
198
|
+
// Format matches the old code exactly: root -> "", a child of `parent` -> `${parentPath}/${name}`
|
|
199
|
+
// (so a top-level dir is "/name", a nested one "/a/b").
|
|
200
|
+
const folderPathByUUID = new Map();
|
|
201
|
+
const resolveFolderPath = (uuid, depth) => {
|
|
202
|
+
const cached = folderPathByUUID.get(uuid);
|
|
203
|
+
if (cached !== undefined) {
|
|
204
|
+
return cached;
|
|
205
|
+
}
|
|
206
|
+
if (depth > dir.folders.length) {
|
|
207
|
+
return undefined;
|
|
208
|
+
}
|
|
209
|
+
const meta = folderMeta.get(uuid);
|
|
210
|
+
if (!meta) {
|
|
211
|
+
return undefined;
|
|
212
|
+
}
|
|
213
|
+
if (meta.parent === "base") {
|
|
214
|
+
folderPathByUUID.set(uuid, "");
|
|
215
|
+
return "";
|
|
216
|
+
}
|
|
217
|
+
const parentPath = resolveFolderPath(meta.parent, depth + 1);
|
|
218
|
+
if (parentPath === undefined) {
|
|
219
|
+
return undefined;
|
|
220
|
+
}
|
|
221
|
+
const path = `${parentPath}/${meta.name}`;
|
|
222
|
+
folderPathByUUID.set(uuid, path);
|
|
223
|
+
return path;
|
|
224
|
+
};
|
|
225
|
+
// Build the folder tree. Iterate the response array (preserving first-occurrence-wins dedup order);
|
|
226
|
+
// only the PATH resolution above is order-independent.
|
|
227
|
+
for (const folder of dir.folders) {
|
|
228
|
+
const meta = folderMeta.get(folder[0]);
|
|
229
|
+
if (!meta) {
|
|
230
|
+
continue;
|
|
231
|
+
}
|
|
232
|
+
const folderPath = resolveFolderPath(folder[0], 0);
|
|
233
|
+
// undefined => orphan (broken parent chain); "" => the sync root. Neither is a tree entry.
|
|
234
|
+
if (folderPath === undefined || folderPath.length === 0) {
|
|
235
|
+
continue;
|
|
236
|
+
}
|
|
237
|
+
const localPath = path_1.default.join(this.sync.syncPair.localPath, folderPath);
|
|
238
|
+
if (folderPath.startsWith(constants_1.LOCAL_TRASH_NAME) || meta.name.startsWith(constants_1.LOCAL_TRASH_NAME) || meta.name.length === 0) {
|
|
239
|
+
continue;
|
|
240
|
+
}
|
|
241
|
+
const lowercasePath = folderPath.toLowerCase();
|
|
242
|
+
if (pathsAdded[lowercasePath]) {
|
|
243
|
+
this.getDirectoryTreeCache.ignored.push({
|
|
244
|
+
localPath,
|
|
201
245
|
relativePath: folderPath,
|
|
202
|
-
|
|
203
|
-
type: "directory"
|
|
246
|
+
reason: "duplicate"
|
|
204
247
|
});
|
|
205
|
-
|
|
206
|
-
this.getDirectoryTreeCache.ignored.push({
|
|
207
|
-
localPath,
|
|
208
|
-
relativePath: folderPath,
|
|
209
|
-
reason: ignored.reason
|
|
210
|
-
});
|
|
211
|
-
continue;
|
|
212
|
-
}
|
|
213
|
-
const item = {
|
|
214
|
-
type: "directory",
|
|
215
|
-
uuid: folder[0],
|
|
216
|
-
name: decrypted.name,
|
|
217
|
-
size: 0,
|
|
218
|
-
path: folderPath
|
|
219
|
-
};
|
|
220
|
-
this.getDirectoryTreeCache.tree[folderPath] = item;
|
|
221
|
-
this.getDirectoryTreeCache.uuids[folder[0]] = item;
|
|
222
|
-
size += 1;
|
|
248
|
+
continue;
|
|
223
249
|
}
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
250
|
+
pathsAdded[lowercasePath] = true;
|
|
251
|
+
const ignored = this.isPathIgnored({
|
|
252
|
+
absolutePath: localPath,
|
|
253
|
+
relativePath: folderPath,
|
|
254
|
+
name: meta.name,
|
|
255
|
+
type: "directory"
|
|
256
|
+
});
|
|
257
|
+
if (ignored.ignored) {
|
|
258
|
+
this.getDirectoryTreeCache.ignored.push({
|
|
259
|
+
localPath,
|
|
260
|
+
relativePath: folderPath,
|
|
261
|
+
reason: ignored.reason
|
|
262
|
+
});
|
|
263
|
+
continue;
|
|
227
264
|
}
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
265
|
+
const item = {
|
|
266
|
+
type: "directory",
|
|
267
|
+
uuid: folder[0],
|
|
268
|
+
name: meta.name,
|
|
269
|
+
size: 0,
|
|
270
|
+
path: folderPath
|
|
271
|
+
};
|
|
272
|
+
this.getDirectoryTreeCache.tree[folderPath] = item;
|
|
273
|
+
this.getDirectoryTreeCache.uuids[folder[0]] = item;
|
|
274
|
+
size += 1;
|
|
275
|
+
}
|
|
276
|
+
// Files: decrypt + place in bounded-concurrency batches (caps peak pending promises/memory; the batch
|
|
277
|
+
// size IS the parallel-decrypt width, replacing the eager `map` + 1024 semaphore that created N
|
|
278
|
+
// promises up front). Folders were fully resolved above, so every file's parent path is available.
|
|
279
|
+
for (let offset = 0; offset < dir.files.length; offset += exports.REMOTE_BUILD_CONCURRENCY) {
|
|
280
|
+
await Promise.all(dir.files.slice(offset, offset + exports.REMOTE_BUILD_CONCURRENCY).map(async (file) => {
|
|
281
|
+
try {
|
|
282
|
+
const decrypted = await this.sync.sdk.crypto().decrypt().fileMetadata({ metadata: file[5] });
|
|
283
|
+
if (decrypted.name.length === 0) {
|
|
284
|
+
return;
|
|
285
|
+
}
|
|
286
|
+
const parentPath = folderPathByUUID.get(file[4]);
|
|
287
|
+
// Orphan file (its parent folder was absent from the response) — skip it.
|
|
288
|
+
if (parentPath === undefined) {
|
|
289
|
+
return;
|
|
290
|
+
}
|
|
291
|
+
const filePath = `${parentPath}/${decrypted.name}`;
|
|
292
|
+
const localPath = path_1.default.join(this.sync.syncPair.localPath, filePath);
|
|
293
|
+
if (filePath.startsWith(constants_1.LOCAL_TRASH_NAME) || decrypted.name.startsWith(constants_1.LOCAL_TRASH_NAME) || filePath.length === 0) {
|
|
294
|
+
return;
|
|
295
|
+
}
|
|
296
|
+
const lowercasePath = filePath.toLowerCase();
|
|
297
|
+
if (pathsAdded[lowercasePath]) {
|
|
298
|
+
this.getDirectoryTreeCache.ignored.push({
|
|
299
|
+
localPath,
|
|
300
|
+
relativePath: filePath,
|
|
301
|
+
reason: "duplicate"
|
|
302
|
+
});
|
|
303
|
+
return;
|
|
304
|
+
}
|
|
305
|
+
pathsAdded[lowercasePath] = true;
|
|
306
|
+
const ignored = this.isPathIgnored({
|
|
307
|
+
absolutePath: localPath,
|
|
258
308
|
relativePath: filePath,
|
|
259
|
-
|
|
309
|
+
name: decrypted.name,
|
|
310
|
+
type: "file"
|
|
260
311
|
});
|
|
261
|
-
|
|
312
|
+
if (ignored.ignored) {
|
|
313
|
+
this.getDirectoryTreeCache.ignored.push({
|
|
314
|
+
localPath,
|
|
315
|
+
relativePath: filePath,
|
|
316
|
+
reason: ignored.reason
|
|
317
|
+
});
|
|
318
|
+
return;
|
|
319
|
+
}
|
|
320
|
+
const item = Object.assign(Object.assign(Object.assign({ type: "file", uuid: file[0], name: decrypted.name, size: decrypted.size, mime: decrypted.mime, lastModified: (0, utils_1.convertTimestampToMs)(decrypted.lastModified), version: file[6], chunks: file[3], key: decrypted.key, bucket: file[1], region: file[2] }, (decrypted.creation !== undefined ? { creation: decrypted.creation } : {})), (decrypted.hash !== undefined ? { hash: decrypted.hash } : {})), { path: filePath });
|
|
321
|
+
this.getDirectoryTreeCache.tree[filePath] = item;
|
|
322
|
+
this.getDirectoryTreeCache.uuids[item.uuid] = item;
|
|
323
|
+
size += 1;
|
|
262
324
|
}
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
name: decrypted.name,
|
|
267
|
-
type: "directory"
|
|
268
|
-
});
|
|
269
|
-
if (ignored.ignored) {
|
|
270
|
-
this.getDirectoryTreeCache.ignored.push({
|
|
271
|
-
localPath,
|
|
272
|
-
relativePath: filePath,
|
|
273
|
-
reason: ignored.reason
|
|
274
|
-
});
|
|
275
|
-
return;
|
|
325
|
+
catch (e) {
|
|
326
|
+
this.sync.worker.logger.log("error", e, "filesystems.remote.getDirectoryTree");
|
|
327
|
+
this.sync.worker.logger.log("error", e);
|
|
276
328
|
}
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
uuid: file[0],
|
|
280
|
-
name: decrypted.name,
|
|
281
|
-
size: decrypted.size,
|
|
282
|
-
mime: decrypted.mime,
|
|
283
|
-
lastModified: (0, utils_1.convertTimestampToMs)(decrypted.lastModified),
|
|
284
|
-
version: file[6],
|
|
285
|
-
chunks: file[3],
|
|
286
|
-
key: decrypted.key,
|
|
287
|
-
bucket: file[1],
|
|
288
|
-
region: file[2],
|
|
289
|
-
creation: decrypted.creation,
|
|
290
|
-
hash: decrypted.hash,
|
|
291
|
-
path: filePath
|
|
292
|
-
};
|
|
293
|
-
this.getDirectoryTreeCache.tree[filePath] = item;
|
|
294
|
-
this.getDirectoryTreeCache.uuids[item.uuid] = item;
|
|
295
|
-
size += 1;
|
|
296
|
-
}
|
|
297
|
-
catch (e) {
|
|
298
|
-
this.sync.worker.logger.log("error", e, "filesystems.remote.getDirectoryTree");
|
|
299
|
-
this.sync.worker.logger.log("error", e);
|
|
300
|
-
}
|
|
301
|
-
finally {
|
|
302
|
-
this.listSemaphore.release();
|
|
303
|
-
}
|
|
304
|
-
}), 10000, false);
|
|
329
|
+
}));
|
|
330
|
+
}
|
|
305
331
|
this.getDirectoryTreeCache.size = size;
|
|
306
332
|
this.getDirectoryTreeCache.timestamp = now;
|
|
307
333
|
return {
|
|
@@ -333,8 +359,9 @@ class RemoteFileSystem {
|
|
|
333
359
|
if (relativePath === "/" || relativePath === "." || relativePath.length <= 0) {
|
|
334
360
|
return this.sync.syncPair.remoteParentUUID;
|
|
335
361
|
}
|
|
336
|
-
|
|
337
|
-
|
|
362
|
+
const item = this.getDirectoryTreeCache.tree[relativePath];
|
|
363
|
+
if (item && acceptedTypes.includes(item.type)) {
|
|
364
|
+
return item.uuid;
|
|
338
365
|
}
|
|
339
366
|
return null;
|
|
340
367
|
}
|
|
@@ -387,18 +414,23 @@ class RemoteFileSystem {
|
|
|
387
414
|
const pathEx = relativePath.split("/");
|
|
388
415
|
let builtPath = "/";
|
|
389
416
|
for (const part of pathEx) {
|
|
390
|
-
|
|
417
|
+
// Skip the empty segments produced by leading/duplicate slashes (e.g. "" from "/x/y").
|
|
418
|
+
if (part.length <= 0) {
|
|
391
419
|
continue;
|
|
392
420
|
}
|
|
393
421
|
builtPath = path_1.default.posix.join(builtPath, part);
|
|
394
422
|
if (!this.getDirectoryTreeCache.tree[builtPath]) {
|
|
395
423
|
const partBasename = path_1.default.posix.basename(builtPath);
|
|
396
424
|
const partParentPath = path_1.default.posix.dirname(builtPath);
|
|
425
|
+
const parentIsBase = partParentPath === "/" || partParentPath === "." || partParentPath === "";
|
|
397
426
|
const parentItem = this.getDirectoryTreeCache.tree[partParentPath];
|
|
398
|
-
|
|
427
|
+
// The sync root is never a cache key, so when this level's parent IS the root we must fall
|
|
428
|
+
// back to remoteParentUUID. Only bail when the parent is a non-root directory we cannot find
|
|
429
|
+
// (a level we failed to build earlier) — checking parentIsBase BEFORE this guard is what lets
|
|
430
|
+
// a directory whose parent is the root get created at all.
|
|
431
|
+
if (!parentIsBase && !parentItem) {
|
|
399
432
|
continue;
|
|
400
433
|
}
|
|
401
|
-
const parentIsBase = partParentPath === "/" || partParentPath === "." || partParentPath === "";
|
|
402
434
|
const parentUUID = parentIsBase ? this.sync.syncPair.remoteParentUUID : parentItem.uuid;
|
|
403
435
|
const uuid = await this.sync.sdk.cloud().createDirectory({
|
|
404
436
|
name: partBasename,
|
|
@@ -406,19 +438,21 @@ class RemoteFileSystem {
|
|
|
406
438
|
renameIfExists: true
|
|
407
439
|
});
|
|
408
440
|
await this.itemsMutex.acquire();
|
|
409
|
-
|
|
441
|
+
// Store each intermediate directory under its OWN built path (not the final relativePath), so
|
|
442
|
+
// the next iteration's parent lookup (tree[partParentPath]) resolves and the chain keeps building.
|
|
443
|
+
this.getDirectoryTreeCache.tree[builtPath] = {
|
|
410
444
|
type: "directory",
|
|
411
445
|
uuid,
|
|
412
446
|
name: partBasename,
|
|
413
447
|
size: 0,
|
|
414
|
-
path:
|
|
448
|
+
path: builtPath
|
|
415
449
|
};
|
|
416
450
|
this.getDirectoryTreeCache.uuids[uuid] = {
|
|
417
451
|
type: "directory",
|
|
418
452
|
uuid,
|
|
419
453
|
name: partBasename,
|
|
420
454
|
size: 0,
|
|
421
|
-
path:
|
|
455
|
+
path: builtPath
|
|
422
456
|
};
|
|
423
457
|
this.itemsMutex.release();
|
|
424
458
|
}
|
|
@@ -532,18 +566,10 @@ class RemoteFileSystem {
|
|
|
532
566
|
const newBasename = path_1.default.posix.basename(toRelativePath);
|
|
533
567
|
const oldBasename = path_1.default.posix.basename(fromRelativePath);
|
|
534
568
|
const itemMetadata = item.type === "file"
|
|
535
|
-
? ({
|
|
536
|
-
|
|
537
|
-
size: item.size,
|
|
538
|
-
mime: item.mime,
|
|
539
|
-
lastModified: item.lastModified,
|
|
540
|
-
creation: item.creation,
|
|
541
|
-
hash: item.hash,
|
|
542
|
-
key: item.key
|
|
543
|
-
})
|
|
544
|
-
: ({
|
|
569
|
+
? Object.assign(Object.assign(Object.assign({ name: newBasename, size: item.size, mime: item.mime, lastModified: item.lastModified }, (item.creation !== undefined ? { creation: item.creation } : {})), (item.hash !== undefined ? { hash: item.hash } : {})), { key: item.key })
|
|
570
|
+
: {
|
|
545
571
|
name: newBasename
|
|
546
|
-
}
|
|
572
|
+
};
|
|
547
573
|
if (newParentPath === currentParentPath) {
|
|
548
574
|
if (toRelativePath === "/" || newBasename.length <= 0) {
|
|
549
575
|
throw new Error("Invalid paths.");
|
|
@@ -659,10 +685,9 @@ class RemoteFileSystem {
|
|
|
659
685
|
* @async
|
|
660
686
|
* @param {{ relativePath: string }} param0
|
|
661
687
|
* @param {string} param0.relativePath
|
|
662
|
-
* @returns {Promise<
|
|
688
|
+
* @returns {Promise<Stats>}
|
|
663
689
|
*/
|
|
664
690
|
async download({ relativePath }) {
|
|
665
|
-
var _a;
|
|
666
691
|
const localPath = path_1.default.posix.join(this.sync.syncPair.localPath, relativePath);
|
|
667
692
|
const tmpLocalPath = path_1.default.join(this.sync.syncPair.localPath, constants_1.LOCAL_TRASH_NAME, (0, uuid_1.v4)());
|
|
668
693
|
const signalKey = `download:${relativePath}`;
|
|
@@ -702,7 +727,7 @@ class RemoteFileSystem {
|
|
|
702
727
|
key: item.key,
|
|
703
728
|
size: item.size,
|
|
704
729
|
pauseSignal: this.sync.pauseSignals[signalKey],
|
|
705
|
-
abortSignal:
|
|
730
|
+
abortSignal: this.sync.abortControllers[signalKey].signal,
|
|
706
731
|
onError: err => {
|
|
707
732
|
this.sync.worker.logger.log("error", err, "filesystems.remote.download");
|
|
708
733
|
this.sync.worker.logger.log("error", err);
|
|
@@ -748,11 +773,25 @@ class RemoteFileSystem {
|
|
|
748
773
|
});
|
|
749
774
|
}
|
|
750
775
|
});
|
|
751
|
-
|
|
776
|
+
// Integrity guard: only commit a download whose staged size matches the expected size. The SDK
|
|
777
|
+
// can RESOLVE (rather than reject) with an incomplete staged file when the transfer is aborted
|
|
778
|
+
// mid-stream — the read stream ends cleanly, so the pipeline reports no error even though fewer
|
|
779
|
+
// bytes were written. Without this check the partial/0-byte file would be moved into place and
|
|
780
|
+
// cached as "synced", leaving local and remote permanently diverged (it never re-downloads,
|
|
781
|
+
// because the cached size now matches the persisted base). Discarding it and throwing turns an
|
|
782
|
+
// aborted download into a normal failed task that the next cycle retries.
|
|
783
|
+
const stagedSize = (await this.sync.environment.fs.stat(tmpLocalPath)).size;
|
|
784
|
+
if (stagedSize !== item.size) {
|
|
785
|
+
await this.sync.environment.fs
|
|
786
|
+
.rm(tmpLocalPath, { force: true, maxRetries: 60 * 10, recursive: true, retryDelay: 100 })
|
|
787
|
+
.catch(() => { });
|
|
788
|
+
throw new Error(`Download of ${relativePath} is incomplete: expected ${item.size} bytes but staged ${stagedSize}.`);
|
|
789
|
+
}
|
|
790
|
+
await this.sync.environment.fs.move(tmpLocalPath, localPath, {
|
|
752
791
|
overwrite: true
|
|
753
792
|
});
|
|
754
|
-
await
|
|
755
|
-
const stats = await
|
|
793
|
+
await this.sync.environment.fs.utimes(localPath, new Date(), new Date((0, utils_1.convertTimestampToMs)(item.lastModified)));
|
|
794
|
+
const stats = await this.sync.environment.fs.stat(localPath);
|
|
756
795
|
const localItem = {
|
|
757
796
|
type: "file",
|
|
758
797
|
inode: stats.ino,
|
|
@@ -779,6 +818,13 @@ class RemoteFileSystem {
|
|
|
779
818
|
return stats;
|
|
780
819
|
}
|
|
781
820
|
catch (e) {
|
|
821
|
+
// Best-effort: discard the staged temp file so a download that failed AFTER staging — a partial
|
|
822
|
+
// write, or a failure committing it into place — does not orphan it in the local trash directory.
|
|
823
|
+
// (force ignores an already-moved/never-created temp; the periodic trash sweep is only a delayed
|
|
824
|
+
// backstop, so reclaim the space immediately.)
|
|
825
|
+
await this.sync.environment.fs
|
|
826
|
+
.rm(tmpLocalPath, { force: true, maxRetries: 3, recursive: true, retryDelay: 100 })
|
|
827
|
+
.catch(() => { });
|
|
782
828
|
this.sync.worker.logger.log("error", e, "filesystems.remote.download");
|
|
783
829
|
this.sync.worker.logger.log("error", e);
|
|
784
830
|
if (e instanceof Error) {
|
|
@@ -834,13 +880,21 @@ class RemoteFileSystem {
|
|
|
834
880
|
async fileExists(relativePath) {
|
|
835
881
|
try {
|
|
836
882
|
const item = this.getDirectoryTreeCache.tree[relativePath];
|
|
837
|
-
|
|
838
|
-
|
|
883
|
+
if (!item) {
|
|
884
|
+
return false;
|
|
885
|
+
}
|
|
886
|
+
const parentPath = path_1.default.posix.dirname(relativePath);
|
|
887
|
+
// The sync root is never a cache key, so a top-level file's parent must resolve to remoteParentUUID.
|
|
888
|
+
// Looking up tree["/"] used to yield undefined and make this return false for EVERY top-level file —
|
|
889
|
+
// which silently swallowed a failed top-level rename/delete task (caller reads false as "vanished").
|
|
890
|
+
const parentIsBase = parentPath === "/" || parentPath === "." || parentPath === "";
|
|
891
|
+
const parent = parentIsBase ? undefined : this.getDirectoryTreeCache.tree[parentPath];
|
|
892
|
+
if (!parentIsBase && !parent) {
|
|
839
893
|
return false;
|
|
840
894
|
}
|
|
841
895
|
const { exists, uuid: existsUUID } = await this.sync.sdk.cloud().fileExists({
|
|
842
896
|
name: item.name,
|
|
843
|
-
parent: parent.uuid
|
|
897
|
+
parent: parentIsBase ? this.sync.syncPair.remoteParentUUID : parent.uuid
|
|
844
898
|
});
|
|
845
899
|
if (!exists || existsUUID !== item.uuid) {
|
|
846
900
|
return false;
|