@filen/sync 0.2.1 → 0.3.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (152) hide show
  1. package/.node-version +1 -1
  2. package/dist/ignorer.d.ts +6 -0
  3. package/dist/ignorer.js +43 -24
  4. package/dist/ignorer.js.map +1 -1
  5. package/dist/index.d.ts +4 -1
  6. package/dist/index.js +3 -1
  7. package/dist/index.js.map +1 -1
  8. package/dist/lib/deltas.d.ts +58 -2
  9. package/dist/lib/deltas.js +693 -108
  10. package/dist/lib/deltas.js.map +1 -1
  11. package/dist/lib/environment.d.ts +47 -0
  12. package/dist/lib/environment.js +71 -0
  13. package/dist/lib/environment.js.map +1 -0
  14. package/dist/lib/filesystems/dirTree.d.ts +70 -0
  15. package/dist/lib/filesystems/dirTree.js +157 -0
  16. package/dist/lib/filesystems/dirTree.js.map +1 -0
  17. package/dist/lib/filesystems/local.d.ts +18 -8
  18. package/dist/lib/filesystems/local.js +166 -160
  19. package/dist/lib/filesystems/local.js.map +1 -1
  20. package/dist/lib/filesystems/remote.d.ts +12 -5
  21. package/dist/lib/filesystems/remote.js +226 -172
  22. package/dist/lib/filesystems/remote.js.map +1 -1
  23. package/dist/lib/ipc.js +1 -2
  24. package/dist/lib/ipc.js.map +1 -1
  25. package/dist/lib/lock.js +19 -12
  26. package/dist/lib/lock.js.map +1 -1
  27. package/dist/lib/logger.js +9 -7
  28. package/dist/lib/logger.js.map +1 -1
  29. package/dist/lib/state.js +159 -63
  30. package/dist/lib/state.js.map +1 -1
  31. package/dist/lib/sync.d.ts +18 -0
  32. package/dist/lib/sync.js +165 -96
  33. package/dist/lib/sync.js.map +1 -1
  34. package/dist/lib/tasks.d.ts +7 -8
  35. package/dist/lib/tasks.js +38 -45
  36. package/dist/lib/tasks.js.map +1 -1
  37. package/dist/semaphore.d.ts +1 -0
  38. package/dist/semaphore.js +22 -5
  39. package/dist/semaphore.js.map +1 -1
  40. package/dist/utils.js +51 -35
  41. package/dist/utils.js.map +1 -1
  42. package/eslint.config.mjs +36 -0
  43. package/package.json +19 -15
  44. package/tests/bench/collapse.bench.ts +114 -0
  45. package/tests/bench/cycle.bench.ts +111 -0
  46. package/tests/bench/deltas.bench.ts +151 -0
  47. package/tests/bench/harness/fake-sync.ts +32 -0
  48. package/tests/bench/harness/measure.ts +276 -0
  49. package/tests/bench/harness/scale-world.ts +160 -0
  50. package/tests/bench/harness/trees.ts +275 -0
  51. package/tests/bench/local-scan.bench.ts +74 -0
  52. package/tests/bench/longrun.bench.ts +130 -0
  53. package/tests/bench/profile-incremental.ts +90 -0
  54. package/tests/bench/remote-build.bench.ts +104 -0
  55. package/tests/bench/render.ts +14 -0
  56. package/tests/bench/semaphore.bench.ts +79 -0
  57. package/tests/bench/state.bench.ts +85 -0
  58. package/tests/bench/tasks-dispatch.bench.ts +156 -0
  59. package/tests/conformance/virtual-fs.test.ts +213 -0
  60. package/tests/e2e/backup.e2e.test.ts +130 -0
  61. package/tests/e2e/confirm.e2e.test.ts +191 -0
  62. package/tests/e2e/conflict.e2e.test.ts +261 -0
  63. package/tests/e2e/edge.e2e.test.ts +339 -0
  64. package/tests/e2e/harness/account.ts +104 -0
  65. package/tests/e2e/harness/assert.ts +127 -0
  66. package/tests/e2e/harness/drive.ts +88 -0
  67. package/tests/e2e/harness/mutations.ts +249 -0
  68. package/tests/e2e/harness/world.ts +222 -0
  69. package/tests/e2e/ignore.e2e.test.ts +123 -0
  70. package/tests/e2e/lifecycle.e2e.test.ts +290 -0
  71. package/tests/e2e/modes.e2e.test.ts +215 -0
  72. package/tests/e2e/platform.e2e.test.ts +157 -0
  73. package/tests/e2e/property.e2e.test.ts +163 -0
  74. package/tests/e2e/races.e2e.test.ts +90 -0
  75. package/tests/e2e/regressions.e2e.test.ts +212 -0
  76. package/tests/e2e/resilience.e2e.test.ts +231 -0
  77. package/tests/e2e/special.e2e.test.ts +185 -0
  78. package/tests/e2e/state.e2e.test.ts +229 -0
  79. package/tests/e2e/sync.e2e.test.ts +222 -0
  80. package/tests/fakes/fake-cloud.test.ts +267 -0
  81. package/tests/fakes/fake-cloud.ts +1094 -0
  82. package/tests/fakes/virtual-fs.ts +354 -0
  83. package/tests/harness/known-bug.ts +17 -0
  84. package/tests/harness/mutations.ts +65 -0
  85. package/tests/harness/runner.ts +141 -0
  86. package/tests/harness/snapshot.ts +113 -0
  87. package/tests/harness/world.ts +187 -0
  88. package/tests/scenarios/a-baseline.test.ts +107 -0
  89. package/tests/scenarios/aa-races.test.ts +258 -0
  90. package/tests/scenarios/ab-mode-property.test.ts +189 -0
  91. package/tests/scenarios/ac-platform.test.ts +320 -0
  92. package/tests/scenarios/ad-unicode-normalization.test.ts +67 -0
  93. package/tests/scenarios/b-additions.test.ts +160 -0
  94. package/tests/scenarios/c-modifications.test.ts +194 -0
  95. package/tests/scenarios/d-deletions.test.ts +259 -0
  96. package/tests/scenarios/e-rename-move.test.ts +288 -0
  97. package/tests/scenarios/f-ignore-filter.test.ts +346 -0
  98. package/tests/scenarios/g-large-deletion.test.ts +277 -0
  99. package/tests/scenarios/h-resilience.test.ts +167 -0
  100. package/tests/scenarios/i-lifecycle.test.ts +353 -0
  101. package/tests/scenarios/j-state-cache.test.ts +264 -0
  102. package/tests/scenarios/k-scale.test.ts +202 -0
  103. package/tests/scenarios/l-property.test.ts +145 -0
  104. package/tests/scenarios/m-golden.test.ts +452 -0
  105. package/tests/scenarios/o-task-errors.test.ts +497 -0
  106. package/tests/scenarios/p-remote-originated.test.ts +306 -0
  107. package/tests/scenarios/q-cycle-lifecycle.test.ts +234 -0
  108. package/tests/scenarios/r-rename-stress.test.ts +208 -0
  109. package/tests/scenarios/s-upgrade-transition.test.ts +171 -0
  110. package/tests/scenarios/t-type-change.test.ts +144 -0
  111. package/tests/scenarios/u-mode-local-to-cloud.test.ts +347 -0
  112. package/tests/scenarios/v-mode-local-backup.test.ts +201 -0
  113. package/tests/scenarios/w-mode-cloud-to-local.test.ts +304 -0
  114. package/tests/scenarios/x-mode-cloud-backup.test.ts +201 -0
  115. package/tests/scenarios/y-conflict-matrix.test.ts +292 -0
  116. package/tests/scenarios/z-cross-ops.test.ts +285 -0
  117. package/tests/scenarios/zb-dir-rename-cross.test.ts +296 -0
  118. package/tests/scenarios/zc-crash-recovery.test.ts +189 -0
  119. package/tests/scenarios/zd-inode-reuse.test.ts +118 -0
  120. package/tests/scenarios/ze-move-into-new-dir.test.ts +130 -0
  121. package/tests/scenarios/zf-remote-change-unchanged-local.test.ts +81 -0
  122. package/tests/scenarios/zg-edit-during-scan.test.ts +68 -0
  123. package/tests/scenarios/zh-dir-delete-vs-child.test.ts +104 -0
  124. package/tests/scenarios/zi-smoke-test-outage.test.ts +78 -0
  125. package/tests/scenarios/zj-trash-cleanup.test.ts +133 -0
  126. package/tests/scenarios/zk-ignore-asymmetry.test.ts +150 -0
  127. package/tests/scenarios/zl-mode-atomicity.test.ts +104 -0
  128. package/tests/scenarios/zm-scan-concurrency.test.ts +78 -0
  129. package/tests/scenarios/zn-delta-ordering.test.ts +130 -0
  130. package/tests/scenarios/zo-download-temp-cleanup.test.ts +65 -0
  131. package/tests/unit/collapse-deltas.test.ts +276 -0
  132. package/tests/unit/dir-tree.test.ts +159 -0
  133. package/tests/unit/icloud.test.ts +115 -0
  134. package/tests/unit/ignorer-cache-regression.test.ts +70 -0
  135. package/tests/unit/ignorer.test.ts +63 -0
  136. package/tests/unit/ipc-lock.test.ts +438 -0
  137. package/tests/unit/lock.test.ts +135 -0
  138. package/tests/unit/n-unit.test.ts +632 -0
  139. package/tests/unit/remote-tree-unordered-regression.test.ts +101 -0
  140. package/tests/unit/semaphore-regression.test.ts +140 -0
  141. package/tests/unit/state-refencode-regression.test.ts +224 -0
  142. package/tests/unit/state.test.ts +809 -0
  143. package/tests/unit/tasks-dispatch-order-regression.test.ts +53 -0
  144. package/tests/unit/worker-api.test.ts +379 -0
  145. package/tsconfig.json +10 -1
  146. package/tsconfig.test.json +12 -0
  147. package/tsconfig.tsbuildinfo +1 -0
  148. package/vitest.bench.config.ts +32 -0
  149. package/vitest.config.ts +27 -0
  150. package/vitest.e2e.config.ts +68 -0
  151. package/.eslintrc +0 -16
  152. package/jest.config.js +0 -5
@@ -3,17 +3,24 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
3
3
  return (mod && mod.__esModule) ? mod : { "default": mod };
4
4
  };
5
5
  Object.defineProperty(exports, "__esModule", { value: true });
6
- exports.RemoteFileSystem = exports.DEVICE_ID_VERSION = void 0;
6
+ exports.RemoteFileSystem = exports.REMOTE_BUILD_CONCURRENCY = exports.DEVICE_ID_VERSION = void 0;
7
7
  const sdk_1 = require("@filen/sdk");
8
8
  const path_1 = __importDefault(require("path"));
9
9
  const semaphore_1 = require("../../semaphore");
10
- const fs_extra_1 = __importDefault(require("fs-extra"));
11
10
  const ipc_1 = require("../ipc");
12
11
  const utils_1 = require("../../utils");
13
12
  const uuid_1 = require("uuid");
14
13
  const constants_1 = require("../../constants");
15
- const write_file_atomic_1 = __importDefault(require("write-file-atomic"));
16
14
  exports.DEVICE_ID_VERSION = 1;
15
+ /**
16
+ * How many remote items (folders, then files) are decrypted concurrently while building the tree. The
17
+ * build decrypts entries in fixed-size batches instead of mapping ALL entries to promises up front — on a
18
+ * tree with millions of entries that eager map allocated millions of pending promises + closures at once
19
+ * (an O(n) peak-memory spike). A batch caps peak concurrency — and therefore peak memory — without
20
+ * reducing throughput (the SDK's decrypt/IO services only a handful in parallel regardless), and the batch
21
+ * size IS the parallel-decrypt width (it replaces the old 1024-wide listSemaphore).
22
+ */
23
+ exports.REMOTE_BUILD_CONCURRENCY = 1024;
17
24
  class RemoteFileSystem {
18
25
  constructor(sync) {
19
26
  this.getDirectoryTreeCache = {
@@ -26,7 +33,6 @@ class RemoteFileSystem {
26
33
  this.mutex = new semaphore_1.Semaphore(1);
27
34
  this.mkdirMutex = new semaphore_1.Semaphore(1);
28
35
  this.itemsMutex = new semaphore_1.Semaphore(1);
29
- this.listSemaphore = new semaphore_1.Semaphore(1024);
30
36
  this.deviceIdCache = "";
31
37
  this.ignoredCache = new Map();
32
38
  this.sync = sync;
@@ -36,16 +42,16 @@ class RemoteFileSystem {
36
42
  return this.deviceIdCache;
37
43
  }
38
44
  const deviceIdFile = path_1.default.join(this.sync.dbPath, "deviceId", `v${exports.DEVICE_ID_VERSION}`, this.sync.syncPair.uuid);
39
- await fs_extra_1.default.ensureDir(path_1.default.dirname(deviceIdFile));
40
- let deviceId = "";
41
- if (!(await fs_extra_1.default.exists(deviceIdFile))) {
45
+ await this.sync.environment.fs.ensureDir(path_1.default.dirname(deviceIdFile));
46
+ let deviceId;
47
+ if (!(await this.sync.environment.fs.exists(deviceIdFile))) {
42
48
  deviceId = (0, uuid_1.v4)();
43
- await (0, write_file_atomic_1.default)(deviceIdFile, deviceId, {
49
+ await this.sync.environment.writeFileAtomic(deviceIdFile, deviceId, {
44
50
  encoding: "utf-8"
45
51
  });
46
52
  }
47
53
  else {
48
- deviceId = await fs_extra_1.default.readFile(deviceIdFile, {
54
+ deviceId = await this.sync.environment.fs.readFile(deviceIdFile, {
49
55
  encoding: "utf-8"
50
56
  });
51
57
  }
@@ -54,8 +60,8 @@ class RemoteFileSystem {
54
60
  }
55
61
  async clearDeviceId() {
56
62
  const deviceIdFile = path_1.default.join(this.sync.dbPath, "deviceId", `v${exports.DEVICE_ID_VERSION}`, this.sync.syncPair.uuid);
57
- await fs_extra_1.default.ensureDir(path_1.default.dirname(deviceIdFile));
58
- await fs_extra_1.default.rm(deviceIdFile, {
63
+ await this.sync.environment.fs.ensureDir(path_1.default.dirname(deviceIdFile));
64
+ await this.sync.environment.fs.rm(deviceIdFile, {
59
65
  force: true,
60
66
  maxRetries: 60 * 10,
61
67
  recursive: true,
@@ -64,8 +70,10 @@ class RemoteFileSystem {
64
70
  }
65
71
  isPathIgnored({ absolutePath, relativePath, name, type }) {
66
72
  const key = absolutePath + ":" + type;
67
- if (this.ignoredCache.get(key)) {
68
- return this.ignoredCache.get(key);
73
+ // One cache lookup, not two (the second redundant get() ran per item on every tree build).
74
+ const cached = this.ignoredCache.get(key);
75
+ if (cached) {
76
+ return cached;
69
77
  }
70
78
  if ((0, utils_1.isPathOverMaxLength)(absolutePath)) {
71
79
  this.ignoredCache.set(key, {
@@ -137,11 +145,10 @@ class RemoteFileSystem {
137
145
  }
138
146
  async getDirectoryTree(skipCache = false) {
139
147
  const deviceId = await this.getDeviceId();
140
- const dir = await this.sync.sdk.api(3).dir().tree({
148
+ const dir = await this.sync.environment.fetchDirTree(this.sync.sdk, {
141
149
  uuid: this.sync.syncPair.remoteParentUUID,
142
150
  deviceId,
143
- skipCache,
144
- includeRaw: false
151
+ skipCache
145
152
  });
146
153
  const now = Date.now();
147
154
  // Data did not change, use cache
@@ -156,152 +163,171 @@ class RemoteFileSystem {
156
163
  changed: false
157
164
  };
158
165
  }
159
- const baseFolder = dir.folders[0];
160
- if (!baseFolder) {
161
- throw new Error("Could not get base folder.");
162
- }
163
- const baseFolderParent = baseFolder[2];
164
- if (baseFolderParent !== "base") {
166
+ // The /v3/dir/tree response can arrive UNORDERED — there is no guarantee a parent folder precedes its
167
+ // children, nor that folders[0] is the sync root. So we decrypt every folder's metadata first (in
168
+ // bounded-concurrency batches, capping peak pending promises/memory), then resolve each folder's full
169
+ // path by walking its parent chain to the sync root (memoized). The previous implementation built
170
+ // paths incrementally in array order and required folders[0].parent === "base", so an out-of-order
171
+ // response corrupted child paths or threw. We still REQUIRE a sync-root folder (parent "base") to
172
+ // exist: a response with none is malformed and must NOT be read as an empty tree (that would look like
173
+ // every remote node was deleted). (perf + unordered correctness)
174
+ if (!dir.folders.some(folder => folder[2] === "base")) {
165
175
  throw new Error("Invalid base folder parent.");
166
176
  }
167
- const folderNames = { base: "/" };
168
177
  const pathsAdded = {};
169
178
  let size = 0;
170
179
  this.getDirectoryTreeCache.ignored = [];
171
180
  this.getDirectoryTreeCache.tree = {};
172
181
  this.getDirectoryTreeCache.uuids = {};
173
182
  this.getDirectoryTreeCache.size = 0;
174
- for (const folder of dir.folders) {
175
- try {
176
- const decrypted = await this.sync.sdk.crypto().decrypt().folderMetadata({ metadata: folder[1] });
177
- const parentPath = folder[2] === "base" ? "" : `${folderNames[folder[2]]}/`;
178
- const folderPath = folder[2] === "base" ? "" : `${parentPath}${decrypted.name}`;
179
- const localPath = path_1.default.join(this.sync.syncPair.localPath, folderPath);
180
- folderNames[folder[0]] = folderPath;
181
- if (folderPath.startsWith(constants_1.LOCAL_TRASH_NAME) ||
182
- decrypted.name.startsWith(constants_1.LOCAL_TRASH_NAME) ||
183
- (folder[2] !== "base" && decrypted.name.length === 0)) {
184
- continue;
183
+ const folderMeta = new Map();
184
+ for (let offset = 0; offset < dir.folders.length; offset += exports.REMOTE_BUILD_CONCURRENCY) {
185
+ await Promise.all(dir.folders.slice(offset, offset + exports.REMOTE_BUILD_CONCURRENCY).map(async (folder) => {
186
+ try {
187
+ const decrypted = await this.sync.sdk.crypto().decrypt().folderMetadata({ metadata: folder[1] });
188
+ folderMeta.set(folder[0], { name: decrypted.name, parent: folder[2] });
185
189
  }
186
- const lowercasePath = folderPath.toLowerCase();
187
- if (pathsAdded[lowercasePath]) {
188
- this.getDirectoryTreeCache.ignored.push({
189
- localPath,
190
- relativePath: folderPath,
191
- reason: "duplicate"
192
- });
193
- continue;
194
- }
195
- pathsAdded[lowercasePath] = true;
196
- if (folderPath.length === 0) {
197
- continue;
190
+ catch (e) {
191
+ this.sync.worker.logger.log("error", e, "filesystems.remote.getDirectoryTree");
192
+ this.sync.worker.logger.log("error", e);
198
193
  }
199
- const ignored = this.isPathIgnored({
200
- absolutePath: localPath,
194
+ }));
195
+ }
196
+ // Resolve a folder uuid to its relative path ("" = the sync root), memoized. Depth-bounded so a
197
+ // malformed cyclic chain can never infinite-loop; returns undefined for an orphan (parent absent).
198
+ // Format matches the old code exactly: root -> "", a child of `parent` -> `${parentPath}/${name}`
199
+ // (so a top-level dir is "/name", a nested one "/a/b").
200
+ const folderPathByUUID = new Map();
201
+ const resolveFolderPath = (uuid, depth) => {
202
+ const cached = folderPathByUUID.get(uuid);
203
+ if (cached !== undefined) {
204
+ return cached;
205
+ }
206
+ if (depth > dir.folders.length) {
207
+ return undefined;
208
+ }
209
+ const meta = folderMeta.get(uuid);
210
+ if (!meta) {
211
+ return undefined;
212
+ }
213
+ if (meta.parent === "base") {
214
+ folderPathByUUID.set(uuid, "");
215
+ return "";
216
+ }
217
+ const parentPath = resolveFolderPath(meta.parent, depth + 1);
218
+ if (parentPath === undefined) {
219
+ return undefined;
220
+ }
221
+ const path = `${parentPath}/${meta.name}`;
222
+ folderPathByUUID.set(uuid, path);
223
+ return path;
224
+ };
225
+ // Build the folder tree. Iterate the response array (preserving first-occurrence-wins dedup order);
226
+ // only the PATH resolution above is order-independent.
227
+ for (const folder of dir.folders) {
228
+ const meta = folderMeta.get(folder[0]);
229
+ if (!meta) {
230
+ continue;
231
+ }
232
+ const folderPath = resolveFolderPath(folder[0], 0);
233
+ // undefined => orphan (broken parent chain); "" => the sync root. Neither is a tree entry.
234
+ if (folderPath === undefined || folderPath.length === 0) {
235
+ continue;
236
+ }
237
+ const localPath = path_1.default.join(this.sync.syncPair.localPath, folderPath);
238
+ if (folderPath.startsWith(constants_1.LOCAL_TRASH_NAME) || meta.name.startsWith(constants_1.LOCAL_TRASH_NAME) || meta.name.length === 0) {
239
+ continue;
240
+ }
241
+ const lowercasePath = folderPath.toLowerCase();
242
+ if (pathsAdded[lowercasePath]) {
243
+ this.getDirectoryTreeCache.ignored.push({
244
+ localPath,
201
245
  relativePath: folderPath,
202
- name: decrypted.name,
203
- type: "directory"
246
+ reason: "duplicate"
204
247
  });
205
- if (ignored.ignored) {
206
- this.getDirectoryTreeCache.ignored.push({
207
- localPath,
208
- relativePath: folderPath,
209
- reason: ignored.reason
210
- });
211
- continue;
212
- }
213
- const item = {
214
- type: "directory",
215
- uuid: folder[0],
216
- name: decrypted.name,
217
- size: 0,
218
- path: folderPath
219
- };
220
- this.getDirectoryTreeCache.tree[folderPath] = item;
221
- this.getDirectoryTreeCache.uuids[folder[0]] = item;
222
- size += 1;
248
+ continue;
223
249
  }
224
- catch (e) {
225
- this.sync.worker.logger.log("error", e, "filesystems.remote.getDirectoryTree");
226
- this.sync.worker.logger.log("error", e);
250
+ pathsAdded[lowercasePath] = true;
251
+ const ignored = this.isPathIgnored({
252
+ absolutePath: localPath,
253
+ relativePath: folderPath,
254
+ name: meta.name,
255
+ type: "directory"
256
+ });
257
+ if (ignored.ignored) {
258
+ this.getDirectoryTreeCache.ignored.push({
259
+ localPath,
260
+ relativePath: folderPath,
261
+ reason: ignored.reason
262
+ });
263
+ continue;
227
264
  }
228
- }
229
- if (Object.keys(folderNames).length === 0) {
230
- throw new Error("Could not build directory tree.");
231
- }
232
- await (0, utils_1.promiseAllChunked)(dir.files.map(async (file) => {
233
- await this.listSemaphore.acquire();
234
- try {
235
- const decrypted = await this.sync.sdk.crypto().decrypt().fileMetadata({ metadata: file[5] });
236
- if (decrypted.name.length === 0) {
237
- return;
238
- }
239
- const parentPath = folderNames[file[4]];
240
- const filePath = `${parentPath}/${decrypted.name}`;
241
- const localPath = path_1.default.join(this.sync.syncPair.localPath, filePath);
242
- if (filePath.startsWith(constants_1.LOCAL_TRASH_NAME) || decrypted.name.startsWith(constants_1.LOCAL_TRASH_NAME) || filePath.length === 0) {
243
- return;
244
- }
245
- const lowercasePath = filePath.toLowerCase();
246
- if (pathsAdded[lowercasePath]) {
247
- this.getDirectoryTreeCache.ignored.push({
248
- localPath,
249
- relativePath: filePath,
250
- reason: "duplicate"
251
- });
252
- return;
253
- }
254
- pathsAdded[lowercasePath] = true;
255
- if (decrypted.size <= 0) {
256
- this.getDirectoryTreeCache.ignored.push({
257
- localPath,
265
+ const item = {
266
+ type: "directory",
267
+ uuid: folder[0],
268
+ name: meta.name,
269
+ size: 0,
270
+ path: folderPath
271
+ };
272
+ this.getDirectoryTreeCache.tree[folderPath] = item;
273
+ this.getDirectoryTreeCache.uuids[folder[0]] = item;
274
+ size += 1;
275
+ }
276
+ // Files: decrypt + place in bounded-concurrency batches (caps peak pending promises/memory; the batch
277
+ // size IS the parallel-decrypt width, replacing the eager `map` + 1024 semaphore that created N
278
+ // promises up front). Folders were fully resolved above, so every file's parent path is available.
279
+ for (let offset = 0; offset < dir.files.length; offset += exports.REMOTE_BUILD_CONCURRENCY) {
280
+ await Promise.all(dir.files.slice(offset, offset + exports.REMOTE_BUILD_CONCURRENCY).map(async (file) => {
281
+ try {
282
+ const decrypted = await this.sync.sdk.crypto().decrypt().fileMetadata({ metadata: file[5] });
283
+ if (decrypted.name.length === 0) {
284
+ return;
285
+ }
286
+ const parentPath = folderPathByUUID.get(file[4]);
287
+ // Orphan file (its parent folder was absent from the response) — skip it.
288
+ if (parentPath === undefined) {
289
+ return;
290
+ }
291
+ const filePath = `${parentPath}/${decrypted.name}`;
292
+ const localPath = path_1.default.join(this.sync.syncPair.localPath, filePath);
293
+ if (filePath.startsWith(constants_1.LOCAL_TRASH_NAME) || decrypted.name.startsWith(constants_1.LOCAL_TRASH_NAME) || filePath.length === 0) {
294
+ return;
295
+ }
296
+ const lowercasePath = filePath.toLowerCase();
297
+ if (pathsAdded[lowercasePath]) {
298
+ this.getDirectoryTreeCache.ignored.push({
299
+ localPath,
300
+ relativePath: filePath,
301
+ reason: "duplicate"
302
+ });
303
+ return;
304
+ }
305
+ pathsAdded[lowercasePath] = true;
306
+ const ignored = this.isPathIgnored({
307
+ absolutePath: localPath,
258
308
  relativePath: filePath,
259
- reason: "empty"
309
+ name: decrypted.name,
310
+ type: "file"
260
311
  });
261
- return;
312
+ if (ignored.ignored) {
313
+ this.getDirectoryTreeCache.ignored.push({
314
+ localPath,
315
+ relativePath: filePath,
316
+ reason: ignored.reason
317
+ });
318
+ return;
319
+ }
320
+ const item = Object.assign(Object.assign(Object.assign({ type: "file", uuid: file[0], name: decrypted.name, size: decrypted.size, mime: decrypted.mime, lastModified: (0, utils_1.convertTimestampToMs)(decrypted.lastModified), version: file[6], chunks: file[3], key: decrypted.key, bucket: file[1], region: file[2] }, (decrypted.creation !== undefined ? { creation: decrypted.creation } : {})), (decrypted.hash !== undefined ? { hash: decrypted.hash } : {})), { path: filePath });
321
+ this.getDirectoryTreeCache.tree[filePath] = item;
322
+ this.getDirectoryTreeCache.uuids[item.uuid] = item;
323
+ size += 1;
262
324
  }
263
- const ignored = this.isPathIgnored({
264
- absolutePath: localPath,
265
- relativePath: filePath,
266
- name: decrypted.name,
267
- type: "directory"
268
- });
269
- if (ignored.ignored) {
270
- this.getDirectoryTreeCache.ignored.push({
271
- localPath,
272
- relativePath: filePath,
273
- reason: ignored.reason
274
- });
275
- return;
325
+ catch (e) {
326
+ this.sync.worker.logger.log("error", e, "filesystems.remote.getDirectoryTree");
327
+ this.sync.worker.logger.log("error", e);
276
328
  }
277
- const item = {
278
- type: "file",
279
- uuid: file[0],
280
- name: decrypted.name,
281
- size: decrypted.size,
282
- mime: decrypted.mime,
283
- lastModified: (0, utils_1.convertTimestampToMs)(decrypted.lastModified),
284
- version: file[6],
285
- chunks: file[3],
286
- key: decrypted.key,
287
- bucket: file[1],
288
- region: file[2],
289
- creation: decrypted.creation,
290
- hash: decrypted.hash,
291
- path: filePath
292
- };
293
- this.getDirectoryTreeCache.tree[filePath] = item;
294
- this.getDirectoryTreeCache.uuids[item.uuid] = item;
295
- size += 1;
296
- }
297
- catch (e) {
298
- this.sync.worker.logger.log("error", e, "filesystems.remote.getDirectoryTree");
299
- this.sync.worker.logger.log("error", e);
300
- }
301
- finally {
302
- this.listSemaphore.release();
303
- }
304
- }), 10000, false);
329
+ }));
330
+ }
305
331
  this.getDirectoryTreeCache.size = size;
306
332
  this.getDirectoryTreeCache.timestamp = now;
307
333
  return {
@@ -333,8 +359,9 @@ class RemoteFileSystem {
333
359
  if (relativePath === "/" || relativePath === "." || relativePath.length <= 0) {
334
360
  return this.sync.syncPair.remoteParentUUID;
335
361
  }
336
- if (this.getDirectoryTreeCache.tree[relativePath] && acceptedTypes.includes(this.getDirectoryTreeCache.tree[relativePath].type)) {
337
- return this.getDirectoryTreeCache.tree[relativePath].uuid;
362
+ const item = this.getDirectoryTreeCache.tree[relativePath];
363
+ if (item && acceptedTypes.includes(item.type)) {
364
+ return item.uuid;
338
365
  }
339
366
  return null;
340
367
  }
@@ -387,18 +414,23 @@ class RemoteFileSystem {
387
414
  const pathEx = relativePath.split("/");
388
415
  let builtPath = "/";
389
416
  for (const part of pathEx) {
390
- if (pathEx.length <= 0) {
417
+ // Skip the empty segments produced by leading/duplicate slashes (e.g. "" from "/x/y").
418
+ if (part.length <= 0) {
391
419
  continue;
392
420
  }
393
421
  builtPath = path_1.default.posix.join(builtPath, part);
394
422
  if (!this.getDirectoryTreeCache.tree[builtPath]) {
395
423
  const partBasename = path_1.default.posix.basename(builtPath);
396
424
  const partParentPath = path_1.default.posix.dirname(builtPath);
425
+ const parentIsBase = partParentPath === "/" || partParentPath === "." || partParentPath === "";
397
426
  const parentItem = this.getDirectoryTreeCache.tree[partParentPath];
398
- if (!parentItem) {
427
+ // The sync root is never a cache key, so when this level's parent IS the root we must fall
428
+ // back to remoteParentUUID. Only bail when the parent is a non-root directory we cannot find
429
+ // (a level we failed to build earlier) — checking parentIsBase BEFORE this guard is what lets
430
+ // a directory whose parent is the root get created at all.
431
+ if (!parentIsBase && !parentItem) {
399
432
  continue;
400
433
  }
401
- const parentIsBase = partParentPath === "/" || partParentPath === "." || partParentPath === "";
402
434
  const parentUUID = parentIsBase ? this.sync.syncPair.remoteParentUUID : parentItem.uuid;
403
435
  const uuid = await this.sync.sdk.cloud().createDirectory({
404
436
  name: partBasename,
@@ -406,19 +438,21 @@ class RemoteFileSystem {
406
438
  renameIfExists: true
407
439
  });
408
440
  await this.itemsMutex.acquire();
409
- this.getDirectoryTreeCache.tree[relativePath] = {
441
+ // Store each intermediate directory under its OWN built path (not the final relativePath), so
442
+ // the next iteration's parent lookup (tree[partParentPath]) resolves and the chain keeps building.
443
+ this.getDirectoryTreeCache.tree[builtPath] = {
410
444
  type: "directory",
411
445
  uuid,
412
446
  name: partBasename,
413
447
  size: 0,
414
- path: relativePath
448
+ path: builtPath
415
449
  };
416
450
  this.getDirectoryTreeCache.uuids[uuid] = {
417
451
  type: "directory",
418
452
  uuid,
419
453
  name: partBasename,
420
454
  size: 0,
421
- path: relativePath
455
+ path: builtPath
422
456
  };
423
457
  this.itemsMutex.release();
424
458
  }
@@ -532,18 +566,10 @@ class RemoteFileSystem {
532
566
  const newBasename = path_1.default.posix.basename(toRelativePath);
533
567
  const oldBasename = path_1.default.posix.basename(fromRelativePath);
534
568
  const itemMetadata = item.type === "file"
535
- ? ({
536
- name: newBasename,
537
- size: item.size,
538
- mime: item.mime,
539
- lastModified: item.lastModified,
540
- creation: item.creation,
541
- hash: item.hash,
542
- key: item.key
543
- })
544
- : ({
569
+ ? Object.assign(Object.assign(Object.assign({ name: newBasename, size: item.size, mime: item.mime, lastModified: item.lastModified }, (item.creation !== undefined ? { creation: item.creation } : {})), (item.hash !== undefined ? { hash: item.hash } : {})), { key: item.key })
570
+ : {
545
571
  name: newBasename
546
- });
572
+ };
547
573
  if (newParentPath === currentParentPath) {
548
574
  if (toRelativePath === "/" || newBasename.length <= 0) {
549
575
  throw new Error("Invalid paths.");
@@ -659,10 +685,9 @@ class RemoteFileSystem {
659
685
  * @async
660
686
  * @param {{ relativePath: string }} param0
661
687
  * @param {string} param0.relativePath
662
- * @returns {Promise<fs.Stats>}
688
+ * @returns {Promise<Stats>}
663
689
  */
664
690
  async download({ relativePath }) {
665
- var _a;
666
691
  const localPath = path_1.default.posix.join(this.sync.syncPair.localPath, relativePath);
667
692
  const tmpLocalPath = path_1.default.join(this.sync.syncPair.localPath, constants_1.LOCAL_TRASH_NAME, (0, uuid_1.v4)());
668
693
  const signalKey = `download:${relativePath}`;
@@ -702,7 +727,7 @@ class RemoteFileSystem {
702
727
  key: item.key,
703
728
  size: item.size,
704
729
  pauseSignal: this.sync.pauseSignals[signalKey],
705
- abortSignal: (_a = this.sync.abortControllers[signalKey]) === null || _a === void 0 ? void 0 : _a.signal,
730
+ abortSignal: this.sync.abortControllers[signalKey].signal,
706
731
  onError: err => {
707
732
  this.sync.worker.logger.log("error", err, "filesystems.remote.download");
708
733
  this.sync.worker.logger.log("error", err);
@@ -748,11 +773,25 @@ class RemoteFileSystem {
748
773
  });
749
774
  }
750
775
  });
751
- await fs_extra_1.default.move(tmpLocalPath, localPath, {
776
+ // Integrity guard: only commit a download whose staged size matches the expected size. The SDK
777
+ // can RESOLVE (rather than reject) with an incomplete staged file when the transfer is aborted
778
+ // mid-stream — the read stream ends cleanly, so the pipeline reports no error even though fewer
779
+ // bytes were written. Without this check the partial/0-byte file would be moved into place and
780
+ // cached as "synced", leaving local and remote permanently diverged (it never re-downloads,
781
+ // because the cached size now matches the persisted base). Discarding it and throwing turns an
782
+ // aborted download into a normal failed task that the next cycle retries.
783
+ const stagedSize = (await this.sync.environment.fs.stat(tmpLocalPath)).size;
784
+ if (stagedSize !== item.size) {
785
+ await this.sync.environment.fs
786
+ .rm(tmpLocalPath, { force: true, maxRetries: 60 * 10, recursive: true, retryDelay: 100 })
787
+ .catch(() => { });
788
+ throw new Error(`Download of ${relativePath} is incomplete: expected ${item.size} bytes but staged ${stagedSize}.`);
789
+ }
790
+ await this.sync.environment.fs.move(tmpLocalPath, localPath, {
752
791
  overwrite: true
753
792
  });
754
- await fs_extra_1.default.utimes(localPath, new Date(), new Date((0, utils_1.convertTimestampToMs)(item.lastModified)));
755
- const stats = await fs_extra_1.default.stat(localPath);
793
+ await this.sync.environment.fs.utimes(localPath, new Date(), new Date((0, utils_1.convertTimestampToMs)(item.lastModified)));
794
+ const stats = await this.sync.environment.fs.stat(localPath);
756
795
  const localItem = {
757
796
  type: "file",
758
797
  inode: stats.ino,
@@ -779,6 +818,13 @@ class RemoteFileSystem {
779
818
  return stats;
780
819
  }
781
820
  catch (e) {
821
+ // Best-effort: discard the staged temp file so a download that failed AFTER staging — a partial
822
+ // write, or a failure committing it into place — does not orphan it in the local trash directory.
823
+ // (force ignores an already-moved/never-created temp; the periodic trash sweep is only a delayed
824
+ // backstop, so reclaim the space immediately.)
825
+ await this.sync.environment.fs
826
+ .rm(tmpLocalPath, { force: true, maxRetries: 3, recursive: true, retryDelay: 100 })
827
+ .catch(() => { });
782
828
  this.sync.worker.logger.log("error", e, "filesystems.remote.download");
783
829
  this.sync.worker.logger.log("error", e);
784
830
  if (e instanceof Error) {
@@ -834,13 +880,21 @@ class RemoteFileSystem {
834
880
  async fileExists(relativePath) {
835
881
  try {
836
882
  const item = this.getDirectoryTreeCache.tree[relativePath];
837
- const parent = this.getDirectoryTreeCache.tree[path_1.default.posix.dirname(relativePath)];
838
- if (!item || !parent) {
883
+ if (!item) {
884
+ return false;
885
+ }
886
+ const parentPath = path_1.default.posix.dirname(relativePath);
887
+ // The sync root is never a cache key, so a top-level file's parent must resolve to remoteParentUUID.
888
+ // Looking up tree["/"] used to yield undefined and make this return false for EVERY top-level file —
889
+ // which silently swallowed a failed top-level rename/delete task (caller reads false as "vanished").
890
+ const parentIsBase = parentPath === "/" || parentPath === "." || parentPath === "";
891
+ const parent = parentIsBase ? undefined : this.getDirectoryTreeCache.tree[parentPath];
892
+ if (!parentIsBase && !parent) {
839
893
  return false;
840
894
  }
841
895
  const { exists, uuid: existsUUID } = await this.sync.sdk.cloud().fileExists({
842
896
  name: item.name,
843
- parent: parent.uuid
897
+ parent: parentIsBase ? this.sync.syncPair.remoteParentUUID : parent.uuid
844
898
  });
845
899
  if (!exists || existsUUID !== item.uuid) {
846
900
  return false;