@muhkoo/cli 0.15.0-alpha.5 → 0.15.0-alpha.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,11 +1,14 @@
1
1
  {
2
2
  "name": "@muhkoo/cli",
3
- "version": "0.15.0-alpha.5",
3
+ "version": "0.15.0-alpha.7",
4
4
  "description": "Command-line interface for the Muhkoo platform — provision backends, deploy apps, and manage agents, functions, databases, and domains.",
5
5
  "type": "module",
6
6
  "license": "MIT",
7
7
  "author": "muhkoo",
8
- "repository": "https://github.com/muhkoo/cli.git",
8
+ "repository": {
9
+ "type": "git",
10
+ "url": "git+https://github.com/muhkoo/cli.git"
11
+ },
9
12
  "homepage": "https://docs.muhkoo.dev",
10
13
  "bugs": "https://github.com/muhkoo/cli/issues",
11
14
  "keywords": [
@@ -40,7 +43,7 @@
40
43
  "release": "npm publish --tag alpha && npm dist-tag add @muhkoo/cli@$npm_package_version latest"
41
44
  },
42
45
  "dependencies": {
43
- "@muhkoo/connect": "0.15.0-alpha.5",
46
+ "@muhkoo/connect": "0.15.0-alpha.7",
44
47
  "snarkjs": "^0.7.0"
45
48
  },
46
49
  "packageManager": "yarn@1.22.22+sha512.a6b2f7906b721bba3d67d4aff083df04dad64c399707841b7acf00f6b133b7ac24255f2652fa22ae3534329dc6180534e98d17432037ff6fd140556e2bb3137e",
package/src/lib/mount.js CHANGED
@@ -42,7 +42,7 @@ import { constants as fsConstants, watch } from "node:fs";
42
42
  import { homedir, tmpdir } from "node:os";
43
43
  import { basename, dirname, isAbsolute, join, parse, relative, resolve, sep } from "node:path";
44
44
 
45
- import { holdMassDeletes, planSync } from "./syncPlan.js";
45
+ import { DIR, dirKey, filesOf, holdMassDeletes, planDirs, planSync } from "./syncPlan.js";
46
46
  import {
47
47
  DEPTH_MAX, IgnoreBudgetError, NAME_MAX, caseFold, ignoreFileProblem, ruleBank, ruleText, withinMatchBounds,
48
48
  } from "./ignore.js";
@@ -77,12 +77,40 @@ const KEEP_HIDDEN = new Set([".vcsignore", ".gitignore"]);
77
77
  const MAX_BYTES = 100 * 1024 * 1024;
78
78
 
79
79
  /**
80
- * How many files to read at once.
80
+ * How many files the scan reads at once.
81
81
  *
82
82
  * Enough to hide per-request latency; low enough not to open a connection per
83
- * file on a large project.
83
+ * file on a large project. A read costs 0.3–0.5 s against production, nearly
84
+ * all of it fixed, so throughput comes from overlapping them: 96 small files
85
+ * took 6.35 s eight at a time, 4.50 s at sixteen, and 4.12 s at thirty-two.
84
86
  */
85
- const READ_CONCURRENCY = 8;
87
+ export const SCAN_CONCURRENCY = 16;
88
+
89
+ /**
90
+ * And the most the files being read may claim between them — by their
91
+ * records' sizes, and their manifests' — before another read waits for one
92
+ * to finish. So raising the count does not raise the memory a run of large
93
+ * files holds at once. A file that claims more than this on its own is read
94
+ * alone: nothing else starts until it is done.
95
+ */
96
+ export const SCAN_WINDOW_BYTES = 64 * 1024 * 1024;
97
+
98
+ /**
99
+ * How many pulls to fetch at once while the oldest lands (see `applyPlan`).
100
+ *
101
+ * A pull the scan did not keep the bytes of — past BLOB_CACHE_BYTES, or a
102
+ * follow-up pass — is a round trip of its own, and one at a time they cost
103
+ * the SUM of the latencies. Only the fetches overlap; every pull still lands
104
+ * one at a time, in plan order.
105
+ */
106
+ const PULL_CONCURRENCY = 8;
107
+
108
+ /**
109
+ * And the most the records of the pulls being fetched may claim between
110
+ * them before the window waits for one to land — so a run of large files is
111
+ * not held in memory eight at a time. One is always let through.
112
+ */
113
+ const PULL_WINDOW_BYTES = 64 * 1024 * 1024;
86
114
 
87
115
  /**
88
116
  * The most one pass pulls: bytes and files. Past it, the rest wait for a
@@ -195,6 +223,50 @@ async function saveState(dir, root, paths) {
195
223
  await writeAtomic(statePath(dir), JSON.stringify({ v: 1, root, paths }, null, 2) + "\n");
196
224
  }
197
225
 
226
+ /**
227
+ * What the scan has hashed, kept between runs, so a restarted mount does not
228
+ * download the whole tree again just to learn it has not changed.
229
+ *
230
+ * It maps a content key — the SHA-256 of a file's manifest, which names the
231
+ * exact shards and the key that decrypts them, so one key is one content — to
232
+ * that content's hash and size. Unlike `mount.json` it decides nothing on its
233
+ * own: an entry only spares a read. So a file that cannot be read, or an entry
234
+ * that is not in the shape written here, is ignored, and that content is read.
235
+ */
236
+ const hashCachePath = (dir) => join(dir, ".muhkoo", "content-hashes.json");
237
+ const HEX64 = /^[0-9a-f]{64}$/;
238
+ /** Entries kept: well past any tree the read budget lets one pass hash. */
239
+ const HASH_CACHE_MAX = 200_000;
240
+
241
+ async function loadHashCache(dir) {
242
+ const hashes = new Map();
243
+ try {
244
+ const saved = JSON.parse(await readFile(hashCachePath(dir), "utf8"));
245
+ if (saved?.v !== 1 || !saved.contents || typeof saved.contents !== "object") return hashes;
246
+ for (const [key, entry] of Object.entries(saved.contents)) {
247
+ if (hashes.size >= HASH_CACHE_MAX) break;
248
+ const [hash, size] = Array.isArray(entry) ? entry : [];
249
+ if (!HEX64.test(key) || !HEX64.test(hash ?? "") || !Number.isSafeInteger(size) || size < 0) return new Map();
250
+ hashes.set(key, { hash, size });
251
+ }
252
+ } catch {
253
+ return new Map(); // none yet, or unreadable: the scan reads, as it always did
254
+ }
255
+ return hashes;
256
+ }
257
+
258
+ function sameHashes(a, b) {
259
+ if (a.size !== b.size) return false;
260
+ for (const [key, { hash }] of a) if (b.get(key)?.hash !== hash) return false;
261
+ return true;
262
+ }
263
+
264
+ async function saveHashCache(dir, hashes) {
265
+ const contents = {};
266
+ for (const [key, { hash, size }] of hashes) contents[key] = [hash, size];
267
+ await writeAtomic(hashCachePath(dir), JSON.stringify({ v: 1, contents }) + "\n");
268
+ }
269
+
198
270
  function sameState(a, b) {
199
271
  const keys = Object.keys(a);
200
272
  return keys.length === Object.keys(b).length && keys.every((k) => a[k] === b[k]);
@@ -941,6 +1013,7 @@ export async function foldsCaseReadOnly(dir) {
941
1013
  */
942
1014
  async function scanLocal(dir, { hidden = false, ignores, sess, signal }) {
943
1015
  const files = {};
1016
+ const dirs = [];
944
1017
  const unknown = new Set();
945
1018
  const held = new Set();
946
1019
  const pace = pacer();
@@ -978,6 +1051,7 @@ async function scanLocal(dir, { hidden = false, ignores, sess, signal }) {
978
1051
  if (verdict === "held") held.add(entry.isDirectory() ? `${key}/` : key);
979
1052
  if (verdict) continue;
980
1053
  if (entry.isDirectory()) {
1054
+ dirs.push(key);
981
1055
  await walk(child, rel);
982
1056
  continue;
983
1057
  }
@@ -1003,7 +1077,7 @@ async function scanLocal(dir, { hidden = false, ignores, sess, signal }) {
1003
1077
  }
1004
1078
  };
1005
1079
  await walk(dir, "");
1006
- return { files, unknown, held };
1080
+ return { files, dirs, unknown, held };
1007
1081
  }
1008
1082
 
1009
1083
  /**
@@ -1016,8 +1090,35 @@ async function scanLocal(dir, { hidden = false, ignores, sess, signal }) {
1016
1090
  * but this CLI may run against either.
1017
1091
  */
1018
1092
  async function walkRemote(vfs, root) {
1019
- if (root !== "/" && !(await vfs.exists(root))) return [];
1020
- return vfs.walk(root);
1093
+ // `dirs` is null when the folders cannot be told: a root that is not there
1094
+ // (missing or renamed — never "every folder deleted"), or an SDK without
1095
+ // `walkEntries`. Folders are then left alone.
1096
+ if (root !== "/" && !(await vfs.exists(root))) return { paths: [], dirs: null };
1097
+ if (typeof vfs.walkEntries !== "function") return { paths: await vfs.walk(root), dirs: null };
1098
+ const entries = await vfs.walkEntries(root);
1099
+ return {
1100
+ paths: entries.filter((e) => e.kind === "file").map((e) => e.path),
1101
+ dirs: entries.filter((e) => e.kind === "dir").map((e) => e.path),
1102
+ };
1103
+ }
1104
+
1105
+ /**
1106
+ * The remote folders, keyed like the local ones, through the same admission
1107
+ * rule: a refused, ignored or hidden folder is not made here, any more than a
1108
+ * file in it would be.
1109
+ */
1110
+ function remoteDirs(dirs, root, { hidden, ignores, ignoreCase }) {
1111
+ if (!dirs) return null;
1112
+ const out = [];
1113
+ for (const abs of dirs) {
1114
+ const key = root === "/" ? abs : abs.slice(root.length);
1115
+ if (!key.startsWith("/") || key === "/") continue;
1116
+ const rel = key.slice(1);
1117
+ if (!withinMatchBounds(rel, ignoreCase)) continue;
1118
+ if (admission(rel, true, hidden, ignores.now, ignores.held)) continue;
1119
+ out.push(key);
1120
+ }
1121
+ return out;
1021
1122
  }
1022
1123
 
1023
1124
  /**
@@ -1051,7 +1152,7 @@ export async function scanRemote(vfs, root, includeHidden = false, ignore = () =
1051
1152
  const unknown = new Set();
1052
1153
  const unreadable = new Set();
1053
1154
  const held = new Set();
1054
- const paths = ctx.paths ?? (await walkRemote(vfs, root));
1155
+ const paths = ctx.paths ?? (await walkRemote(vfs, root)).paths;
1055
1156
 
1056
1157
  // Decide what to read BEFORE reading anything, so the filters are applied
1057
1158
  // once and the reads that survive them can go out together.
@@ -1110,12 +1211,18 @@ export async function scanRemote(vfs, root, includeHidden = false, ignore = () =
1110
1211
  const readPace = pacer(); // reads from a cache resolve at once, and never yield on their own
1111
1212
  const seen = new Map(); // content key → { hash, size }, for the next pass
1112
1213
  const reading = new Map(); // content key → the read under way
1214
+ const oversize = new Map(); // content key → its length, for a read that came back over MAX_BYTES
1113
1215
  const blobs = ctx.blobs ?? blobCache(BLOB_CACHE_BYTES);
1114
1216
  const budget = sess.readBudget;
1115
1217
  const spent = ctx.spent ?? { bytes: 0, files: 0 };
1116
1218
  const deferred = new Set();
1219
+ // Bytes in flight are bounded as well as reads (SCAN_WINDOW_BYTES), by the
1220
+ // same claim the budget is charged. A read waiting for room is not
1221
+ // started once the scan is stopped.
1222
+ const window = byteWindow(SCAN_WINDOW_BYTES);
1223
+ const STOPPED = Symbol("stopped");
1117
1224
  await Promise.all(
1118
- Array.from({ length: Math.min(READ_CONCURRENCY, queue.length) }, async () => {
1225
+ Array.from({ length: Math.min(SCAN_CONCURRENCY, queue.length) }, async () => {
1119
1226
  for (let item = queue.shift(); item !== undefined && !ctx.signal?.aborted; item = queue.shift()) {
1120
1227
  const breather = readPace();
1121
1228
  if (breather) await breather;
@@ -1140,6 +1247,12 @@ export async function scanRemote(vfs, root, includeHidden = false, ignore = () =
1140
1247
  sizes[key] = known.size;
1141
1248
  continue;
1142
1249
  }
1250
+ const over = content.key && oversize.get(content.key);
1251
+ if (over) {
1252
+ tooBig(sess, key, over);
1253
+ unknown.add(key);
1254
+ continue;
1255
+ }
1143
1256
  let read = content.key && reading.get(content.key);
1144
1257
  if (!read) {
1145
1258
  // Ignore files included: a 256 KB file of comments is
@@ -1154,16 +1267,37 @@ export async function scanRemote(vfs, root, includeHidden = false, ignore = () =
1154
1267
  spent.files++;
1155
1268
  spent.bytes += claim;
1156
1269
  }
1157
- read = content.read().then((bytes) => {
1158
- spent.bytes += bytes.length - claim;
1159
- return bytes;
1160
- });
1270
+ read = (async () => {
1271
+ const admitted = await window.acquire(claim, ctx.signal);
1272
+ if (admitted && ctx.signal?.aborted) window.release(claim); // let in, but stopped since
1273
+ if (!admitted || ctx.signal?.aborted) {
1274
+ if (budget) { // never read: give back what it was charged
1275
+ spent.files--;
1276
+ spent.bytes -= claim;
1277
+ }
1278
+ throw STOPPED;
1279
+ }
1280
+ try {
1281
+ const bytes = await content.read();
1282
+ spent.bytes += bytes.length - claim;
1283
+ return bytes;
1284
+ } finally {
1285
+ window.release(claim);
1286
+ }
1287
+ })();
1161
1288
  if (content.key) reading.set(content.key, read);
1162
1289
  }
1163
1290
  const bytes = await read;
1164
1291
  if (bytes.length > MAX_BYTES) {
1165
1292
  tooBig(sess, key, bytes.length);
1166
1293
  unknown.add(key);
1294
+ if (content.key) {
1295
+ // Only its length is kept: entries sharing this
1296
+ // content get the same answer without the read
1297
+ // (up to 100 MB) staying referenced.
1298
+ oversize.set(content.key, bytes.length);
1299
+ reading.delete(content.key);
1300
+ }
1167
1301
  continue;
1168
1302
  }
1169
1303
  const hash = sha(bytes);
@@ -1172,8 +1306,18 @@ export async function scanRemote(vfs, root, includeHidden = false, ignore = () =
1172
1306
  if (content.key) {
1173
1307
  seen.set(content.key, { hash, size: bytes.length });
1174
1308
  blobs.put(content.key, bytes);
1309
+ // `seen` answers for this content from here on. Left
1310
+ // in `reading`, the settled read held every byte the
1311
+ // pass read, past the blob cache's limit and the window.
1312
+ reading.delete(content.key);
1175
1313
  }
1176
1314
  } catch (err) {
1315
+ // Stopped before its read began: not read, and nothing to
1316
+ // warn of. The caller does not plan from a stopped scan.
1317
+ if (err === STOPPED) {
1318
+ unknown.add(key);
1319
+ continue;
1320
+ }
1177
1321
  // One unreadable entry — a lost shard, a planted manifest, a
1178
1322
  // dropped request — must not stop the whole sync, and must
1179
1323
  // not read as a deletion either.
@@ -1213,10 +1357,11 @@ export async function scanRemote(vfs, root, includeHidden = false, ignore = () =
1213
1357
  */
1214
1358
  async function remoteContent(vfs, abs) {
1215
1359
  if (typeof vfs.statManifest !== "function") {
1216
- const { size } = await vfs.stat(abs);
1360
+ const size = recordSize((await vfs.stat(abs)).size);
1217
1361
  return { size, key: null, bound: null, read: () => vfs.readFile(abs) };
1218
1362
  }
1219
- const { manifest, size } = await vfs.statManifest(abs);
1363
+ const { manifest, size: claimed } = await vfs.statManifest(abs);
1364
+ const size = recordSize(claimed);
1220
1365
  const exact = typeof vfs.readByManifest === "function";
1221
1366
  return {
1222
1367
  size,
@@ -1226,6 +1371,15 @@ async function remoteContent(vfs, abs) {
1226
1371
  };
1227
1372
  }
1228
1373
 
1374
+ /**
1375
+ * A record's size, as a number the budgets can add. The record is written by
1376
+ * the remote party, so the size may be absent, a string, negative or NaN; any
1377
+ * of those counts as 0 (the bytes that come back are what is held to the limit).
1378
+ */
1379
+ function recordSize(size) {
1380
+ return Number.isSafeInteger(size) && size >= 0 ? size : 0;
1381
+ }
1382
+
1229
1383
  /**
1230
1384
  * What a manifest lets a read bring back (see `remoteContent`), or null when
1231
1385
  * it is not in a shape this knows. Each chunk decrypts to its ciphertext less
@@ -1246,6 +1400,63 @@ function manifestBound(manifest) {
1246
1400
  return { plain, fetched };
1247
1401
  }
1248
1402
 
1403
+ /**
1404
+ * A first-come, first-served allowance of `limit` bytes for reads in flight.
1405
+ *
1406
+ * `acquire(n, signal)` resolves true once `n` more bytes fit — or at once
1407
+ * when nothing is held, so one claim larger than the whole window is still
1408
+ * let through, alone — and false, without taking anything, when `signal`
1409
+ * has fired. Waiters are served in order, so a large claim is not starved by
1410
+ * small ones arriving behind it. `release(n)` gives back what an acquire took.
1411
+ */
1412
+ function byteWindow(limit) {
1413
+ let held = 0;
1414
+ // A claim that is not a finite, non-negative number is 0: NaN in `held`
1415
+ // would make nothing fit again, and the scan would never finish.
1416
+ const sane = (n) => (Number.isFinite(n) && n > 0 ? n : 0);
1417
+ const waiting = [];
1418
+ const fits = (n) => held === 0 || held + n <= limit;
1419
+ const wake = () => {
1420
+ while (waiting.length && fits(waiting[0].n)) {
1421
+ const w = waiting.shift();
1422
+ held += w.n;
1423
+ w.done(true);
1424
+ }
1425
+ };
1426
+ return {
1427
+ acquire(claim, signal) {
1428
+ const n = sane(claim);
1429
+ if (signal?.aborted) return Promise.resolve(false);
1430
+ if (!waiting.length && fits(n)) {
1431
+ held += n;
1432
+ return Promise.resolve(true);
1433
+ }
1434
+ return new Promise((resolve) => {
1435
+ const onAbort = () => {
1436
+ const i = waiting.indexOf(w);
1437
+ if (i < 0) return;
1438
+ waiting.splice(i, 1);
1439
+ resolve(false);
1440
+ wake(); // the head may have been what held the rest
1441
+ };
1442
+ const w = {
1443
+ n,
1444
+ done: (ok) => {
1445
+ signal?.removeEventListener("abort", onAbort);
1446
+ resolve(ok);
1447
+ },
1448
+ };
1449
+ waiting.push(w);
1450
+ signal?.addEventListener("abort", onAbort, { once: true });
1451
+ });
1452
+ },
1453
+ release(claim) {
1454
+ held -= sane(claim);
1455
+ wake();
1456
+ },
1457
+ };
1458
+ }
1459
+
1249
1460
  /**
1250
1461
  * The bytes a pass has read, by content key, up to `limit` bytes: the oldest
1251
1462
  * go first. What lets a pull of content the scan already read — two hundred
@@ -1461,7 +1672,7 @@ export function win32NameProblem(name) {
1461
1672
  * directories to create, in order, and `existing` the stats of the file being
1462
1673
  * replaced — or `{ ok: false, reason }`. `listing` caches `readdir` for a pass.
1463
1674
  */
1464
- export async function writeTarget(dir, key, { localKeys = new Set(), listing = new Map(), platform = process.platform } = {}) {
1675
+ export async function writeTarget(dir, key, { localKeys = new Set(), listing = new Map(), platform = process.platform, kind = "file" } = {}) {
1465
1676
  const refuse = (reason) => ({ ok: false, reason });
1466
1677
  const names = key.split("/").filter(Boolean);
1467
1678
  if (!names.length) return refuse("it names the mount directory itself");
@@ -1487,7 +1698,8 @@ export async function writeTarget(dir, key, { localKeys = new Set(), listing = n
1487
1698
  if (!entries) return refuse(`${parent} could not be listed`);
1488
1699
  if (!entries.has(name)) {
1489
1700
  if (await present(abs)) return alias(name);
1490
- return { ok: true, abs: join(parent, ...names.slice(i)), parent, missing: names.slice(i, -1), existing: null };
1701
+ // A folder is made whole, its own name included; a file's last name is the file.
1702
+ return { ok: true, abs: join(parent, ...names.slice(i)), parent, missing: kind === "dir" ? names.slice(i) : names.slice(i, -1), existing: null };
1491
1703
  }
1492
1704
  const info = await lstat(abs).catch(() => null);
1493
1705
  if (!last) {
@@ -1495,6 +1707,10 @@ export async function writeTarget(dir, key, { localKeys = new Set(), listing = n
1495
1707
  parent = abs;
1496
1708
  continue;
1497
1709
  }
1710
+ if (kind === "dir") {
1711
+ if (!info?.isDirectory()) return refuse(`${relative(dir, abs)} is ${info?.isSymbolicLink() ? "a symlink" : "not a directory"}`);
1712
+ return { ok: true, abs, parent, missing: [], existing: info };
1713
+ }
1498
1714
  if (!info?.isFile()) return refuse(`${relative(dir, abs)} is a symlink or not a regular file`);
1499
1715
  if (!localKeys.has(key)) return refuse(`${relative(dir, abs)} exists here but is not synced (ignored, hidden or refused), so it may exist nowhere else`);
1500
1716
  return { ok: true, abs, parent, missing: [], existing: info };
@@ -1514,14 +1730,7 @@ export async function writeTarget(dir, key, { localKeys = new Set(), listing = n
1514
1730
  * default mode, never an execute bit.
1515
1731
  */
1516
1732
  async function writeAt(target, bytes, listing) {
1517
- let parent = target.parent;
1518
- for (const name of target.missing) {
1519
- const next = join(parent, name);
1520
- await mkdir(next); // not recursive: each component is created here, never followed
1521
- listing.get(parent)?.add(name);
1522
- listing.set(next, new Set());
1523
- parent = next;
1524
- }
1733
+ const parent = await makeMissing(target, listing);
1525
1734
  const tmp = join(parent, TMP_PREFIX + randomBytes(6).toString("hex"));
1526
1735
  try {
1527
1736
  await writeFile(tmp, bytes, { flag: "wx" });
@@ -1534,6 +1743,114 @@ async function writeAt(target, bytes, listing) {
1534
1743
  listing.get(parent)?.add(target.abs.slice(parent.length + 1));
1535
1744
  }
1536
1745
 
1746
+ /**
1747
+ * Apply the folder plan (`planDirs`), after the files: folders are made
1748
+ * shallowest first and removed deepest first, so a folder emptied by this
1749
+ * pass's file deletes is gone by its end.
1750
+ *
1751
+ * A folder is only ever removed EMPTY — `rmdir` here, a delete that is not
1752
+ * recursive there — so removing one can never take a file with it, synced or
1753
+ * not. One that still holds something (an ignored or refused file, or one that
1754
+ * arrived since the scan) is kept, said once, and left out of the state: the
1755
+ * next pass sees it on one side only and makes it on the other again, which is
1756
+ * the truth, since it is not empty. Removals are gated by `--allow-delete` as
1757
+ * file deletes are; a removal not made is forgotten, as theirs is.
1758
+ */
1759
+ async function applyDirPlan({ vfs, dir, root, state, plan, opts, sess, signal }) {
1760
+ const next = { ...state };
1761
+ const listing = new Map();
1762
+ const remoteAbs = (path) => (root === "/" ? path : root + path);
1763
+ const depth = (path) => path.split("/").length;
1764
+ const makes = plan.filter(({ action }) => !action.startsWith("rmdir")).sort((a, b) => depth(a.path) - depth(b.path));
1765
+ const removes = plan.filter(({ action }) => action.startsWith("rmdir")).sort((a, b) => depth(b.path) - depth(a.path));
1766
+ const kept = (path, where) => warnOnce(sess, `dir-kept:${where}:${path}`, `kept the folder ${path}/ ${where} — it still holds files that are not synced`);
1767
+ for (const { path, action } of [...makes, ...removes]) {
1768
+ if (signal?.aborted) break;
1769
+ const key = dirKey(path);
1770
+ try {
1771
+ switch (action) {
1772
+ case "record":
1773
+ next[key] = DIR;
1774
+ break;
1775
+ case "forget":
1776
+ delete next[key];
1777
+ break;
1778
+ case "mkdir-remote":
1779
+ await vfs.mkdir(remoteAbs(path), { recursive: true });
1780
+ next[key] = DIR;
1781
+ info(`${SIGN.push} ${path}/`);
1782
+ break;
1783
+ case "mkdir-local": {
1784
+ const target = await writeTarget(dir, path, { listing, kind: "dir" });
1785
+ if (!target.ok) {
1786
+ warnOnce(sess, `declined:${action}:${path}`, `skipping ${path}/ — ${target.reason}`);
1787
+ break;
1788
+ }
1789
+ if (!target.existing) await makeMissing(target, listing);
1790
+ next[key] = DIR;
1791
+ info(`${SIGN.pull} ${path}/`);
1792
+ break;
1793
+ }
1794
+ case "rmdir-remote": {
1795
+ delete next[key];
1796
+ if (!opts.allowDelete) {
1797
+ warn(
1798
+ `${path}/ is gone locally but still in your filesystem — not deleting it.`,
1799
+ ` Pass --allow-delete to propagate deletions, or remove it with: muhkoo vfs rm -r ${remoteAbs(path)}`,
1800
+ );
1801
+ break;
1802
+ }
1803
+ const found = await vfs.stat(remoteAbs(path)).catch(() => null);
1804
+ if (found?.kind !== "dir") break;
1805
+ try {
1806
+ await vfs.delete(remoteAbs(path)); // not recursive: refused unless empty
1807
+ info(`${SIGN["delete-remote"]} ${path}/ (deleted from your filesystem)`);
1808
+ } catch {
1809
+ kept(path, "in your filesystem");
1810
+ }
1811
+ break;
1812
+ }
1813
+ case "rmdir-local": {
1814
+ delete next[key];
1815
+ if (!opts.allowDelete) {
1816
+ warn(`${path}/ is gone from your filesystem but still here — not deleting it locally.`, ` Pass --allow-delete to propagate deletions.`);
1817
+ break;
1818
+ }
1819
+ const target = await writeTarget(dir, path, { listing, kind: "dir" });
1820
+ if (!target.ok || !target.existing) break;
1821
+ try {
1822
+ await rmdir(target.abs);
1823
+ listing.get(target.parent)?.delete(target.abs.slice(target.parent.length + 1));
1824
+ listing.delete(target.abs);
1825
+ info(`${SIGN["delete-local"]} ${path}/ (deleted from this disk)`);
1826
+ } catch (err) {
1827
+ if (err.code !== "ENOTEMPTY" && err.code !== "EEXIST") throw err;
1828
+ kept(path, "on this disk");
1829
+ }
1830
+ break;
1831
+ }
1832
+ }
1833
+ } catch (err) {
1834
+ const message = err?.message ?? String(err);
1835
+ warnOnce(sess, `error:${path}/:${message}`, `${path}/: ${message}`);
1836
+ }
1837
+ }
1838
+ return next;
1839
+ }
1840
+
1841
+ /** Make the directories `writeTarget` found missing, in order; returns the deepest. */
1842
+ async function makeMissing(target, listing) {
1843
+ let parent = target.parent;
1844
+ for (const name of target.missing) {
1845
+ const next = join(parent, name);
1846
+ await mkdir(next); // not recursive: each component is created here, never followed
1847
+ listing.get(parent)?.add(name);
1848
+ listing.set(next, new Set());
1849
+ parent = next;
1850
+ }
1851
+ return parent;
1852
+ }
1853
+
1537
1854
  // ---- planning and applying ------------------------------------------------------
1538
1855
 
1539
1856
  /**
@@ -1556,7 +1873,7 @@ async function preparePass({ vfs, dir, rulesDir = dir, root, state, opts, sess,
1556
1873
  // The remote walk is the slow part; start it while the local scan runs.
1557
1874
  // The remote FILTER has to wait for the local scan, which is what finds
1558
1875
  // the nested ignore files.
1559
- const [local, paths] = await Promise.all([
1876
+ const [local, listed] = await Promise.all([
1560
1877
  scanLocal(dir, { hidden: opts.hidden, ignores, sess, signal }),
1561
1878
  walkRemote(vfs, root),
1562
1879
  ]);
@@ -1564,7 +1881,7 @@ async function preparePass({ vfs, dir, rulesDir = dir, root, state, opts, sess,
1564
1881
  // The ignore files as they now stand are known; nothing has been pulled.
1565
1882
  await ignores.record(rulesDir, root, save);
1566
1883
  const remote = await scanRemote(vfs, root, opts.hidden, ignores.now, {
1567
- paths, held: ignores.held, session: sess, ignoreCase: sess.ignoreCase,
1884
+ paths: listed.paths, held: ignores.held, session: sess, ignoreCase: sess.ignoreCase,
1568
1885
  localKeys: new Set(Object.keys(local.files)), signal, blobs, spent: read,
1569
1886
  });
1570
1887
  if (signal?.aborted) return null;
@@ -1585,13 +1902,24 @@ async function preparePass({ vfs, dir, rulesDir = dir, root, state, opts, sess,
1585
1902
  if (keptBack.length) warnOnce(sess, `kept-back:${keptBack.join("\0")}`, ...heldNotice(keptBack, rulesDir));
1586
1903
 
1587
1904
  const unknown = new Set([...local.unknown, ...remote.unknown]);
1588
- const planned = planSync({ last: state, local: local.files, remote: remote.files, unknown });
1905
+ const files = filesOf(state);
1906
+ const planned = planSync({ last: files, local: local.files, remote: remote.files, unknown });
1589
1907
  const views = { local: Object.keys(local.files).length, remote: Object.keys(remote.files).length };
1590
- sess.peak = Math.max(sess.peak ?? 0, Object.keys(state).length);
1591
- const { plan, held } = holdMassDeletes(planned, state, views, sess.peak);
1908
+ sess.peak = Math.max(sess.peak ?? 0, Object.keys(files).length);
1909
+ const { plan, held } = holdMassDeletes(planned, files, views, sess.peak);
1592
1910
  for (const h of held) warnOnce(sess, `held:${h.action}:${h.count}`, ...breakerMessage(h, dir, root));
1911
+ // Folders after files, by the same rule. When the breaker held file
1912
+ // deletes, no folder is removed either: the same pass would empty them.
1913
+ let dirPlan = planDirs({
1914
+ last: state,
1915
+ local: local.dirs,
1916
+ remote: remoteDirs(listed.dirs, root, { hidden: opts.hidden, ignores, ignoreCase: sess.ignoreCase }),
1917
+ unknown,
1918
+ fold: sess.ignoreCase ? caseFold : null,
1919
+ });
1920
+ if (held.length) dirPlan = dirPlan.filter(({ action }) => !action.startsWith("rmdir"));
1593
1921
  return {
1594
- plan, local: local.files, remote: remote.files, sizes: remote.sizes, blobs: remote.blobs, ignores,
1922
+ plan, dirPlan, local: local.files, remote: remote.files, sizes: remote.sizes, blobs: remote.blobs, ignores,
1595
1923
  unreadable: remote.unreadable, deferred: remote.deferred,
1596
1924
  landed: new Set(), // ignore files pulled in this pass
1597
1925
  notLanded: new Map(), // and why the others were not
@@ -1617,10 +1945,22 @@ function breakerMessage({ action, count, tracked, empty }, dir, root) {
1617
1945
  /**
1618
1946
  * Apply a plan, and return the new last-synced view.
1619
1947
  *
1620
- * Applied sequentially rather than in parallel: writes touch one directory
1621
- * record each, and two concurrent writes to the same record would have one
1622
- * silently lose ([[VfsNamespace]] consistency note). `signal` stops it between
1623
- * steps — never inside one — and what was done by then is what it returns.
1948
+ * Every step that changes something — a write to the filesystem, a delete, a
1949
+ * file landing on this disk — is made one at a time, in plan order: writes
1950
+ * touch one directory record each, and two concurrent writes to the same
1951
+ * record would have one silently lose ([[VfsNamespace]] consistency note).
1952
+ * Pushes, deletes and conflicts are applied exactly as before, in sequence.
1953
+ *
1954
+ * Only the FETCH of a pull overlaps: up to PULL_CONCURRENCY pulls, in plan
1955
+ * order, are fetched while the oldest lands. A pull reads nothing but the
1956
+ * network and the pass's blob cache, so overlapping them changes no state;
1957
+ * landing stays in plan order, on the same checks, so what lands — and which
1958
+ * of two spellings a folding volume refuses — is what a sequential pass would
1959
+ * land. A step that is not a pull first lands every pull before it.
1960
+ *
1961
+ * `signal` stops it between steps — never inside one — and what was done by
1962
+ * then is what it returns. Pulls fetched but not yet landed when it fires are
1963
+ * dropped, unrecorded, and the next pass plans them again.
1624
1964
  *
1625
1965
  * A step this disk declines (a pull onto a symlink, a remote file where a
1626
1966
  * directory stands) is warned about once and left out of the result, so the
@@ -1635,9 +1975,139 @@ async function applyPlan({ vfs, dir, root, state, plan, pass, opts, sess, signal
1635
1975
  const listing = new Map();
1636
1976
  const localKeys = new Set(Object.keys(pass.local));
1637
1977
  const decline = (action, path, reason) => warnOnce(sess, `declined:${action}:${path}`, `skipping ${path} — ${reason}`);
1978
+ const failed = (action, path, err) => {
1979
+ const message = err?.message ?? String(err);
1980
+ warnOnce(sess, `error:${path}:${message}`, `${path}: ${message}`);
1981
+ if (action === "pull" && isIgnoreFile(path.slice(1))) pass.notLanded?.set(path, message);
1982
+ };
1638
1983
  let short = 0; // pulls held for want of disk space
1639
1984
  let over = 0; // and past the pass's budget
1640
1985
 
1986
+ // ---- pulls: admitted and fetched ahead, landed in order ----
1987
+ //
1988
+ // Held pulls change nothing, so the next pass plans them again. Each limit
1989
+ // is checked twice: on the size the scan read, so a pull that cannot fit is
1990
+ // not fetched at all, and then on the bytes actually fetched — the file may
1991
+ // have changed since, and they are what land on this disk.
1992
+ //
1993
+ // A pull is RESERVED against both limits when it is admitted, before it is
1994
+ // fetched: `inFlight` is what has been admitted and not yet landed. Without
1995
+ // it, pulls checked together, before any had landed, would each see the
1996
+ // same free space and the same budget, and together overshoot both.
1997
+ //
1998
+ // Ignore files are held to the same limits. They were exempt as "small",
1999
+ // and a 256 KB file of comments, copied to thousands of directories, put
2000
+ // 7 GiB on a disk already under its reserve in one pass. One held here
2001
+ // holds this pass's uploads instead (see `reconcile`).
2002
+ const limit = sess.pullBudget;
2003
+ const inFlight = { files: 0, bytes: 0 };
2004
+ const window = []; // pulls being fetched, in plan order
2005
+ const fetching = new Map(); // content key → its fetch, so two entries sharing it fetch it once
2006
+ /**
2007
+ * Why a pull of `size` more bytes must wait, or null. `admitted`: the pull
2008
+ * was let in already, so a budget that has filled behind it — with pulls
2009
+ * admitted after it, which it lands ahead of — does not stop it; only its
2010
+ * own bytes, with what is still in flight, going over does.
2011
+ */
2012
+ const holdFor = async (size, admitted = false) => {
2013
+ const files = budget.files + inFlight.files;
2014
+ if ((budget.full && !admitted) || (files && (files + 1 > limit.files || budget.bytes + inFlight.bytes + size > limit.bytes))) {
2015
+ budget.full = true;
2016
+ over++;
2017
+ return "this pass has pulled all it may; the next pass carries on";
2018
+ }
2019
+ if (await roomFor(dir, inFlight.bytes + size, sess)) return null;
2020
+ short++;
2021
+ return "the disk would be left with less than its reserve free";
2022
+ };
2023
+ /** The bytes of `remoteAbs`. Content the scan read — or an earlier pull, of an entry sharing it — is not fetched again. */
2024
+ const fetchBytes = async (remoteAbs) => {
2025
+ const content = await remoteContent(vfs, remoteAbs);
2026
+ const kept = pass.blobs?.get(content.key);
2027
+ if (kept) return kept;
2028
+ let read = content.key ? fetching.get(content.key) : undefined;
2029
+ if (!read) {
2030
+ read = content.read();
2031
+ if (content.key) {
2032
+ fetching.set(content.key, read);
2033
+ read.then(() => fetching.delete(content.key), () => fetching.delete(content.key));
2034
+ }
2035
+ }
2036
+ const bytes = await read;
2037
+ pass.blobs?.put(content.key, bytes);
2038
+ return bytes;
2039
+ };
2040
+ /** Admit a pull and start fetching it; null when it is held or declined here. */
2041
+ const startPull = async (path, remoteAbs) => {
2042
+ const claim = pass.sizes?.[path] ?? 0;
2043
+ const hold = await holdFor(claim);
2044
+ if (hold) {
2045
+ pass.notLanded?.set(path, hold);
2046
+ return null;
2047
+ }
2048
+ // A target this disk refuses is not fetched at all. Checked again when
2049
+ // it lands: the pulls landing ahead of it may have changed the answer.
2050
+ const target = await writeTarget(dir, path, { localKeys, listing });
2051
+ if (!target.ok) {
2052
+ decline("pull", path, target.reason);
2053
+ pass.notLanded?.set(path, target.reason);
2054
+ return null;
2055
+ }
2056
+ inFlight.files++;
2057
+ inFlight.bytes += claim;
2058
+ const bytes = fetchBytes(remoteAbs);
2059
+ bytes.catch(() => {}); // its failure is reported when it lands
2060
+ return { path, remoteAbs, claim, bytes };
2061
+ };
2062
+ /** Land a fetched pull: the same checks a sequential pass made, then the write. */
2063
+ const land = async ({ path, remoteAbs, claim, bytes: fetched }) => {
2064
+ inFlight.files--;
2065
+ inFlight.bytes -= claim;
2066
+ const bytes = await fetched;
2067
+ if (bytes.length > MAX_BYTES) {
2068
+ tooBig(sess, path, bytes.length);
2069
+ pass.notLanded?.set(path, "it is over the mount's size limit");
2070
+ return;
2071
+ }
2072
+ const hold = await holdFor(bytes.length, true);
2073
+ if (hold) {
2074
+ pass.notLanded?.set(path, hold);
2075
+ return;
2076
+ }
2077
+ // An ignore file that cannot be read as rules is not pulled: landed, it
2078
+ // would stop every pass (see `see`), and uploads wait for it instead
2079
+ // (see `reconcile`).
2080
+ const rules = isIgnoreFile(path.slice(1));
2081
+ const problem = rules && ignoreFileProblem(bytes);
2082
+ if (problem) {
2083
+ decline("pull", path, `it cannot be read as ignore rules: ${problem}. If it is not yours, remove it: muhkoo vfs rm ${remoteAbs}`);
2084
+ pass.notLanded?.set(path, `it cannot be read as ignore rules: ${problem}`);
2085
+ return;
2086
+ }
2087
+ const target = await writeTarget(dir, path, { localKeys, listing });
2088
+ if (!target.ok) {
2089
+ decline("pull", path, target.reason);
2090
+ pass.notLanded?.set(path, target.reason);
2091
+ return;
2092
+ }
2093
+ await writeAt(target, bytes, listing);
2094
+ next[path] = sha(bytes);
2095
+ if (rules) pass.landed?.add(path);
2096
+ budget.files++;
2097
+ budget.bytes += bytes.length;
2098
+ info(`${SIGN.pull} ${path}`);
2099
+ };
2100
+ const landOldest = async () => {
2101
+ const pull = window.shift();
2102
+ try {
2103
+ await land(pull);
2104
+ } catch (err) {
2105
+ failed("pull", pull.path, err);
2106
+ }
2107
+ };
2108
+ /** Room for another pull in the window? Bounded in files, and — by what the records claim — in bytes held. */
2109
+ const windowFull = () => window.length >= PULL_CONCURRENCY || (window.length > 0 && inFlight.bytes >= PULL_WINDOW_BYTES);
2110
+
1641
2111
  for (const { path, action } of plan) {
1642
2112
  if (signal?.aborted) break;
1643
2113
  const localAbs = localPathFor(dir, path);
@@ -1649,6 +2119,21 @@ async function applyPlan({ vfs, dir, root, state, plan, pass, opts, sess, signal
1649
2119
  continue;
1650
2120
  }
1651
2121
  const remoteAbs = root === "/" ? path : root + path;
2122
+ if (action === "pull") {
2123
+ while (windowFull() && !signal?.aborted) await landOldest();
2124
+ if (signal?.aborted) break;
2125
+ try {
2126
+ const pull = await startPull(path, remoteAbs);
2127
+ if (pull) window.push(pull);
2128
+ } catch (err) {
2129
+ failed(action, path, err);
2130
+ }
2131
+ continue;
2132
+ }
2133
+ // Anything else waits for the pulls before it, so steps still take
2134
+ // effect in plan order.
2135
+ while (window.length && !signal?.aborted) await landOldest();
2136
+ if (signal?.aborted) break;
1652
2137
  try {
1653
2138
  switch (action) {
1654
2139
  case "push": {
@@ -1658,77 +2143,6 @@ async function applyPlan({ vfs, dir, root, state, plan, pass, opts, sess, signal
1658
2143
  info(`${SIGN.push} ${path}`);
1659
2144
  break;
1660
2145
  }
1661
- case "pull": {
1662
- // Held pulls change nothing, so the next pass plans them
1663
- // again. Each limit is checked twice: on the size the scan
1664
- // read, so a pull that cannot fit is not fetched at all, and
1665
- // then on the bytes actually fetched — the file may have
1666
- // changed since, and they are what land on this disk.
1667
- //
1668
- // Ignore files are held to the same limits. They were
1669
- // exempt as "small", and a 256 KB file of comments, copied
1670
- // to thousands of directories, put 7 GiB on a disk already
1671
- // under its reserve in one pass. One held here holds this
1672
- // pass's uploads instead (see `reconcile`).
1673
- const limit = sess.pullBudget;
1674
- /** Why a pull of `size` bytes must wait, or null. */
1675
- const holdFor = async (size) => {
1676
- if (budget.full || (budget.files && (budget.files + 1 > limit.files || budget.bytes + size > limit.bytes))) {
1677
- budget.full = true;
1678
- over++;
1679
- return "this pass has pulled all it may; the next pass carries on";
1680
- }
1681
- if (await roomFor(dir, size, sess)) return null;
1682
- short++;
1683
- return "the disk would be left with less than its reserve free";
1684
- };
1685
- let hold = await holdFor(pass.sizes?.[path] ?? 0);
1686
- if (hold) {
1687
- pass.notLanded?.set(path, hold);
1688
- break;
1689
- }
1690
- const target = await writeTarget(dir, path, { localKeys, listing });
1691
- if (!target.ok) {
1692
- decline(action, path, target.reason);
1693
- pass.notLanded?.set(path, target.reason);
1694
- break;
1695
- }
1696
- // Content the scan read — or an earlier pull, of an entry
1697
- // sharing it — is not fetched again.
1698
- const content = await remoteContent(vfs, remoteAbs);
1699
- let bytes = pass.blobs?.get(content.key);
1700
- if (!bytes) {
1701
- bytes = await content.read();
1702
- pass.blobs?.put(content.key, bytes);
1703
- }
1704
- if (bytes.length > MAX_BYTES) {
1705
- tooBig(sess, path, bytes.length);
1706
- pass.notLanded?.set(path, "it is over the mount's size limit");
1707
- break;
1708
- }
1709
- hold = await holdFor(bytes.length);
1710
- if (hold) {
1711
- pass.notLanded?.set(path, hold);
1712
- break;
1713
- }
1714
- // An ignore file that cannot be read as rules is not
1715
- // pulled: landed, it would stop every pass (see `see`), and
1716
- // uploads wait for it instead (see `reconcile`).
1717
- const rules = isIgnoreFile(path.slice(1));
1718
- const problem = rules && ignoreFileProblem(bytes);
1719
- if (problem) {
1720
- decline(action, path, `it cannot be read as ignore rules: ${problem}. If it is not yours, remove it: muhkoo vfs rm ${remoteAbs}`);
1721
- pass.notLanded?.set(path, `it cannot be read as ignore rules: ${problem}`);
1722
- break;
1723
- }
1724
- await writeAt(target, bytes, listing);
1725
- next[path] = sha(bytes);
1726
- if (rules) pass.landed?.add(path);
1727
- budget.files++;
1728
- budget.bytes += bytes.length;
1729
- info(`${SIGN.pull} ${path}`);
1730
- break;
1731
- }
1732
2146
  case "delete-remote":
1733
2147
  // Deletes do NOT propagate unless asked for.
1734
2148
  //
@@ -1795,11 +2209,17 @@ async function applyPlan({ vfs, dir, root, state, plan, pass, opts, sess, signal
1795
2209
  }
1796
2210
  }
1797
2211
  } catch (err) {
1798
- const message = err?.message ?? String(err);
1799
- warnOnce(sess, `error:${path}:${message}`, `${path}: ${message}`);
1800
- if (action === "pull" && isIgnoreFile(path.slice(1))) pass.notLanded?.set(path, message);
2212
+ failed(action, path, err);
1801
2213
  }
1802
2214
  }
2215
+ // Land what is still fetching — or, stopped, drop it: a fetch is not a
2216
+ // step, and nothing is recorded for a pull that did not land. Its fetch is
2217
+ // still waited for, so none runs on after the pass has returned.
2218
+ while (window.length && !signal?.aborted) await landOldest();
2219
+ if (window.length) {
2220
+ await Promise.allSettled(window.map((p) => p.bytes));
2221
+ window.length = 0;
2222
+ }
1803
2223
  if (budget.full) {
1804
2224
  sess.more = true;
1805
2225
  warnOnce(
@@ -1914,6 +2334,24 @@ function unreadRules(pass, state) {
1914
2334
  export async function reconcile({ vfs, dir, root, state, opts = {}, session: s, signal }) {
1915
2335
  const sess = session(s);
1916
2336
  if (signal?.aborted) return state;
2337
+ // A restart begins with what earlier runs hashed (see `loadHashCache`).
2338
+ if (!sess.hashCacheLoaded) {
2339
+ sess.hashCacheLoaded = true;
2340
+ if (sess.hashes.size === 0) sess.hashes = await loadHashCache(dir);
2341
+ sess.hashCacheSaved = new Map(sess.hashes);
2342
+ }
2343
+ try {
2344
+ return await reconcilePass({ vfs, dir, root, state, opts, sess, signal });
2345
+ } finally {
2346
+ // Written only when it changed: most passes hash nothing new.
2347
+ if (!sameHashes(sess.hashes, sess.hashCacheSaved)) {
2348
+ await saveHashCache(dir, sess.hashes).catch((err) => warn(`could not save the content hash cache: ${err?.message ?? err}`));
2349
+ sess.hashCacheSaved = new Map(sess.hashes);
2350
+ }
2351
+ }
2352
+ }
2353
+
2354
+ async function reconcilePass({ vfs, dir, root, state, opts, sess, signal }) {
1917
2355
  const budget = { bytes: 0, files: 0 }; // what the whole pass may pull, both rounds
1918
2356
  const read = { bytes: 0, files: 0 }; // and read to compare
1919
2357
  let pass = await preparePass({ vfs, dir, root, state, opts, sess, signal, read });
@@ -1958,7 +2396,10 @@ export async function reconcile({ vfs, dir, root, state, opts = {}, session: s,
1958
2396
  );
1959
2397
  }
1960
2398
  const agreed = withAgreed(state, pass);
1961
- const next = plan.length ? await applyPlan({ vfs, dir, root, state: agreed, plan, pass, opts, sess, signal, budget }) : agreed;
2399
+ let next = plan.length ? await applyPlan({ vfs, dir, root, state: agreed, plan, pass, opts, sess, signal, budget }) : agreed;
2400
+ // While uploads wait on ignore rules, so do the folders going up.
2401
+ const dirPlan = waiting.size ? pass.dirPlan.filter(({ action }) => !action.endsWith("-remote")) : pass.dirPlan;
2402
+ if (dirPlan.length && !signal?.aborted) next = await applyDirPlan({ vfs, dir, root, state: next, plan: dirPlan, opts, sess, signal });
1962
2403
  await persist(dir, root, state, next);
1963
2404
  const short = pass.outstanding?.space ?? 0;
1964
2405
  if (short) {
@@ -67,6 +67,64 @@ export function planSync({ last, local, remote, unknown = [] }) {
67
67
  return plan;
68
68
  }
69
69
 
70
+ /** What a folder is in the state file, in place of a content hash. */
71
+ export const DIR = "dir";
72
+
73
+ /** A folder's key in the state file: its path with a trailing slash, so it can never be a file's. */
74
+ export const dirKey = (path) => `${path}/`;
75
+
76
+ /** The state without its folders: what the file planner, and its breaker, count. */
77
+ export function filesOf(state) {
78
+ const out = {};
79
+ for (const [key, hash] of Object.entries(state)) if (!key.endsWith("/")) out[key] = hash;
80
+ return out;
81
+ }
82
+
83
+ /**
84
+ * Plan the folders, by the same three-way rule as the files ({@link planPath}),
85
+ * with {@link DIR} standing in for a hash. `local` and `remote` are the folder
86
+ * paths each side holds (`remote` null when the folders could not be listed:
87
+ * then nothing is planned, since a folder not listed is not a folder gone).
88
+ *
89
+ * Returns `{ path, action }` with action one of: "record" (on both sides, not
90
+ * yet in the state), "mkdir-local", "mkdir-remote", "rmdir-local",
91
+ * "rmdir-remote", "forget". Paths in `unknown` are left alone, and so, when
92
+ * `fold` is given (a volume that folds names), is every folder whose folded
93
+ * name more than one spelling shares: which of them this disk holds cannot be
94
+ * told apart, and reading one as missing would plan its removal.
95
+ */
96
+ export function planDirs({ last, local, remote, unknown = [], fold = null }) {
97
+ if (!remote) return [];
98
+ const isUnknown = coveredBy(unknown);
99
+ const before = new Set(Object.keys(last).filter((k) => k.endsWith("/")).map((k) => k.slice(0, -1)));
100
+ const here = new Set(local);
101
+ const there = new Set(remote);
102
+ const paths = new Set([...before, ...here, ...there]);
103
+ const clash = new Set();
104
+ if (fold) {
105
+ const spellings = new Map();
106
+ for (const path of paths) {
107
+ const folded = fold(path);
108
+ if (!spellings.has(folded)) spellings.set(folded, []);
109
+ spellings.get(folded).push(path);
110
+ }
111
+ for (const group of spellings.values()) if (group.length > 1) for (const path of group) clash.add(path);
112
+ }
113
+ const ACTIONS = { push: "mkdir-remote", pull: "mkdir-local", "delete-remote": "rmdir-remote", "delete-local": "rmdir-local", forget: "forget" };
114
+ const plan = [];
115
+ for (const path of [...paths].sort()) {
116
+ if (isUnknown(path) || clash.has(path)) continue;
117
+ const was = before.has(path) ? DIR : undefined;
118
+ const action = planPath({ last: was, local: here.has(path) ? DIR : undefined, remote: there.has(path) ? DIR : undefined });
119
+ if (action === "none") {
120
+ if (!was) plan.push({ path, action: "record" });
121
+ } else if (ACTIONS[action]) {
122
+ plan.push({ path, action: ACTIONS[action] });
123
+ }
124
+ }
125
+ return plan;
126
+ }
127
+
70
128
  /**
71
129
  * `(path) => boolean`: is `path`, or a directory above it, in `prefixes`?
72
130
  *