@muhkoo/cli 0.14.0-alpha.8 → 0.14.0-alpha.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +106 -5
- package/bin/muhkoo.js +8 -1
- package/package.json +2 -2
- package/src/commands/domains.js +1 -1
- package/src/commands/functions.js +2 -2
- package/src/commands/login.js +2 -1
- package/src/commands/promote.js +2 -2
- package/src/commands/provision.js +2 -2
- package/src/commands/tokens.js +1 -1
- package/src/commands/vcs.js +89 -6
- package/src/commands/vfs.js +264 -51
- package/src/index.js +11 -6
- package/src/lib/auth.js +3 -3
- package/src/lib/ignore.js +900 -94
- package/src/lib/mount.js +2049 -164
- package/src/lib/syncPlan.js +105 -2
- package/src/lib/ui.js +92 -23
- package/src/lib/vfs.js +39 -16
- package/src/lib/webLogin.js +1 -1
package/src/lib/mount.js
CHANGED
|
@@ -14,6 +14,19 @@
|
|
|
14
14
|
* its own. Suppression by flag or by timing window is the usual approach and it
|
|
15
15
|
* is fragile — an editor that rewrites the file 20ms late defeats it.
|
|
16
16
|
*
|
|
17
|
+
* ## What comes down is untrusted
|
|
18
|
+
*
|
|
19
|
+
* Anything able to write a user's VFS — today every app they sign into, since
|
|
20
|
+
* hosted auth hands the app the master seed — chooses the paths, names and
|
|
21
|
+
* bytes that arrive here. It cannot touch this disk directly, so everything it
|
|
22
|
+
* can do goes through three places, and those are where the care is:
|
|
23
|
+
*
|
|
24
|
+
* - `admission`, the one rule for what syncs at all, applied identically to
|
|
25
|
+
* both sides and to the watcher;
|
|
26
|
+
* - `writeTarget`, the one gate every write and delete on this disk passes;
|
|
27
|
+
* - the ignore ratchet, which stops any edit to an ignore file — pulled or
|
|
28
|
+
* local — from widening what leaves this machine.
|
|
29
|
+
*
|
|
17
30
|
* ## What is deliberately not here
|
|
18
31
|
*
|
|
19
32
|
* No daemon. `mount` runs in the foreground until interrupted, like a dev
|
|
@@ -21,23 +34,21 @@
|
|
|
21
34
|
* story for "which mounts are running" that is worth more than a first version.
|
|
22
35
|
*/
|
|
23
36
|
|
|
24
|
-
import { createHash } from "node:crypto";
|
|
25
|
-
import {
|
|
26
|
-
|
|
27
|
-
|
|
37
|
+
import { createHash, randomBytes } from "node:crypto";
|
|
38
|
+
import {
|
|
39
|
+
access, chmod, lstat, mkdir, mkdtemp, readFile, readdir, realpath, rename, rm, rmdir, stat, statfs, unlink, writeFile,
|
|
40
|
+
} from "node:fs/promises";
|
|
41
|
+
import { constants as fsConstants, watch } from "node:fs";
|
|
42
|
+
import { homedir, tmpdir } from "node:os";
|
|
43
|
+
import { basename, dirname, isAbsolute, join, parse, relative, resolve, sep } from "node:path";
|
|
28
44
|
|
|
29
|
-
import { planSync } from "./syncPlan.js";
|
|
30
|
-
import {
|
|
45
|
+
import { holdMassDeletes, planSync } from "./syncPlan.js";
|
|
46
|
+
import {
|
|
47
|
+
DEPTH_MAX, IgnoreBudgetError, NAME_MAX, caseFold, ignoreFileProblem, ruleBank, ruleText, withinMatchBounds,
|
|
48
|
+
} from "./ignore.js";
|
|
49
|
+
import { configDir } from "./config.js";
|
|
31
50
|
import { info, ok, warn } from "./ui.js";
|
|
32
51
|
|
|
33
|
-
/**
|
|
34
|
-
* Last-resort deny-list, used only for the watcher's fast path.
|
|
35
|
-
*
|
|
36
|
-
* The real rules come from `.vcsignore` (see `./ignore.js`); this exists so a
|
|
37
|
-
* filesystem event under `node_modules` can be dropped without a scan.
|
|
38
|
-
*/
|
|
39
|
-
const IGNORE = new Set(["node_modules", "dist", "build", "coverage", "target", "vendor", ".git"]);
|
|
40
|
-
|
|
41
52
|
/**
|
|
42
53
|
* Skip dot-entries by default.
|
|
43
54
|
*
|
|
@@ -48,9 +59,11 @@ const IGNORE = new Set(["node_modules", "dist", "build", "coverage", "target", "
|
|
|
48
59
|
* have contained, because it did not exist when the list was written.
|
|
49
60
|
*
|
|
50
61
|
* Dot-entries are machine state until proven otherwise. `--hidden` opts back in
|
|
51
|
-
* for a project that genuinely keeps source there.
|
|
62
|
+
* for a project that genuinely keeps source there. Compared on the folded name
|
|
63
|
+
* as well, so a fullwidth or zero-width-prefixed dot is still a dot — and on
|
|
64
|
+
* the raw one, because folding strips trailing dots and `...` must stay hidden.
|
|
52
65
|
*/
|
|
53
|
-
const isHidden = (name) => name.startsWith(".");
|
|
66
|
+
const isHidden = (name) => name.startsWith(".") || foldName(name).startsWith(".");
|
|
54
67
|
|
|
55
68
|
/** Dot-files that describe the project rather than the machine. */
|
|
56
69
|
const KEEP_HIDDEN = new Set([".vcsignore", ".gitignore"]);
|
|
@@ -71,11 +84,89 @@ const MAX_BYTES = 100 * 1024 * 1024;
|
|
|
71
84
|
*/
|
|
72
85
|
const READ_CONCURRENCY = 8;
|
|
73
86
|
|
|
87
|
+
/**
|
|
88
|
+
* The most one pass pulls: bytes and files. Past it, the rest wait for a
|
|
89
|
+
* follow-up pass, which a running mount starts at once.
|
|
90
|
+
*
|
|
91
|
+
* MAX_BYTES bounds one file, and nothing bounded their number. The VFS lets
|
|
92
|
+
* many entries share one stored blob — a copy moves no bytes — so a remote
|
|
93
|
+
* party that stored one 100MB file and made a million cheap entries pointing
|
|
94
|
+
* at it had this disk write it a million times, in one pass. The budget turns
|
|
95
|
+
* that into steady passes the free-space reserve below can stop.
|
|
96
|
+
*/
|
|
97
|
+
export const PULL_BUDGET = { bytes: 2 * 1024 * 1024 * 1024, files: 10_000 };
|
|
98
|
+
|
|
99
|
+
/**
|
|
100
|
+
* The most one pass of a mount READS to compare with this disk, the same
|
|
101
|
+
* sizes. Past it, the files not yet read are left alone for the pass —
|
|
102
|
+
* neither pulled nor deleted — and a running mount carries on in a
|
|
103
|
+
* follow-up pass.
|
|
104
|
+
*
|
|
105
|
+
* A file whose content was read before is not read again (`remoteContent`),
|
|
106
|
+
* so this bounds a first pass over a large project, and a remote party that
|
|
107
|
+
* keeps storing new content. Every pass used to read and decrypt every
|
|
108
|
+
* remote file to hash it: two hundred entries sharing one 4 MB blob were
|
|
109
|
+
* 868 MB a pass, steady passes included. `import` reads what it has to
|
|
110
|
+
* compare, once, and is not held to this.
|
|
111
|
+
*/
|
|
112
|
+
export const READ_BUDGET = { bytes: 2 * 1024 * 1024 * 1024, files: 10_000 };
|
|
113
|
+
|
|
114
|
+
/**
|
|
115
|
+
* The most a read may fetch, going by the file's manifest: MAX_BYTES and the
|
|
116
|
+
* Reed–Solomon parity stored beside it (half as much again, by default).
|
|
117
|
+
*/
|
|
118
|
+
const FETCH_MAX = 2 * MAX_BYTES;
|
|
119
|
+
|
|
120
|
+
/** Bytes a pass keeps of what it has read, so entries sharing content are fetched once. */
|
|
121
|
+
const BLOB_CACHE_BYTES = MAX_BYTES;
|
|
122
|
+
|
|
123
|
+
/**
|
|
124
|
+
* Space a pull must leave free on the volume holding the mount: 5% of it,
|
|
125
|
+
* but at least 1 GiB and at most 10 GiB. A pull that would cut into it waits
|
|
126
|
+
* — with one warning — until something frees space; filling a disk breaks
|
|
127
|
+
* everything else on the machine, which is a worse failure than a file
|
|
128
|
+
* arriving late. Capped because 5% of a 1 TB disk is 50 GB, and many laptops
|
|
129
|
+
* live below that for good: past it the mount pushed and never pulled.
|
|
130
|
+
*/
|
|
131
|
+
const RESERVE_MIN = 1024 * 1024 * 1024;
|
|
132
|
+
const RESERVE_MAX = 10 * RESERVE_MIN;
|
|
133
|
+
const RESERVE_SHARE = 0.05;
|
|
134
|
+
|
|
74
135
|
/** Coalesce editor churn — a save is often several events in a few ms. */
|
|
75
136
|
const SETTLE_MS = 250;
|
|
76
137
|
|
|
138
|
+
/** Temp files for replace-by-rename. Never synced, never watched. */
|
|
139
|
+
const TMP_PREFIX = ".muhkoo-tmp-";
|
|
140
|
+
|
|
141
|
+
/**
|
|
142
|
+
* The longest a scan computes before handing the event loop back.
|
|
143
|
+
*
|
|
144
|
+
* Filtering is pure CPU over every path and every rule, and while it runs
|
|
145
|
+
* nothing else does — Ctrl-C included. Counting paths between yields was not
|
|
146
|
+
* enough: ten thousand planted rules made each path slow, and 256 slow paths
|
|
147
|
+
* in a row held the loop for 42 seconds. So yield by the clock.
|
|
148
|
+
*/
|
|
149
|
+
const SLICE_MS = 25;
|
|
150
|
+
|
|
151
|
+
/** `() => Promise | null`: a promise to await when the current slice is used up. */
|
|
152
|
+
function pacer() {
|
|
153
|
+
let since = performance.now();
|
|
154
|
+
return () => {
|
|
155
|
+
if (performance.now() - since < SLICE_MS) return null;
|
|
156
|
+
return new Promise((resolve) => setImmediate(resolve)).then(() => {
|
|
157
|
+
since = performance.now();
|
|
158
|
+
});
|
|
159
|
+
};
|
|
160
|
+
}
|
|
161
|
+
|
|
77
162
|
const sha = (bytes) => createHash("sha256").update(bytes).digest("hex");
|
|
78
163
|
|
|
164
|
+
/**
|
|
165
|
+
* Bytes as text. `vfs.readFile` returns a plain Uint8Array, whose own
|
|
166
|
+
* `toString` lists the byte values — never call that on bytes that came down.
|
|
167
|
+
*/
|
|
168
|
+
const utf8 = (bytes) => Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength).toString("utf8");
|
|
169
|
+
|
|
79
170
|
/** Where the last-synced view lives, inside the mount. */
|
|
80
171
|
const statePath = (dir) => join(dir, ".muhkoo", "mount.json");
|
|
81
172
|
|
|
@@ -101,122 +192,1213 @@ async function loadState(dir, root) {
|
|
|
101
192
|
}
|
|
102
193
|
|
|
103
194
|
async function saveState(dir, root, paths) {
|
|
104
|
-
await
|
|
105
|
-
|
|
195
|
+
await writeAtomic(statePath(dir), JSON.stringify({ v: 1, root, paths }, null, 2) + "\n");
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
function sameState(a, b) {
|
|
199
|
+
const keys = Object.keys(a);
|
|
200
|
+
return keys.length === Object.keys(b).length && keys.every((k) => a[k] === b[k]);
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
/**
|
|
204
|
+
* What one running mount remembers between passes — in memory, on purpose.
|
|
205
|
+
*
|
|
206
|
+
* `warned` holds the warnings already given, so a refusal is said once rather
|
|
207
|
+
* than every pass; `ignoreCase` is the volume probe; `more` says the last pass
|
|
208
|
+
* stopped at a budget; `hashes` maps each remote content read so far to its
|
|
209
|
+
* hash and size, so an unchanged file is not read again. None is persisted:
|
|
210
|
+
* a restart says everything once more, and a warning saved to disk would
|
|
211
|
+
* hide a problem for good. Tests may set `pullBudget`, `readBudget` (null:
|
|
212
|
+
* none), `pathBudget` (ignore.js PATH_BUDGET) and `statfs`.
|
|
213
|
+
*/
|
|
214
|
+
function session(s = {}) {
|
|
215
|
+
s.warned ??= new Set();
|
|
216
|
+
s.pullBudget ??= PULL_BUDGET;
|
|
217
|
+
if (s.readBudget === undefined) s.readBudget = READ_BUDGET;
|
|
218
|
+
s.statfs ??= statfs;
|
|
219
|
+
s.hashes ??= new Map();
|
|
220
|
+
return s;
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
/** Warn once per session for each `key`: a refusal is news once, then noise. */
|
|
224
|
+
function warnOnce(sess, key, ...lines) {
|
|
225
|
+
if (sess.warned.has(key)) return;
|
|
226
|
+
sess.warned.add(key);
|
|
227
|
+
warn(...lines);
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
// ---- what syncs at all --------------------------------------------------------
|
|
231
|
+
|
|
232
|
+
/**
|
|
233
|
+
* Paths no flag may bring down, and no ignore file may negate, ever.
|
|
234
|
+
*
|
|
235
|
+
* `--hidden` means "sync dot-files", not "let whatever can write my filesystem
|
|
236
|
+
* rewrite my git config". Each of these either holds credentials that must
|
|
237
|
+
* never leave, or reaches code execution without needing an execute bit: git
|
|
238
|
+
* runs `core.pager` and `core.fsmonitor` from `.git/config` itself, a shell
|
|
239
|
+
* sources its startup files, `.husky` is git's hooks path, and the editor and
|
|
240
|
+
* agent directories hold tasks and hooks that run on open. Matched against
|
|
241
|
+
* every segment, and upward as firmly as downward.
|
|
242
|
+
*
|
|
243
|
+
* Deliberately NOT here: `bin/`, `Library/`, and the yarn/pnpm/cargo/bun
|
|
244
|
+
* configs. Those are ordinary committed project files — a package's `bin/` is
|
|
245
|
+
* its entry point — and code the user runs on purpose is the surface an app
|
|
246
|
+
* holding the seed can already change by editing source (SEC-3). Refusing them
|
|
247
|
+
* would stop real projects syncing without closing anything.
|
|
248
|
+
*/
|
|
249
|
+
const REFUSED_DIRS = new Set([
|
|
250
|
+
".git", ".ssh", ".muhkoo", ".vscode", ".idea", ".config", "node_modules",
|
|
251
|
+
".husky", ".claude", ".cursor", ".zed", ".fleet",
|
|
252
|
+
]);
|
|
253
|
+
const REFUSED_FILES = new Set([
|
|
254
|
+
".envrc", ".npmrc", ".netrc", ".gitconfig", ".pam_environment", ".xprofile",
|
|
255
|
+
".profile", ".bashrc", ".bash_profile", ".bash_login", ".bash_logout",
|
|
256
|
+
".zshrc", ".zshenv", ".zprofile", ".zlogin", ".zlogout", ".kshrc",
|
|
257
|
+
]);
|
|
258
|
+
const REFUSED = new Set([...REFUSED_DIRS, ...REFUSED_FILES]);
|
|
259
|
+
|
|
260
|
+
/**
|
|
261
|
+
* A name as a folding filesystem might see it — for COMPARING, never writing.
|
|
262
|
+
*
|
|
263
|
+
* APFS and NTFS fold case, and APFS folds Unicode normalization, so a remote
|
|
264
|
+
* `Node_Modules/` or `.GIT/` opens the real `node_modules/` or `.git/`. HFS+
|
|
265
|
+
* also drops zero-width and direction marks; Win32 strips trailing dots and
|
|
266
|
+
* spaces and reads `name:stream` as a stream of `name`. `caseFold` is the
|
|
267
|
+
* ignore matcher's own fold — full case folding (`ß` to ss, as APFS does) plus
|
|
268
|
+
* NFKC (`ſ` to s, the Kelvin sign to k, a fullwidth dot to a dot) — so what
|
|
269
|
+
* this refuses and what the rules exclude agree. This folds further than any
|
|
270
|
+
* one filesystem does. Over-folding only ever refuses more, which is safe
|
|
271
|
+
* here; the written name is always the exact one the write boundary verified.
|
|
272
|
+
*/
|
|
273
|
+
export function foldName(name) {
|
|
274
|
+
return caseFold(String(name).split(":")[0]).replace(/[. ]+$/, "");
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
/**
|
|
278
|
+
* Windows 8.3 short names reach the long name they abbreviate — `GIT~1` opens
|
|
279
|
+
* `.git`, `NODE_M~1` opens `node_modules` (git's CVE-2014-9390 class). The
|
|
280
|
+
* short form is the first six characters of the name without its leading dots,
|
|
281
|
+
* or, after a few collisions, two characters and a four-hex-digit hash.
|
|
282
|
+
*/
|
|
283
|
+
const SHORT_NAME = /^([^~]{1,6})~[0-9]+(?:\..*)?$/;
|
|
284
|
+
const REFUSED_STEMS = [...REFUSED].map((n) => n.replace(/^\.+/, "").replace(/[^a-z0-9_-]/g, ""));
|
|
285
|
+
|
|
286
|
+
function isShortFormOfRefused(folded) {
|
|
287
|
+
const m = SHORT_NAME.exec(folded);
|
|
288
|
+
if (!m) return false;
|
|
289
|
+
const p = m[1];
|
|
290
|
+
const hashed = p.length === 6 && /^[0-9a-f]{4}$/.test(p.slice(2));
|
|
291
|
+
return REFUSED_STEMS.some((stem) => p === stem.slice(0, 6) || (hashed && stem.startsWith(p.slice(0, 2))));
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
// Exported for tests: these ARE the security boundary, so they are worth
|
|
295
|
+
// asserting on directly rather than only through a full sync.
|
|
296
|
+
export function isRefused(rel) {
|
|
297
|
+
return rel.split("/").some((part) => {
|
|
298
|
+
const folded = foldName(part);
|
|
299
|
+
return REFUSED.has(folded) || isShortFormOfRefused(folded);
|
|
300
|
+
});
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
/** The names an ignore file goes by — these spellings exactly. */
|
|
304
|
+
const IGNORE_NAMES = new Set([".vcsignore", ".gitignore"]);
|
|
305
|
+
|
|
306
|
+
/**
|
|
307
|
+
* A name a folding volume would open AS an ignore file, spelled otherwise:
|
|
308
|
+
* `.VCSIGNORE`, `.GitIgnore`, `.gitignore.`. Refused both ways, like an alias
|
|
309
|
+
* of anything else. With --hidden a remote `/.VCSIGNORE` was pulled under that
|
|
310
|
+
* spelling, so nothing marked it as pulled — and the next pass read it back as
|
|
311
|
+
* `.vcsignore`, on a volume that folds case, as a file the user had written.
|
|
312
|
+
*/
|
|
313
|
+
const ignoreAlias = (name) => !IGNORE_NAMES.has(name) && IGNORE_NAMES.has(foldName(name));
|
|
314
|
+
|
|
315
|
+
/**
|
|
316
|
+
* The one admission rule: what syncs, in either direction, and what the
|
|
317
|
+
* watcher wakes for. Returns "refused" or "alias" (never synced, and never
|
|
318
|
+
* mistaken for absent), "excluded", "held" (excluded, but only by ignore rules
|
|
319
|
+
* this directory used earlier — see `ignoreContext`), "unknown" (too
|
|
320
|
+
* costly to match, so not decided either way), or null (admitted).
|
|
321
|
+
*
|
|
322
|
+
* The order is the point. Refusal first, so no ignore file can negate it —
|
|
323
|
+
* `!node_modules/` in a pulled `.vcsignore` used to be enough. Then the fixed
|
|
324
|
+
* exclusions no rule can lift: our own temp files, and dot-entries without
|
|
325
|
+
* --hidden. Then the ignore rules, whose matchers answer for every ANCESTOR
|
|
326
|
+
* too, as git does: a file under an excluded directory is excluded, whatever
|
|
327
|
+
* a later rule says.
|
|
328
|
+
*
|
|
329
|
+
* `ignore` is the rules as the files now stand; `held`, when given, the rules
|
|
330
|
+
* as they stood before.
|
|
331
|
+
*
|
|
332
|
+
* The two scans used to filter differently — refusal on one side only, a size
|
|
333
|
+
* limit on one side only, pruning on one side only — and the planner read each
|
|
334
|
+
* difference as a deletion.
|
|
335
|
+
*/
|
|
336
|
+
function admission(rel, isDir, hidden, ignore, held) {
|
|
337
|
+
if (isRefused(rel)) return "refused";
|
|
338
|
+
const parts = rel.split("/");
|
|
339
|
+
const name = parts.at(-1);
|
|
340
|
+
if (ignoreAlias(name)) return "alias";
|
|
341
|
+
// A replace-by-rename left behind by a process that died mid-write.
|
|
342
|
+
if (name.startsWith(TMP_PREFIX)) return "excluded";
|
|
343
|
+
if (!hidden && parts.some((part) => isHidden(part) && !KEEP_HIDDEN.has(part))) return "excluded";
|
|
344
|
+
try {
|
|
345
|
+
if (ignore(rel, isDir)) return "excluded";
|
|
346
|
+
if (held?.(rel, isDir)) return "held";
|
|
347
|
+
} catch (err) {
|
|
348
|
+
// Too costly to decide (ignore.js PATH_BUDGET): left alone, like a
|
|
349
|
+
// path that cannot be read — never synced, and never read as gone.
|
|
350
|
+
if (err instanceof IgnoreBudgetError) return "unknown";
|
|
351
|
+
throw err;
|
|
352
|
+
}
|
|
353
|
+
return null;
|
|
354
|
+
}
|
|
355
|
+
|
|
356
|
+
// ---- the ignore ratchet ---------------------------------------------------------
|
|
357
|
+
|
|
358
|
+
/** Plan steps that change your filesystem to match this disk. */
|
|
359
|
+
const UPLOADS = new Set(["push", "conflict", "delete-remote"]);
|
|
360
|
+
|
|
361
|
+
/** Paths whose text is ignore rules (relative, no leading slash). */
|
|
362
|
+
const isIgnoreFile = (rel) => rel === ".vcsignore" || rel === ".gitignore" || rel.endsWith("/.gitignore");
|
|
363
|
+
|
|
364
|
+
/** Inside a refused directory, so never synced and never remote — but rules all the same. */
|
|
365
|
+
const EXCLUDE_FILE = ".git/info/exclude";
|
|
366
|
+
|
|
367
|
+
/** Where the ratchet keeps its snapshots (see `ignoreContext`). */
|
|
368
|
+
const rulesPath = (dir) => join(dir, ".muhkoo", "ignore-rules.json");
|
|
369
|
+
|
|
370
|
+
/**
|
|
371
|
+
* And a copy outside the project, keyed by where it is and the remote root
|
|
372
|
+
* mounted there: `git clean -fdX` removes `.muhkoo/` — the CLI suggests
|
|
373
|
+
* git-ignoring it — and with no rules file a pass starts afresh from the
|
|
374
|
+
* ignore files on disk, a pulled one included. The root is in the key so
|
|
375
|
+
* that another project checked out at the same path, from another root,
|
|
376
|
+
* starts afresh rather than being held by the old one's rules. The copy in
|
|
377
|
+
* the project is the one that counts; this one is read only when that is
|
|
378
|
+
* gone, and written as best it can be. `root` null: no copy.
|
|
379
|
+
*/
|
|
380
|
+
async function mirrorPath(dir, root) {
|
|
381
|
+
const real = await plannedRealpath(dir);
|
|
382
|
+
const key = createHash("sha256").update(`${real}\0${root}`).digest("hex").slice(0, 32);
|
|
383
|
+
return join(configDir(), "mounts", key, "ignore-rules.json");
|
|
384
|
+
}
|
|
385
|
+
|
|
386
|
+
/**
|
|
387
|
+
* `dir`'s realpath — or, for a directory not created yet, its nearest existing
|
|
388
|
+
* ancestor's realpath with the rest appended: what the realpath will be once a
|
|
389
|
+
* mount creates it. A dry run of a mount on a directory that is gone must find
|
|
390
|
+
* the copy of the rules that the mount, having created it, will find.
|
|
391
|
+
*/
|
|
392
|
+
async function plannedRealpath(dir) {
|
|
393
|
+
const rest = [];
|
|
394
|
+
for (let at = resolve(dir); ; at = dirname(at)) {
|
|
395
|
+
try {
|
|
396
|
+
return join(await realpath(at), ...rest.reverse());
|
|
397
|
+
} catch {
|
|
398
|
+
if (dirname(at) === at) return resolve(dir);
|
|
399
|
+
rest.push(basename(at));
|
|
400
|
+
}
|
|
401
|
+
}
|
|
402
|
+
}
|
|
403
|
+
|
|
404
|
+
/**
|
|
405
|
+
* How many snapshots, and how many bytes of them, the ratchet keeps before a
|
|
406
|
+
* pass refuses. A snapshot is added only when the RULES change — not a
|
|
407
|
+
* comment, a blank line or a line ending, and not a set of files the list
|
|
408
|
+
* already holds, so switching branches back and forth adds nothing. Anything
|
|
409
|
+
* else adds one, a rule added included. Each version of a file is stored
|
|
410
|
+
* once, however many snapshots hold it. At either cap the pass FAILS CLOSED:
|
|
411
|
+
* dropping the oldest snapshot would drop exactly the exclusions only it
|
|
412
|
+
* still holds.
|
|
413
|
+
*/
|
|
414
|
+
const MAX_SNAPSHOTS = 64;
|
|
415
|
+
const MAX_SNAPSHOT_BYTES = 2 * 1024 * 1024;
|
|
416
|
+
|
|
417
|
+
/**
|
|
418
|
+
* The ignore rules for one pass: a sticky exclusion ratchet.
|
|
419
|
+
*
|
|
420
|
+
* Ignore files travel with the project, so anything able to write the user's
|
|
421
|
+
* filesystem can write them — and the rules they hold decide what leaves this
|
|
422
|
+
* machine. A pulled `.vcsignore` reading `# tidy-up` replaced a project's
|
|
423
|
+
* `.gitignore` outright, and the next pass uploaded `config/master.key`.
|
|
424
|
+
*
|
|
425
|
+
* Two earlier designs tried to tell a pulled file from the user's own edit of
|
|
426
|
+
* it, and each was undone by one ordinary local edit — a blank line, a rule
|
|
427
|
+
* added, CRLF line endings — to a file that had arrived from elsewhere. So
|
|
428
|
+
* nothing here asks where a file came from.
|
|
429
|
+
*
|
|
430
|
+
* `.muhkoo/ignore-rules.json` (never synced: `.muhkoo` is refused) keeps a
|
|
431
|
+
* list of SNAPSHOTS. A snapshot is every ignore file a pass reads — the root
|
|
432
|
+
* `.vcsignore` and `.gitignore`, every nested `.gitignore` in a directory the
|
|
433
|
+
* scan enters, and `.git/info/exclude` — path to rules (`ruleText`), as they
|
|
434
|
+
* stood. The first is the set last accepted: on the first pass, the files on
|
|
435
|
+
* disk BEFORE anything is pulled. Every pass after that appends the files as
|
|
436
|
+
* they now stand whenever the list does not already hold them. Only an accept
|
|
437
|
+
* ever removes or replaces a snapshot: a check that let rules "only added" to
|
|
438
|
+
* a file replace the snapshot they extended was fooled by a byte-order mark
|
|
439
|
+
* hiding a `!`, and dropped the one snapshot keeping `config/master.key` home.
|
|
440
|
+
*
|
|
441
|
+
* A path is ignored if ANY snapshot, or the files as they now stand, ignore
|
|
442
|
+
* it — each read on its own, with the usual rules (a `.vcsignore` wins
|
|
443
|
+
* outright within its own snapshot). The same test decides both directions.
|
|
444
|
+
* So once a path has been kept home, no edit — pulled or local, a rule or a
|
|
445
|
+
* blank line — makes it sync. Tightening applies at once, since it is in the
|
|
446
|
+
* files as they now stand; loosening waits for `muhkoo vfs ignore <dir>
|
|
447
|
+
* --accept`, which shows what it lets through, the ignore files changed since
|
|
448
|
+
* the last accept, and a digest of the files as they now stand;
|
|
449
|
+
* `--accept <digest>` resets the snapshots to exactly those files, and refuses
|
|
450
|
+
* when they have changed since. A pass that holds files back says so, once.
|
|
451
|
+
*
|
|
452
|
+
* It fails closed. At either cap, or when the rules file cannot be read, the
|
|
453
|
+
* pass stops with a message to review and accept — never a fresh start, which
|
|
454
|
+
* would adopt whatever ignore files are on disk, a pulled one included. Ignore
|
|
455
|
+
* files from elsewhere count toward the caps like any others, so a remote
|
|
456
|
+
* party can force that review: a refusal, never a leak.
|
|
457
|
+
*
|
|
458
|
+
* `stored` is what `loadSnapshots` gave: null on the first pass here,
|
|
459
|
+
* undefined when it could not be read and the caller (an accept) is replacing
|
|
460
|
+
* it anyway. `hidden` is whether this pass reads dot-directories, recorded
|
|
461
|
+
* beside the snapshots so that `vfs ignore` reads the tree as the mount does.
|
|
462
|
+
*/
|
|
463
|
+
function ignoreContext(sess, stored, { hidden = false } = {}) {
|
|
464
|
+
const bank = ruleBank({
|
|
465
|
+
ignoreCase: sess.ignoreCase,
|
|
466
|
+
budget: sess.pathBudget,
|
|
467
|
+
onBadLine: (line, reason, file) =>
|
|
468
|
+
warnOnce(sess, `bad-line:${file}:${line}`, `ignoring "${line}" in ${file ?? "an ignore file"}: ${reason}`),
|
|
469
|
+
});
|
|
470
|
+
const current = bank.view(); // the ignore files as they now stand, as the scan finds them
|
|
471
|
+
|
|
472
|
+
const snapshots = []; // oldest first; the first is the set last accepted
|
|
473
|
+
const holds = (v) => snapshots.some((s) => s.same(v));
|
|
474
|
+
// Is every snapshot the files as they now stand — the steady state, where
|
|
475
|
+
// the ratchet has nothing to add? Worked out again only when either changes.
|
|
476
|
+
let steadyAt = null;
|
|
477
|
+
let steady = false;
|
|
478
|
+
let recorded = 0;
|
|
479
|
+
const isSteady = () => {
|
|
480
|
+
const at = `${current.changes}:${recorded}`;
|
|
481
|
+
if (steadyAt !== at) {
|
|
482
|
+
steadyAt = at;
|
|
483
|
+
steady = snapshots.every((s) => s.same(current));
|
|
484
|
+
}
|
|
485
|
+
return steady;
|
|
486
|
+
};
|
|
487
|
+
for (const files of stored?.snapshots ?? []) {
|
|
488
|
+
const v = bank.view();
|
|
489
|
+
for (const [rel, text] of files) v.put(rel, bank.version(rel, text));
|
|
490
|
+
snapshots.push(v);
|
|
491
|
+
}
|
|
492
|
+
|
|
493
|
+
const ctx = {
|
|
494
|
+
current,
|
|
495
|
+
source: () => current.source,
|
|
496
|
+
/** Ignored by the files as they now stand. */
|
|
497
|
+
now: (path, isDir = false) => current.excludes(bank.subject(path), isDir),
|
|
498
|
+
/**
|
|
499
|
+
* Ignored by a snapshot — rules other than the files as they now
|
|
500
|
+
* stand. One whose files governing the path are those now on disk
|
|
501
|
+
* gives the same verdict, and is not asked: the latest snapshot almost
|
|
502
|
+
* always IS the files as they now stand, so this is what keeps the
|
|
503
|
+
* ratchet free in the steady state.
|
|
504
|
+
*/
|
|
505
|
+
held(path, isDir = false) {
|
|
506
|
+
if (isSteady()) return false;
|
|
507
|
+
const subj = bank.subject(path);
|
|
508
|
+
return snapshots.some((s) => !s.alike(current, subj) && s.excludes(subj, isDir));
|
|
509
|
+
},
|
|
510
|
+
/** Are rules other than the files as they now stand still in force? */
|
|
511
|
+
inForce: () => !isSteady(),
|
|
512
|
+
/** The ignore files as they now stand against the last accepted, as "path (new|changed|removed)". */
|
|
513
|
+
changed: () => (snapshots.length ? changedFiles(snapshots[0], current) : []),
|
|
514
|
+
/** The files as they now stand, path to rules. */
|
|
515
|
+
texts: () => new Map([...current.files].map(([rel, v]) => [rel, v.text])),
|
|
516
|
+
|
|
517
|
+
/**
|
|
518
|
+
* Record what an ignore file on disk holds now (`bytes` null: absent).
|
|
519
|
+
* A file too big to be rules stops the pass: reading part of it, or
|
|
520
|
+
* none, would drop exclusions it holds.
|
|
521
|
+
*/
|
|
522
|
+
see(rel, bytes) {
|
|
523
|
+
const problem = bytes && ignoreFileProblem(bytes);
|
|
524
|
+
if (problem) throw new Error(`${rel} cannot be read as ignore rules: ${problem}. Nothing was synced — shorten it.`);
|
|
525
|
+
current.put(rel, bytes ? bank.version(rel, ruleText(utf8(bytes))) : null);
|
|
526
|
+
},
|
|
527
|
+
|
|
528
|
+
/**
|
|
529
|
+
* After the local scan, before anything is pulled: append the files
|
|
530
|
+
* as they now stand, unless a snapshot already holds them. `save`
|
|
531
|
+
* false (a dry run) checks the caps without writing.
|
|
532
|
+
*/
|
|
533
|
+
async record(dir, root, save = true) {
|
|
534
|
+
// Rules read from the copy outside the project go back into it
|
|
535
|
+
// first: `vfs ignore` finds that copy only by the root in the
|
|
536
|
+
// state file, which `git clean -fdX` took too, and a pass that
|
|
537
|
+
// refuses below never writes again.
|
|
538
|
+
if (save && stored?.mirrored) await saveSnapshots(dir, root, serializeRules(snapshots, hidden));
|
|
539
|
+
if (holds(current)) {
|
|
540
|
+
// Rewritten only when the mode changed, so that `vfs ignore`
|
|
541
|
+
// reads the tree as the last pass did. Writing it otherwise
|
|
542
|
+
// would wake a running mount into another pass.
|
|
543
|
+
if (save && stored && Boolean(stored.hidden) !== hidden) await saveSnapshots(dir, root, serializeRules(snapshots, hidden));
|
|
544
|
+
return;
|
|
545
|
+
}
|
|
546
|
+
const latest = bank.view(current.files);
|
|
547
|
+
const text = serializeRules([...snapshots, latest], hidden);
|
|
548
|
+
const bytes = Buffer.byteLength(text);
|
|
549
|
+
const why = snapshots.length + 1 > MAX_SNAPSHOTS
|
|
550
|
+
? `they have changed ${snapshots.length} times since they were last accepted, and at most ${MAX_SNAPSHOTS - 1} changes are kept`
|
|
551
|
+
: bytes > MAX_SNAPSHOT_BYTES
|
|
552
|
+
? `the rules they have held come to ${Math.round(bytes / 1024)} KB, and at most ${MAX_SNAPSHOT_BYTES / 1024} KB are kept`
|
|
553
|
+
: null;
|
|
554
|
+
if (why) {
|
|
555
|
+
const changed = ctx.changed();
|
|
556
|
+
throw new Error(
|
|
557
|
+
`the ignore files in ${dir} cannot all be kept: ${why}. Nothing was synced.` +
|
|
558
|
+
(changed.length ? ` Changed since they were last accepted: ${changed.join(", ")} — check them, since one that came from your filesystem may not be yours.` : "") +
|
|
559
|
+
` Then see what accepting them would let through with \`muhkoo vfs ignore ${dir} --accept\`.`,
|
|
560
|
+
);
|
|
561
|
+
}
|
|
562
|
+
snapshots.push(latest);
|
|
563
|
+
recorded++;
|
|
564
|
+
if (save) await saveSnapshots(dir, root, text);
|
|
565
|
+
},
|
|
566
|
+
};
|
|
567
|
+
|
|
568
|
+
return ctx;
|
|
106
569
|
}
|
|
107
570
|
|
|
108
|
-
/**
|
|
109
|
-
|
|
110
|
-
const out =
|
|
111
|
-
const
|
|
571
|
+
/** Each ignore file that differs between two views, as "path (new|changed|removed)". */
|
|
572
|
+
function changedFiles(before, after) {
|
|
573
|
+
const out = [];
|
|
574
|
+
for (const rel of [...new Set([...before.files.keys(), ...after.files.keys()])].sort()) {
|
|
575
|
+
if (before.files.get(rel) === after.files.get(rel)) continue;
|
|
576
|
+
out.push(`${rel} (${!before.files.has(rel) ? "new" : !after.files.has(rel) ? "removed" : "changed"})`);
|
|
577
|
+
}
|
|
578
|
+
return out;
|
|
579
|
+
}
|
|
580
|
+
|
|
581
|
+
/**
|
|
582
|
+
* Read the root ignore files into `ignores` — only under their exact names: on
|
|
583
|
+
* a volume that folds case, `readFile(".vcsignore")` opens a `.VCSIGNORE` too,
|
|
584
|
+
* and the rules in whatever it opened would be read as the project's own.
|
|
585
|
+
*/
|
|
586
|
+
async function seeRootFiles(dir, ignores, skip) {
|
|
587
|
+
const names = new Set(await readdir(dir).catch(() => []));
|
|
588
|
+
for (const name of IGNORE_NAMES) {
|
|
589
|
+
if (name !== skip) ignores.see(name, names.has(name) ? await readIgnoreFile(join(dir, name)) : null);
|
|
590
|
+
}
|
|
591
|
+
ignores.see(EXCLUDE_FILE, await readIgnoreFile(join(dir, EXCLUDE_FILE)));
|
|
592
|
+
}
|
|
593
|
+
|
|
594
|
+
/**
|
|
595
|
+
* The rules file: every version of a file once, and each snapshot as the
|
|
596
|
+
* list of versions it holds, so a file many snapshots share — a heavy
|
|
597
|
+
* `.vcsignore` among them — is stored, and counted against the byte cap,
|
|
598
|
+
* once.
|
|
599
|
+
*/
|
|
600
|
+
function serializeRules(snapshots, hidden) {
|
|
601
|
+
const index = new Map();
|
|
602
|
+
const files = [];
|
|
603
|
+
const sets = snapshots.map((s) =>
|
|
604
|
+
[...s.files.values()].map((v) => {
|
|
605
|
+
let i = index.get(v);
|
|
606
|
+
if (i === undefined) {
|
|
607
|
+
index.set(v, (i = files.length));
|
|
608
|
+
files.push([v.file, v.text]);
|
|
609
|
+
}
|
|
610
|
+
return i;
|
|
611
|
+
}),
|
|
612
|
+
);
|
|
613
|
+
return JSON.stringify({ v: 2, hidden, files, snapshots: sets }) + "\n";
|
|
614
|
+
}
|
|
615
|
+
|
|
616
|
+
/** Write the rules file: in the project, and — as best it can — the copy outside it. */
|
|
617
|
+
async function saveSnapshots(dir, root, text) {
|
|
618
|
+
await writeAtomic(rulesPath(dir), text);
|
|
619
|
+
if (root !== null) await writeAtomic(await mirrorPath(dir, root), text).catch(() => {});
|
|
620
|
+
}
|
|
621
|
+
|
|
622
|
+
/**
|
|
623
|
+
* The stored snapshots, as `{ hidden, snapshots }` — each snapshot a Map of
|
|
624
|
+
* path to rules: null when there are none yet (the first pass here), or
|
|
625
|
+
* thrown when the file cannot be read — never an empty start, which would
|
|
626
|
+
* adopt whatever ignore files are on disk now, a pulled one included. The
|
|
627
|
+
* copy in the project counts; the one outside it (for `root`) only when that
|
|
628
|
+
* is gone, and is then `mirrored`.
|
|
629
|
+
*/
|
|
630
|
+
async function loadSnapshots(dir, root) {
|
|
631
|
+
const own = await readSnapshots(rulesPath(dir), dir);
|
|
632
|
+
if (own || root === null) return own;
|
|
633
|
+
const copy = await readSnapshots(await mirrorPath(dir, root), dir);
|
|
634
|
+
return copy && { ...copy, mirrored: true };
|
|
635
|
+
}
|
|
636
|
+
|
|
637
|
+
async function readSnapshots(path, dir) {
|
|
638
|
+
let text;
|
|
639
|
+
try {
|
|
640
|
+
text = await readFile(path, "utf8");
|
|
641
|
+
} catch (err) {
|
|
642
|
+
if (err.code === "ENOENT" || err.code === "ENOTDIR") return null;
|
|
643
|
+
throw rulesUnreadable(dir, path, err.code ?? err.message);
|
|
644
|
+
}
|
|
645
|
+
let saved;
|
|
646
|
+
try {
|
|
647
|
+
saved = JSON.parse(text);
|
|
648
|
+
} catch {
|
|
649
|
+
throw rulesUnreadable(dir, path, "it is not valid JSON — a write cut short, perhaps");
|
|
650
|
+
}
|
|
651
|
+
const snapshots = snapshotsFrom(saved);
|
|
652
|
+
if (!snapshots) throw rulesUnreadable(dir, path, "it is not in the format this version writes");
|
|
653
|
+
return { hidden: saved.hidden === true, snapshots };
|
|
654
|
+
}
|
|
655
|
+
|
|
656
|
+
/**
|
|
657
|
+
* The snapshots in a parsed rules file, as Maps of path to rules, or null when
|
|
658
|
+
* it is not one. Each version's rules are taken as stored — they are what
|
|
659
|
+
* `ruleText` gave. Read through it again, a byte-order mark that began a
|
|
660
|
+
* later line began the text, and was stripped: rules unlike any stored, on
|
|
661
|
+
* every pass.
|
|
662
|
+
*/
|
|
663
|
+
function snapshotsFrom(saved) {
|
|
664
|
+
const named = (rel, text) => typeof rel === "string" && typeof text === "string" && (isIgnoreFile(rel) || rel === EXCLUDE_FILE);
|
|
665
|
+
if (saved?.v !== 2 || !Array.isArray(saved.files) || !Array.isArray(saved.snapshots) || !saved.snapshots.length) return null;
|
|
666
|
+
if (!saved.files.every((f) => Array.isArray(f) && f.length === 2 && named(f[0], f[1]))) return null;
|
|
667
|
+
const out = [];
|
|
668
|
+
for (const set of saved.snapshots) {
|
|
669
|
+
if (!Array.isArray(set)) return null;
|
|
670
|
+
const files = new Map();
|
|
671
|
+
for (const i of set) {
|
|
672
|
+
const entry = Number.isInteger(i) ? saved.files[i] : undefined;
|
|
673
|
+
if (!entry || files.has(entry[0])) return null;
|
|
674
|
+
files.set(entry[0], entry[1]);
|
|
675
|
+
}
|
|
676
|
+
out.push(files);
|
|
677
|
+
}
|
|
678
|
+
return out;
|
|
679
|
+
}
|
|
680
|
+
|
|
681
|
+
function rulesUnreadable(dir, path, why) {
|
|
682
|
+
return new Error(
|
|
683
|
+
`${path} cannot be read (${why}). It holds every set of ignore rules ${dir} has used, and without it ` +
|
|
684
|
+
`nothing can be shown to stay home, so nothing was synced. Check the ignore files as they now stand, then ` +
|
|
685
|
+
`accept them: \`muhkoo vfs ignore ${dir} --accept\` shows how.`,
|
|
686
|
+
);
|
|
687
|
+
}
|
|
688
|
+
|
|
689
|
+
/** Replace `path` by rename, so a crash leaves the old file or the new one — never half of either. */
|
|
690
|
+
async function writeAtomic(path, text) {
|
|
691
|
+
await mkdir(dirname(path), { recursive: true });
|
|
692
|
+
const tmp = `${path}.${TMP_PREFIX}${randomBytes(6).toString("hex")}`;
|
|
693
|
+
try {
|
|
694
|
+
await writeFile(tmp, text, { flag: "wx" });
|
|
695
|
+
await rename(tmp, path);
|
|
696
|
+
} catch (err) {
|
|
697
|
+
await rm(tmp, { force: true }).catch(() => {});
|
|
698
|
+
throw err;
|
|
699
|
+
}
|
|
700
|
+
}
|
|
701
|
+
|
|
702
|
+
/**
|
|
703
|
+
* The files the ignore rules hold back, as a warning's lines: named, and with
|
|
704
|
+
* the command that shows what accepting would let through. "Earlier" rules,
|
|
705
|
+
* not "dropped" ones: on a fresh machine the first snapshot is the defaults
|
|
706
|
+
* alone, and what holds a file there is a pulled rule re-including it.
|
|
707
|
+
*/
|
|
708
|
+
function heldNotice(paths, dir) {
|
|
709
|
+
const shown = paths.slice(0, 5).map((p) => p.replace(/^\//, "")).join(", ");
|
|
710
|
+
return [
|
|
711
|
+
`${paths.length} path${paths.length === 1 ? " is" : "s are"} held back — the ignore rules as they now stand would sync ` +
|
|
712
|
+
`${paths.length === 1 ? "it" : "them"}, but rules this directory used earlier (when last accepted, or since) exclude ` +
|
|
713
|
+
`${paths.length === 1 ? "it" : "them"}: ${shown}${paths.length > 5 ? ", …" : ""}`,
|
|
714
|
+
` Nothing is synced for ${paths.length === 1 ? "it" : "them"} until you accept the ignore files as they now stand. ` +
|
|
715
|
+
`See what that would let through: muhkoo vfs ignore ${dir} --accept`,
|
|
716
|
+
];
|
|
717
|
+
}
|
|
718
|
+
|
|
719
|
+
/** Is `relDir` in, or under, a dot-directory? */
|
|
720
|
+
const inDotDir = (relDir) => relDir.split("/").some((part) => isHidden(part));
|
|
721
|
+
|
|
722
|
+
/**
|
|
723
|
+
* The ignore files on disk, and the local files only the stored snapshots
|
|
724
|
+
* hold back. Changes nothing. Walks the way git reads: exact names, and never
|
|
725
|
+
* inside a directory the files as they now stand exclude — the pruning a pass
|
|
726
|
+
* will do once the snapshots are accepted.
|
|
727
|
+
*
|
|
728
|
+
* Nested `.gitignore` files in dot-directories count only where the mount
|
|
729
|
+
* reads them — with --hidden, as the rules file records (`hidden`). Reading
|
|
730
|
+
* them regardless put a pytest cache's own `*` among the files as they now
|
|
731
|
+
* stand, which no pass records, so `vfs ignore` said for good that older
|
|
732
|
+
* rules were in force, and flagged a tool's file as possibly planted. HELD
|
|
733
|
+
* files are listed in dot-directories whatever the mode: listing one costs
|
|
734
|
+
* nothing, and leaving out one a --hidden mount would release did. `draft`
|
|
735
|
+
* reads the tree as git would, with no `.vcsignore` — the file a draft
|
|
736
|
+
* replaces.
|
|
737
|
+
*/
|
|
738
|
+
async function diskRules(dir, { draft = false, stored, hidden = false } = {}) {
|
|
739
|
+
const sess = session();
|
|
740
|
+
sess.ignoreCase = await foldsCase(dir);
|
|
741
|
+
const ignores = ignoreContext(sess, stored, { hidden });
|
|
742
|
+
await seeRootFiles(dir, ignores, draft ? ".vcsignore" : null);
|
|
743
|
+
const held = [];
|
|
744
|
+
const walk = async (abs, relDir) => {
|
|
745
|
+
const entries = await readdir(abs, { withFileTypes: true }).catch(() => []);
|
|
746
|
+
if (relDir && (hidden || !inDotDir(relDir))) {
|
|
747
|
+
// As scanLocal reads it, so an accept records what a pass will see.
|
|
748
|
+
const file = entries.find((e) => e.name === ".gitignore" && (e.isFile() || e.isSymbolicLink()));
|
|
749
|
+
ignores.see(`${relDir}/.gitignore`, file ? await readIgnoreFile(join(abs, ".gitignore")) : null);
|
|
750
|
+
}
|
|
751
|
+
for (const entry of entries) {
|
|
752
|
+
const rel = relDir ? `${relDir}/${entry.name}` : entry.name;
|
|
753
|
+
const isDir = entry.isDirectory();
|
|
754
|
+
if (admission(rel, isDir, true, ignores.now)) continue;
|
|
755
|
+
if (isDir) await walk(join(abs, entry.name), rel);
|
|
756
|
+
else if (entry.isFile() && heldBy(ignores, rel)) held.push("/" + rel);
|
|
757
|
+
}
|
|
758
|
+
};
|
|
759
|
+
await walk(dir, "");
|
|
760
|
+
return { ignores, held: held.sort(), changed: ignores.changed(), inForce: ignores.inForce() };
|
|
761
|
+
}
|
|
762
|
+
|
|
763
|
+
/** Does only a snapshot exclude `rel`? A path too costly to decide is not listed. */
|
|
764
|
+
function heldBy(ignores, rel) {
|
|
765
|
+
try {
|
|
766
|
+
return ignores.held(rel);
|
|
767
|
+
} catch (err) {
|
|
768
|
+
if (err instanceof IgnoreBudgetError) return false;
|
|
769
|
+
throw err;
|
|
770
|
+
}
|
|
771
|
+
}
|
|
772
|
+
|
|
773
|
+
/**
|
|
774
|
+
* What `muhkoo vfs ignore <dir>` shows, and `--accept` previews, changing
|
|
775
|
+
* nothing: whether rules other than the ignore files as they now stand still
|
|
776
|
+
* apply (`inForce`), which ignore files changed since the last accept
|
|
777
|
+
* (`changed`), the local files those rules hold back, which an accept lets
|
|
778
|
+
* through (`held`), and `digest`, naming exactly the ignore files an accept
|
|
779
|
+
* would adopt. Files arriving from elsewhere may be held too, and are not
|
|
780
|
+
* listed: that needs the filesystem, and this is local. `mounted` is false
|
|
781
|
+
* where no rules were ever stored; `unreadable` is the error when the stored
|
|
782
|
+
* rules cannot be read, so that nobody can say what they hold.
|
|
783
|
+
*/
|
|
784
|
+
export async function heldBack(dir) {
|
|
785
|
+
return (await toAccept(dir)).shown;
|
|
786
|
+
}
|
|
787
|
+
|
|
788
|
+
/**
|
|
789
|
+
* `muhkoo vfs ignore <dir> --accept <digest>`: make the ignore files as they
|
|
790
|
+
* now stand the only rules — if they are still the ones `heldBack` showed
|
|
791
|
+
* under `digest`. Otherwise it throws, and accepts nothing: an ignore file
|
|
792
|
+
* that arrived between the preview and the accept was never shown to anyone.
|
|
793
|
+
* Returns the local files that lets through (`eligible`; null when the old
|
|
794
|
+
* rules could not be read, so nobody can say).
|
|
795
|
+
*/
|
|
796
|
+
export async function acceptIgnores(dir, digest) {
|
|
797
|
+
const { root, text, shown } = await toAccept(dir);
|
|
798
|
+
if (digest !== shown.digest) {
|
|
799
|
+
throw new Error(
|
|
800
|
+
`the ignore files in ${dir} have changed since that preview (it showed ${digest}; they are now ${shown.digest}), ` +
|
|
801
|
+
`so nothing was accepted. See what accepting them now would let through: muhkoo vfs ignore ${dir} --accept`,
|
|
802
|
+
);
|
|
803
|
+
}
|
|
804
|
+
await saveSnapshots(dir, root, text);
|
|
805
|
+
return { eligible: shown.unreadable ? null : shown.held };
|
|
806
|
+
}
|
|
807
|
+
|
|
808
|
+
/** The rules file an accept would write, and what `heldBack` shows of it. */
|
|
809
|
+
async function toAccept(dir) {
|
|
810
|
+
const root = await syncedRoot(dir);
|
|
811
|
+
let stored; // undefined when it cannot be read: the accept replaces it
|
|
812
|
+
let unreadable = null;
|
|
813
|
+
try {
|
|
814
|
+
stored = await loadSnapshots(dir, root);
|
|
815
|
+
} catch (err) {
|
|
816
|
+
unreadable = err;
|
|
817
|
+
}
|
|
818
|
+
const hidden = Boolean(stored?.hidden);
|
|
819
|
+
const { ignores, held, changed, inForce } = await diskRules(dir, { stored, hidden });
|
|
820
|
+
// Of the rules themselves, in a fixed order: the order the walk met the
|
|
821
|
+
// files in is not part of what is being accepted.
|
|
822
|
+
const rules = [...ignores.current.files.values()].map((v) => [v.file, v.text]).sort((a, b) => (a[0] < b[0] ? -1 : 1));
|
|
823
|
+
const digest = sha(JSON.stringify({ hidden, rules })).slice(0, 16);
|
|
824
|
+
const text = serializeRules([ignores.current], hidden);
|
|
825
|
+
return { root, text, shown: { mounted: stored !== null, unreadable, inForce, changed, held, digest } };
|
|
826
|
+
}
|
|
827
|
+
|
|
828
|
+
/** The remote root `dir` was last synced with, from its state file, or null. */
|
|
829
|
+
async function syncedRoot(dir) {
|
|
830
|
+
const saved = await readFile(statePath(dir), "utf8").then(JSON.parse).catch(() => null);
|
|
831
|
+
return saved?.v === 1 && typeof saved.root === "string" ? saved.root : null;
|
|
832
|
+
}
|
|
833
|
+
|
|
834
|
+
/**
|
|
835
|
+
* Every ignore file git would read under `dir`, path to rules, as it stands
|
|
836
|
+
* on disk — what `vfs ignore --write` drafts a `.vcsignore` from. A
|
|
837
|
+
* `.vcsignore` already there is left out: the draft replaces it.
|
|
838
|
+
*/
|
|
839
|
+
export async function diskIgnoreTexts(dir) {
|
|
840
|
+
const { ignores } = await diskRules(dir, { draft: true, stored: null, hidden: true });
|
|
841
|
+
return ignores.texts();
|
|
842
|
+
}
|
|
843
|
+
|
|
844
|
+
/** An ignore file's bytes, or null when it is absent or unreadable. */
|
|
845
|
+
async function readIgnoreFile(abs) {
|
|
846
|
+
try {
|
|
847
|
+
return await readFile(abs);
|
|
848
|
+
} catch {
|
|
849
|
+
return null;
|
|
850
|
+
}
|
|
851
|
+
}
|
|
852
|
+
|
|
853
|
+
/**
|
|
854
|
+
* Does the volume holding `dir` fold case? Mirrors git's `core.ignorecase`.
|
|
855
|
+
*
|
|
856
|
+
* Probed rather than assumed from the platform: macOS volumes can be
|
|
857
|
+
* case-sensitive, and Linux can mount case-insensitive ones. If the probe
|
|
858
|
+
* cannot run, fold: that only ever excludes and refuses more.
|
|
859
|
+
*/
|
|
860
|
+
async function foldsCase(dir) {
|
|
861
|
+
const probeDir = join(dir, ".muhkoo");
|
|
862
|
+
let made;
|
|
863
|
+
const name = `case-probe-${process.pid}-${randomBytes(4).toString("hex")}`;
|
|
864
|
+
try {
|
|
865
|
+
made = await mkdir(probeDir, { recursive: true });
|
|
866
|
+
await writeFile(join(probeDir, name), "", { flag: "wx" });
|
|
867
|
+
return await lstat(join(probeDir, name.toUpperCase())).then(() => true, () => false);
|
|
868
|
+
} catch {
|
|
869
|
+
return true;
|
|
870
|
+
} finally {
|
|
871
|
+
await rm(join(probeDir, name), { force: true }).catch(() => {});
|
|
872
|
+
if (made) await rmdir(probeDir).catch(() => {}); // leave no .muhkoo/ made only for this
|
|
873
|
+
}
|
|
874
|
+
}
|
|
875
|
+
|
|
876
|
+
/**
|
|
877
|
+
* The same question, asked without writing anything — for a dry run, which
|
|
878
|
+
* promises to change nothing, not even for a moment. `foldsCase` writes a
|
|
879
|
+
* probe file and removes it: no name is left behind, but the directory (or
|
|
880
|
+
* `.muhkoo/`, or wherever a symlinked `.muhkoo` points) has changed.
|
|
881
|
+
*
|
|
882
|
+
* So look an entry already there up under its other spelling: found, and not
|
|
883
|
+
* a second entry of that name, means the volume folded the name. Nothing in
|
|
884
|
+
* `dir` with a letter in its name, and `dir`'s own name is asked of its parent,
|
|
885
|
+
* when that is on the same volume. With neither, fold — as a directory that is
|
|
886
|
+
* not there yet does — which only ever refuses more.
|
|
887
|
+
*/
|
|
888
|
+
export async function foldsCaseReadOnly(dir) {
|
|
889
|
+
const swapped = (name) => name.replace(/[a-z]/gi, (ch) => (ch === ch.toLowerCase() ? ch.toUpperCase() : ch.toLowerCase()));
|
|
890
|
+
// true or false when `name` in `parent` tells, undefined when it does not.
|
|
891
|
+
const ask = async (parent, name, listed) => {
|
|
892
|
+
const other = swapped(name);
|
|
893
|
+
if (other === name) return undefined;
|
|
894
|
+
if (listed.has(other)) return false; // both spellings, side by side: nothing folded
|
|
895
|
+
const a = await lstat(join(parent, name), { bigint: true }).catch(() => null);
|
|
896
|
+
if (!a) return undefined; // gone since it was listed
|
|
897
|
+
let b;
|
|
898
|
+
try {
|
|
899
|
+
b = await lstat(join(parent, other), { bigint: true });
|
|
900
|
+
} catch (err) {
|
|
901
|
+
return err.code === "ENOENT" ? false : undefined;
|
|
902
|
+
}
|
|
903
|
+
if (!a.ino || !b.ino) return true; // found under a spelling not listed; no inodes to compare
|
|
904
|
+
return a.dev === b.dev && a.ino === b.ino ? true : undefined;
|
|
905
|
+
};
|
|
906
|
+
try {
|
|
907
|
+
const names = await readdir(dir);
|
|
908
|
+
const listed = new Set(names);
|
|
909
|
+
for (const name of names) {
|
|
910
|
+
const answer = await ask(dir, name, listed);
|
|
911
|
+
if (answer !== undefined) return answer;
|
|
912
|
+
}
|
|
913
|
+
} catch {
|
|
914
|
+
// unreadable: the parent may still tell
|
|
915
|
+
}
|
|
916
|
+
try {
|
|
917
|
+
const parent = dirname(dir);
|
|
918
|
+
const [here, above] = await Promise.all([lstat(dir, { bigint: true }), lstat(parent, { bigint: true })]);
|
|
919
|
+
if (parent !== dir && here.dev === above.dev) {
|
|
920
|
+
const answer = await ask(parent, basename(dir), new Set(await readdir(parent)));
|
|
921
|
+
if (answer !== undefined) return answer;
|
|
922
|
+
}
|
|
923
|
+
} catch {
|
|
924
|
+
// nothing more to ask
|
|
925
|
+
}
|
|
926
|
+
return true;
|
|
927
|
+
}
|
|
928
|
+
|
|
929
|
+
// ---- the scans ------------------------------------------------------------------
|
|
930
|
+
|
|
931
|
+
/**
|
|
932
|
+
* Hash every syncable file under `dir`, keyed by VFS-style relative path.
|
|
933
|
+
*
|
|
934
|
+
* Also returns the paths it deliberately did not look at — refused, over the
|
|
935
|
+
* size limit, unreadable, or not a regular file — as `unknown`, so the planner
|
|
936
|
+
* leaves them alone rather than reading them as deleted, and as `held` the
|
|
937
|
+
* ones only earlier ignore rules keep out (a directory with a trailing slash),
|
|
938
|
+
* for the notice. Discovers nested `.gitignore` files on the way down,
|
|
939
|
+
* registering each before anything in its directory is matched, and never
|
|
940
|
+
* descends into an excluded directory to look for one (neither does git).
|
|
941
|
+
*/
|
|
942
|
+
async function scanLocal(dir, { hidden = false, ignores, sess, signal }) {
|
|
943
|
+
const files = {};
|
|
944
|
+
const unknown = new Set();
|
|
945
|
+
const held = new Set();
|
|
946
|
+
const pace = pacer();
|
|
947
|
+
|
|
948
|
+
const walk = async (abs, relDir) => {
|
|
112
949
|
let entries;
|
|
113
950
|
try {
|
|
114
951
|
entries = await readdir(abs, { withFileTypes: true });
|
|
115
|
-
} catch {
|
|
116
|
-
|
|
952
|
+
} catch (err) {
|
|
953
|
+
if (!relDir) throw err; // the mount itself: fail the pass loudly
|
|
954
|
+
if (err.code === "ENOENT") return; // vanished mid-scan; the next pass will see it
|
|
955
|
+
warnOnce(sess, `unreadable:/${relDir}`, `could not read ${relDir}/ (${err.code ?? err.message}) — leaving it alone`);
|
|
956
|
+
unknown.add("/" + relDir);
|
|
957
|
+
return;
|
|
958
|
+
}
|
|
959
|
+
if (relDir) {
|
|
960
|
+
const file = entries.find((e) => e.name === ".gitignore" && (e.isFile() || e.isSymbolicLink()));
|
|
961
|
+
ignores.see(`${relDir}/.gitignore`, file ? await readIgnoreFile(join(abs, ".gitignore")) : null);
|
|
117
962
|
}
|
|
118
963
|
for (const entry of entries) {
|
|
964
|
+
const breather = pace();
|
|
965
|
+
if (breather) await breather;
|
|
966
|
+
if (signal?.aborted) return; // the caller must not plan from half a scan
|
|
119
967
|
const child = join(abs, entry.name);
|
|
120
|
-
const rel =
|
|
968
|
+
const rel = relDir ? `${relDir}/${entry.name}` : entry.name;
|
|
969
|
+
const key = "/" + rel;
|
|
121
970
|
// Pruned at the directory, not per file: never descend into
|
|
122
971
|
// node_modules to decide, file by file, not to sync it.
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
972
|
+
const verdict = admission(rel, entry.isDirectory(), hidden, ignores.now, ignores.held);
|
|
973
|
+
if (verdict === "unknown") costly(sess, key);
|
|
974
|
+
if (verdict === "refused" || verdict === "alias" || verdict === "unknown") {
|
|
975
|
+
unknown.add(key);
|
|
976
|
+
continue;
|
|
977
|
+
}
|
|
978
|
+
if (verdict === "held") held.add(entry.isDirectory() ? `${key}/` : key);
|
|
979
|
+
if (verdict) continue;
|
|
128
980
|
if (entry.isDirectory()) {
|
|
129
|
-
await walk(child);
|
|
981
|
+
await walk(child, rel);
|
|
982
|
+
continue;
|
|
983
|
+
}
|
|
984
|
+
// Sockets, fifos, symlinks: not ours to read — or to replace, so
|
|
985
|
+
// they are unknown rather than absent.
|
|
986
|
+
if (!entry.isFile()) {
|
|
987
|
+
unknown.add(key);
|
|
130
988
|
continue;
|
|
131
989
|
}
|
|
132
|
-
if (!entry.isFile()) continue; // sockets, fifos, symlinks: not ours
|
|
133
990
|
try {
|
|
134
|
-
const info = await
|
|
991
|
+
const info = await lstat(child);
|
|
135
992
|
if (info.size > MAX_BYTES) {
|
|
136
|
-
|
|
993
|
+
warnOnce(sess, `big:${key}`, `skipping ${rel} — ${Math.round(info.size / 1e6)} MB is over the mount limit`);
|
|
994
|
+
unknown.add(key);
|
|
137
995
|
continue;
|
|
138
996
|
}
|
|
139
|
-
|
|
140
|
-
} catch {
|
|
141
|
-
|
|
997
|
+
files[key] = sha(await readFile(child));
|
|
998
|
+
} catch (err) {
|
|
999
|
+
if (err.code === "ENOENT") continue; // deleted mid-scan: genuinely gone
|
|
1000
|
+
warnOnce(sess, `unreadable:${key}`, `could not read ${rel} (${err.code ?? err.message}) — leaving it alone`);
|
|
1001
|
+
unknown.add(key);
|
|
142
1002
|
}
|
|
143
1003
|
}
|
|
144
1004
|
};
|
|
145
|
-
await walk(dir);
|
|
146
|
-
return
|
|
1005
|
+
await walk(dir, "");
|
|
1006
|
+
return { files, unknown, held };
|
|
147
1007
|
}
|
|
148
1008
|
|
|
149
1009
|
/**
|
|
150
|
-
*
|
|
151
|
-
*
|
|
1010
|
+
* Every file path under the remote root, or none when it does not exist yet.
|
|
1011
|
+
*
|
|
1012
|
+
* `/` is the one root that must not be asked: through 0.14.0-alpha.8 the SDK
|
|
1013
|
+
* answered `exists("/")` with false (the root has no entry of its own), and
|
|
1014
|
+
* asking made every pass see an empty filesystem — pass one uploaded the
|
|
1015
|
+
* project, pass two planned deleting all of it locally. Newer SDKs answer true,
|
|
1016
|
+
* but this CLI may run against either.
|
|
1017
|
+
*/
|
|
1018
|
+
async function walkRemote(vfs, root) {
|
|
1019
|
+
if (root !== "/" && !(await vfs.exists(root))) return [];
|
|
1020
|
+
return vfs.walk(root);
|
|
1021
|
+
}
|
|
1022
|
+
|
|
1023
|
+
/**
|
|
1024
|
+
* Hash every file under the mounted VFS subtree — through the SAME admission
|
|
1025
|
+
* rule as the local scan.
|
|
152
1026
|
*
|
|
153
1027
|
* These filters used to run in one direction only. They decided what left the
|
|
154
1028
|
* machine and nothing decided what arrived, so a remote `.git/config` was
|
|
155
1029
|
* hashed here, found no local counterpart, and was written straight into the
|
|
156
|
-
* developer's repository.
|
|
157
|
-
*
|
|
158
|
-
*
|
|
1030
|
+
* developer's repository. Anything that can write to a user's VFS is a party
|
|
1031
|
+
* that can write files onto the machines they mount from.
|
|
1032
|
+
*
|
|
1033
|
+
* Returns `{ files, unknown, held }`: `unknown` is every path deliberately
|
|
1034
|
+
* not read — refused, over the size limit, unreadable, left for the next pass
|
|
1035
|
+
* (`deferred`), or an alias of another remote path — which the planner must
|
|
1036
|
+
* not mistake for absent; `held` the paths only earlier ignore rules keep out.
|
|
159
1037
|
*
|
|
160
|
-
*
|
|
161
|
-
*
|
|
1038
|
+
* `ctx`: `paths` (an already-walked listing), `held` (the earlier rules; see
|
|
1039
|
+
* `ignoreContext`), `ignoreCase` (the local volume folds names), `localKeys`
|
|
1040
|
+
* (for resolving fold collisions), `session`, `spent` (what the pass has read
|
|
1041
|
+
* so far, against the session's `readBudget`), and `signal`, which stops the
|
|
1042
|
+
* reads early — the caller must then not plan.
|
|
1043
|
+
*
|
|
1044
|
+
* `ignore` must answer for a path's ancestors as well as the path, as every
|
|
1045
|
+
* matcher in ignore.js does.
|
|
162
1046
|
*/
|
|
163
|
-
export async function scanRemote(vfs, root, includeHidden = false, ignore = () => false) {
|
|
164
|
-
const
|
|
165
|
-
|
|
1047
|
+
export async function scanRemote(vfs, root, includeHidden = false, ignore = () => false, ctx = {}) {
|
|
1048
|
+
const sess = session(ctx.session);
|
|
1049
|
+
const files = {};
|
|
1050
|
+
const sizes = {};
|
|
1051
|
+
const unknown = new Set();
|
|
1052
|
+
const unreadable = new Set();
|
|
1053
|
+
const held = new Set();
|
|
1054
|
+
const paths = ctx.paths ?? (await walkRemote(vfs, root));
|
|
166
1055
|
|
|
167
1056
|
// Decide what to read BEFORE reading anything, so the filters are applied
|
|
168
1057
|
// once and the reads that survive them can go out together.
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
const
|
|
173
|
-
if (
|
|
174
|
-
if (
|
|
175
|
-
|
|
1058
|
+
let wanted = [];
|
|
1059
|
+
const pace = pacer();
|
|
1060
|
+
for (const abs of paths) {
|
|
1061
|
+
const breather = pace();
|
|
1062
|
+
if (breather) await breather;
|
|
1063
|
+
if (ctx.signal?.aborted) return { files, sizes, unknown, unreadable, held }; // not to be planned from
|
|
1064
|
+
// Always with a leading slash, to match the local keys: at root "/" the
|
|
1065
|
+
// old `abs.slice(root.length)` dropped it, and every file became two
|
|
1066
|
+
// paths that pushed, pulled and deleted each other in turn.
|
|
1067
|
+
const key = root === "/" ? abs : abs.slice(root.length);
|
|
1068
|
+
if (!key.startsWith("/") || key === "/") continue;
|
|
1069
|
+
const rel = key.slice(1);
|
|
1070
|
+
// Setting a path up to be matched costs its depth squared before any
|
|
1071
|
+
// rule runs (ignore.js `withinMatchBounds`), and nothing deeper, or
|
|
1072
|
+
// with longer names, than a filesystem holds could land here anyway,
|
|
1073
|
+
// so such a path is not matched at all.
|
|
1074
|
+
if (!withinMatchBounds(rel, ctx.ignoreCase)) {
|
|
1075
|
+
warnOnce(sess, `bounds:${key}`, `skipping ${key} from your filesystem — it is more than ${DEPTH_MAX} directories deep, or has a name longer than ${NAME_MAX} characters`);
|
|
1076
|
+
unknown.add(key);
|
|
1077
|
+
continue;
|
|
1078
|
+
}
|
|
1079
|
+
const verdict = admission(rel, false, includeHidden, ignore, ctx.held);
|
|
1080
|
+
if (verdict === "held") held.add(key);
|
|
1081
|
+
if (verdict === "refused") {
|
|
1082
|
+
warnOnce(sess, `refused:${key}`, `refusing ${key} from your filesystem — that path can execute code on this machine`);
|
|
1083
|
+
unknown.add(key);
|
|
176
1084
|
continue;
|
|
177
1085
|
}
|
|
178
|
-
if (
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
1086
|
+
if (verdict === "alias") {
|
|
1087
|
+
warnOnce(sess, `alias:${key}`, `refusing ${key} from your filesystem — a volume that folds names would read it as an ignore file`);
|
|
1088
|
+
unknown.add(key);
|
|
1089
|
+
continue;
|
|
1090
|
+
}
|
|
1091
|
+
if (verdict === "unknown") {
|
|
1092
|
+
costly(sess, key);
|
|
1093
|
+
unknown.add(key);
|
|
1094
|
+
continue;
|
|
1095
|
+
}
|
|
1096
|
+
if (verdict) continue;
|
|
182
1097
|
wanted.push([key, abs]);
|
|
183
1098
|
}
|
|
1099
|
+
if (ctx.ignoreCase) wanted = await refuseFoldCollisions(wanted, ctx.localKeys, unknown, sess, ctx.signal, { ignoreCase: true });
|
|
184
1100
|
|
|
185
1101
|
// Read in PARALLEL. Each file is an independent round trip - a manifest
|
|
186
1102
|
// lookup plus its shards - so reading them one after another cost the SUM
|
|
187
1103
|
// of the latencies. On a project of any size that is the difference between
|
|
188
1104
|
// `mount` taking a moment and looking like it has hung: doodledottie is 38
|
|
189
1105
|
// files and 16.5MB, which took over twenty seconds in series.
|
|
1106
|
+
//
|
|
1107
|
+
// And only what has not been read before: content read in an earlier pass
|
|
1108
|
+
// (`sess.hashes`) or earlier in this one (`reading`) is not fetched again.
|
|
190
1109
|
const queue = [...wanted];
|
|
1110
|
+
const readPace = pacer(); // reads from a cache resolve at once, and never yield on their own
|
|
1111
|
+
const seen = new Map(); // content key → { hash, size }, for the next pass
|
|
1112
|
+
const reading = new Map(); // content key → the read under way
|
|
1113
|
+
const blobs = ctx.blobs ?? blobCache(BLOB_CACHE_BYTES);
|
|
1114
|
+
const budget = sess.readBudget;
|
|
1115
|
+
const spent = ctx.spent ?? { bytes: 0, files: 0 };
|
|
1116
|
+
const deferred = new Set();
|
|
191
1117
|
await Promise.all(
|
|
192
1118
|
Array.from({ length: Math.min(READ_CONCURRENCY, queue.length) }, async () => {
|
|
193
|
-
for (let item = queue.shift(); item !== undefined; item = queue.shift()) {
|
|
194
|
-
|
|
1119
|
+
for (let item = queue.shift(); item !== undefined && !ctx.signal?.aborted; item = queue.shift()) {
|
|
1120
|
+
const breather = readPace();
|
|
1121
|
+
if (breather) await breather;
|
|
1122
|
+
const [key, abs] = item;
|
|
1123
|
+
try {
|
|
1124
|
+
// The record's size, and the manifest's chunk sizes, skip a
|
|
1125
|
+
// file they admit is too big BEFORE fetching it. The record
|
|
1126
|
+
// is only a hint — the remote party writes it, and one that
|
|
1127
|
+
// says 0 can hold anything — so the limit is also the bytes
|
|
1128
|
+
// that come back. The manifest is what a read is held to.
|
|
1129
|
+
const content = await remoteContent(vfs, abs);
|
|
1130
|
+
const { bound } = content;
|
|
1131
|
+
if (content.size > MAX_BYTES || (bound && (bound.plain > MAX_BYTES || bound.fetched > FETCH_MAX))) {
|
|
1132
|
+
tooBig(sess, key, Math.max(content.size, bound?.plain ?? 0, bound?.fetched ?? 0));
|
|
1133
|
+
unknown.add(key);
|
|
1134
|
+
continue;
|
|
1135
|
+
}
|
|
1136
|
+
const known = content.key && (seen.get(content.key) ?? sess.hashes.get(content.key));
|
|
1137
|
+
if (known) {
|
|
1138
|
+
seen.set(content.key, known);
|
|
1139
|
+
files[key] = known.hash;
|
|
1140
|
+
sizes[key] = known.size;
|
|
1141
|
+
continue;
|
|
1142
|
+
}
|
|
1143
|
+
let read = content.key && reading.get(content.key);
|
|
1144
|
+
if (!read) {
|
|
1145
|
+
// Ignore files included: a 256 KB file of comments is
|
|
1146
|
+
// 256 KB to fetch however little it adds to the rules.
|
|
1147
|
+
const claim = Math.max(content.size, content.bound?.plain ?? 0);
|
|
1148
|
+
if (budget) {
|
|
1149
|
+
if (spent.files && (spent.files + 1 > budget.files || spent.bytes + claim > budget.bytes)) {
|
|
1150
|
+
deferred.add(key);
|
|
1151
|
+
unknown.add(key);
|
|
1152
|
+
continue;
|
|
1153
|
+
}
|
|
1154
|
+
spent.files++;
|
|
1155
|
+
spent.bytes += claim;
|
|
1156
|
+
}
|
|
1157
|
+
read = content.read().then((bytes) => {
|
|
1158
|
+
spent.bytes += bytes.length - claim;
|
|
1159
|
+
return bytes;
|
|
1160
|
+
});
|
|
1161
|
+
if (content.key) reading.set(content.key, read);
|
|
1162
|
+
}
|
|
1163
|
+
const bytes = await read;
|
|
1164
|
+
if (bytes.length > MAX_BYTES) {
|
|
1165
|
+
tooBig(sess, key, bytes.length);
|
|
1166
|
+
unknown.add(key);
|
|
1167
|
+
continue;
|
|
1168
|
+
}
|
|
1169
|
+
const hash = sha(bytes);
|
|
1170
|
+
files[key] = hash;
|
|
1171
|
+
sizes[key] = bytes.length;
|
|
1172
|
+
if (content.key) {
|
|
1173
|
+
seen.set(content.key, { hash, size: bytes.length });
|
|
1174
|
+
blobs.put(content.key, bytes);
|
|
1175
|
+
}
|
|
1176
|
+
} catch (err) {
|
|
1177
|
+
// One unreadable entry — a lost shard, a planted manifest, a
|
|
1178
|
+
// dropped request — must not stop the whole sync, and must
|
|
1179
|
+
// not read as a deletion either.
|
|
1180
|
+
warnOnce(sess, `unreadable:${key}`, `could not read ${key} from your filesystem (${err?.message ?? err}) — leaving it alone`);
|
|
1181
|
+
unknown.add(key);
|
|
1182
|
+
unreadable.add(key);
|
|
1183
|
+
}
|
|
195
1184
|
}
|
|
196
1185
|
}),
|
|
197
1186
|
);
|
|
198
|
-
|
|
1187
|
+
// What this pass read or recognised is what the next one starts from; a
|
|
1188
|
+
// pass cut short keeps what came before as well.
|
|
1189
|
+
if (ctx.signal?.aborted) for (const [k, v] of seen) sess.hashes.set(k, v);
|
|
1190
|
+
else sess.hashes = seen;
|
|
1191
|
+
if (deferred.size) {
|
|
1192
|
+
sess.more = true;
|
|
1193
|
+
warnOnce(
|
|
1194
|
+
sess,
|
|
1195
|
+
"read-budget",
|
|
1196
|
+
`read ${spent.files} files (${human(spent.bytes)}) from your filesystem in one pass, the most one pass reads; ` +
|
|
1197
|
+
`the other ${deferred.size} are compared in the next pass, and left alone until then.`,
|
|
1198
|
+
);
|
|
1199
|
+
}
|
|
1200
|
+
return { files, sizes, unknown, unreadable, deferred, held, blobs };
|
|
199
1201
|
}
|
|
200
1202
|
|
|
201
1203
|
/**
|
|
202
|
-
*
|
|
1204
|
+
* How to fetch a remote file, and a key for its content.
|
|
203
1205
|
*
|
|
204
|
-
*
|
|
205
|
-
*
|
|
206
|
-
*
|
|
207
|
-
*
|
|
1206
|
+
* The key is the file's manifest, which fixes the bytes — shards addressed by
|
|
1207
|
+
* their hashes, and the keys that open them — so one key is one content,
|
|
1208
|
+
* however many entries point at it and however many passes ask. `read`
|
|
1209
|
+
* fetches exactly that content. `bound` is the most a read of it can bring
|
|
1210
|
+
* back, from the manifest's own chunk sizes, which the reader holds it to:
|
|
1211
|
+
* `plain` bytes out, `fetched` bytes of shards in. Where this vfs cannot say,
|
|
1212
|
+
* the key and bound are null and the path is read as it stands.
|
|
208
1213
|
*/
|
|
209
|
-
|
|
210
|
-
|
|
1214
|
+
async function remoteContent(vfs, abs) {
|
|
1215
|
+
if (typeof vfs.statManifest !== "function") {
|
|
1216
|
+
const { size } = await vfs.stat(abs);
|
|
1217
|
+
return { size, key: null, bound: null, read: () => vfs.readFile(abs) };
|
|
1218
|
+
}
|
|
1219
|
+
const { manifest, size } = await vfs.statManifest(abs);
|
|
1220
|
+
const exact = typeof vfs.readByManifest === "function";
|
|
1221
|
+
return {
|
|
1222
|
+
size,
|
|
1223
|
+
key: exact ? sha(JSON.stringify(manifest)) : null,
|
|
1224
|
+
bound: manifestBound(manifest),
|
|
1225
|
+
read: exact ? () => vfs.readByManifest(manifest) : () => vfs.readFile(abs),
|
|
1226
|
+
};
|
|
1227
|
+
}
|
|
211
1228
|
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
1229
|
+
/**
|
|
1230
|
+
* What a manifest lets a read bring back (see `remoteContent`), or null when
|
|
1231
|
+
* it is not in a shape this knows. Each chunk decrypts to its ciphertext less
|
|
1232
|
+
* the 16-byte tag, after the reader has fetched every shard at exactly the
|
|
1233
|
+
* size the manifest gives — it refuses one that is not.
|
|
1234
|
+
*/
|
|
1235
|
+
function manifestBound(manifest) {
|
|
1236
|
+
const chunks = manifest?.chunks;
|
|
1237
|
+
if (!Array.isArray(chunks)) return null;
|
|
1238
|
+
let plain = 0;
|
|
1239
|
+
let fetched = 0;
|
|
1240
|
+
for (const chunk of chunks) {
|
|
1241
|
+
const { ciphertextSize, shardSize, shardHashes } = chunk ?? {};
|
|
1242
|
+
if (!Number.isFinite(ciphertextSize) || !Number.isFinite(shardSize) || !Array.isArray(shardHashes)) return null;
|
|
1243
|
+
plain += Math.max(0, ciphertextSize - 16);
|
|
1244
|
+
fetched += shardHashes.length * shardSize;
|
|
1245
|
+
}
|
|
1246
|
+
return { plain, fetched };
|
|
1247
|
+
}
|
|
1248
|
+
|
|
1249
|
+
/**
|
|
1250
|
+
* The bytes a pass has read, by content key, up to `limit` bytes: the oldest
|
|
1251
|
+
* go first. What lets a pull of content the scan already read — two hundred
|
|
1252
|
+
* entries sharing one blob — write it without fetching it again.
|
|
1253
|
+
*/
|
|
1254
|
+
function blobCache(limit) {
|
|
1255
|
+
const kept = new Map();
|
|
1256
|
+
let total = 0;
|
|
1257
|
+
return {
|
|
1258
|
+
get(key) {
|
|
1259
|
+
return key ? kept.get(key) : undefined;
|
|
1260
|
+
},
|
|
1261
|
+
put(key, bytes) {
|
|
1262
|
+
if (!key || kept.has(key) || bytes.length > limit) return;
|
|
1263
|
+
kept.set(key, bytes);
|
|
1264
|
+
total += bytes.length;
|
|
1265
|
+
for (const [old, b] of kept) {
|
|
1266
|
+
if (total <= limit) break;
|
|
1267
|
+
kept.delete(old);
|
|
1268
|
+
total -= b.length;
|
|
1269
|
+
}
|
|
1270
|
+
},
|
|
1271
|
+
};
|
|
1272
|
+
}
|
|
1273
|
+
|
|
1274
|
+
/**
|
|
1275
|
+
* A name as THIS volume compares names — for spotting two remote spellings
|
|
1276
|
+
* that would land on one file here.
|
|
1277
|
+
*
|
|
1278
|
+
* Not `foldName`, which folds further than any one filesystem does because
|
|
1279
|
+
* over-folding only refuses more there. Here over-folding costs files: APFS
|
|
1280
|
+
* holds `run-10:30.txt` beside `run-10:45.txt`, `report` beside `report.`, and
|
|
1281
|
+
* `v2.md` beside `v².md`, and the old fold refused all six on every Mac — a
|
|
1282
|
+
* directory of timestamped logs stopped syncing. So: canonical normalization
|
|
1283
|
+
* (NFC; APFS ignores the difference, but not compatibility forms), full case
|
|
1284
|
+
* folding only when the volume folds case, and Win32's stream and trailing
|
|
1285
|
+
* dot rules only on Windows. `writeTarget`'s exact-spelling check is what
|
|
1286
|
+
* keeps a write off the wrong file; this only keeps two spellings from taking
|
|
1287
|
+
* turns at one name.
|
|
1288
|
+
*/
|
|
1289
|
+
function volumeFold(name, { ignoreCase = true, platform = process.platform } = {}) {
|
|
1290
|
+
let s = String(name);
|
|
1291
|
+
if (platform === "win32") s = s.split(":")[0].replace(/[. ]+$/, "");
|
|
1292
|
+
s = s.normalize("NFC");
|
|
1293
|
+
return ignoreCase ? s.toLowerCase().toUpperCase().toLowerCase().normalize("NFC") : s;
|
|
1294
|
+
}
|
|
1295
|
+
|
|
1296
|
+
/**
|
|
1297
|
+
* On a volume that folds names, two remote paths that fold alike cannot both
|
|
1298
|
+
* exist here, and whichever landed second would overwrite the first. Refuse
|
|
1299
|
+
* the aliases: where exactly one spelling matches what is already on disk,
|
|
1300
|
+
* that one is the real file and syncs; where every spelling is on disk, the
|
|
1301
|
+
* disk evidently holds them all and they all sync; otherwise nothing in the
|
|
1302
|
+
* group does. Checked on every directory prefix too, since `Src/a.ts` and
|
|
1303
|
+
* `src/b.ts` collide on the directory.
|
|
1304
|
+
*
|
|
1305
|
+
* Prefixes are numbered in two tries — by exact spelling and by fold — so a
|
|
1306
|
+
* path costs one step per segment, and each distinct segment is folded once.
|
|
1307
|
+
* Refolding every prefix of every path from scratch was quadratic in depth:
|
|
1308
|
+
* 20,000 remote paths 60 deep held the event loop for 11.8 seconds.
|
|
1309
|
+
*/
|
|
1310
|
+
export async function refuseFoldCollisions(wanted, localKeys = new Set(), unknown, sess, signal, fold = {}) {
|
|
1311
|
+
const pace = pacer();
|
|
1312
|
+
const folds = new Map();
|
|
1313
|
+
const foldOf = (seg) => {
|
|
1314
|
+
let f = folds.get(seg);
|
|
1315
|
+
if (f === undefined) folds.set(seg, (f = volumeFold(seg, fold)));
|
|
1316
|
+
return f;
|
|
1317
|
+
};
|
|
1318
|
+
const exact = new Map(); // "parent id/segment" → id, by spelling
|
|
1319
|
+
const folded = new Map(); // the same, by fold
|
|
1320
|
+
const parentOf = [0];
|
|
1321
|
+
const segOf = [""];
|
|
1322
|
+
const step = (trie, parent, seg) => {
|
|
1323
|
+
const k = `${parent}/${seg}`;
|
|
1324
|
+
let id = trie.get(k);
|
|
1325
|
+
if (id === undefined) {
|
|
1326
|
+
trie.set(k, (id = trie.size + 1));
|
|
1327
|
+
if (trie === exact) {
|
|
1328
|
+
parentOf[id] = parent;
|
|
1329
|
+
segOf[id] = seg;
|
|
1330
|
+
}
|
|
1331
|
+
}
|
|
1332
|
+
return id;
|
|
1333
|
+
};
|
|
1334
|
+
const spelled = (id) => {
|
|
1335
|
+
const parts = [];
|
|
1336
|
+
for (; id; id = parentOf[id]) parts.push(segOf[id]);
|
|
1337
|
+
return parts.reverse().join("/");
|
|
1338
|
+
};
|
|
1339
|
+
|
|
1340
|
+
const onDisk = new Set();
|
|
1341
|
+
for (const key of localKeys) {
|
|
1342
|
+
const breather = pace();
|
|
1343
|
+
if (breather) await breather;
|
|
1344
|
+
if (signal?.aborted) return [];
|
|
1345
|
+
let e = 0;
|
|
1346
|
+
for (const seg of key.slice(1).split("/")) onDisk.add((e = step(exact, e, seg)));
|
|
1347
|
+
}
|
|
1348
|
+
const spellings = new Map(); // fold id → Set of exact ids
|
|
1349
|
+
const chains = [];
|
|
1350
|
+
for (const [key] of wanted) {
|
|
1351
|
+
const breather = pace();
|
|
1352
|
+
if (breather) await breather;
|
|
1353
|
+
if (signal?.aborted) return [];
|
|
1354
|
+
const chain = [];
|
|
1355
|
+
let e = 0;
|
|
1356
|
+
let f = 0;
|
|
1357
|
+
for (const seg of key.slice(1).split("/")) {
|
|
1358
|
+
e = step(exact, e, seg);
|
|
1359
|
+
f = step(folded, f, foldOf(seg));
|
|
1360
|
+
chain.push(e, f);
|
|
1361
|
+
if (!spellings.has(f)) spellings.set(f, new Set());
|
|
1362
|
+
spellings.get(f).add(e);
|
|
1363
|
+
}
|
|
1364
|
+
chains.push(chain);
|
|
1365
|
+
}
|
|
1366
|
+
// Per group, once: every spelling on disk, or the one that is.
|
|
1367
|
+
const verdicts = new Map();
|
|
1368
|
+
const verdictOf = (f) => {
|
|
1369
|
+
let v = verdicts.get(f);
|
|
1370
|
+
if (!v) {
|
|
1371
|
+
const here = [...spellings.get(f)].filter((e) => onDisk.has(e));
|
|
1372
|
+
verdicts.set(f, (v = { all: here.length === spellings.get(f).size, only: here.length === 1 ? here[0] : 0 }));
|
|
1373
|
+
}
|
|
1374
|
+
return v;
|
|
1375
|
+
};
|
|
1376
|
+
const kept = [];
|
|
1377
|
+
for (let i = 0; i < wanted.length; i++) {
|
|
1378
|
+
const breather = pace();
|
|
1379
|
+
if (breather) await breather;
|
|
1380
|
+
if (signal?.aborted) return [];
|
|
1381
|
+
const key = wanted[i][0];
|
|
1382
|
+
const chain = chains[i];
|
|
1383
|
+
let refused = false;
|
|
1384
|
+
for (let k = 0; k < chain.length && !refused; k += 2) {
|
|
1385
|
+
const [e, f] = [chain[k], chain[k + 1]];
|
|
1386
|
+
const group = spellings.get(f);
|
|
1387
|
+
if (group.size < 2) continue;
|
|
1388
|
+
const v = verdictOf(f);
|
|
1389
|
+
if (v.all || v.only === e) continue;
|
|
1390
|
+
const other = [...group].find((s) => s !== e);
|
|
1391
|
+
warnOnce(sess, `fold:${key}`, `refusing ${key} — it folds to the same name as /${spelled(other)} on this disk, which cannot hold both`);
|
|
1392
|
+
unknown.add(key);
|
|
1393
|
+
refused = true;
|
|
1394
|
+
}
|
|
1395
|
+
if (!refused) kept.push(wanted[i]);
|
|
1396
|
+
}
|
|
1397
|
+
return kept;
|
|
218
1398
|
}
|
|
219
1399
|
|
|
1400
|
+
// ---- the write boundary ---------------------------------------------------------
|
|
1401
|
+
|
|
220
1402
|
/**
|
|
221
1403
|
* The local path for a synced path, or null when it escapes the mount.
|
|
222
1404
|
*
|
|
@@ -233,44 +1415,231 @@ export function localPathFor(dir, path) {
|
|
|
233
1415
|
return candidate;
|
|
234
1416
|
}
|
|
235
1417
|
|
|
1418
|
+
/** Windows device names, with or without an extension: `nul.txt` opens NUL. */
|
|
1419
|
+
const WIN32_RESERVED = /^(con|prn|aux|nul|conin\$|conout\$|com[0-9¹²³]|lpt[0-9¹²³])$/i;
|
|
1420
|
+
|
|
1421
|
+
/**
|
|
1422
|
+
* Why Windows cannot hold `name` as itself, or null. Used only on win32: these
|
|
1423
|
+
* are ordinary names on macOS and Linux, and a VFS is shared across machines,
|
|
1424
|
+
* so refusing them everywhere would hide real files from their owners.
|
|
1425
|
+
*/
|
|
1426
|
+
export function win32NameProblem(name) {
|
|
1427
|
+
// A backslash is a separator on Windows: the name would be two path
|
|
1428
|
+
// components, and the one-at-a-time checks in writeTarget would miss the
|
|
1429
|
+
// inner one. connect refuses it today; nothing here depends on that.
|
|
1430
|
+
if (name.includes("\\")) return "contains a backslash, which Windows reads as a directory separator";
|
|
1431
|
+
if (name.includes(":")) return "contains ':', which Windows reads as a stream of another file";
|
|
1432
|
+
if (/[<>"|?*]/.test(name)) return "contains a character Windows does not allow in a name (<>\"|?*)";
|
|
1433
|
+
if (/[. ]$/.test(name)) return "ends in a dot or a space, which Windows strips";
|
|
1434
|
+
if (WIN32_RESERVED.test(name.split(".")[0].replace(/ +$/, ""))) return "is a Windows device name";
|
|
1435
|
+
return null;
|
|
1436
|
+
}
|
|
1437
|
+
|
|
1438
|
+
/**
|
|
1439
|
+
* The one gate every write and delete on this disk passes.
|
|
1440
|
+
*
|
|
1441
|
+
* Walks the key from the mount directory, one component at a time. Every
|
|
1442
|
+
* component that already exists must be listed by `readdir` of its parent
|
|
1443
|
+
* under EXACTLY that spelling, and be a real directory — not a symlink. A
|
|
1444
|
+
* component that is not listed but that `lstat` still finds is an alias: the
|
|
1445
|
+
* volume folded case, normalization, an ignorable code point, a trailing dot
|
|
1446
|
+
* or an 8.3 short name onto something else. Refuse it. This one check covers
|
|
1447
|
+
* every folding rule a filesystem has, including ones nobody has listed yet,
|
|
1448
|
+
* because it asks the filesystem rather than predicting it.
|
|
1449
|
+
*
|
|
1450
|
+
* The final file may be created when absent, or replaced only when it exists
|
|
1451
|
+
* under exactly that spelling as a regular file AND the local scan reported it
|
|
1452
|
+
* — a synced file. Anything else on disk is local-only (ignored, hidden,
|
|
1453
|
+
* refused, over the limit), exists nowhere else, and is never overwritten.
|
|
1454
|
+
*
|
|
1455
|
+
* This replaces a check of the final component alone. `lstat` follows every
|
|
1456
|
+
* other component, so a symlinked directory one level up redirected the write
|
|
1457
|
+
* outside the mount; a hard-linked file was truncated in place, changing its
|
|
1458
|
+
* twin; and a remote `src/App.ts` overwrote the unsynced `src/app.ts`.
|
|
1459
|
+
*
|
|
1460
|
+
* Returns `{ ok: true, abs, parent, missing, existing }` — `missing` being the
|
|
1461
|
+
* directories to create, in order, and `existing` the stats of the file being
|
|
1462
|
+
* replaced — or `{ ok: false, reason }`. `listing` caches `readdir` for a pass.
|
|
1463
|
+
*/
|
|
1464
|
+
export async function writeTarget(dir, key, { localKeys = new Set(), listing = new Map(), platform = process.platform } = {}) {
|
|
1465
|
+
const refuse = (reason) => ({ ok: false, reason });
|
|
1466
|
+
const names = key.split("/").filter(Boolean);
|
|
1467
|
+
if (!names.length) return refuse("it names the mount directory itself");
|
|
1468
|
+
if (platform === "win32") {
|
|
1469
|
+
for (const name of names) {
|
|
1470
|
+
const problem = win32NameProblem(name);
|
|
1471
|
+
if (problem) return refuse(`${JSON.stringify(name)} ${problem}`);
|
|
1472
|
+
}
|
|
1473
|
+
}
|
|
1474
|
+
const list = async (d) => {
|
|
1475
|
+
if (!listing.has(d)) listing.set(d, await readdir(d).then((n) => new Set(n), () => null));
|
|
1476
|
+
return listing.get(d);
|
|
1477
|
+
};
|
|
1478
|
+
const present = (p) => lstat(p).then(() => true, () => false);
|
|
1479
|
+
const alias = (name) => refuse(`${JSON.stringify(name)} is spelled differently from an entry already here that this volume treats as the same name`);
|
|
1480
|
+
|
|
1481
|
+
let parent = dir;
|
|
1482
|
+
for (let i = 0; i < names.length; i++) {
|
|
1483
|
+
const name = names[i];
|
|
1484
|
+
const abs = join(parent, name);
|
|
1485
|
+
const last = i === names.length - 1;
|
|
1486
|
+
const entries = await list(parent);
|
|
1487
|
+
if (!entries) return refuse(`${parent} could not be listed`);
|
|
1488
|
+
if (!entries.has(name)) {
|
|
1489
|
+
if (await present(abs)) return alias(name);
|
|
1490
|
+
return { ok: true, abs: join(parent, ...names.slice(i)), parent, missing: names.slice(i, -1), existing: null };
|
|
1491
|
+
}
|
|
1492
|
+
const info = await lstat(abs).catch(() => null);
|
|
1493
|
+
if (!last) {
|
|
1494
|
+
if (!info?.isDirectory()) return refuse(`${relative(dir, abs)} is ${info?.isSymbolicLink() ? "a symlink" : "not a directory"}`);
|
|
1495
|
+
parent = abs;
|
|
1496
|
+
continue;
|
|
1497
|
+
}
|
|
1498
|
+
if (!info?.isFile()) return refuse(`${relative(dir, abs)} is a symlink or not a regular file`);
|
|
1499
|
+
if (!localKeys.has(key)) return refuse(`${relative(dir, abs)} exists here but is not synced (ignored, hidden or refused), so it may exist nowhere else`);
|
|
1500
|
+
return { ok: true, abs, parent, missing: [], existing: info };
|
|
1501
|
+
}
|
|
1502
|
+
return refuse("unreachable");
|
|
1503
|
+
}
|
|
1504
|
+
|
|
236
1505
|
/**
|
|
237
|
-
*
|
|
1506
|
+
* Write `bytes` where `writeTarget` said: missing directories one component at
|
|
1507
|
+
* a time, then a temp file in the verified parent renamed over the target.
|
|
238
1508
|
*
|
|
239
|
-
*
|
|
240
|
-
*
|
|
241
|
-
*
|
|
242
|
-
*
|
|
243
|
-
*
|
|
1509
|
+
* Rename replaces the directory ENTRY, so a hard-linked twin keeps its bytes;
|
|
1510
|
+
* `wx` means the temp name can never be an existing link. The replaced file's
|
|
1511
|
+
* permission bits are copied across, so an edit to `scripts/deploy.sh` or
|
|
1512
|
+
* `gradlew` made on another machine still runs here — stripping the execute
|
|
1513
|
+
* bit would break every such edit, and a freshly created file gets the
|
|
1514
|
+
* default mode, never an execute bit.
|
|
244
1515
|
*/
|
|
245
|
-
async function
|
|
1516
|
+
async function writeAt(target, bytes, listing) {
|
|
1517
|
+
let parent = target.parent;
|
|
1518
|
+
for (const name of target.missing) {
|
|
1519
|
+
const next = join(parent, name);
|
|
1520
|
+
await mkdir(next); // not recursive: each component is created here, never followed
|
|
1521
|
+
listing.get(parent)?.add(name);
|
|
1522
|
+
listing.set(next, new Set());
|
|
1523
|
+
parent = next;
|
|
1524
|
+
}
|
|
1525
|
+
const tmp = join(parent, TMP_PREFIX + randomBytes(6).toString("hex"));
|
|
246
1526
|
try {
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
1527
|
+
await writeFile(tmp, bytes, { flag: "wx" });
|
|
1528
|
+
if (target.existing) await chmod(tmp, target.existing.mode & 0o777);
|
|
1529
|
+
await rename(tmp, target.abs);
|
|
1530
|
+
} catch (err) {
|
|
1531
|
+
await rm(tmp, { force: true }).catch(() => {});
|
|
1532
|
+
throw err;
|
|
251
1533
|
}
|
|
1534
|
+
listing.get(parent)?.add(target.abs.slice(parent.length + 1));
|
|
252
1535
|
}
|
|
253
1536
|
|
|
1537
|
+
// ---- planning and applying ------------------------------------------------------
|
|
1538
|
+
|
|
254
1539
|
/**
|
|
255
|
-
*
|
|
1540
|
+
* Scan both sides and plan, without changing anything.
|
|
256
1541
|
*
|
|
257
|
-
*
|
|
258
|
-
*
|
|
259
|
-
*
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
1542
|
+
* Shared by `mount` (via `reconcile`) and `import`, so the rules about what is
|
|
1543
|
+
* skipped and what a delete means are defined in exactly one place. Returns
|
|
1544
|
+
* null when `signal` stopped it: a half-finished scan must never be planned
|
|
1545
|
+
* from, since what it did not reach would read as deleted.
|
|
1546
|
+
*
|
|
1547
|
+
* `rulesDir` is the directory whose stored ignore rules apply, when that is not
|
|
1548
|
+
* the one scanned: a dry run of a mount on a directory that is not there yet
|
|
1549
|
+
* scans an empty stand-in, but the rules are the ones recorded for the real
|
|
1550
|
+
* directory, which the mount will find once it has created it.
|
|
1551
|
+
*/
|
|
1552
|
+
async function preparePass({ vfs, dir, rulesDir = dir, root, state, opts, sess, save = true, signal, blobs, read }) {
|
|
1553
|
+
if (sess.ignoreCase === undefined) sess.ignoreCase = await foldsCase(dir);
|
|
1554
|
+
const ignores = ignoreContext(sess, await loadSnapshots(rulesDir, root), { hidden: Boolean(opts.hidden) });
|
|
1555
|
+
await seeRootFiles(dir, ignores);
|
|
1556
|
+
// The remote walk is the slow part; start it while the local scan runs.
|
|
1557
|
+
// The remote FILTER has to wait for the local scan, which is what finds
|
|
1558
|
+
// the nested ignore files.
|
|
1559
|
+
const [local, paths] = await Promise.all([
|
|
1560
|
+
scanLocal(dir, { hidden: opts.hidden, ignores, sess, signal }),
|
|
1561
|
+
walkRemote(vfs, root),
|
|
268
1562
|
]);
|
|
269
|
-
|
|
270
|
-
|
|
1563
|
+
if (signal?.aborted) return null;
|
|
1564
|
+
// The ignore files as they now stand are known; nothing has been pulled.
|
|
1565
|
+
await ignores.record(rulesDir, root, save);
|
|
1566
|
+
const remote = await scanRemote(vfs, root, opts.hidden, ignores.now, {
|
|
1567
|
+
paths, held: ignores.held, session: sess, ignoreCase: sess.ignoreCase,
|
|
1568
|
+
localKeys: new Set(Object.keys(local.files)), signal, blobs, spent: read,
|
|
1569
|
+
});
|
|
1570
|
+
if (signal?.aborted) return null;
|
|
1571
|
+
sess.admit = (rel, isDir) => admission(rel, isDir, opts.hidden, ignores.now, ignores.held);
|
|
1572
|
+
// One warning for them all: a hostile ignore file can make thousands.
|
|
1573
|
+
const costlyPaths = [...(sess.costly ?? [])].sort();
|
|
1574
|
+
sess.costly = null;
|
|
1575
|
+
if (costlyPaths.length) {
|
|
1576
|
+
warnOnce(
|
|
1577
|
+
sess,
|
|
1578
|
+
`costly:${costlyPaths.join("\0")}`,
|
|
1579
|
+
`leaving ${costlyPaths.length} path${costlyPaths.length === 1 ? "" : "s"} alone — matching ${costlyPaths.length === 1 ? "it" : "them"} ` +
|
|
1580
|
+
`against the ignore rules would take too long: ${costlyPaths.slice(0, 5).join(", ")}${costlyPaths.length > 5 ? ", …" : ""}. ` +
|
|
1581
|
+
`Nothing is synced or deleted for ${costlyPaths.length === 1 ? "it" : "them"}; simpler ignore rules, or shorter names, let them sync.`,
|
|
1582
|
+
);
|
|
1583
|
+
}
|
|
1584
|
+
const keptBack = [...new Set([...local.held, ...remote.held])].sort();
|
|
1585
|
+
if (keptBack.length) warnOnce(sess, `kept-back:${keptBack.join("\0")}`, ...heldNotice(keptBack, rulesDir));
|
|
1586
|
+
|
|
1587
|
+
const unknown = new Set([...local.unknown, ...remote.unknown]);
|
|
1588
|
+
const planned = planSync({ last: state, local: local.files, remote: remote.files, unknown });
|
|
1589
|
+
const views = { local: Object.keys(local.files).length, remote: Object.keys(remote.files).length };
|
|
1590
|
+
sess.peak = Math.max(sess.peak ?? 0, Object.keys(state).length);
|
|
1591
|
+
const { plan, held } = holdMassDeletes(planned, state, views, sess.peak);
|
|
1592
|
+
for (const h of held) warnOnce(sess, `held:${h.action}:${h.count}`, ...breakerMessage(h, dir, root));
|
|
1593
|
+
return {
|
|
1594
|
+
plan, local: local.files, remote: remote.files, sizes: remote.sizes, blobs: remote.blobs, ignores,
|
|
1595
|
+
unreadable: remote.unreadable, deferred: remote.deferred,
|
|
1596
|
+
landed: new Set(), // ignore files pulled in this pass
|
|
1597
|
+
notLanded: new Map(), // and why the others were not
|
|
1598
|
+
};
|
|
1599
|
+
}
|
|
1600
|
+
|
|
1601
|
+
/** The breaker's warning, as lines. */
|
|
1602
|
+
function breakerMessage({ action, count, tracked, empty }, dir, root) {
|
|
1603
|
+
if (action === "delete-local") {
|
|
1604
|
+
return [
|
|
1605
|
+
`refusing to delete ${count} of ${tracked} files from ${dir} in one pass — they are gone from ${root}, ` +
|
|
1606
|
+
`which ${empty ? "now holds nothing at all" : "looks like a missing or renamed root, not an edit"}. Nothing was deleted.`,
|
|
1607
|
+
` If you meant it, delete them here yourself; to upload them all again instead, remove ${statePath(dir)}.`,
|
|
1608
|
+
];
|
|
1609
|
+
}
|
|
1610
|
+
return [
|
|
1611
|
+
`refusing to delete ${count} of ${tracked} files from your filesystem (${root}) in one pass — they are gone from ${dir}, ` +
|
|
1612
|
+
`which ${empty ? "now holds nothing at all" : "looks like a wiped or switched directory, not an edit"}. Nothing was deleted.`,
|
|
1613
|
+
` If you meant it, use: muhkoo vfs rm -r <path>; to download them all again instead, remove ${statePath(dir)}.`,
|
|
1614
|
+
];
|
|
1615
|
+
}
|
|
271
1616
|
|
|
1617
|
+
/**
|
|
1618
|
+
* Apply a plan, and return the new last-synced view.
|
|
1619
|
+
*
|
|
1620
|
+
* Applied sequentially rather than in parallel: writes touch one directory
|
|
1621
|
+
* record each, and two concurrent writes to the same record would have one
|
|
1622
|
+
* silently lose ([[VfsNamespace]] consistency note). `signal` stops it between
|
|
1623
|
+
* steps — never inside one — and what was done by then is what it returns.
|
|
1624
|
+
*
|
|
1625
|
+
* A step this disk declines (a pull onto a symlink, a remote file where a
|
|
1626
|
+
* directory stands) is warned about once and left out of the result, so the
|
|
1627
|
+
* state does not change and nothing is written — which is what keeps a plan
|
|
1628
|
+
* that cannot converge from waking the watcher into another pass.
|
|
1629
|
+
*
|
|
1630
|
+
* What it held back is left in `pass.outstanding`: pulls waiting for disk
|
|
1631
|
+
* space, and pulls past the pass's budget.
|
|
1632
|
+
*/
|
|
1633
|
+
async function applyPlan({ vfs, dir, root, state, plan, pass, opts, sess, signal, budget = { bytes: 0, files: 0 } }) {
|
|
272
1634
|
const next = { ...state };
|
|
1635
|
+
const listing = new Map();
|
|
1636
|
+
const localKeys = new Set(Object.keys(pass.local));
|
|
1637
|
+
const decline = (action, path, reason) => warnOnce(sess, `declined:${action}:${path}`, `skipping ${path} — ${reason}`);
|
|
1638
|
+
let short = 0; // pulls held for want of disk space
|
|
1639
|
+
let over = 0; // and past the pass's budget
|
|
1640
|
+
|
|
273
1641
|
for (const { path, action } of plan) {
|
|
1642
|
+
if (signal?.aborted) break;
|
|
274
1643
|
const localAbs = localPathFor(dir, path);
|
|
275
1644
|
if (!localAbs) {
|
|
276
1645
|
// Warn and skip rather than throw: one bad path must not abandon
|
|
@@ -279,26 +1648,85 @@ export async function reconcile({ vfs, dir, root, state, opts = {} }) {
|
|
|
279
1648
|
delete next[path];
|
|
280
1649
|
continue;
|
|
281
1650
|
}
|
|
282
|
-
const remoteAbs = root + path;
|
|
1651
|
+
const remoteAbs = root === "/" ? path : root + path;
|
|
283
1652
|
try {
|
|
284
1653
|
switch (action) {
|
|
285
1654
|
case "push": {
|
|
286
1655
|
const bytes = await readFile(localAbs);
|
|
287
1656
|
await vfs.writeFile(remoteAbs, new Uint8Array(bytes));
|
|
288
1657
|
next[path] = sha(bytes);
|
|
289
|
-
info(
|
|
1658
|
+
info(`${SIGN.push} ${path}`);
|
|
290
1659
|
break;
|
|
291
1660
|
}
|
|
292
1661
|
case "pull": {
|
|
293
|
-
|
|
294
|
-
|
|
1662
|
+
// Held pulls change nothing, so the next pass plans them
|
|
1663
|
+
// again. Each limit is checked twice: on the size the scan
|
|
1664
|
+
// read, so a pull that cannot fit is not fetched at all, and
|
|
1665
|
+
// then on the bytes actually fetched — the file may have
|
|
1666
|
+
// changed since, and they are what land on this disk.
|
|
1667
|
+
//
|
|
1668
|
+
// Ignore files are held to the same limits. They were
|
|
1669
|
+
// exempt as "small", and a 256 KB file of comments, copied
|
|
1670
|
+
// to thousands of directories, put 7 GiB on a disk already
|
|
1671
|
+
// under its reserve in one pass. One held here holds this
|
|
1672
|
+
// pass's uploads instead (see `reconcile`).
|
|
1673
|
+
const limit = sess.pullBudget;
|
|
1674
|
+
/** Why a pull of `size` bytes must wait, or null. */
|
|
1675
|
+
const holdFor = async (size) => {
|
|
1676
|
+
if (budget.full || (budget.files && (budget.files + 1 > limit.files || budget.bytes + size > limit.bytes))) {
|
|
1677
|
+
budget.full = true;
|
|
1678
|
+
over++;
|
|
1679
|
+
return "this pass has pulled all it may; the next pass carries on";
|
|
1680
|
+
}
|
|
1681
|
+
if (await roomFor(dir, size, sess)) return null;
|
|
1682
|
+
short++;
|
|
1683
|
+
return "the disk would be left with less than its reserve free";
|
|
1684
|
+
};
|
|
1685
|
+
let hold = await holdFor(pass.sizes?.[path] ?? 0);
|
|
1686
|
+
if (hold) {
|
|
1687
|
+
pass.notLanded?.set(path, hold);
|
|
1688
|
+
break;
|
|
1689
|
+
}
|
|
1690
|
+
const target = await writeTarget(dir, path, { localKeys, listing });
|
|
1691
|
+
if (!target.ok) {
|
|
1692
|
+
decline(action, path, target.reason);
|
|
1693
|
+
pass.notLanded?.set(path, target.reason);
|
|
1694
|
+
break;
|
|
1695
|
+
}
|
|
1696
|
+
// Content the scan read — or an earlier pull, of an entry
|
|
1697
|
+
// sharing it — is not fetched again.
|
|
1698
|
+
const content = await remoteContent(vfs, remoteAbs);
|
|
1699
|
+
let bytes = pass.blobs?.get(content.key);
|
|
1700
|
+
if (!bytes) {
|
|
1701
|
+
bytes = await content.read();
|
|
1702
|
+
pass.blobs?.put(content.key, bytes);
|
|
1703
|
+
}
|
|
1704
|
+
if (bytes.length > MAX_BYTES) {
|
|
1705
|
+
tooBig(sess, path, bytes.length);
|
|
1706
|
+
pass.notLanded?.set(path, "it is over the mount's size limit");
|
|
295
1707
|
break;
|
|
296
1708
|
}
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
1709
|
+
hold = await holdFor(bytes.length);
|
|
1710
|
+
if (hold) {
|
|
1711
|
+
pass.notLanded?.set(path, hold);
|
|
1712
|
+
break;
|
|
1713
|
+
}
|
|
1714
|
+
// An ignore file that cannot be read as rules is not
|
|
1715
|
+
// pulled: landed, it would stop every pass (see `see`), and
|
|
1716
|
+
// uploads wait for it instead (see `reconcile`).
|
|
1717
|
+
const rules = isIgnoreFile(path.slice(1));
|
|
1718
|
+
const problem = rules && ignoreFileProblem(bytes);
|
|
1719
|
+
if (problem) {
|
|
1720
|
+
decline(action, path, `it cannot be read as ignore rules: ${problem}. If it is not yours, remove it: muhkoo vfs rm ${remoteAbs}`);
|
|
1721
|
+
pass.notLanded?.set(path, `it cannot be read as ignore rules: ${problem}`);
|
|
1722
|
+
break;
|
|
1723
|
+
}
|
|
1724
|
+
await writeAt(target, bytes, listing);
|
|
300
1725
|
next[path] = sha(bytes);
|
|
301
|
-
|
|
1726
|
+
if (rules) pass.landed?.add(path);
|
|
1727
|
+
budget.files++;
|
|
1728
|
+
budget.bytes += bytes.length;
|
|
1729
|
+
info(`${SIGN.pull} ${path}`);
|
|
302
1730
|
break;
|
|
303
1731
|
}
|
|
304
1732
|
case "delete-remote":
|
|
@@ -313,7 +1741,7 @@ export async function reconcile({ vfs, dir, root, state, opts = {} }) {
|
|
|
313
1741
|
// high a price for the convenience.
|
|
314
1742
|
if (!opts.allowDelete) {
|
|
315
1743
|
warn(
|
|
316
|
-
`${path} is gone locally but still in your filesystem — not deleting it
|
|
1744
|
+
`${path} is gone locally but still in your filesystem — not deleting it.`,
|
|
317
1745
|
` Pass --allow-delete to propagate deletions, or remove it with: muhkoo vfs rm ${remoteAbs}`,
|
|
318
1746
|
);
|
|
319
1747
|
delete next[path]; // stop re-reporting it every pass
|
|
@@ -321,9 +1749,9 @@ export async function reconcile({ vfs, dir, root, state, opts = {} }) {
|
|
|
321
1749
|
}
|
|
322
1750
|
await vfs.delete(remoteAbs).catch(() => {});
|
|
323
1751
|
delete next[path];
|
|
324
|
-
info(
|
|
1752
|
+
info(`${SIGN["delete-remote"]} ${path} (deleted from your filesystem)`);
|
|
325
1753
|
break;
|
|
326
|
-
case "delete-local":
|
|
1754
|
+
case "delete-local": {
|
|
327
1755
|
// Gated like the remote direction. `--allow-delete` used to
|
|
328
1756
|
// cover only deletions going UP, so anything able to remove
|
|
329
1757
|
// a file from the filesystem removed it from the
|
|
@@ -331,20 +1759,23 @@ export async function reconcile({ vfs, dir, root, state, opts = {} }) {
|
|
|
331
1759
|
// unrecoverable case the remote branch takes care over.
|
|
332
1760
|
if (!opts.allowDelete) {
|
|
333
1761
|
warn(
|
|
334
|
-
`${path} is gone from your filesystem but still here — not deleting it locally
|
|
1762
|
+
`${path} is gone from your filesystem but still here — not deleting it locally.`,
|
|
335
1763
|
` Pass --allow-delete to propagate deletions.`,
|
|
336
1764
|
);
|
|
337
1765
|
delete next[path];
|
|
338
1766
|
break;
|
|
339
1767
|
}
|
|
340
|
-
|
|
341
|
-
|
|
1768
|
+
const target = await writeTarget(dir, path, { localKeys, listing });
|
|
1769
|
+
if (!target.ok || !target.existing) {
|
|
1770
|
+
decline(action, path, target.reason ?? "it is not on disk as a synced file");
|
|
342
1771
|
break;
|
|
343
1772
|
}
|
|
344
|
-
await
|
|
1773
|
+
await unlink(target.abs);
|
|
1774
|
+
listing.get(target.parent)?.delete(target.abs.slice(target.parent.length + 1));
|
|
345
1775
|
delete next[path];
|
|
346
|
-
info(
|
|
1776
|
+
info(`${SIGN["delete-local"]} ${path} (deleted from this disk)`);
|
|
347
1777
|
break;
|
|
1778
|
+
}
|
|
348
1779
|
case "forget":
|
|
349
1780
|
delete next[path];
|
|
350
1781
|
break;
|
|
@@ -357,41 +1788,252 @@ export async function reconcile({ vfs, dir, root, state, opts = {} }) {
|
|
|
357
1788
|
await vfs.writeFile(remoteAbs, new Uint8Array(bytes));
|
|
358
1789
|
next[path] = sha(bytes);
|
|
359
1790
|
warn(
|
|
360
|
-
`conflict on ${path} — kept your local copy
|
|
1791
|
+
`conflict on ${path} — kept your local copy.`,
|
|
361
1792
|
` The other version is still there: muhkoo vfs history ${remoteAbs}`,
|
|
362
1793
|
);
|
|
363
1794
|
break;
|
|
364
1795
|
}
|
|
365
1796
|
}
|
|
366
1797
|
} catch (err) {
|
|
367
|
-
|
|
1798
|
+
const message = err?.message ?? String(err);
|
|
1799
|
+
warnOnce(sess, `error:${path}:${message}`, `${path}: ${message}`);
|
|
1800
|
+
if (action === "pull" && isIgnoreFile(path.slice(1))) pass.notLanded?.set(path, message);
|
|
1801
|
+
}
|
|
1802
|
+
}
|
|
1803
|
+
if (budget.full) {
|
|
1804
|
+
sess.more = true;
|
|
1805
|
+
warnOnce(
|
|
1806
|
+
sess,
|
|
1807
|
+
"pull-budget",
|
|
1808
|
+
`pulled ${budget.files} files (${human(budget.bytes)}) in one pass, the most one pass takes; ` +
|
|
1809
|
+
`the rest follow in the next pass.`,
|
|
1810
|
+
);
|
|
1811
|
+
}
|
|
1812
|
+
pass.outstanding = { space: short, budget: over };
|
|
1813
|
+
return next;
|
|
1814
|
+
}
|
|
1815
|
+
|
|
1816
|
+
/** What a pass left undone (`session.outstanding`), as a phrase: "" when nothing. */
|
|
1817
|
+
function undone({ space = 0, budget = 0, uploads = 0 } = {}) {
|
|
1818
|
+
const n = (count, one, many) => `${count} ${count === 1 ? one : many}`;
|
|
1819
|
+
return [
|
|
1820
|
+
space && `${n(space, "file waits", "files wait")} for disk space`,
|
|
1821
|
+
budget && `${n(budget, "file follows", "files follow")} in the next pass`,
|
|
1822
|
+
uploads && `${n(uploads, "change waits", "changes wait")} to upload until ignore rules come down`,
|
|
1823
|
+
].filter(Boolean).join("; ");
|
|
1824
|
+
}
|
|
1825
|
+
|
|
1826
|
+
/** Note a path too costly to match against the ignore rules; `preparePass` names them all at once. */
|
|
1827
|
+
function costly(sess, key) {
|
|
1828
|
+
(sess.costly ??= new Set()).add(key);
|
|
1829
|
+
}
|
|
1830
|
+
|
|
1831
|
+
/** The warning for a file over MAX_BYTES, once. */
|
|
1832
|
+
function tooBig(sess, key, size) {
|
|
1833
|
+
warnOnce(sess, `big:${key}`, `skipping ${key} — ${Math.round(size / 1e6)} MB is over the mount limit`);
|
|
1834
|
+
}
|
|
1835
|
+
|
|
1836
|
+
/** Would pulling `size` bytes leave the mount's volume its reserve? Unknown: yes. */
|
|
1837
|
+
async function roomFor(dir, size, sess) {
|
|
1838
|
+
let fsinfo;
|
|
1839
|
+
try {
|
|
1840
|
+
fsinfo = await sess.statfs(dir);
|
|
1841
|
+
} catch {
|
|
1842
|
+
return true; // no answer from this platform; the pass budget still bounds a pass
|
|
1843
|
+
}
|
|
1844
|
+
const total = Number(fsinfo.blocks) * Number(fsinfo.bsize);
|
|
1845
|
+
const free = Number(fsinfo.bavail) * Number(fsinfo.bsize);
|
|
1846
|
+
return free - size >= Math.min(RESERVE_MAX, Math.max(RESERVE_MIN, total * RESERVE_SHARE));
|
|
1847
|
+
}
|
|
1848
|
+
|
|
1849
|
+
/**
|
|
1850
|
+
* `state` with every path the two sides already agree on recorded as synced.
|
|
1851
|
+
*
|
|
1852
|
+
* The planner has nothing to do for such a path, so nothing else recorded it,
|
|
1853
|
+
* and a checkout mounted onto the project it came from — a git clone, or a
|
|
1854
|
+
* directory whose `.muhkoo/` was cleaned — stayed out of the state for good.
|
|
1855
|
+
* A teammate's next edit then read as a change on BOTH sides: a conflict,
|
|
1856
|
+
* which this machine's stale copy won, overwriting the edit.
|
|
1857
|
+
*/
|
|
1858
|
+
function withAgreed(state, pass) {
|
|
1859
|
+
let next = state;
|
|
1860
|
+
for (const [path, hash] of Object.entries(pass.local)) {
|
|
1861
|
+
if (pass.remote[path] !== hash || state[path] === hash) continue;
|
|
1862
|
+
if (next === state) next = { ...state };
|
|
1863
|
+
next[path] = hash;
|
|
1864
|
+
}
|
|
1865
|
+
return next;
|
|
1866
|
+
}
|
|
1867
|
+
|
|
1868
|
+
/**
|
|
1869
|
+
* Persist what a pass changed, and nothing it did not.
|
|
1870
|
+
*
|
|
1871
|
+
* Writing the state file on a pass that changed nothing is what kept an
|
|
1872
|
+
* unconvergeable plan alive: the write woke the watcher, the watcher ran
|
|
1873
|
+
* another pass, and that pass planned the same refused action again.
|
|
1874
|
+
*/
|
|
1875
|
+
async function persist(dir, root, state, next) {
|
|
1876
|
+
if (!sameState(state, next)) await saveState(dir, root, next);
|
|
1877
|
+
}
|
|
1878
|
+
|
|
1879
|
+
/**
|
|
1880
|
+
* Ignore files from elsewhere that `pass` could not read, or has not read yet —
|
|
1881
|
+
* path to why — leaving out any whose copy on this disk is the one last synced.
|
|
1882
|
+
* Their rules are missing from the plan, and may keep some of its uploads home.
|
|
1883
|
+
*/
|
|
1884
|
+
function unreadRules(pass, state) {
|
|
1885
|
+
const out = new Map();
|
|
1886
|
+
for (const path of [...pass.unreadable, ...pass.deferred]) {
|
|
1887
|
+
if (!isIgnoreFile(path.slice(1)) || (path in pass.local && pass.local[path] === state[path])) continue;
|
|
1888
|
+
out.set(path, pass.unreadable.has(path) ? "it could not be read" : "this pass has read all it may; the next pass reads it");
|
|
1889
|
+
}
|
|
1890
|
+
return out;
|
|
1891
|
+
}
|
|
1892
|
+
|
|
1893
|
+
/**
|
|
1894
|
+
* Reconcile once, and return the new last-synced view.
|
|
1895
|
+
*
|
|
1896
|
+
* Ignore files that are coming down come down FIRST, and the pass is planned
|
|
1897
|
+
* again under them. Otherwise the pushes were decided before the project's own
|
|
1898
|
+
* rules arrived: a fresh directory with a local `config/master.key`, mounted
|
|
1899
|
+
* onto a project whose `.gitignore` lists it, uploaded the key in the very
|
|
1900
|
+
* pass that brought the `.gitignore`. Rules from elsewhere can only add
|
|
1901
|
+
* exclusions, so applying them early can only keep more at home.
|
|
1902
|
+
*
|
|
1903
|
+
* One that did not come down — held for disk space or by the pass's budget,
|
|
1904
|
+
* unreadable, not yet read, or too large to be read as rules — means this plan
|
|
1905
|
+
* was made without rules that may keep some of its uploads home, so nothing
|
|
1906
|
+
* uploads until it is here; pulls go ahead. Both rounds share the pass's pull
|
|
1907
|
+
* and read budgets.
|
|
1908
|
+
*
|
|
1909
|
+
* What the pass left undone is left in `session.outstanding`, for `mount` to
|
|
1910
|
+
* say. `session` is what a running mount keeps between passes (see
|
|
1911
|
+
* `session`); omitted, each call starts fresh. `signal` stops the pass: before
|
|
1912
|
+
* planning, or between steps, and what was done by then is saved.
|
|
1913
|
+
*/
|
|
1914
|
+
export async function reconcile({ vfs, dir, root, state, opts = {}, session: s, signal }) {
|
|
1915
|
+
const sess = session(s);
|
|
1916
|
+
if (signal?.aborted) return state;
|
|
1917
|
+
const budget = { bytes: 0, files: 0 }; // what the whole pass may pull, both rounds
|
|
1918
|
+
const read = { bytes: 0, files: 0 }; // and read to compare
|
|
1919
|
+
let pass = await preparePass({ vfs, dir, root, state, opts, sess, signal, read });
|
|
1920
|
+
if (!pass) return state;
|
|
1921
|
+
const rulesDue = (plan) => plan.filter(({ path, action }) => action === "pull" && isIgnoreFile(path.slice(1)));
|
|
1922
|
+
const rules = rulesDue(pass.plan);
|
|
1923
|
+
const waiting = new Map(); // ignore files from elsewhere that have not come down → why
|
|
1924
|
+
if (rules.length && rules.length < pass.plan.length) {
|
|
1925
|
+
const next = await applyPlan({ vfs, dir, root, state, plan: rules, pass, opts, sess, signal, budget });
|
|
1926
|
+
await persist(dir, root, state, next);
|
|
1927
|
+
for (const { path } of rules) {
|
|
1928
|
+
if (!pass.landed.has(path)) waiting.set(path, pass.notLanded.get(path) ?? "it could not be pulled");
|
|
368
1929
|
}
|
|
1930
|
+
state = next;
|
|
1931
|
+
if (signal?.aborted) return state;
|
|
1932
|
+
pass = await preparePass({ vfs, dir, root, state, opts, sess, signal, blobs: pass.blobs, read });
|
|
1933
|
+
if (!pass) return state;
|
|
1934
|
+
}
|
|
1935
|
+
// Still due in the plan just made; or one the scan could not read, or has
|
|
1936
|
+
// not read yet, unless this disk holds the copy last synced.
|
|
1937
|
+
for (const { path } of rulesDue(pass.plan)) {
|
|
1938
|
+
if (!waiting.has(path)) waiting.set(path, "it has not come down yet");
|
|
1939
|
+
}
|
|
1940
|
+
for (const [path, why] of unreadRules(pass, state)) waiting.set(path, why);
|
|
1941
|
+
let plan = pass.plan;
|
|
1942
|
+
const uploads = plan.filter(({ action }) => UPLOADS.has(action));
|
|
1943
|
+
if (waiting.size && uploads.length) {
|
|
1944
|
+
plan = plan.filter(({ action }) => !UPLOADS.has(action));
|
|
1945
|
+
// Said again whenever what is held changes: once per session let a
|
|
1946
|
+
// user go on editing, and nothing uploading, with no word after the
|
|
1947
|
+
// first pass.
|
|
1948
|
+
const names = [...waiting.keys()].sort();
|
|
1949
|
+
const held = uploads.map(({ path }) => path);
|
|
1950
|
+
warnOnce(
|
|
1951
|
+
sess,
|
|
1952
|
+
`rules-missing:${names.join("\0")}:${held.join("\0")}`,
|
|
1953
|
+
`not uploading ${held.length} change${held.length === 1 ? "" : "s"} this pass (${held.slice(0, 5).join(", ")}${held.length > 5 ? ", …" : ""}): ` +
|
|
1954
|
+
`ignore rules from your filesystem have not come down here, and they may keep some of these files home.`,
|
|
1955
|
+
...names.slice(0, 5).map((path) => ` ${path}: ${waiting.get(path)}`),
|
|
1956
|
+
...(names.length > 5 ? [` …and ${names.length - 5} more ignore files`] : []),
|
|
1957
|
+
" Pulls go on, and the next pass tries again. An ignore file that is not yours can be removed there: muhkoo vfs rm <path>",
|
|
1958
|
+
);
|
|
1959
|
+
}
|
|
1960
|
+
const agreed = withAgreed(state, pass);
|
|
1961
|
+
const next = plan.length ? await applyPlan({ vfs, dir, root, state: agreed, plan, pass, opts, sess, signal, budget }) : agreed;
|
|
1962
|
+
await persist(dir, root, state, next);
|
|
1963
|
+
const short = pass.outstanding?.space ?? 0;
|
|
1964
|
+
if (short) {
|
|
1965
|
+
warnOnce(
|
|
1966
|
+
sess,
|
|
1967
|
+
"disk-reserve",
|
|
1968
|
+
`not pulling ${short} file${short === 1 ? "" : "s"}: the disk holding ${dir} would be left with less than ` +
|
|
1969
|
+
`its reserve free (5% of the volume, at least 1 GiB and at most 10 GiB).`,
|
|
1970
|
+
" Free some space, and they come down in the next pass — a change here or in your filesystem starts one, and so does restarting the mount.",
|
|
1971
|
+
);
|
|
369
1972
|
}
|
|
370
|
-
|
|
1973
|
+
sess.outstanding = {
|
|
1974
|
+
space: short,
|
|
1975
|
+
budget: (pass.outstanding?.budget ?? 0) + pass.deferred.size,
|
|
1976
|
+
uploads: plan === pass.plan ? 0 : uploads.length,
|
|
1977
|
+
};
|
|
371
1978
|
return next;
|
|
372
1979
|
}
|
|
373
1980
|
|
|
1981
|
+
// ---- import -----------------------------------------------------------------------
|
|
1982
|
+
|
|
1983
|
+
/**
|
|
1984
|
+
* `import` only uploads.
|
|
1985
|
+
*
|
|
1986
|
+
* It used to run the full two-way reconcile, so a command documented as
|
|
1987
|
+
* "upload whole projects, once" wrote remote files into the project — the
|
|
1988
|
+
* `.vcsignore` that then widened what uploads among them — and, with
|
|
1989
|
+
* `--allow-delete`, deleted local files a remote party had removed. Now:
|
|
1990
|
+
*
|
|
1991
|
+
* push applied
|
|
1992
|
+
* conflict applied as a push: local wins, as `mount` does, and the
|
|
1993
|
+
* other version stays in `vfs history`
|
|
1994
|
+
* delete-remote applied only with --allow-delete
|
|
1995
|
+
* pull, delete-local, and delete-remote without the flag: skipped, reported,
|
|
1996
|
+
* and left out of state — so a later `mount` still sees them
|
|
1997
|
+
* for what they are.
|
|
1998
|
+
*/
|
|
1999
|
+
function importPlan(plan, allowDelete) {
|
|
2000
|
+
const apply = [];
|
|
2001
|
+
const skipped = [];
|
|
2002
|
+
for (const step of plan) {
|
|
2003
|
+
const allowed = step.action === "push" || step.action === "conflict" || step.action === "forget"
|
|
2004
|
+
|| (step.action === "delete-remote" && allowDelete);
|
|
2005
|
+
(allowed ? apply : skipped).push(step);
|
|
2006
|
+
}
|
|
2007
|
+
return { apply, skipped };
|
|
2008
|
+
}
|
|
2009
|
+
|
|
2010
|
+
const SKIP_REASON = {
|
|
2011
|
+
pull: "changed only in your filesystem — import only uploads; `muhkoo vfs mount` brings it down",
|
|
2012
|
+
"delete-local": "gone from your filesystem — import never deletes here",
|
|
2013
|
+
"delete-remote": "gone locally — pass --allow-delete to delete it from your filesystem",
|
|
2014
|
+
};
|
|
2015
|
+
|
|
374
2016
|
/**
|
|
375
2017
|
* Push a directory up once and stop — no watcher.
|
|
376
2018
|
*
|
|
377
|
-
* The same scan,
|
|
378
|
-
*
|
|
379
|
-
*
|
|
380
|
-
* that way".
|
|
2019
|
+
* The same scan, admission rule and planner `mount` uses; see `importPlan` for
|
|
2020
|
+
* what it does with the plan. This is the command for "get this project into
|
|
2021
|
+
* my filesystem"; `mount` is for "and keep it that way".
|
|
381
2022
|
*
|
|
382
2023
|
* One process means ONE vault unlock, which is the practical reason this exists
|
|
383
2024
|
* rather than a shell loop over `vfs put`: unlocking per file trips the auth
|
|
384
2025
|
* rate limiter well before a real project finishes uploading.
|
|
385
2026
|
*/
|
|
386
2027
|
export async function importOnce({ client, dir, root, opts = {} }) {
|
|
2028
|
+
await assertMountable(dir);
|
|
387
2029
|
const vfs = client.vfs;
|
|
388
|
-
const {
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
scanRemote(vfs, root, opts.hidden, match),
|
|
392
|
-
]);
|
|
2030
|
+
const sess = session({ readBudget: null }); // one pass, which has to compare everything
|
|
2031
|
+
// A dry run writes nothing, not even the probe file `foldsCase` writes and removes.
|
|
2032
|
+
if (opts.dryRun) sess.ignoreCase = await foldsCaseReadOnly(dir);
|
|
393
2033
|
const state = await loadState(dir, root);
|
|
394
|
-
const
|
|
2034
|
+
const pass = await preparePass({ vfs, dir, root, state, opts, sess, save: !opts.dryRun });
|
|
2035
|
+
const { apply, skipped } = importPlan(pass.plan, opts.allowDelete);
|
|
2036
|
+
const shown = apply.filter((p) => p.action !== "forget");
|
|
395
2037
|
|
|
396
2038
|
// Say what will happen before spending metered bytes. A project directory
|
|
397
2039
|
// is easy to point at the wrong place, and the bytes are billed either way.
|
|
@@ -399,10 +2041,10 @@ export async function importOnce({ client, dir, root, opts = {} }) {
|
|
|
399
2041
|
// Sizes are stat'd here rather than carried out of `scanLocal`, which
|
|
400
2042
|
// returns hashes — the scan is shared with `mount`, and widening its return
|
|
401
2043
|
// shape to serve one caller's progress line is not worth the churn.
|
|
402
|
-
const
|
|
2044
|
+
const uploads = apply.filter((p) => p.action === "push" || p.action === "conflict");
|
|
403
2045
|
const bytes = (
|
|
404
2046
|
await Promise.all(
|
|
405
|
-
|
|
2047
|
+
uploads.map(async ({ path }) => {
|
|
406
2048
|
try {
|
|
407
2049
|
return (await stat(join(dir, path.slice(1).split("/").join(sep)))).size;
|
|
408
2050
|
} catch {
|
|
@@ -411,21 +2053,153 @@ export async function importOnce({ client, dir, root, opts = {} }) {
|
|
|
411
2053
|
}),
|
|
412
2054
|
)
|
|
413
2055
|
).reduce((n, size) => n + size, 0);
|
|
2056
|
+
const reportSkipped = () => {
|
|
2057
|
+
for (const { path, action } of skipped) info(`${SIGN.skip} ${path} (skipped: ${SKIP_REASON[action]})`);
|
|
2058
|
+
};
|
|
414
2059
|
|
|
415
2060
|
if (opts.dryRun) {
|
|
416
|
-
if (!
|
|
417
|
-
for (const { path, action } of
|
|
418
|
-
|
|
419
|
-
|
|
2061
|
+
if (!shown.length && !skipped.length) return ok(`${root} already matches ${dir}.`);
|
|
2062
|
+
for (const { path, action } of shown) info(`${SIGN[action] ?? "?"} ${path}${DESCRIBE[action] ?? ""}`);
|
|
2063
|
+
reportSkipped();
|
|
2064
|
+
info("", `${shown.length} change(s); ${uploads.length} file(s) to upload, ${human(bytes)}.`);
|
|
2065
|
+
return info(`Ignoring per ${describeSource(pass.ignores.source())}.`);
|
|
420
2066
|
}
|
|
421
2067
|
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
2068
|
+
const agreed = withAgreed(state, pass);
|
|
2069
|
+
if (!apply.length) {
|
|
2070
|
+
await persist(dir, root, state, agreed);
|
|
2071
|
+
reportSkipped();
|
|
2072
|
+
// Not "already matches" when differences were just listed: they are
|
|
2073
|
+
// ones import leaves alone, and the two sides still differ.
|
|
2074
|
+
if (skipped.length) return ok(`Nothing to upload from ${dir} — import leaves the ${skipped.length} difference(s) above alone.`);
|
|
2075
|
+
return ok(`${root} already matches ${dir} — nothing to upload.`);
|
|
2076
|
+
}
|
|
2077
|
+
info(`Importing ${dir} → ${root} (${uploads.length} file(s), ${human(bytes)}; ignoring per ${describeSource(pass.ignores.source())})`);
|
|
2078
|
+
const next = await applyPlan({ vfs, dir, root, state: agreed, plan: apply, pass, opts, sess });
|
|
2079
|
+
await persist(dir, root, state, next);
|
|
2080
|
+
reportSkipped();
|
|
425
2081
|
ok(`Imported into ${root}.`);
|
|
426
2082
|
await noteStateFile(dir);
|
|
427
2083
|
}
|
|
428
2084
|
|
|
2085
|
+
/**
|
|
2086
|
+
* What `mount` would do, without doing any of it.
|
|
2087
|
+
*
|
|
2088
|
+
* `--dry-run` used to be read by import alone, so `vfs mount <dir> --dry-run`
|
|
2089
|
+
* dropped the flag and mounted for real: pulling, pushing and, with
|
|
2090
|
+
* --allow-delete, deleting — the opposite of what was asked. This is the same
|
|
2091
|
+
* scan and planner a pass runs, printed in both directions, and then it stops:
|
|
2092
|
+
* no watcher.
|
|
2093
|
+
*
|
|
2094
|
+
* It writes nothing, anywhere: no state, no rules (nor their copy under
|
|
2095
|
+
* ~/.muhkoo), and not even for a moment — the volume's case folding is asked
|
|
2096
|
+
* of the names already there, not probed with a file. It plans everything, with
|
|
2097
|
+
* none of a pass's budgets, so for a large project it shows what the passes a
|
|
2098
|
+
* mount runs until it is done would do, not what the first one alone would.
|
|
2099
|
+
* Uploads a pass would hold, because an ignore file from elsewhere cannot be
|
|
2100
|
+
* read, are marked held and left out of the count.
|
|
2101
|
+
*
|
|
2102
|
+
* A directory that does not exist yet is not created: the pass is planned
|
|
2103
|
+
* against an empty stand-in, which is what the mount would start from, under
|
|
2104
|
+
* the ignore rules recorded for the real directory. Its volume cannot be asked
|
|
2105
|
+
* about case folding, so it is taken to fold — which only ever refuses more. A
|
|
2106
|
+
* path the mount could not create, or use, is refused here as the mount would
|
|
2107
|
+
* refuse it.
|
|
2108
|
+
*/
|
|
2109
|
+
export async function previewMount({ client, dir, root, opts = {} }) {
|
|
2110
|
+
const vfs = client.vfs;
|
|
2111
|
+
const kind = await previewTarget(dir);
|
|
2112
|
+
if (kind === "dir") await assertMountable(dir);
|
|
2113
|
+
const sess = session({ readBudget: null }); // no budgets: everything the mount would do
|
|
2114
|
+
sess.ignoreCase = kind === "dir" ? await foldsCaseReadOnly(dir) : true;
|
|
2115
|
+
const state = kind === "dir" ? await loadState(dir, root) : {};
|
|
2116
|
+
// The scan fails loudly on a mount directory it cannot read, as a real
|
|
2117
|
+
// pass must; a directory that is not there yet reads as an empty one.
|
|
2118
|
+
const standIn = kind === "missing" ? await mkdtemp(join(tmpdir(), "muhkoo-preview-")) : null;
|
|
2119
|
+
let pass;
|
|
2120
|
+
try {
|
|
2121
|
+
pass = await preparePass({ vfs, dir: standIn ?? dir, rulesDir: dir, root, state, opts, sess, save: false });
|
|
2122
|
+
} finally {
|
|
2123
|
+
if (standIn) await rm(standIn, { recursive: true, force: true });
|
|
2124
|
+
}
|
|
2125
|
+
const shown = pass.plan.filter((p) => p.action !== "forget");
|
|
2126
|
+
|
|
2127
|
+
if (!shown.length) return ok(`${root} already matches ${dir} — mounting would change nothing.`);
|
|
2128
|
+
// As a pass holds them: while an ignore file from elsewhere cannot be read,
|
|
2129
|
+
// its rules may keep some of these files home, so nothing uploads.
|
|
2130
|
+
const waiting = unreadRules(pass, state);
|
|
2131
|
+
const gated = (action) => (action === "delete-local" || action === "delete-remote") && !opts.allowDelete;
|
|
2132
|
+
const held = (action) => waiting.size > 0 && UPLOADS.has(action) && !gated(action);
|
|
2133
|
+
info(`Mounting ${root} → ${dir} would:`);
|
|
2134
|
+
for (const { path, action } of shown) {
|
|
2135
|
+
const note = gated(action) ? " — only with --allow-delete" : held(action) ? " — held until the ignore files below can be read" : "";
|
|
2136
|
+
info(`${SIGN[action] ?? "?"} ${path}${DESCRIBE[action] ?? ""}${note}`);
|
|
2137
|
+
}
|
|
2138
|
+
const goes = (p) => !gated(p.action) && !held(p.action);
|
|
2139
|
+
const count = (...actions) => shown.filter((p) => actions.includes(p.action) && goes(p)).length;
|
|
2140
|
+
const deletes = shown.filter((p) => p.action.startsWith("delete-") && goes(p)).length;
|
|
2141
|
+
const holding = shown.filter((p) => held(p.action)).length;
|
|
2142
|
+
info(
|
|
2143
|
+
"",
|
|
2144
|
+
`${shown.length} change(s): ${count("pull")} to download, ${count("push", "conflict")} to upload` +
|
|
2145
|
+
`${deletes ? `, ${deletes} to delete` : ""}${holding ? `, ${holding} held` : ""}. Nothing was changed.`,
|
|
2146
|
+
);
|
|
2147
|
+
if (holding) {
|
|
2148
|
+
const names = [...waiting.keys()].sort();
|
|
2149
|
+
info(
|
|
2150
|
+
"Nothing uploads until these ignore files from your filesystem can be read, since their rules may keep some of those files home:",
|
|
2151
|
+
...names.slice(0, 5).map((path) => ` ${path}: ${waiting.get(path)}`),
|
|
2152
|
+
...(names.length > 5 ? [` …and ${names.length - 5} more ignore files`] : []),
|
|
2153
|
+
);
|
|
2154
|
+
}
|
|
2155
|
+
// A pass pulls ignore files first and plans the rest again with them.
|
|
2156
|
+
// Their rules can only exclude more, so it can only do less.
|
|
2157
|
+
if (shown.some(({ path, action }) => action === "pull" && isIgnoreFile(path.slice(1)))) {
|
|
2158
|
+
info("Ignore files come down first and the rest is planned again with them, so the mount may do less than this.");
|
|
2159
|
+
}
|
|
2160
|
+
return info(`Ignoring per ${describeSource(pass.ignores.source())}.`);
|
|
2161
|
+
}
|
|
2162
|
+
|
|
2163
|
+
/**
|
|
2164
|
+
* What is at `dir`, for a preview: "dir", or "missing" when a mount would
|
|
2165
|
+
* create it. A mount runs `mkdir -p`, so anything that would fail there is
|
|
2166
|
+
* refused here — a path under a file, a dangling symlink, a parent that cannot
|
|
2167
|
+
* be searched or written — rather than given a plan ending "Nothing was
|
|
2168
|
+
* changed." that the mount then cannot carry out.
|
|
2169
|
+
*/
|
|
2170
|
+
async function previewTarget(dir) {
|
|
2171
|
+
let found;
|
|
2172
|
+
try {
|
|
2173
|
+
found = await stat(dir);
|
|
2174
|
+
} catch (err) {
|
|
2175
|
+
if (err.code !== "ENOENT") throw err; // ENOTDIR, EACCES, ELOOP: the mount fails on these too
|
|
2176
|
+
const link = await lstat(dir).catch((e) => {
|
|
2177
|
+
if (e.code === "ENOENT") return null;
|
|
2178
|
+
throw e;
|
|
2179
|
+
});
|
|
2180
|
+
if (link) throw new Error(`${dir} is a symbolic link to something that does not exist, so a mount cannot create a directory there.`);
|
|
2181
|
+
// The nearest ancestor that exists is where mkdir -p starts.
|
|
2182
|
+
for (let up = dirname(resolve(dir)); ; up = dirname(up)) {
|
|
2183
|
+
let above;
|
|
2184
|
+
try {
|
|
2185
|
+
above = await stat(up);
|
|
2186
|
+
} catch (e) {
|
|
2187
|
+
if (e.code === "ENOENT" && dirname(up) !== up) continue;
|
|
2188
|
+
throw e;
|
|
2189
|
+
}
|
|
2190
|
+
if (!above.isDirectory()) throw new Error(`${dir} cannot be created: ${up} is not a directory.`);
|
|
2191
|
+
try {
|
|
2192
|
+
await access(up, fsConstants.W_OK | fsConstants.X_OK);
|
|
2193
|
+
} catch {
|
|
2194
|
+
throw new Error(`${dir} does not exist, and cannot be created: ${up} is not writable.`);
|
|
2195
|
+
}
|
|
2196
|
+
return "missing";
|
|
2197
|
+
}
|
|
2198
|
+
}
|
|
2199
|
+
if (!found.isDirectory()) throw new Error(`${dir} is not a directory.`);
|
|
2200
|
+
return "dir";
|
|
2201
|
+
}
|
|
2202
|
+
|
|
429
2203
|
/**
|
|
430
2204
|
* Mention the state file, once, in a repository that is not ignoring it.
|
|
431
2205
|
*
|
|
@@ -449,14 +2223,27 @@ async function noteStateFile(dir) {
|
|
|
449
2223
|
info(" Tracking state is in .muhkoo/ — add it to .gitignore.");
|
|
450
2224
|
}
|
|
451
2225
|
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
2226
|
+
/**
|
|
2227
|
+
* Dry-run symbols. The arrow says which side a change lands on — ↑ your
|
|
2228
|
+
* filesystem, ↓ this disk — including for deletes, which used to share one ✕
|
|
2229
|
+
* so a dry run could not tell "delete in the VFS" from "delete on my disk".
|
|
2230
|
+
*/
|
|
2231
|
+
export const SIGN = {
|
|
2232
|
+
push: "↑", pull: "↓", conflict: "!",
|
|
2233
|
+
"delete-remote": "✕↑", "delete-local": "✕↓", forget: "·", skip: "–",
|
|
2234
|
+
};
|
|
2235
|
+
|
|
2236
|
+
const DESCRIBE = {
|
|
2237
|
+
conflict: " (changed on both sides — your copy replaces the other, which stays in `vfs history`)",
|
|
2238
|
+
"delete-remote": " (delete from your filesystem)",
|
|
2239
|
+
"delete-local": " (delete from this disk)",
|
|
455
2240
|
};
|
|
456
2241
|
|
|
457
2242
|
function describeSource(source) {
|
|
458
2243
|
if (source === "defaults") return "the built-in Node defaults (write a .vcsignore to change them)";
|
|
459
|
-
if (source === ".gitignore")
|
|
2244
|
+
if (source === ".gitignore") {
|
|
2245
|
+
return "your .gitignore files and .git/info/exclude, as git reads them (`muhkoo vfs ignore --write` drafts a .vcsignore from them all, to diverge)";
|
|
2246
|
+
}
|
|
460
2247
|
return source;
|
|
461
2248
|
}
|
|
462
2249
|
|
|
@@ -466,62 +2253,160 @@ function human(bytes) {
|
|
|
466
2253
|
return `${(bytes / 1024 / 1024).toFixed(1)} MB`;
|
|
467
2254
|
}
|
|
468
2255
|
|
|
2256
|
+
// ---- mount ------------------------------------------------------------------------
|
|
2257
|
+
|
|
2258
|
+
/**
|
|
2259
|
+
* Refuse to sync the filesystem root, the home directory, or anything that
|
|
2260
|
+
* contains the home directory.
|
|
2261
|
+
*
|
|
2262
|
+
* A home directory is not a project. Every non-hidden file in it — Documents,
|
|
2263
|
+
* Desktop, Downloads — would upload, and a remote `Library/LaunchAgents/x.plist`
|
|
2264
|
+
* would land where launchd loads it at the next login, with no flag needed.
|
|
2265
|
+
* Mounting a project directory instead makes the same remote file inert. There
|
|
2266
|
+
* is no override: nothing a project needs is only reachable this way.
|
|
2267
|
+
*/
|
|
2268
|
+
export async function assertMountable(dir) {
|
|
2269
|
+
const real = await realpath(dir);
|
|
2270
|
+
const home = await realpath(homedir()).catch(() => resolve(homedir()));
|
|
2271
|
+
if (real === parse(real).root) {
|
|
2272
|
+
throw new Error(`refusing to sync ${dir}: it is the filesystem root, which contains your home directory. Point this at a project directory.`);
|
|
2273
|
+
}
|
|
2274
|
+
const rel = relative(real, home);
|
|
2275
|
+
if (rel === "" || !(rel === ".." || rel.startsWith(".." + sep) || isAbsolute(rel)) || (await isHomeOrAbove(real, home))) {
|
|
2276
|
+
throw new Error(
|
|
2277
|
+
`refusing to sync ${dir}: it is your home directory, or contains it. Everything in it would upload, ` +
|
|
2278
|
+
`and a remote file could land anywhere in it. Point this at a project directory.`,
|
|
2279
|
+
);
|
|
2280
|
+
}
|
|
2281
|
+
}
|
|
2282
|
+
|
|
2283
|
+
/**
|
|
2284
|
+
* Does `real` hold the home directory — is it home, or above it — compared as
|
|
2285
|
+
* DIRECTORIES, by device and inode, rather than as spellings?
|
|
2286
|
+
*
|
|
2287
|
+
* realpath resolves symlinks but not macOS firmlinks: `/System/Volumes/Data`
|
|
2288
|
+
* holds `/Users/<you>` under a spelling no string comparison recognises, and
|
|
2289
|
+
* it is not above it by inode either, since `/` is a separate system volume.
|
|
2290
|
+
* So ask the directory itself: for each directory above home, does `real`
|
|
2291
|
+
* reach home by the same remaining path? Inodes are read as bigints, which is
|
|
2292
|
+
* exact everywhere; a filesystem that reports none is left to the string
|
|
2293
|
+
* check above.
|
|
2294
|
+
*/
|
|
2295
|
+
async function isHomeOrAbove(real, home) {
|
|
2296
|
+
const target = await lstat(home, { bigint: true }).catch(() => null);
|
|
2297
|
+
if (!target?.ino) return false;
|
|
2298
|
+
for (let above = home; ; above = dirname(above)) {
|
|
2299
|
+
const s = await lstat(join(real, relative(above, home)), { bigint: true }).catch(() => null);
|
|
2300
|
+
if (s && s.dev === target.dev && s.ino === target.ino) return true;
|
|
2301
|
+
if (dirname(above) === above) return false;
|
|
2302
|
+
}
|
|
2303
|
+
}
|
|
2304
|
+
|
|
469
2305
|
/**
|
|
470
2306
|
* Mount `root` at `dir` and keep them in step until `signal` aborts.
|
|
2307
|
+
*
|
|
2308
|
+
* Stopping is prompt and clean. `signal` stops a pass between steps — the
|
|
2309
|
+
* first pass included, which used to ignore Ctrl-C altogether and left kill
|
|
2310
|
+
* as the only way out — and `mount` does not return while a pass is still
|
|
2311
|
+
* writing, since the CLI exits the moment it does.
|
|
471
2312
|
*/
|
|
472
|
-
export async function mount({ client, dir, root, signal, opts = {} }) {
|
|
2313
|
+
export async function mount({ client, dir, root, signal, opts = {}, session: s }) {
|
|
473
2314
|
const vfs = client.vfs;
|
|
474
2315
|
await mkdir(dir, { recursive: true });
|
|
2316
|
+
await assertMountable(dir);
|
|
475
2317
|
|
|
2318
|
+
const sess = session(s);
|
|
476
2319
|
let state = await loadState(dir, root);
|
|
477
2320
|
info(`Mounting ${root} → ${dir}`);
|
|
478
|
-
|
|
479
|
-
|
|
2321
|
+
// Deliberately not caught: a mount that cannot complete its first pass —
|
|
2322
|
+
// the vault locked, the root unreadable — must fail the command, not sit
|
|
2323
|
+
// "watching" something broken.
|
|
2324
|
+
state = await reconcile({ vfs, dir, root, state, opts, session: sess, signal });
|
|
2325
|
+
if (signal.aborted) return; // what the pass did is saved; nothing is watching yet
|
|
2326
|
+
// Not "in sync" when it is not: a fresh mount on a laptop short of its
|
|
2327
|
+
// disk reserve pulled nothing, and said it was.
|
|
2328
|
+
const behind = undone(sess.outstanding);
|
|
2329
|
+
if (behind) warn(`Not in sync yet: ${behind}.`, " Watching for changes — press Ctrl-C to stop.");
|
|
2330
|
+
else ok("In sync. Watching for changes — press Ctrl-C to stop.");
|
|
480
2331
|
|
|
481
2332
|
// One reconcile at a time, with a trailing pass if anything arrived while we
|
|
482
2333
|
// were busy. Without the trailing pass a change landing mid-sync would wait
|
|
483
2334
|
// for the NEXT unrelated event to be noticed.
|
|
484
|
-
let running =
|
|
2335
|
+
let running = null;
|
|
485
2336
|
let queued = false;
|
|
486
2337
|
let timer = null;
|
|
487
|
-
const sync =
|
|
2338
|
+
const sync = () => {
|
|
2339
|
+
if (signal.aborted) return;
|
|
488
2340
|
if (running) {
|
|
489
2341
|
queued = true;
|
|
490
2342
|
return;
|
|
491
2343
|
}
|
|
492
|
-
running =
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
2344
|
+
running = reconcile({ vfs, dir, root, state, opts, session: sess, signal })
|
|
2345
|
+
.then((next) => {
|
|
2346
|
+
state = next;
|
|
2347
|
+
}, (err) => warn(`sync failed: ${err?.message ?? err}`))
|
|
2348
|
+
.finally(() => {
|
|
2349
|
+
running = null;
|
|
2350
|
+
// A pass that stopped at its pull budget goes straight on.
|
|
2351
|
+
if (sess.more) {
|
|
2352
|
+
sess.more = false;
|
|
2353
|
+
queued = true;
|
|
2354
|
+
}
|
|
2355
|
+
if (queued) {
|
|
2356
|
+
queued = false;
|
|
2357
|
+
sync();
|
|
2358
|
+
}
|
|
2359
|
+
});
|
|
504
2360
|
};
|
|
505
2361
|
const schedule = () => {
|
|
506
2362
|
clearTimeout(timer);
|
|
507
|
-
timer = setTimeout(
|
|
2363
|
+
timer = setTimeout(sync, SETTLE_MS);
|
|
2364
|
+
};
|
|
2365
|
+
|
|
2366
|
+
/**
|
|
2367
|
+
* Would a change at `filename` matter? The same admission rule the scans
|
|
2368
|
+
* use, from the last pass — not a separate hard-coded list, which dropped
|
|
2369
|
+
* edits under any directory named vendor/, target/ or build/ even where
|
|
2370
|
+
* the project syncs it. `.muhkoo` is dropped: a pass writes its state
|
|
2371
|
+
* there, and waking for that write is how one pass scheduled the next,
|
|
2372
|
+
* forever. Except the ignore rules, which `vfs ignore --accept` writes:
|
|
2373
|
+
* the files it lets through have not changed, so nothing else would wake
|
|
2374
|
+
* the mount to send them. A pass writes that file only when the ignore
|
|
2375
|
+
* files change, so the pass it wakes writes nothing and it settles.
|
|
2376
|
+
*/
|
|
2377
|
+
const matters = (filename) => {
|
|
2378
|
+
const rel = filename.split(sep).join("/");
|
|
2379
|
+
if (rel === ".muhkoo/ignore-rules.json") return true;
|
|
2380
|
+
if (rel === ".muhkoo" || rel.startsWith(".muhkoo/")) return false;
|
|
2381
|
+
if (rel.split("/").pop().startsWith(TMP_PREFIX)) return false;
|
|
2382
|
+
if (rel === ".git/info/exclude") return true; // refused, but its rules apply
|
|
2383
|
+
return !sess.admit || sess.admit(rel, false) === null;
|
|
508
2384
|
};
|
|
509
2385
|
|
|
510
2386
|
const watcher = watch(dir, { recursive: true }, (_event, filename) => {
|
|
511
2387
|
// `filename` can be null on some platforms; a bare reconcile is correct
|
|
512
2388
|
// either way, so this never has to trust it.
|
|
513
|
-
if (filename && filename
|
|
2389
|
+
if (filename && !matters(filename)) return;
|
|
514
2390
|
schedule();
|
|
515
2391
|
});
|
|
516
2392
|
|
|
517
2393
|
const unwatchRemote = vfs.watch(() => schedule());
|
|
2394
|
+
if (sess.more) {
|
|
2395
|
+
sess.more = false;
|
|
2396
|
+
sync();
|
|
2397
|
+
}
|
|
518
2398
|
|
|
519
2399
|
await new Promise((resolve) => {
|
|
520
|
-
signal
|
|
2400
|
+
// An already-aborted signal never fires "abort" again.
|
|
2401
|
+
if (signal.aborted) resolve();
|
|
2402
|
+
else signal.addEventListener("abort", resolve, { once: true });
|
|
521
2403
|
});
|
|
522
2404
|
|
|
523
2405
|
clearTimeout(timer);
|
|
524
2406
|
watcher.close();
|
|
525
2407
|
unwatchRemote();
|
|
526
|
-
|
|
2408
|
+
// A pass under way stops at its next step and saves what it did. Wait for
|
|
2409
|
+
// it: returning now let the CLI exit in the middle of a write.
|
|
2410
|
+
while (running) await running;
|
|
2411
|
+
await saveState(dir, root, state);
|
|
527
2412
|
}
|