@muhkoo/cli 0.15.0-alpha.6 → 0.15.0-alpha.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +6 -3
- package/src/lib/mount.js +253 -23
- package/src/lib/syncPlan.js +73 -3
package/package.json
CHANGED
|
@@ -1,11 +1,14 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@muhkoo/cli",
|
|
3
|
-
"version": "0.15.0-alpha.
|
|
3
|
+
"version": "0.15.0-alpha.8",
|
|
4
4
|
"description": "Command-line interface for the Muhkoo platform — provision backends, deploy apps, and manage agents, functions, databases, and domains.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|
|
7
7
|
"author": "muhkoo",
|
|
8
|
-
"repository":
|
|
8
|
+
"repository": {
|
|
9
|
+
"type": "git",
|
|
10
|
+
"url": "git+https://github.com/muhkoo/cli.git"
|
|
11
|
+
},
|
|
9
12
|
"homepage": "https://docs.muhkoo.dev",
|
|
10
13
|
"bugs": "https://github.com/muhkoo/cli/issues",
|
|
11
14
|
"keywords": [
|
|
@@ -40,7 +43,7 @@
|
|
|
40
43
|
"release": "npm publish --tag alpha && npm dist-tag add @muhkoo/cli@$npm_package_version latest"
|
|
41
44
|
},
|
|
42
45
|
"dependencies": {
|
|
43
|
-
"@muhkoo/connect": "0.15.0-alpha.
|
|
46
|
+
"@muhkoo/connect": "0.15.0-alpha.8",
|
|
44
47
|
"snarkjs": "^0.7.0"
|
|
45
48
|
},
|
|
46
49
|
"packageManager": "yarn@1.22.22+sha512.a6b2f7906b721bba3d67d4aff083df04dad64c399707841b7acf00f6b133b7ac24255f2652fa22ae3534329dc6180534e98d17432037ff6fd140556e2bb3137e",
|
package/src/lib/mount.js
CHANGED
|
@@ -42,7 +42,7 @@ import { constants as fsConstants, watch } from "node:fs";
|
|
|
42
42
|
import { homedir, tmpdir } from "node:os";
|
|
43
43
|
import { basename, dirname, isAbsolute, join, parse, relative, resolve, sep } from "node:path";
|
|
44
44
|
|
|
45
|
-
import { holdMassDeletes, planSync } from "./syncPlan.js";
|
|
45
|
+
import { DIR, dirKey, filesOf, holdMassDeletes, planDirs, planSync } from "./syncPlan.js";
|
|
46
46
|
import {
|
|
47
47
|
DEPTH_MAX, IgnoreBudgetError, NAME_MAX, caseFold, ignoreFileProblem, ruleBank, ruleText, withinMatchBounds,
|
|
48
48
|
} from "./ignore.js";
|
|
@@ -223,6 +223,50 @@ async function saveState(dir, root, paths) {
|
|
|
223
223
|
await writeAtomic(statePath(dir), JSON.stringify({ v: 1, root, paths }, null, 2) + "\n");
|
|
224
224
|
}
|
|
225
225
|
|
|
226
|
+
/**
|
|
227
|
+
* What the scan has hashed, kept between runs, so a restarted mount does not
|
|
228
|
+
* download the whole tree again just to learn it has not changed.
|
|
229
|
+
*
|
|
230
|
+
* It maps a content key — the SHA-256 of a file's manifest, which names the
|
|
231
|
+
* exact shards and the key that decrypts them, so one key is one content — to
|
|
232
|
+
* that content's hash and size. Unlike `mount.json` it decides nothing on its
|
|
233
|
+
* own: an entry only spares a read. So a file that cannot be read, or an entry
|
|
234
|
+
* that is not in the shape written here, is ignored, and that content is read.
|
|
235
|
+
*/
|
|
236
|
+
const hashCachePath = (dir) => join(dir, ".muhkoo", "content-hashes.json");
|
|
237
|
+
const HEX64 = /^[0-9a-f]{64}$/;
|
|
238
|
+
/** Entries kept: well past any tree the read budget lets one pass hash. */
|
|
239
|
+
const HASH_CACHE_MAX = 200_000;
|
|
240
|
+
|
|
241
|
+
async function loadHashCache(dir) {
|
|
242
|
+
const hashes = new Map();
|
|
243
|
+
try {
|
|
244
|
+
const saved = JSON.parse(await readFile(hashCachePath(dir), "utf8"));
|
|
245
|
+
if (saved?.v !== 1 || !saved.contents || typeof saved.contents !== "object") return hashes;
|
|
246
|
+
for (const [key, entry] of Object.entries(saved.contents)) {
|
|
247
|
+
if (hashes.size >= HASH_CACHE_MAX) break;
|
|
248
|
+
const [hash, size] = Array.isArray(entry) ? entry : [];
|
|
249
|
+
if (!HEX64.test(key) || !HEX64.test(hash ?? "") || !Number.isSafeInteger(size) || size < 0) return new Map();
|
|
250
|
+
hashes.set(key, { hash, size });
|
|
251
|
+
}
|
|
252
|
+
} catch {
|
|
253
|
+
return new Map(); // none yet, or unreadable: the scan reads, as it always did
|
|
254
|
+
}
|
|
255
|
+
return hashes;
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
function sameHashes(a, b) {
|
|
259
|
+
if (a.size !== b.size) return false;
|
|
260
|
+
for (const [key, { hash }] of a) if (b.get(key)?.hash !== hash) return false;
|
|
261
|
+
return true;
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
async function saveHashCache(dir, hashes) {
|
|
265
|
+
const contents = {};
|
|
266
|
+
for (const [key, { hash, size }] of hashes) contents[key] = [hash, size];
|
|
267
|
+
await writeAtomic(hashCachePath(dir), JSON.stringify({ v: 1, contents }) + "\n");
|
|
268
|
+
}
|
|
269
|
+
|
|
226
270
|
function sameState(a, b) {
|
|
227
271
|
const keys = Object.keys(a);
|
|
228
272
|
return keys.length === Object.keys(b).length && keys.every((k) => a[k] === b[k]);
|
|
@@ -969,6 +1013,7 @@ export async function foldsCaseReadOnly(dir) {
|
|
|
969
1013
|
*/
|
|
970
1014
|
async function scanLocal(dir, { hidden = false, ignores, sess, signal }) {
|
|
971
1015
|
const files = {};
|
|
1016
|
+
const dirs = [];
|
|
972
1017
|
const unknown = new Set();
|
|
973
1018
|
const held = new Set();
|
|
974
1019
|
const pace = pacer();
|
|
@@ -1006,6 +1051,7 @@ async function scanLocal(dir, { hidden = false, ignores, sess, signal }) {
|
|
|
1006
1051
|
if (verdict === "held") held.add(entry.isDirectory() ? `${key}/` : key);
|
|
1007
1052
|
if (verdict) continue;
|
|
1008
1053
|
if (entry.isDirectory()) {
|
|
1054
|
+
dirs.push(key);
|
|
1009
1055
|
await walk(child, rel);
|
|
1010
1056
|
continue;
|
|
1011
1057
|
}
|
|
@@ -1031,7 +1077,7 @@ async function scanLocal(dir, { hidden = false, ignores, sess, signal }) {
|
|
|
1031
1077
|
}
|
|
1032
1078
|
};
|
|
1033
1079
|
await walk(dir, "");
|
|
1034
|
-
return { files, unknown, held };
|
|
1080
|
+
return { files, dirs, unknown, held };
|
|
1035
1081
|
}
|
|
1036
1082
|
|
|
1037
1083
|
/**
|
|
@@ -1044,8 +1090,35 @@ async function scanLocal(dir, { hidden = false, ignores, sess, signal }) {
|
|
|
1044
1090
|
* but this CLI may run against either.
|
|
1045
1091
|
*/
|
|
1046
1092
|
async function walkRemote(vfs, root) {
|
|
1047
|
-
|
|
1048
|
-
|
|
1093
|
+
// `dirs` is null when the folders cannot be told: a root that is not there
|
|
1094
|
+
// (missing or renamed — never "every folder deleted"), or an SDK without
|
|
1095
|
+
// `walkEntries`. Folders are then left alone.
|
|
1096
|
+
if (root !== "/" && !(await vfs.exists(root))) return { paths: [], dirs: null };
|
|
1097
|
+
if (typeof vfs.walkEntries !== "function") return { paths: await vfs.walk(root), dirs: null };
|
|
1098
|
+
const entries = await vfs.walkEntries(root);
|
|
1099
|
+
return {
|
|
1100
|
+
paths: entries.filter((e) => e.kind === "file").map((e) => e.path),
|
|
1101
|
+
dirs: entries.filter((e) => e.kind === "dir").map((e) => e.path),
|
|
1102
|
+
};
|
|
1103
|
+
}
|
|
1104
|
+
|
|
1105
|
+
/**
|
|
1106
|
+
* The remote folders, keyed like the local ones, through the same admission
|
|
1107
|
+
* rule: a refused, ignored or hidden folder is not made here, any more than a
|
|
1108
|
+
* file in it would be.
|
|
1109
|
+
*/
|
|
1110
|
+
function remoteDirs(dirs, root, { hidden, ignores, ignoreCase }) {
|
|
1111
|
+
if (!dirs) return null;
|
|
1112
|
+
const out = [];
|
|
1113
|
+
for (const abs of dirs) {
|
|
1114
|
+
const key = root === "/" ? abs : abs.slice(root.length);
|
|
1115
|
+
if (!key.startsWith("/") || key === "/") continue;
|
|
1116
|
+
const rel = key.slice(1);
|
|
1117
|
+
if (!withinMatchBounds(rel, ignoreCase)) continue;
|
|
1118
|
+
if (admission(rel, true, hidden, ignores.now, ignores.held)) continue;
|
|
1119
|
+
out.push(key);
|
|
1120
|
+
}
|
|
1121
|
+
return out;
|
|
1049
1122
|
}
|
|
1050
1123
|
|
|
1051
1124
|
/**
|
|
@@ -1079,7 +1152,7 @@ export async function scanRemote(vfs, root, includeHidden = false, ignore = () =
|
|
|
1079
1152
|
const unknown = new Set();
|
|
1080
1153
|
const unreadable = new Set();
|
|
1081
1154
|
const held = new Set();
|
|
1082
|
-
const paths = ctx.paths ?? (await walkRemote(vfs, root));
|
|
1155
|
+
const paths = ctx.paths ?? (await walkRemote(vfs, root)).paths;
|
|
1083
1156
|
|
|
1084
1157
|
// Decide what to read BEFORE reading anything, so the filters are applied
|
|
1085
1158
|
// once and the reads that survive them can go out together.
|
|
@@ -1599,7 +1672,7 @@ export function win32NameProblem(name) {
|
|
|
1599
1672
|
* directories to create, in order, and `existing` the stats of the file being
|
|
1600
1673
|
* replaced — or `{ ok: false, reason }`. `listing` caches `readdir` for a pass.
|
|
1601
1674
|
*/
|
|
1602
|
-
export async function writeTarget(dir, key, { localKeys = new Set(), listing = new Map(), platform = process.platform } = {}) {
|
|
1675
|
+
export async function writeTarget(dir, key, { localKeys = new Set(), listing = new Map(), platform = process.platform, kind = "file" } = {}) {
|
|
1603
1676
|
const refuse = (reason) => ({ ok: false, reason });
|
|
1604
1677
|
const names = key.split("/").filter(Boolean);
|
|
1605
1678
|
if (!names.length) return refuse("it names the mount directory itself");
|
|
@@ -1625,7 +1698,8 @@ export async function writeTarget(dir, key, { localKeys = new Set(), listing = n
|
|
|
1625
1698
|
if (!entries) return refuse(`${parent} could not be listed`);
|
|
1626
1699
|
if (!entries.has(name)) {
|
|
1627
1700
|
if (await present(abs)) return alias(name);
|
|
1628
|
-
|
|
1701
|
+
// A folder is made whole, its own name included; a file's last name is the file.
|
|
1702
|
+
return { ok: true, abs: join(parent, ...names.slice(i)), parent, missing: kind === "dir" ? names.slice(i) : names.slice(i, -1), existing: null };
|
|
1629
1703
|
}
|
|
1630
1704
|
const info = await lstat(abs).catch(() => null);
|
|
1631
1705
|
if (!last) {
|
|
@@ -1633,6 +1707,10 @@ export async function writeTarget(dir, key, { localKeys = new Set(), listing = n
|
|
|
1633
1707
|
parent = abs;
|
|
1634
1708
|
continue;
|
|
1635
1709
|
}
|
|
1710
|
+
if (kind === "dir") {
|
|
1711
|
+
if (!info?.isDirectory()) return refuse(`${relative(dir, abs)} is ${info?.isSymbolicLink() ? "a symlink" : "not a directory"}`);
|
|
1712
|
+
return { ok: true, abs, parent, missing: [], existing: info };
|
|
1713
|
+
}
|
|
1636
1714
|
if (!info?.isFile()) return refuse(`${relative(dir, abs)} is a symlink or not a regular file`);
|
|
1637
1715
|
if (!localKeys.has(key)) return refuse(`${relative(dir, abs)} exists here but is not synced (ignored, hidden or refused), so it may exist nowhere else`);
|
|
1638
1716
|
return { ok: true, abs, parent, missing: [], existing: info };
|
|
@@ -1652,14 +1730,7 @@ export async function writeTarget(dir, key, { localKeys = new Set(), listing = n
|
|
|
1652
1730
|
* default mode, never an execute bit.
|
|
1653
1731
|
*/
|
|
1654
1732
|
async function writeAt(target, bytes, listing) {
|
|
1655
|
-
|
|
1656
|
-
for (const name of target.missing) {
|
|
1657
|
-
const next = join(parent, name);
|
|
1658
|
-
await mkdir(next); // not recursive: each component is created here, never followed
|
|
1659
|
-
listing.get(parent)?.add(name);
|
|
1660
|
-
listing.set(next, new Set());
|
|
1661
|
-
parent = next;
|
|
1662
|
-
}
|
|
1733
|
+
const parent = await makeMissing(target, listing);
|
|
1663
1734
|
const tmp = join(parent, TMP_PREFIX + randomBytes(6).toString("hex"));
|
|
1664
1735
|
try {
|
|
1665
1736
|
await writeFile(tmp, bytes, { flag: "wx" });
|
|
@@ -1672,6 +1743,114 @@ async function writeAt(target, bytes, listing) {
|
|
|
1672
1743
|
listing.get(parent)?.add(target.abs.slice(parent.length + 1));
|
|
1673
1744
|
}
|
|
1674
1745
|
|
|
1746
|
+
/**
|
|
1747
|
+
* Apply the folder plan (`planDirs`), after the files: folders are made
|
|
1748
|
+
* shallowest first and removed deepest first, so a folder emptied by this
|
|
1749
|
+
* pass's file deletes is gone by its end.
|
|
1750
|
+
*
|
|
1751
|
+
* A folder is only ever removed EMPTY — `rmdir` here, a delete that is not
|
|
1752
|
+
* recursive there — so removing one can never take a file with it, synced or
|
|
1753
|
+
* not. One that still holds something (an ignored or refused file, or one that
|
|
1754
|
+
* arrived since the scan) is kept, said once, and left out of the state: the
|
|
1755
|
+
* next pass sees it on one side only and makes it on the other again, which is
|
|
1756
|
+
* the truth, since it is not empty. Removals are gated by `--allow-delete` as
|
|
1757
|
+
* file deletes are; a removal not made is forgotten, as theirs is.
|
|
1758
|
+
*/
|
|
1759
|
+
async function applyDirPlan({ vfs, dir, root, state, plan, opts, sess, signal }) {
|
|
1760
|
+
const next = { ...state };
|
|
1761
|
+
const listing = new Map();
|
|
1762
|
+
const remoteAbs = (path) => (root === "/" ? path : root + path);
|
|
1763
|
+
const depth = (path) => path.split("/").length;
|
|
1764
|
+
const makes = plan.filter(({ action }) => !action.startsWith("rmdir")).sort((a, b) => depth(a.path) - depth(b.path));
|
|
1765
|
+
const removes = plan.filter(({ action }) => action.startsWith("rmdir")).sort((a, b) => depth(b.path) - depth(a.path));
|
|
1766
|
+
const kept = (path, where) => warnOnce(sess, `dir-kept:${where}:${path}`, `kept the folder ${path}/ ${where} — it still holds files that are not synced`);
|
|
1767
|
+
for (const { path, action } of [...makes, ...removes]) {
|
|
1768
|
+
if (signal?.aborted) break;
|
|
1769
|
+
const key = dirKey(path);
|
|
1770
|
+
try {
|
|
1771
|
+
switch (action) {
|
|
1772
|
+
case "record":
|
|
1773
|
+
next[key] = DIR;
|
|
1774
|
+
break;
|
|
1775
|
+
case "forget":
|
|
1776
|
+
delete next[key];
|
|
1777
|
+
break;
|
|
1778
|
+
case "mkdir-remote":
|
|
1779
|
+
await vfs.mkdir(remoteAbs(path), { recursive: true });
|
|
1780
|
+
next[key] = DIR;
|
|
1781
|
+
info(`${SIGN.push} ${path}/`);
|
|
1782
|
+
break;
|
|
1783
|
+
case "mkdir-local": {
|
|
1784
|
+
const target = await writeTarget(dir, path, { listing, kind: "dir" });
|
|
1785
|
+
if (!target.ok) {
|
|
1786
|
+
warnOnce(sess, `declined:${action}:${path}`, `skipping ${path}/ — ${target.reason}`);
|
|
1787
|
+
break;
|
|
1788
|
+
}
|
|
1789
|
+
if (!target.existing) await makeMissing(target, listing);
|
|
1790
|
+
next[key] = DIR;
|
|
1791
|
+
info(`${SIGN.pull} ${path}/`);
|
|
1792
|
+
break;
|
|
1793
|
+
}
|
|
1794
|
+
case "rmdir-remote": {
|
|
1795
|
+
delete next[key];
|
|
1796
|
+
if (!opts.allowDelete) {
|
|
1797
|
+
warn(
|
|
1798
|
+
`${path}/ is gone locally but still in your filesystem — not deleting it.`,
|
|
1799
|
+
` Pass --allow-delete to propagate deletions, or remove it with: muhkoo vfs rm -r ${remoteAbs(path)}`,
|
|
1800
|
+
);
|
|
1801
|
+
break;
|
|
1802
|
+
}
|
|
1803
|
+
const found = await vfs.stat(remoteAbs(path)).catch(() => null);
|
|
1804
|
+
if (found?.kind !== "dir") break;
|
|
1805
|
+
try {
|
|
1806
|
+
await vfs.delete(remoteAbs(path)); // not recursive: refused unless empty
|
|
1807
|
+
info(`${SIGN["delete-remote"]} ${path}/ (deleted from your filesystem)`);
|
|
1808
|
+
} catch {
|
|
1809
|
+
kept(path, "in your filesystem");
|
|
1810
|
+
}
|
|
1811
|
+
break;
|
|
1812
|
+
}
|
|
1813
|
+
case "rmdir-local": {
|
|
1814
|
+
delete next[key];
|
|
1815
|
+
if (!opts.allowDelete) {
|
|
1816
|
+
warn(`${path}/ is gone from your filesystem but still here — not deleting it locally.`, ` Pass --allow-delete to propagate deletions.`);
|
|
1817
|
+
break;
|
|
1818
|
+
}
|
|
1819
|
+
const target = await writeTarget(dir, path, { listing, kind: "dir" });
|
|
1820
|
+
if (!target.ok || !target.existing) break;
|
|
1821
|
+
try {
|
|
1822
|
+
await rmdir(target.abs);
|
|
1823
|
+
listing.get(target.parent)?.delete(target.abs.slice(target.parent.length + 1));
|
|
1824
|
+
listing.delete(target.abs);
|
|
1825
|
+
info(`${SIGN["delete-local"]} ${path}/ (deleted from this disk)`);
|
|
1826
|
+
} catch (err) {
|
|
1827
|
+
if (err.code !== "ENOTEMPTY" && err.code !== "EEXIST") throw err;
|
|
1828
|
+
kept(path, "on this disk");
|
|
1829
|
+
}
|
|
1830
|
+
break;
|
|
1831
|
+
}
|
|
1832
|
+
}
|
|
1833
|
+
} catch (err) {
|
|
1834
|
+
const message = err?.message ?? String(err);
|
|
1835
|
+
warnOnce(sess, `error:${path}/:${message}`, `${path}/: ${message}`);
|
|
1836
|
+
}
|
|
1837
|
+
}
|
|
1838
|
+
return next;
|
|
1839
|
+
}
|
|
1840
|
+
|
|
1841
|
+
/** Make the directories `writeTarget` found missing, in order; returns the deepest. */
|
|
1842
|
+
async function makeMissing(target, listing) {
|
|
1843
|
+
let parent = target.parent;
|
|
1844
|
+
for (const name of target.missing) {
|
|
1845
|
+
const next = join(parent, name);
|
|
1846
|
+
await mkdir(next); // not recursive: each component is created here, never followed
|
|
1847
|
+
listing.get(parent)?.add(name);
|
|
1848
|
+
listing.set(next, new Set());
|
|
1849
|
+
parent = next;
|
|
1850
|
+
}
|
|
1851
|
+
return parent;
|
|
1852
|
+
}
|
|
1853
|
+
|
|
1675
1854
|
// ---- planning and applying ------------------------------------------------------
|
|
1676
1855
|
|
|
1677
1856
|
/**
|
|
@@ -1694,7 +1873,7 @@ async function preparePass({ vfs, dir, rulesDir = dir, root, state, opts, sess,
|
|
|
1694
1873
|
// The remote walk is the slow part; start it while the local scan runs.
|
|
1695
1874
|
// The remote FILTER has to wait for the local scan, which is what finds
|
|
1696
1875
|
// the nested ignore files.
|
|
1697
|
-
const [local,
|
|
1876
|
+
const [local, listed] = await Promise.all([
|
|
1698
1877
|
scanLocal(dir, { hidden: opts.hidden, ignores, sess, signal }),
|
|
1699
1878
|
walkRemote(vfs, root),
|
|
1700
1879
|
]);
|
|
@@ -1702,7 +1881,7 @@ async function preparePass({ vfs, dir, rulesDir = dir, root, state, opts, sess,
|
|
|
1702
1881
|
// The ignore files as they now stand are known; nothing has been pulled.
|
|
1703
1882
|
await ignores.record(rulesDir, root, save);
|
|
1704
1883
|
const remote = await scanRemote(vfs, root, opts.hidden, ignores.now, {
|
|
1705
|
-
paths, held: ignores.held, session: sess, ignoreCase: sess.ignoreCase,
|
|
1884
|
+
paths: listed.paths, held: ignores.held, session: sess, ignoreCase: sess.ignoreCase,
|
|
1706
1885
|
localKeys: new Set(Object.keys(local.files)), signal, blobs, spent: read,
|
|
1707
1886
|
});
|
|
1708
1887
|
if (signal?.aborted) return null;
|
|
@@ -1723,13 +1902,43 @@ async function preparePass({ vfs, dir, rulesDir = dir, root, state, opts, sess,
|
|
|
1723
1902
|
if (keptBack.length) warnOnce(sess, `kept-back:${keptBack.join("\0")}`, ...heldNotice(keptBack, rulesDir));
|
|
1724
1903
|
|
|
1725
1904
|
const unknown = new Set([...local.unknown, ...remote.unknown]);
|
|
1726
|
-
const
|
|
1727
|
-
const
|
|
1728
|
-
|
|
1729
|
-
|
|
1905
|
+
const files = filesOf(state);
|
|
1906
|
+
const planned = planSync({ last: files, local: local.files, remote: remote.files, unknown });
|
|
1907
|
+
const views = {
|
|
1908
|
+
local: Object.keys(local.files).length,
|
|
1909
|
+
remote: Object.keys(remote.files).length,
|
|
1910
|
+
// No files and no folders here: a mount folder emptied to start
|
|
1911
|
+
// again, and its files come back down. What the scan left alone (the
|
|
1912
|
+
// mount's own .muhkoo, .DS_Store, a symlink) does not count, as no
|
|
1913
|
+
// pull is planned over it; a path the ignore rules hold back does.
|
|
1914
|
+
bare: !Object.keys(local.files).length && !local.dirs.length && !local.held.size,
|
|
1915
|
+
};
|
|
1916
|
+
sess.peak = Math.max(sess.peak ?? 0, Object.keys(files).length);
|
|
1917
|
+
const { plan, held, restored } = holdMassDeletes(planned, files, views, sess.peak);
|
|
1730
1918
|
for (const h of held) warnOnce(sess, `held:${h.action}:${h.count}`, ...breakerMessage(h, dir, root));
|
|
1919
|
+
if (restored) {
|
|
1920
|
+
warnOnce(
|
|
1921
|
+
sess,
|
|
1922
|
+
`restored:${restored}`,
|
|
1923
|
+
`${dir} holds nothing at all, so its ${restored} file${restored === 1 ? "" : "s"} come${restored === 1 ? "s" : ""} back down from ${root} rather than being deleted there.`,
|
|
1924
|
+
` To delete them from your filesystem instead, use: muhkoo vfs rm -r <path>`,
|
|
1925
|
+
);
|
|
1926
|
+
}
|
|
1927
|
+
// Folders after files, by the same rule. When the breaker held file
|
|
1928
|
+
// deletes, no folder is removed either: the same pass would empty them.
|
|
1929
|
+
let dirPlan = planDirs({
|
|
1930
|
+
last: state,
|
|
1931
|
+
local: local.dirs,
|
|
1932
|
+
remote: remoteDirs(listed.dirs, root, { hidden: opts.hidden, ignores, ignoreCase: sess.ignoreCase }),
|
|
1933
|
+
unknown,
|
|
1934
|
+
fold: sess.ignoreCase ? caseFold : null,
|
|
1935
|
+
});
|
|
1936
|
+
if (held.length) dirPlan = dirPlan.filter(({ action }) => !action.startsWith("rmdir"));
|
|
1937
|
+
// A bare mount folder gets its folders back too, rather than taking them
|
|
1938
|
+
// from the remote.
|
|
1939
|
+
else if (views.bare) dirPlan = dirPlan.map((p) => (p.action === "rmdir-remote" ? { path: p.path, action: "mkdir-local" } : p));
|
|
1731
1940
|
return {
|
|
1732
|
-
plan, local: local.files, remote: remote.files, sizes: remote.sizes, blobs: remote.blobs, ignores,
|
|
1941
|
+
plan, dirPlan, local: local.files, remote: remote.files, sizes: remote.sizes, blobs: remote.blobs, ignores,
|
|
1733
1942
|
unreadable: remote.unreadable, deferred: remote.deferred,
|
|
1734
1943
|
landed: new Set(), // ignore files pulled in this pass
|
|
1735
1944
|
notLanded: new Map(), // and why the others were not
|
|
@@ -2144,6 +2353,24 @@ function unreadRules(pass, state) {
|
|
|
2144
2353
|
export async function reconcile({ vfs, dir, root, state, opts = {}, session: s, signal }) {
|
|
2145
2354
|
const sess = session(s);
|
|
2146
2355
|
if (signal?.aborted) return state;
|
|
2356
|
+
// A restart begins with what earlier runs hashed (see `loadHashCache`).
|
|
2357
|
+
if (!sess.hashCacheLoaded) {
|
|
2358
|
+
sess.hashCacheLoaded = true;
|
|
2359
|
+
if (sess.hashes.size === 0) sess.hashes = await loadHashCache(dir);
|
|
2360
|
+
sess.hashCacheSaved = new Map(sess.hashes);
|
|
2361
|
+
}
|
|
2362
|
+
try {
|
|
2363
|
+
return await reconcilePass({ vfs, dir, root, state, opts, sess, signal });
|
|
2364
|
+
} finally {
|
|
2365
|
+
// Written only when it changed: most passes hash nothing new.
|
|
2366
|
+
if (!sameHashes(sess.hashes, sess.hashCacheSaved)) {
|
|
2367
|
+
await saveHashCache(dir, sess.hashes).catch((err) => warn(`could not save the content hash cache: ${err?.message ?? err}`));
|
|
2368
|
+
sess.hashCacheSaved = new Map(sess.hashes);
|
|
2369
|
+
}
|
|
2370
|
+
}
|
|
2371
|
+
}
|
|
2372
|
+
|
|
2373
|
+
async function reconcilePass({ vfs, dir, root, state, opts, sess, signal }) {
|
|
2147
2374
|
const budget = { bytes: 0, files: 0 }; // what the whole pass may pull, both rounds
|
|
2148
2375
|
const read = { bytes: 0, files: 0 }; // and read to compare
|
|
2149
2376
|
let pass = await preparePass({ vfs, dir, root, state, opts, sess, signal, read });
|
|
@@ -2188,7 +2415,10 @@ export async function reconcile({ vfs, dir, root, state, opts = {}, session: s,
|
|
|
2188
2415
|
);
|
|
2189
2416
|
}
|
|
2190
2417
|
const agreed = withAgreed(state, pass);
|
|
2191
|
-
|
|
2418
|
+
let next = plan.length ? await applyPlan({ vfs, dir, root, state: agreed, plan, pass, opts, sess, signal, budget }) : agreed;
|
|
2419
|
+
// While uploads wait on ignore rules, so do the folders going up.
|
|
2420
|
+
const dirPlan = waiting.size ? pass.dirPlan.filter(({ action }) => !action.endsWith("-remote")) : pass.dirPlan;
|
|
2421
|
+
if (dirPlan.length && !signal?.aborted) next = await applyDirPlan({ vfs, dir, root, state: next, plan: dirPlan, opts, sess, signal });
|
|
2192
2422
|
await persist(dir, root, state, next);
|
|
2193
2423
|
const short = pass.outstanding?.space ?? 0;
|
|
2194
2424
|
if (short) {
|
package/src/lib/syncPlan.js
CHANGED
|
@@ -67,6 +67,64 @@ export function planSync({ last, local, remote, unknown = [] }) {
|
|
|
67
67
|
return plan;
|
|
68
68
|
}
|
|
69
69
|
|
|
70
|
+
/** What a folder is in the state file, in place of a content hash. */
|
|
71
|
+
export const DIR = "dir";
|
|
72
|
+
|
|
73
|
+
/** A folder's key in the state file: its path with a trailing slash, so it can never be a file's. */
|
|
74
|
+
export const dirKey = (path) => `${path}/`;
|
|
75
|
+
|
|
76
|
+
/** The state without its folders: what the file planner, and its breaker, count. */
|
|
77
|
+
export function filesOf(state) {
|
|
78
|
+
const out = {};
|
|
79
|
+
for (const [key, hash] of Object.entries(state)) if (!key.endsWith("/")) out[key] = hash;
|
|
80
|
+
return out;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/**
|
|
84
|
+
* Plan the folders, by the same three-way rule as the files ({@link planPath}),
|
|
85
|
+
* with {@link DIR} standing in for a hash. `local` and `remote` are the folder
|
|
86
|
+
* paths each side holds (`remote` null when the folders could not be listed:
|
|
87
|
+
* then nothing is planned, since a folder not listed is not a folder gone).
|
|
88
|
+
*
|
|
89
|
+
* Returns `{ path, action }` with action one of: "record" (on both sides, not
|
|
90
|
+
* yet in the state), "mkdir-local", "mkdir-remote", "rmdir-local",
|
|
91
|
+
* "rmdir-remote", "forget". Paths in `unknown` are left alone, and so, when
|
|
92
|
+
* `fold` is given (a volume that folds names), is every folder whose folded
|
|
93
|
+
* name more than one spelling shares: which of them this disk holds cannot be
|
|
94
|
+
* told apart, and reading one as missing would plan its removal.
|
|
95
|
+
*/
|
|
96
|
+
export function planDirs({ last, local, remote, unknown = [], fold = null }) {
|
|
97
|
+
if (!remote) return [];
|
|
98
|
+
const isUnknown = coveredBy(unknown);
|
|
99
|
+
const before = new Set(Object.keys(last).filter((k) => k.endsWith("/")).map((k) => k.slice(0, -1)));
|
|
100
|
+
const here = new Set(local);
|
|
101
|
+
const there = new Set(remote);
|
|
102
|
+
const paths = new Set([...before, ...here, ...there]);
|
|
103
|
+
const clash = new Set();
|
|
104
|
+
if (fold) {
|
|
105
|
+
const spellings = new Map();
|
|
106
|
+
for (const path of paths) {
|
|
107
|
+
const folded = fold(path);
|
|
108
|
+
if (!spellings.has(folded)) spellings.set(folded, []);
|
|
109
|
+
spellings.get(folded).push(path);
|
|
110
|
+
}
|
|
111
|
+
for (const group of spellings.values()) if (group.length > 1) for (const path of group) clash.add(path);
|
|
112
|
+
}
|
|
113
|
+
const ACTIONS = { push: "mkdir-remote", pull: "mkdir-local", "delete-remote": "rmdir-remote", "delete-local": "rmdir-local", forget: "forget" };
|
|
114
|
+
const plan = [];
|
|
115
|
+
for (const path of [...paths].sort()) {
|
|
116
|
+
if (isUnknown(path) || clash.has(path)) continue;
|
|
117
|
+
const was = before.has(path) ? DIR : undefined;
|
|
118
|
+
const action = planPath({ last: was, local: here.has(path) ? DIR : undefined, remote: there.has(path) ? DIR : undefined });
|
|
119
|
+
if (action === "none") {
|
|
120
|
+
if (!was) plan.push({ path, action: "record" });
|
|
121
|
+
} else if (ACTIONS[action]) {
|
|
122
|
+
plan.push({ path, action: ACTIONS[action] });
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
return plan;
|
|
126
|
+
}
|
|
127
|
+
|
|
70
128
|
/**
|
|
71
129
|
* `(path) => boolean`: is `path`, or a directory above it, in `prefixes`?
|
|
72
130
|
*
|
|
@@ -133,6 +191,15 @@ function coveredBy(prefixes) {
|
|
|
133
191
|
* as they were: the next pass sees the same picture rather than "forgotten
|
|
134
192
|
* files" to copy back.
|
|
135
193
|
*
|
|
194
|
+
* One broken view is answered rather than held: a local side that holds
|
|
195
|
+
* nothing at all (`views.bare`: no files, no folders, nothing refused or
|
|
196
|
+
* unreadable) — a mount folder emptied to start again. Its remote deletes
|
|
197
|
+
* become pulls and `restored` counts them, so the files come back instead of
|
|
198
|
+
* the user deleting the state file by hand. Nothing local can be overwritten,
|
|
199
|
+
* as there is nothing local, and nothing remote is deleted. An empty REMOTE is
|
|
200
|
+
* never answered this way: there an empty view more likely means a renamed or
|
|
201
|
+
* missing root, and uploading everything again would put it in the wrong place.
|
|
202
|
+
*
|
|
136
203
|
* What this is NOT: a defence against whatever can write the filesystem. It
|
|
137
204
|
* catches broken views — a renamed root, an empty walk, a wiped checkout. A
|
|
138
205
|
* remote party can add filler files first, so that its deletions are a
|
|
@@ -145,13 +212,16 @@ export function holdMassDeletes(plan, last, views = {}, peak = 0) {
|
|
|
145
212
|
const tracked = Object.keys(last).length;
|
|
146
213
|
const most = Math.max(tracked, peak);
|
|
147
214
|
const held = [];
|
|
215
|
+
let restored = 0;
|
|
148
216
|
for (const [action, other] of [["delete-local", "remote"], ["delete-remote", "local"]]) {
|
|
149
217
|
const count = plan.filter((p) => p.action === action).length;
|
|
150
218
|
if (!count) continue;
|
|
151
|
-
if (views
|
|
219
|
+
if (action === "delete-remote" && views.local === 0 && views.bare === true) restored = count;
|
|
220
|
+
else if (views[other] === 0) held.push({ action, count, tracked, empty: true });
|
|
152
221
|
else if (count > MASS_DELETE_FLOOR && tracked - count < most / 2) held.push({ action, count, tracked });
|
|
153
222
|
}
|
|
154
|
-
if (
|
|
223
|
+
if (restored) plan = plan.map((p) => (p.action === "delete-remote" ? { path: p.path, action: "pull" } : p));
|
|
224
|
+
if (!held.length) return { plan, held, restored };
|
|
155
225
|
const holding = new Set(held.map((h) => h.action));
|
|
156
|
-
return { plan: plan.filter((p) => !holding.has(p.action)), held };
|
|
226
|
+
return { plan: plan.filter((p) => !holding.has(p.action)), held, restored };
|
|
157
227
|
}
|