@muhkoo/cli 0.15.0-alpha.5 → 0.15.0-alpha.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +6 -3
- package/src/lib/mount.js +551 -110
- package/src/lib/syncPlan.js +58 -0
package/package.json
CHANGED
|
@@ -1,11 +1,14 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@muhkoo/cli",
|
|
3
|
-
"version": "0.15.0-alpha.
|
|
3
|
+
"version": "0.15.0-alpha.7",
|
|
4
4
|
"description": "Command-line interface for the Muhkoo platform — provision backends, deploy apps, and manage agents, functions, databases, and domains.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|
|
7
7
|
"author": "muhkoo",
|
|
8
|
-
"repository":
|
|
8
|
+
"repository": {
|
|
9
|
+
"type": "git",
|
|
10
|
+
"url": "git+https://github.com/muhkoo/cli.git"
|
|
11
|
+
},
|
|
9
12
|
"homepage": "https://docs.muhkoo.dev",
|
|
10
13
|
"bugs": "https://github.com/muhkoo/cli/issues",
|
|
11
14
|
"keywords": [
|
|
@@ -40,7 +43,7 @@
|
|
|
40
43
|
"release": "npm publish --tag alpha && npm dist-tag add @muhkoo/cli@$npm_package_version latest"
|
|
41
44
|
},
|
|
42
45
|
"dependencies": {
|
|
43
|
-
"@muhkoo/connect": "0.15.0-alpha.
|
|
46
|
+
"@muhkoo/connect": "0.15.0-alpha.7",
|
|
44
47
|
"snarkjs": "^0.7.0"
|
|
45
48
|
},
|
|
46
49
|
"packageManager": "yarn@1.22.22+sha512.a6b2f7906b721bba3d67d4aff083df04dad64c399707841b7acf00f6b133b7ac24255f2652fa22ae3534329dc6180534e98d17432037ff6fd140556e2bb3137e",
|
package/src/lib/mount.js
CHANGED
|
@@ -42,7 +42,7 @@ import { constants as fsConstants, watch } from "node:fs";
|
|
|
42
42
|
import { homedir, tmpdir } from "node:os";
|
|
43
43
|
import { basename, dirname, isAbsolute, join, parse, relative, resolve, sep } from "node:path";
|
|
44
44
|
|
|
45
|
-
import { holdMassDeletes, planSync } from "./syncPlan.js";
|
|
45
|
+
import { DIR, dirKey, filesOf, holdMassDeletes, planDirs, planSync } from "./syncPlan.js";
|
|
46
46
|
import {
|
|
47
47
|
DEPTH_MAX, IgnoreBudgetError, NAME_MAX, caseFold, ignoreFileProblem, ruleBank, ruleText, withinMatchBounds,
|
|
48
48
|
} from "./ignore.js";
|
|
@@ -77,12 +77,40 @@ const KEEP_HIDDEN = new Set([".vcsignore", ".gitignore"]);
|
|
|
77
77
|
const MAX_BYTES = 100 * 1024 * 1024;
|
|
78
78
|
|
|
79
79
|
/**
|
|
80
|
-
* How many files
|
|
80
|
+
* How many files the scan reads at once.
|
|
81
81
|
*
|
|
82
82
|
* Enough to hide per-request latency; low enough not to open a connection per
|
|
83
|
-
* file on a large project.
|
|
83
|
+
* file on a large project. A read costs 0.3–0.5 s against production, nearly
|
|
84
|
+
* all of it fixed, so throughput comes from overlapping them: 96 small files
|
|
85
|
+
* took 6.35 s eight at a time, 4.50 s at sixteen, and 4.12 s at thirty-two.
|
|
84
86
|
*/
|
|
85
|
-
const
|
|
87
|
+
export const SCAN_CONCURRENCY = 16;
|
|
88
|
+
|
|
89
|
+
/**
|
|
90
|
+
* And the most the files being read may claim between them — by their
|
|
91
|
+
* records' sizes, and their manifests' — before another read waits for one
|
|
92
|
+
* to finish. So raising the count does not raise the memory a run of large
|
|
93
|
+
* files holds at once. A file that claims more than this on its own is read
|
|
94
|
+
* alone: nothing else starts until it is done.
|
|
95
|
+
*/
|
|
96
|
+
export const SCAN_WINDOW_BYTES = 64 * 1024 * 1024;
|
|
97
|
+
|
|
98
|
+
/**
|
|
99
|
+
* How many pulls to fetch at once while the oldest lands (see `applyPlan`).
|
|
100
|
+
*
|
|
101
|
+
* A pull the scan did not keep the bytes of — past BLOB_CACHE_BYTES, or a
|
|
102
|
+
* follow-up pass — is a round trip of its own, and one at a time they cost
|
|
103
|
+
* the SUM of the latencies. Only the fetches overlap; every pull still lands
|
|
104
|
+
* one at a time, in plan order.
|
|
105
|
+
*/
|
|
106
|
+
const PULL_CONCURRENCY = 8;
|
|
107
|
+
|
|
108
|
+
/**
|
|
109
|
+
* And the most the records of the pulls being fetched may claim between
|
|
110
|
+
* them before the window waits for one to land — so a run of large files is
|
|
111
|
+
* not held in memory eight at a time. One is always let through.
|
|
112
|
+
*/
|
|
113
|
+
const PULL_WINDOW_BYTES = 64 * 1024 * 1024;
|
|
86
114
|
|
|
87
115
|
/**
|
|
88
116
|
* The most one pass pulls: bytes and files. Past it, the rest wait for a
|
|
@@ -195,6 +223,50 @@ async function saveState(dir, root, paths) {
|
|
|
195
223
|
await writeAtomic(statePath(dir), JSON.stringify({ v: 1, root, paths }, null, 2) + "\n");
|
|
196
224
|
}
|
|
197
225
|
|
|
226
|
+
/**
|
|
227
|
+
* What the scan has hashed, kept between runs, so a restarted mount does not
|
|
228
|
+
* download the whole tree again just to learn it has not changed.
|
|
229
|
+
*
|
|
230
|
+
* It maps a content key — the SHA-256 of a file's manifest, which names the
|
|
231
|
+
* exact shards and the key that decrypts them, so one key is one content — to
|
|
232
|
+
* that content's hash and size. Unlike `mount.json` it decides nothing on its
|
|
233
|
+
* own: an entry only spares a read. So a file that cannot be read, or an entry
|
|
234
|
+
* that is not in the shape written here, is ignored, and that content is read.
|
|
235
|
+
*/
|
|
236
|
+
const hashCachePath = (dir) => join(dir, ".muhkoo", "content-hashes.json");
|
|
237
|
+
const HEX64 = /^[0-9a-f]{64}$/;
|
|
238
|
+
/** Entries kept: well past any tree the read budget lets one pass hash. */
|
|
239
|
+
const HASH_CACHE_MAX = 200_000;
|
|
240
|
+
|
|
241
|
+
async function loadHashCache(dir) {
|
|
242
|
+
const hashes = new Map();
|
|
243
|
+
try {
|
|
244
|
+
const saved = JSON.parse(await readFile(hashCachePath(dir), "utf8"));
|
|
245
|
+
if (saved?.v !== 1 || !saved.contents || typeof saved.contents !== "object") return hashes;
|
|
246
|
+
for (const [key, entry] of Object.entries(saved.contents)) {
|
|
247
|
+
if (hashes.size >= HASH_CACHE_MAX) break;
|
|
248
|
+
const [hash, size] = Array.isArray(entry) ? entry : [];
|
|
249
|
+
if (!HEX64.test(key) || !HEX64.test(hash ?? "") || !Number.isSafeInteger(size) || size < 0) return new Map();
|
|
250
|
+
hashes.set(key, { hash, size });
|
|
251
|
+
}
|
|
252
|
+
} catch {
|
|
253
|
+
return new Map(); // none yet, or unreadable: the scan reads, as it always did
|
|
254
|
+
}
|
|
255
|
+
return hashes;
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
function sameHashes(a, b) {
|
|
259
|
+
if (a.size !== b.size) return false;
|
|
260
|
+
for (const [key, { hash }] of a) if (b.get(key)?.hash !== hash) return false;
|
|
261
|
+
return true;
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
async function saveHashCache(dir, hashes) {
|
|
265
|
+
const contents = {};
|
|
266
|
+
for (const [key, { hash, size }] of hashes) contents[key] = [hash, size];
|
|
267
|
+
await writeAtomic(hashCachePath(dir), JSON.stringify({ v: 1, contents }) + "\n");
|
|
268
|
+
}
|
|
269
|
+
|
|
198
270
|
function sameState(a, b) {
|
|
199
271
|
const keys = Object.keys(a);
|
|
200
272
|
return keys.length === Object.keys(b).length && keys.every((k) => a[k] === b[k]);
|
|
@@ -941,6 +1013,7 @@ export async function foldsCaseReadOnly(dir) {
|
|
|
941
1013
|
*/
|
|
942
1014
|
async function scanLocal(dir, { hidden = false, ignores, sess, signal }) {
|
|
943
1015
|
const files = {};
|
|
1016
|
+
const dirs = [];
|
|
944
1017
|
const unknown = new Set();
|
|
945
1018
|
const held = new Set();
|
|
946
1019
|
const pace = pacer();
|
|
@@ -978,6 +1051,7 @@ async function scanLocal(dir, { hidden = false, ignores, sess, signal }) {
|
|
|
978
1051
|
if (verdict === "held") held.add(entry.isDirectory() ? `${key}/` : key);
|
|
979
1052
|
if (verdict) continue;
|
|
980
1053
|
if (entry.isDirectory()) {
|
|
1054
|
+
dirs.push(key);
|
|
981
1055
|
await walk(child, rel);
|
|
982
1056
|
continue;
|
|
983
1057
|
}
|
|
@@ -1003,7 +1077,7 @@ async function scanLocal(dir, { hidden = false, ignores, sess, signal }) {
|
|
|
1003
1077
|
}
|
|
1004
1078
|
};
|
|
1005
1079
|
await walk(dir, "");
|
|
1006
|
-
return { files, unknown, held };
|
|
1080
|
+
return { files, dirs, unknown, held };
|
|
1007
1081
|
}
|
|
1008
1082
|
|
|
1009
1083
|
/**
|
|
@@ -1016,8 +1090,35 @@ async function scanLocal(dir, { hidden = false, ignores, sess, signal }) {
|
|
|
1016
1090
|
* but this CLI may run against either.
|
|
1017
1091
|
*/
|
|
1018
1092
|
async function walkRemote(vfs, root) {
|
|
1019
|
-
|
|
1020
|
-
|
|
1093
|
+
// `dirs` is null when the folders cannot be told: a root that is not there
|
|
1094
|
+
// (missing or renamed — never "every folder deleted"), or an SDK without
|
|
1095
|
+
// `walkEntries`. Folders are then left alone.
|
|
1096
|
+
if (root !== "/" && !(await vfs.exists(root))) return { paths: [], dirs: null };
|
|
1097
|
+
if (typeof vfs.walkEntries !== "function") return { paths: await vfs.walk(root), dirs: null };
|
|
1098
|
+
const entries = await vfs.walkEntries(root);
|
|
1099
|
+
return {
|
|
1100
|
+
paths: entries.filter((e) => e.kind === "file").map((e) => e.path),
|
|
1101
|
+
dirs: entries.filter((e) => e.kind === "dir").map((e) => e.path),
|
|
1102
|
+
};
|
|
1103
|
+
}
|
|
1104
|
+
|
|
1105
|
+
/**
|
|
1106
|
+
* The remote folders, keyed like the local ones, through the same admission
|
|
1107
|
+
* rule: a refused, ignored or hidden folder is not made here, any more than a
|
|
1108
|
+
* file in it would be.
|
|
1109
|
+
*/
|
|
1110
|
+
function remoteDirs(dirs, root, { hidden, ignores, ignoreCase }) {
|
|
1111
|
+
if (!dirs) return null;
|
|
1112
|
+
const out = [];
|
|
1113
|
+
for (const abs of dirs) {
|
|
1114
|
+
const key = root === "/" ? abs : abs.slice(root.length);
|
|
1115
|
+
if (!key.startsWith("/") || key === "/") continue;
|
|
1116
|
+
const rel = key.slice(1);
|
|
1117
|
+
if (!withinMatchBounds(rel, ignoreCase)) continue;
|
|
1118
|
+
if (admission(rel, true, hidden, ignores.now, ignores.held)) continue;
|
|
1119
|
+
out.push(key);
|
|
1120
|
+
}
|
|
1121
|
+
return out;
|
|
1021
1122
|
}
|
|
1022
1123
|
|
|
1023
1124
|
/**
|
|
@@ -1051,7 +1152,7 @@ export async function scanRemote(vfs, root, includeHidden = false, ignore = () =
|
|
|
1051
1152
|
const unknown = new Set();
|
|
1052
1153
|
const unreadable = new Set();
|
|
1053
1154
|
const held = new Set();
|
|
1054
|
-
const paths = ctx.paths ?? (await walkRemote(vfs, root));
|
|
1155
|
+
const paths = ctx.paths ?? (await walkRemote(vfs, root)).paths;
|
|
1055
1156
|
|
|
1056
1157
|
// Decide what to read BEFORE reading anything, so the filters are applied
|
|
1057
1158
|
// once and the reads that survive them can go out together.
|
|
@@ -1110,12 +1211,18 @@ export async function scanRemote(vfs, root, includeHidden = false, ignore = () =
|
|
|
1110
1211
|
const readPace = pacer(); // reads from a cache resolve at once, and never yield on their own
|
|
1111
1212
|
const seen = new Map(); // content key → { hash, size }, for the next pass
|
|
1112
1213
|
const reading = new Map(); // content key → the read under way
|
|
1214
|
+
const oversize = new Map(); // content key → its length, for a read that came back over MAX_BYTES
|
|
1113
1215
|
const blobs = ctx.blobs ?? blobCache(BLOB_CACHE_BYTES);
|
|
1114
1216
|
const budget = sess.readBudget;
|
|
1115
1217
|
const spent = ctx.spent ?? { bytes: 0, files: 0 };
|
|
1116
1218
|
const deferred = new Set();
|
|
1219
|
+
// Bytes in flight are bounded as well as reads (SCAN_WINDOW_BYTES), by the
|
|
1220
|
+
// same claim the budget is charged. A read waiting for room is not
|
|
1221
|
+
// started once the scan is stopped.
|
|
1222
|
+
const window = byteWindow(SCAN_WINDOW_BYTES);
|
|
1223
|
+
const STOPPED = Symbol("stopped");
|
|
1117
1224
|
await Promise.all(
|
|
1118
|
-
Array.from({ length: Math.min(
|
|
1225
|
+
Array.from({ length: Math.min(SCAN_CONCURRENCY, queue.length) }, async () => {
|
|
1119
1226
|
for (let item = queue.shift(); item !== undefined && !ctx.signal?.aborted; item = queue.shift()) {
|
|
1120
1227
|
const breather = readPace();
|
|
1121
1228
|
if (breather) await breather;
|
|
@@ -1140,6 +1247,12 @@ export async function scanRemote(vfs, root, includeHidden = false, ignore = () =
|
|
|
1140
1247
|
sizes[key] = known.size;
|
|
1141
1248
|
continue;
|
|
1142
1249
|
}
|
|
1250
|
+
const over = content.key && oversize.get(content.key);
|
|
1251
|
+
if (over) {
|
|
1252
|
+
tooBig(sess, key, over);
|
|
1253
|
+
unknown.add(key);
|
|
1254
|
+
continue;
|
|
1255
|
+
}
|
|
1143
1256
|
let read = content.key && reading.get(content.key);
|
|
1144
1257
|
if (!read) {
|
|
1145
1258
|
// Ignore files included: a 256 KB file of comments is
|
|
@@ -1154,16 +1267,37 @@ export async function scanRemote(vfs, root, includeHidden = false, ignore = () =
|
|
|
1154
1267
|
spent.files++;
|
|
1155
1268
|
spent.bytes += claim;
|
|
1156
1269
|
}
|
|
1157
|
-
read =
|
|
1158
|
-
|
|
1159
|
-
|
|
1160
|
-
|
|
1270
|
+
read = (async () => {
|
|
1271
|
+
const admitted = await window.acquire(claim, ctx.signal);
|
|
1272
|
+
if (admitted && ctx.signal?.aborted) window.release(claim); // let in, but stopped since
|
|
1273
|
+
if (!admitted || ctx.signal?.aborted) {
|
|
1274
|
+
if (budget) { // never read: give back what it was charged
|
|
1275
|
+
spent.files--;
|
|
1276
|
+
spent.bytes -= claim;
|
|
1277
|
+
}
|
|
1278
|
+
throw STOPPED;
|
|
1279
|
+
}
|
|
1280
|
+
try {
|
|
1281
|
+
const bytes = await content.read();
|
|
1282
|
+
spent.bytes += bytes.length - claim;
|
|
1283
|
+
return bytes;
|
|
1284
|
+
} finally {
|
|
1285
|
+
window.release(claim);
|
|
1286
|
+
}
|
|
1287
|
+
})();
|
|
1161
1288
|
if (content.key) reading.set(content.key, read);
|
|
1162
1289
|
}
|
|
1163
1290
|
const bytes = await read;
|
|
1164
1291
|
if (bytes.length > MAX_BYTES) {
|
|
1165
1292
|
tooBig(sess, key, bytes.length);
|
|
1166
1293
|
unknown.add(key);
|
|
1294
|
+
if (content.key) {
|
|
1295
|
+
// Only its length is kept: entries sharing this
|
|
1296
|
+
// content get the same answer without the read
|
|
1297
|
+
// (up to 100 MB) staying referenced.
|
|
1298
|
+
oversize.set(content.key, bytes.length);
|
|
1299
|
+
reading.delete(content.key);
|
|
1300
|
+
}
|
|
1167
1301
|
continue;
|
|
1168
1302
|
}
|
|
1169
1303
|
const hash = sha(bytes);
|
|
@@ -1172,8 +1306,18 @@ export async function scanRemote(vfs, root, includeHidden = false, ignore = () =
|
|
|
1172
1306
|
if (content.key) {
|
|
1173
1307
|
seen.set(content.key, { hash, size: bytes.length });
|
|
1174
1308
|
blobs.put(content.key, bytes);
|
|
1309
|
+
// `seen` answers for this content from here on. Left
|
|
1310
|
+
// in `reading`, the settled read held every byte the
|
|
1311
|
+
// pass read, past the blob cache's limit and the window.
|
|
1312
|
+
reading.delete(content.key);
|
|
1175
1313
|
}
|
|
1176
1314
|
} catch (err) {
|
|
1315
|
+
// Stopped before its read began: not read, and nothing to
|
|
1316
|
+
// warn of. The caller does not plan from a stopped scan.
|
|
1317
|
+
if (err === STOPPED) {
|
|
1318
|
+
unknown.add(key);
|
|
1319
|
+
continue;
|
|
1320
|
+
}
|
|
1177
1321
|
// One unreadable entry — a lost shard, a planted manifest, a
|
|
1178
1322
|
// dropped request — must not stop the whole sync, and must
|
|
1179
1323
|
// not read as a deletion either.
|
|
@@ -1213,10 +1357,11 @@ export async function scanRemote(vfs, root, includeHidden = false, ignore = () =
|
|
|
1213
1357
|
*/
|
|
1214
1358
|
async function remoteContent(vfs, abs) {
|
|
1215
1359
|
if (typeof vfs.statManifest !== "function") {
|
|
1216
|
-
const
|
|
1360
|
+
const size = recordSize((await vfs.stat(abs)).size);
|
|
1217
1361
|
return { size, key: null, bound: null, read: () => vfs.readFile(abs) };
|
|
1218
1362
|
}
|
|
1219
|
-
const { manifest, size } = await vfs.statManifest(abs);
|
|
1363
|
+
const { manifest, size: claimed } = await vfs.statManifest(abs);
|
|
1364
|
+
const size = recordSize(claimed);
|
|
1220
1365
|
const exact = typeof vfs.readByManifest === "function";
|
|
1221
1366
|
return {
|
|
1222
1367
|
size,
|
|
@@ -1226,6 +1371,15 @@ async function remoteContent(vfs, abs) {
|
|
|
1226
1371
|
};
|
|
1227
1372
|
}
|
|
1228
1373
|
|
|
1374
|
+
/**
|
|
1375
|
+
* A record's size, as a number the budgets can add. The record is written by
|
|
1376
|
+
* the remote party, so the size may be absent, a string, negative or NaN; any
|
|
1377
|
+
* of those counts as 0 (the bytes that come back are what is held to the limit).
|
|
1378
|
+
*/
|
|
1379
|
+
function recordSize(size) {
|
|
1380
|
+
return Number.isSafeInteger(size) && size >= 0 ? size : 0;
|
|
1381
|
+
}
|
|
1382
|
+
|
|
1229
1383
|
/**
|
|
1230
1384
|
* What a manifest lets a read bring back (see `remoteContent`), or null when
|
|
1231
1385
|
* it is not in a shape this knows. Each chunk decrypts to its ciphertext less
|
|
@@ -1246,6 +1400,63 @@ function manifestBound(manifest) {
|
|
|
1246
1400
|
return { plain, fetched };
|
|
1247
1401
|
}
|
|
1248
1402
|
|
|
1403
|
+
/**
|
|
1404
|
+
* A first-come, first-served allowance of `limit` bytes for reads in flight.
|
|
1405
|
+
*
|
|
1406
|
+
* `acquire(n, signal)` resolves true once `n` more bytes fit — or at once
|
|
1407
|
+
* when nothing is held, so one claim larger than the whole window is still
|
|
1408
|
+
* let through, alone — and false, without taking anything, when `signal`
|
|
1409
|
+
* has fired. Waiters are served in order, so a large claim is not starved by
|
|
1410
|
+
* small ones arriving behind it. `release(n)` gives back what an acquire took.
|
|
1411
|
+
*/
|
|
1412
|
+
function byteWindow(limit) {
|
|
1413
|
+
let held = 0;
|
|
1414
|
+
// A claim that is not a finite, non-negative number is 0: NaN in `held`
|
|
1415
|
+
// would make nothing fit again, and the scan would never finish.
|
|
1416
|
+
const sane = (n) => (Number.isFinite(n) && n > 0 ? n : 0);
|
|
1417
|
+
const waiting = [];
|
|
1418
|
+
const fits = (n) => held === 0 || held + n <= limit;
|
|
1419
|
+
const wake = () => {
|
|
1420
|
+
while (waiting.length && fits(waiting[0].n)) {
|
|
1421
|
+
const w = waiting.shift();
|
|
1422
|
+
held += w.n;
|
|
1423
|
+
w.done(true);
|
|
1424
|
+
}
|
|
1425
|
+
};
|
|
1426
|
+
return {
|
|
1427
|
+
acquire(claim, signal) {
|
|
1428
|
+
const n = sane(claim);
|
|
1429
|
+
if (signal?.aborted) return Promise.resolve(false);
|
|
1430
|
+
if (!waiting.length && fits(n)) {
|
|
1431
|
+
held += n;
|
|
1432
|
+
return Promise.resolve(true);
|
|
1433
|
+
}
|
|
1434
|
+
return new Promise((resolve) => {
|
|
1435
|
+
const onAbort = () => {
|
|
1436
|
+
const i = waiting.indexOf(w);
|
|
1437
|
+
if (i < 0) return;
|
|
1438
|
+
waiting.splice(i, 1);
|
|
1439
|
+
resolve(false);
|
|
1440
|
+
wake(); // the head may have been what held the rest
|
|
1441
|
+
};
|
|
1442
|
+
const w = {
|
|
1443
|
+
n,
|
|
1444
|
+
done: (ok) => {
|
|
1445
|
+
signal?.removeEventListener("abort", onAbort);
|
|
1446
|
+
resolve(ok);
|
|
1447
|
+
},
|
|
1448
|
+
};
|
|
1449
|
+
waiting.push(w);
|
|
1450
|
+
signal?.addEventListener("abort", onAbort, { once: true });
|
|
1451
|
+
});
|
|
1452
|
+
},
|
|
1453
|
+
release(claim) {
|
|
1454
|
+
held -= sane(claim);
|
|
1455
|
+
wake();
|
|
1456
|
+
},
|
|
1457
|
+
};
|
|
1458
|
+
}
|
|
1459
|
+
|
|
1249
1460
|
/**
|
|
1250
1461
|
* The bytes a pass has read, by content key, up to `limit` bytes: the oldest
|
|
1251
1462
|
* go first. What lets a pull of content the scan already read — two hundred
|
|
@@ -1461,7 +1672,7 @@ export function win32NameProblem(name) {
|
|
|
1461
1672
|
* directories to create, in order, and `existing` the stats of the file being
|
|
1462
1673
|
* replaced — or `{ ok: false, reason }`. `listing` caches `readdir` for a pass.
|
|
1463
1674
|
*/
|
|
1464
|
-
export async function writeTarget(dir, key, { localKeys = new Set(), listing = new Map(), platform = process.platform } = {}) {
|
|
1675
|
+
export async function writeTarget(dir, key, { localKeys = new Set(), listing = new Map(), platform = process.platform, kind = "file" } = {}) {
|
|
1465
1676
|
const refuse = (reason) => ({ ok: false, reason });
|
|
1466
1677
|
const names = key.split("/").filter(Boolean);
|
|
1467
1678
|
if (!names.length) return refuse("it names the mount directory itself");
|
|
@@ -1487,7 +1698,8 @@ export async function writeTarget(dir, key, { localKeys = new Set(), listing = n
|
|
|
1487
1698
|
if (!entries) return refuse(`${parent} could not be listed`);
|
|
1488
1699
|
if (!entries.has(name)) {
|
|
1489
1700
|
if (await present(abs)) return alias(name);
|
|
1490
|
-
|
|
1701
|
+
// A folder is made whole, its own name included; a file's last name is the file.
|
|
1702
|
+
return { ok: true, abs: join(parent, ...names.slice(i)), parent, missing: kind === "dir" ? names.slice(i) : names.slice(i, -1), existing: null };
|
|
1491
1703
|
}
|
|
1492
1704
|
const info = await lstat(abs).catch(() => null);
|
|
1493
1705
|
if (!last) {
|
|
@@ -1495,6 +1707,10 @@ export async function writeTarget(dir, key, { localKeys = new Set(), listing = n
|
|
|
1495
1707
|
parent = abs;
|
|
1496
1708
|
continue;
|
|
1497
1709
|
}
|
|
1710
|
+
if (kind === "dir") {
|
|
1711
|
+
if (!info?.isDirectory()) return refuse(`${relative(dir, abs)} is ${info?.isSymbolicLink() ? "a symlink" : "not a directory"}`);
|
|
1712
|
+
return { ok: true, abs, parent, missing: [], existing: info };
|
|
1713
|
+
}
|
|
1498
1714
|
if (!info?.isFile()) return refuse(`${relative(dir, abs)} is a symlink or not a regular file`);
|
|
1499
1715
|
if (!localKeys.has(key)) return refuse(`${relative(dir, abs)} exists here but is not synced (ignored, hidden or refused), so it may exist nowhere else`);
|
|
1500
1716
|
return { ok: true, abs, parent, missing: [], existing: info };
|
|
@@ -1514,14 +1730,7 @@ export async function writeTarget(dir, key, { localKeys = new Set(), listing = n
|
|
|
1514
1730
|
* default mode, never an execute bit.
|
|
1515
1731
|
*/
|
|
1516
1732
|
async function writeAt(target, bytes, listing) {
|
|
1517
|
-
|
|
1518
|
-
for (const name of target.missing) {
|
|
1519
|
-
const next = join(parent, name);
|
|
1520
|
-
await mkdir(next); // not recursive: each component is created here, never followed
|
|
1521
|
-
listing.get(parent)?.add(name);
|
|
1522
|
-
listing.set(next, new Set());
|
|
1523
|
-
parent = next;
|
|
1524
|
-
}
|
|
1733
|
+
const parent = await makeMissing(target, listing);
|
|
1525
1734
|
const tmp = join(parent, TMP_PREFIX + randomBytes(6).toString("hex"));
|
|
1526
1735
|
try {
|
|
1527
1736
|
await writeFile(tmp, bytes, { flag: "wx" });
|
|
@@ -1534,6 +1743,114 @@ async function writeAt(target, bytes, listing) {
|
|
|
1534
1743
|
listing.get(parent)?.add(target.abs.slice(parent.length + 1));
|
|
1535
1744
|
}
|
|
1536
1745
|
|
|
1746
|
+
/**
|
|
1747
|
+
* Apply the folder plan (`planDirs`), after the files: folders are made
|
|
1748
|
+
* shallowest first and removed deepest first, so a folder emptied by this
|
|
1749
|
+
* pass's file deletes is gone by its end.
|
|
1750
|
+
*
|
|
1751
|
+
* A folder is only ever removed EMPTY — `rmdir` here, a delete that is not
|
|
1752
|
+
* recursive there — so removing one can never take a file with it, synced or
|
|
1753
|
+
* not. One that still holds something (an ignored or refused file, or one that
|
|
1754
|
+
* arrived since the scan) is kept, said once, and left out of the state: the
|
|
1755
|
+
* next pass sees it on one side only and makes it on the other again, which is
|
|
1756
|
+
* the truth, since it is not empty. Removals are gated by `--allow-delete` as
|
|
1757
|
+
* file deletes are; a removal not made is forgotten, as theirs is.
|
|
1758
|
+
*/
|
|
1759
|
+
async function applyDirPlan({ vfs, dir, root, state, plan, opts, sess, signal }) {
|
|
1760
|
+
const next = { ...state };
|
|
1761
|
+
const listing = new Map();
|
|
1762
|
+
const remoteAbs = (path) => (root === "/" ? path : root + path);
|
|
1763
|
+
const depth = (path) => path.split("/").length;
|
|
1764
|
+
const makes = plan.filter(({ action }) => !action.startsWith("rmdir")).sort((a, b) => depth(a.path) - depth(b.path));
|
|
1765
|
+
const removes = plan.filter(({ action }) => action.startsWith("rmdir")).sort((a, b) => depth(b.path) - depth(a.path));
|
|
1766
|
+
const kept = (path, where) => warnOnce(sess, `dir-kept:${where}:${path}`, `kept the folder ${path}/ ${where} — it still holds files that are not synced`);
|
|
1767
|
+
for (const { path, action } of [...makes, ...removes]) {
|
|
1768
|
+
if (signal?.aborted) break;
|
|
1769
|
+
const key = dirKey(path);
|
|
1770
|
+
try {
|
|
1771
|
+
switch (action) {
|
|
1772
|
+
case "record":
|
|
1773
|
+
next[key] = DIR;
|
|
1774
|
+
break;
|
|
1775
|
+
case "forget":
|
|
1776
|
+
delete next[key];
|
|
1777
|
+
break;
|
|
1778
|
+
case "mkdir-remote":
|
|
1779
|
+
await vfs.mkdir(remoteAbs(path), { recursive: true });
|
|
1780
|
+
next[key] = DIR;
|
|
1781
|
+
info(`${SIGN.push} ${path}/`);
|
|
1782
|
+
break;
|
|
1783
|
+
case "mkdir-local": {
|
|
1784
|
+
const target = await writeTarget(dir, path, { listing, kind: "dir" });
|
|
1785
|
+
if (!target.ok) {
|
|
1786
|
+
warnOnce(sess, `declined:${action}:${path}`, `skipping ${path}/ — ${target.reason}`);
|
|
1787
|
+
break;
|
|
1788
|
+
}
|
|
1789
|
+
if (!target.existing) await makeMissing(target, listing);
|
|
1790
|
+
next[key] = DIR;
|
|
1791
|
+
info(`${SIGN.pull} ${path}/`);
|
|
1792
|
+
break;
|
|
1793
|
+
}
|
|
1794
|
+
case "rmdir-remote": {
|
|
1795
|
+
delete next[key];
|
|
1796
|
+
if (!opts.allowDelete) {
|
|
1797
|
+
warn(
|
|
1798
|
+
`${path}/ is gone locally but still in your filesystem — not deleting it.`,
|
|
1799
|
+
` Pass --allow-delete to propagate deletions, or remove it with: muhkoo vfs rm -r ${remoteAbs(path)}`,
|
|
1800
|
+
);
|
|
1801
|
+
break;
|
|
1802
|
+
}
|
|
1803
|
+
const found = await vfs.stat(remoteAbs(path)).catch(() => null);
|
|
1804
|
+
if (found?.kind !== "dir") break;
|
|
1805
|
+
try {
|
|
1806
|
+
await vfs.delete(remoteAbs(path)); // not recursive: refused unless empty
|
|
1807
|
+
info(`${SIGN["delete-remote"]} ${path}/ (deleted from your filesystem)`);
|
|
1808
|
+
} catch {
|
|
1809
|
+
kept(path, "in your filesystem");
|
|
1810
|
+
}
|
|
1811
|
+
break;
|
|
1812
|
+
}
|
|
1813
|
+
case "rmdir-local": {
|
|
1814
|
+
delete next[key];
|
|
1815
|
+
if (!opts.allowDelete) {
|
|
1816
|
+
warn(`${path}/ is gone from your filesystem but still here — not deleting it locally.`, ` Pass --allow-delete to propagate deletions.`);
|
|
1817
|
+
break;
|
|
1818
|
+
}
|
|
1819
|
+
const target = await writeTarget(dir, path, { listing, kind: "dir" });
|
|
1820
|
+
if (!target.ok || !target.existing) break;
|
|
1821
|
+
try {
|
|
1822
|
+
await rmdir(target.abs);
|
|
1823
|
+
listing.get(target.parent)?.delete(target.abs.slice(target.parent.length + 1));
|
|
1824
|
+
listing.delete(target.abs);
|
|
1825
|
+
info(`${SIGN["delete-local"]} ${path}/ (deleted from this disk)`);
|
|
1826
|
+
} catch (err) {
|
|
1827
|
+
if (err.code !== "ENOTEMPTY" && err.code !== "EEXIST") throw err;
|
|
1828
|
+
kept(path, "on this disk");
|
|
1829
|
+
}
|
|
1830
|
+
break;
|
|
1831
|
+
}
|
|
1832
|
+
}
|
|
1833
|
+
} catch (err) {
|
|
1834
|
+
const message = err?.message ?? String(err);
|
|
1835
|
+
warnOnce(sess, `error:${path}/:${message}`, `${path}/: ${message}`);
|
|
1836
|
+
}
|
|
1837
|
+
}
|
|
1838
|
+
return next;
|
|
1839
|
+
}
|
|
1840
|
+
|
|
1841
|
+
/** Make the directories `writeTarget` found missing, in order; returns the deepest. */
|
|
1842
|
+
async function makeMissing(target, listing) {
|
|
1843
|
+
let parent = target.parent;
|
|
1844
|
+
for (const name of target.missing) {
|
|
1845
|
+
const next = join(parent, name);
|
|
1846
|
+
await mkdir(next); // not recursive: each component is created here, never followed
|
|
1847
|
+
listing.get(parent)?.add(name);
|
|
1848
|
+
listing.set(next, new Set());
|
|
1849
|
+
parent = next;
|
|
1850
|
+
}
|
|
1851
|
+
return parent;
|
|
1852
|
+
}
|
|
1853
|
+
|
|
1537
1854
|
// ---- planning and applying ------------------------------------------------------
|
|
1538
1855
|
|
|
1539
1856
|
/**
|
|
@@ -1556,7 +1873,7 @@ async function preparePass({ vfs, dir, rulesDir = dir, root, state, opts, sess,
|
|
|
1556
1873
|
// The remote walk is the slow part; start it while the local scan runs.
|
|
1557
1874
|
// The remote FILTER has to wait for the local scan, which is what finds
|
|
1558
1875
|
// the nested ignore files.
|
|
1559
|
-
const [local,
|
|
1876
|
+
const [local, listed] = await Promise.all([
|
|
1560
1877
|
scanLocal(dir, { hidden: opts.hidden, ignores, sess, signal }),
|
|
1561
1878
|
walkRemote(vfs, root),
|
|
1562
1879
|
]);
|
|
@@ -1564,7 +1881,7 @@ async function preparePass({ vfs, dir, rulesDir = dir, root, state, opts, sess,
|
|
|
1564
1881
|
// The ignore files as they now stand are known; nothing has been pulled.
|
|
1565
1882
|
await ignores.record(rulesDir, root, save);
|
|
1566
1883
|
const remote = await scanRemote(vfs, root, opts.hidden, ignores.now, {
|
|
1567
|
-
paths, held: ignores.held, session: sess, ignoreCase: sess.ignoreCase,
|
|
1884
|
+
paths: listed.paths, held: ignores.held, session: sess, ignoreCase: sess.ignoreCase,
|
|
1568
1885
|
localKeys: new Set(Object.keys(local.files)), signal, blobs, spent: read,
|
|
1569
1886
|
});
|
|
1570
1887
|
if (signal?.aborted) return null;
|
|
@@ -1585,13 +1902,24 @@ async function preparePass({ vfs, dir, rulesDir = dir, root, state, opts, sess,
|
|
|
1585
1902
|
if (keptBack.length) warnOnce(sess, `kept-back:${keptBack.join("\0")}`, ...heldNotice(keptBack, rulesDir));
|
|
1586
1903
|
|
|
1587
1904
|
const unknown = new Set([...local.unknown, ...remote.unknown]);
|
|
1588
|
-
const
|
|
1905
|
+
const files = filesOf(state);
|
|
1906
|
+
const planned = planSync({ last: files, local: local.files, remote: remote.files, unknown });
|
|
1589
1907
|
const views = { local: Object.keys(local.files).length, remote: Object.keys(remote.files).length };
|
|
1590
|
-
sess.peak = Math.max(sess.peak ?? 0, Object.keys(
|
|
1591
|
-
const { plan, held } = holdMassDeletes(planned,
|
|
1908
|
+
sess.peak = Math.max(sess.peak ?? 0, Object.keys(files).length);
|
|
1909
|
+
const { plan, held } = holdMassDeletes(planned, files, views, sess.peak);
|
|
1592
1910
|
for (const h of held) warnOnce(sess, `held:${h.action}:${h.count}`, ...breakerMessage(h, dir, root));
|
|
1911
|
+
// Folders after files, by the same rule. When the breaker held file
|
|
1912
|
+
// deletes, no folder is removed either: the same pass would empty them.
|
|
1913
|
+
let dirPlan = planDirs({
|
|
1914
|
+
last: state,
|
|
1915
|
+
local: local.dirs,
|
|
1916
|
+
remote: remoteDirs(listed.dirs, root, { hidden: opts.hidden, ignores, ignoreCase: sess.ignoreCase }),
|
|
1917
|
+
unknown,
|
|
1918
|
+
fold: sess.ignoreCase ? caseFold : null,
|
|
1919
|
+
});
|
|
1920
|
+
if (held.length) dirPlan = dirPlan.filter(({ action }) => !action.startsWith("rmdir"));
|
|
1593
1921
|
return {
|
|
1594
|
-
plan, local: local.files, remote: remote.files, sizes: remote.sizes, blobs: remote.blobs, ignores,
|
|
1922
|
+
plan, dirPlan, local: local.files, remote: remote.files, sizes: remote.sizes, blobs: remote.blobs, ignores,
|
|
1595
1923
|
unreadable: remote.unreadable, deferred: remote.deferred,
|
|
1596
1924
|
landed: new Set(), // ignore files pulled in this pass
|
|
1597
1925
|
notLanded: new Map(), // and why the others were not
|
|
@@ -1617,10 +1945,22 @@ function breakerMessage({ action, count, tracked, empty }, dir, root) {
|
|
|
1617
1945
|
/**
|
|
1618
1946
|
* Apply a plan, and return the new last-synced view.
|
|
1619
1947
|
*
|
|
1620
|
-
*
|
|
1621
|
-
*
|
|
1622
|
-
*
|
|
1623
|
-
*
|
|
1948
|
+
* Every step that changes something — a write to the filesystem, a delete, a
|
|
1949
|
+
* file landing on this disk — is made one at a time, in plan order: writes
|
|
1950
|
+
* touch one directory record each, and two concurrent writes to the same
|
|
1951
|
+
* record would have one silently lose ([[VfsNamespace]] consistency note).
|
|
1952
|
+
* Pushes, deletes and conflicts are applied exactly as before, in sequence.
|
|
1953
|
+
*
|
|
1954
|
+
* Only the FETCH of a pull overlaps: up to PULL_CONCURRENCY pulls, in plan
|
|
1955
|
+
* order, are fetched while the oldest lands. A pull reads nothing but the
|
|
1956
|
+
* network and the pass's blob cache, so overlapping them changes no state;
|
|
1957
|
+
* landing stays in plan order, on the same checks, so what lands — and which
|
|
1958
|
+
* of two spellings a folding volume refuses — is what a sequential pass would
|
|
1959
|
+
* land. A step that is not a pull first lands every pull before it.
|
|
1960
|
+
*
|
|
1961
|
+
* `signal` stops it between steps — never inside one — and what was done by
|
|
1962
|
+
* then is what it returns. Pulls fetched but not yet landed when it fires are
|
|
1963
|
+
* dropped, unrecorded, and the next pass plans them again.
|
|
1624
1964
|
*
|
|
1625
1965
|
* A step this disk declines (a pull onto a symlink, a remote file where a
|
|
1626
1966
|
* directory stands) is warned about once and left out of the result, so the
|
|
@@ -1635,9 +1975,139 @@ async function applyPlan({ vfs, dir, root, state, plan, pass, opts, sess, signal
|
|
|
1635
1975
|
const listing = new Map();
|
|
1636
1976
|
const localKeys = new Set(Object.keys(pass.local));
|
|
1637
1977
|
const decline = (action, path, reason) => warnOnce(sess, `declined:${action}:${path}`, `skipping ${path} — ${reason}`);
|
|
1978
|
+
const failed = (action, path, err) => {
|
|
1979
|
+
const message = err?.message ?? String(err);
|
|
1980
|
+
warnOnce(sess, `error:${path}:${message}`, `${path}: ${message}`);
|
|
1981
|
+
if (action === "pull" && isIgnoreFile(path.slice(1))) pass.notLanded?.set(path, message);
|
|
1982
|
+
};
|
|
1638
1983
|
let short = 0; // pulls held for want of disk space
|
|
1639
1984
|
let over = 0; // and past the pass's budget
|
|
1640
1985
|
|
|
1986
|
+
// ---- pulls: admitted and fetched ahead, landed in order ----
|
|
1987
|
+
//
|
|
1988
|
+
// Held pulls change nothing, so the next pass plans them again. Each limit
|
|
1989
|
+
// is checked twice: on the size the scan read, so a pull that cannot fit is
|
|
1990
|
+
// not fetched at all, and then on the bytes actually fetched — the file may
|
|
1991
|
+
// have changed since, and they are what land on this disk.
|
|
1992
|
+
//
|
|
1993
|
+
// A pull is RESERVED against both limits when it is admitted, before it is
|
|
1994
|
+
// fetched: `inFlight` is what has been admitted and not yet landed. Without
|
|
1995
|
+
// it, pulls checked together, before any had landed, would each see the
|
|
1996
|
+
// same free space and the same budget, and together overshoot both.
|
|
1997
|
+
//
|
|
1998
|
+
// Ignore files are held to the same limits. They were exempt as "small",
|
|
1999
|
+
// and a 256 KB file of comments, copied to thousands of directories, put
|
|
2000
|
+
// 7 GiB on a disk already under its reserve in one pass. One held here
|
|
2001
|
+
// holds this pass's uploads instead (see `reconcile`).
|
|
2002
|
+
const limit = sess.pullBudget;
|
|
2003
|
+
const inFlight = { files: 0, bytes: 0 };
|
|
2004
|
+
const window = []; // pulls being fetched, in plan order
|
|
2005
|
+
const fetching = new Map(); // content key → its fetch, so two entries sharing it fetch it once
|
|
2006
|
+
/**
|
|
2007
|
+
* Why a pull of `size` more bytes must wait, or null. `admitted`: the pull
|
|
2008
|
+
* was let in already, so a budget that has filled behind it — with pulls
|
|
2009
|
+
* admitted after it, which it lands ahead of — does not stop it; only its
|
|
2010
|
+
* own bytes, with what is still in flight, going over does.
|
|
2011
|
+
*/
|
|
2012
|
+
const holdFor = async (size, admitted = false) => {
|
|
2013
|
+
const files = budget.files + inFlight.files;
|
|
2014
|
+
if ((budget.full && !admitted) || (files && (files + 1 > limit.files || budget.bytes + inFlight.bytes + size > limit.bytes))) {
|
|
2015
|
+
budget.full = true;
|
|
2016
|
+
over++;
|
|
2017
|
+
return "this pass has pulled all it may; the next pass carries on";
|
|
2018
|
+
}
|
|
2019
|
+
if (await roomFor(dir, inFlight.bytes + size, sess)) return null;
|
|
2020
|
+
short++;
|
|
2021
|
+
return "the disk would be left with less than its reserve free";
|
|
2022
|
+
};
|
|
2023
|
+
/** The bytes of `remoteAbs`. Content the scan read — or an earlier pull, of an entry sharing it — is not fetched again. */
|
|
2024
|
+
const fetchBytes = async (remoteAbs) => {
|
|
2025
|
+
const content = await remoteContent(vfs, remoteAbs);
|
|
2026
|
+
const kept = pass.blobs?.get(content.key);
|
|
2027
|
+
if (kept) return kept;
|
|
2028
|
+
let read = content.key ? fetching.get(content.key) : undefined;
|
|
2029
|
+
if (!read) {
|
|
2030
|
+
read = content.read();
|
|
2031
|
+
if (content.key) {
|
|
2032
|
+
fetching.set(content.key, read);
|
|
2033
|
+
read.then(() => fetching.delete(content.key), () => fetching.delete(content.key));
|
|
2034
|
+
}
|
|
2035
|
+
}
|
|
2036
|
+
const bytes = await read;
|
|
2037
|
+
pass.blobs?.put(content.key, bytes);
|
|
2038
|
+
return bytes;
|
|
2039
|
+
};
|
|
2040
|
+
/** Admit a pull and start fetching it; null when it is held or declined here. */
|
|
2041
|
+
const startPull = async (path, remoteAbs) => {
|
|
2042
|
+
const claim = pass.sizes?.[path] ?? 0;
|
|
2043
|
+
const hold = await holdFor(claim);
|
|
2044
|
+
if (hold) {
|
|
2045
|
+
pass.notLanded?.set(path, hold);
|
|
2046
|
+
return null;
|
|
2047
|
+
}
|
|
2048
|
+
// A target this disk refuses is not fetched at all. Checked again when
|
|
2049
|
+
// it lands: the pulls landing ahead of it may have changed the answer.
|
|
2050
|
+
const target = await writeTarget(dir, path, { localKeys, listing });
|
|
2051
|
+
if (!target.ok) {
|
|
2052
|
+
decline("pull", path, target.reason);
|
|
2053
|
+
pass.notLanded?.set(path, target.reason);
|
|
2054
|
+
return null;
|
|
2055
|
+
}
|
|
2056
|
+
inFlight.files++;
|
|
2057
|
+
inFlight.bytes += claim;
|
|
2058
|
+
const bytes = fetchBytes(remoteAbs);
|
|
2059
|
+
bytes.catch(() => {}); // its failure is reported when it lands
|
|
2060
|
+
return { path, remoteAbs, claim, bytes };
|
|
2061
|
+
};
|
|
2062
|
+
/** Land a fetched pull: the same checks a sequential pass made, then the write. */
|
|
2063
|
+
const land = async ({ path, remoteAbs, claim, bytes: fetched }) => {
|
|
2064
|
+
inFlight.files--;
|
|
2065
|
+
inFlight.bytes -= claim;
|
|
2066
|
+
const bytes = await fetched;
|
|
2067
|
+
if (bytes.length > MAX_BYTES) {
|
|
2068
|
+
tooBig(sess, path, bytes.length);
|
|
2069
|
+
pass.notLanded?.set(path, "it is over the mount's size limit");
|
|
2070
|
+
return;
|
|
2071
|
+
}
|
|
2072
|
+
const hold = await holdFor(bytes.length, true);
|
|
2073
|
+
if (hold) {
|
|
2074
|
+
pass.notLanded?.set(path, hold);
|
|
2075
|
+
return;
|
|
2076
|
+
}
|
|
2077
|
+
// An ignore file that cannot be read as rules is not pulled: landed, it
|
|
2078
|
+
// would stop every pass (see `see`), and uploads wait for it instead
|
|
2079
|
+
// (see `reconcile`).
|
|
2080
|
+
const rules = isIgnoreFile(path.slice(1));
|
|
2081
|
+
const problem = rules && ignoreFileProblem(bytes);
|
|
2082
|
+
if (problem) {
|
|
2083
|
+
decline("pull", path, `it cannot be read as ignore rules: ${problem}. If it is not yours, remove it: muhkoo vfs rm ${remoteAbs}`);
|
|
2084
|
+
pass.notLanded?.set(path, `it cannot be read as ignore rules: ${problem}`);
|
|
2085
|
+
return;
|
|
2086
|
+
}
|
|
2087
|
+
const target = await writeTarget(dir, path, { localKeys, listing });
|
|
2088
|
+
if (!target.ok) {
|
|
2089
|
+
decline("pull", path, target.reason);
|
|
2090
|
+
pass.notLanded?.set(path, target.reason);
|
|
2091
|
+
return;
|
|
2092
|
+
}
|
|
2093
|
+
await writeAt(target, bytes, listing);
|
|
2094
|
+
next[path] = sha(bytes);
|
|
2095
|
+
if (rules) pass.landed?.add(path);
|
|
2096
|
+
budget.files++;
|
|
2097
|
+
budget.bytes += bytes.length;
|
|
2098
|
+
info(`${SIGN.pull} ${path}`);
|
|
2099
|
+
};
|
|
2100
|
+
const landOldest = async () => {
|
|
2101
|
+
const pull = window.shift();
|
|
2102
|
+
try {
|
|
2103
|
+
await land(pull);
|
|
2104
|
+
} catch (err) {
|
|
2105
|
+
failed("pull", pull.path, err);
|
|
2106
|
+
}
|
|
2107
|
+
};
|
|
2108
|
+
/** Room for another pull in the window? Bounded in files, and — by what the records claim — in bytes held. */
|
|
2109
|
+
const windowFull = () => window.length >= PULL_CONCURRENCY || (window.length > 0 && inFlight.bytes >= PULL_WINDOW_BYTES);
|
|
2110
|
+
|
|
1641
2111
|
for (const { path, action } of plan) {
|
|
1642
2112
|
if (signal?.aborted) break;
|
|
1643
2113
|
const localAbs = localPathFor(dir, path);
|
|
@@ -1649,6 +2119,21 @@ async function applyPlan({ vfs, dir, root, state, plan, pass, opts, sess, signal
|
|
|
1649
2119
|
continue;
|
|
1650
2120
|
}
|
|
1651
2121
|
const remoteAbs = root === "/" ? path : root + path;
|
|
2122
|
+
if (action === "pull") {
|
|
2123
|
+
while (windowFull() && !signal?.aborted) await landOldest();
|
|
2124
|
+
if (signal?.aborted) break;
|
|
2125
|
+
try {
|
|
2126
|
+
const pull = await startPull(path, remoteAbs);
|
|
2127
|
+
if (pull) window.push(pull);
|
|
2128
|
+
} catch (err) {
|
|
2129
|
+
failed(action, path, err);
|
|
2130
|
+
}
|
|
2131
|
+
continue;
|
|
2132
|
+
}
|
|
2133
|
+
// Anything else waits for the pulls before it, so steps still take
|
|
2134
|
+
// effect in plan order.
|
|
2135
|
+
while (window.length && !signal?.aborted) await landOldest();
|
|
2136
|
+
if (signal?.aborted) break;
|
|
1652
2137
|
try {
|
|
1653
2138
|
switch (action) {
|
|
1654
2139
|
case "push": {
|
|
@@ -1658,77 +2143,6 @@ async function applyPlan({ vfs, dir, root, state, plan, pass, opts, sess, signal
|
|
|
1658
2143
|
info(`${SIGN.push} ${path}`);
|
|
1659
2144
|
break;
|
|
1660
2145
|
}
|
|
1661
|
-
case "pull": {
|
|
1662
|
-
// Held pulls change nothing, so the next pass plans them
|
|
1663
|
-
// again. Each limit is checked twice: on the size the scan
|
|
1664
|
-
// read, so a pull that cannot fit is not fetched at all, and
|
|
1665
|
-
// then on the bytes actually fetched — the file may have
|
|
1666
|
-
// changed since, and they are what land on this disk.
|
|
1667
|
-
//
|
|
1668
|
-
// Ignore files are held to the same limits. They were
|
|
1669
|
-
// exempt as "small", and a 256 KB file of comments, copied
|
|
1670
|
-
// to thousands of directories, put 7 GiB on a disk already
|
|
1671
|
-
// under its reserve in one pass. One held here holds this
|
|
1672
|
-
// pass's uploads instead (see `reconcile`).
|
|
1673
|
-
const limit = sess.pullBudget;
|
|
1674
|
-
/** Why a pull of `size` bytes must wait, or null. */
|
|
1675
|
-
const holdFor = async (size) => {
|
|
1676
|
-
if (budget.full || (budget.files && (budget.files + 1 > limit.files || budget.bytes + size > limit.bytes))) {
|
|
1677
|
-
budget.full = true;
|
|
1678
|
-
over++;
|
|
1679
|
-
return "this pass has pulled all it may; the next pass carries on";
|
|
1680
|
-
}
|
|
1681
|
-
if (await roomFor(dir, size, sess)) return null;
|
|
1682
|
-
short++;
|
|
1683
|
-
return "the disk would be left with less than its reserve free";
|
|
1684
|
-
};
|
|
1685
|
-
let hold = await holdFor(pass.sizes?.[path] ?? 0);
|
|
1686
|
-
if (hold) {
|
|
1687
|
-
pass.notLanded?.set(path, hold);
|
|
1688
|
-
break;
|
|
1689
|
-
}
|
|
1690
|
-
const target = await writeTarget(dir, path, { localKeys, listing });
|
|
1691
|
-
if (!target.ok) {
|
|
1692
|
-
decline(action, path, target.reason);
|
|
1693
|
-
pass.notLanded?.set(path, target.reason);
|
|
1694
|
-
break;
|
|
1695
|
-
}
|
|
1696
|
-
// Content the scan read — or an earlier pull, of an entry
|
|
1697
|
-
// sharing it — is not fetched again.
|
|
1698
|
-
const content = await remoteContent(vfs, remoteAbs);
|
|
1699
|
-
let bytes = pass.blobs?.get(content.key);
|
|
1700
|
-
if (!bytes) {
|
|
1701
|
-
bytes = await content.read();
|
|
1702
|
-
pass.blobs?.put(content.key, bytes);
|
|
1703
|
-
}
|
|
1704
|
-
if (bytes.length > MAX_BYTES) {
|
|
1705
|
-
tooBig(sess, path, bytes.length);
|
|
1706
|
-
pass.notLanded?.set(path, "it is over the mount's size limit");
|
|
1707
|
-
break;
|
|
1708
|
-
}
|
|
1709
|
-
hold = await holdFor(bytes.length);
|
|
1710
|
-
if (hold) {
|
|
1711
|
-
pass.notLanded?.set(path, hold);
|
|
1712
|
-
break;
|
|
1713
|
-
}
|
|
1714
|
-
// An ignore file that cannot be read as rules is not
|
|
1715
|
-
// pulled: landed, it would stop every pass (see `see`), and
|
|
1716
|
-
// uploads wait for it instead (see `reconcile`).
|
|
1717
|
-
const rules = isIgnoreFile(path.slice(1));
|
|
1718
|
-
const problem = rules && ignoreFileProblem(bytes);
|
|
1719
|
-
if (problem) {
|
|
1720
|
-
decline(action, path, `it cannot be read as ignore rules: ${problem}. If it is not yours, remove it: muhkoo vfs rm ${remoteAbs}`);
|
|
1721
|
-
pass.notLanded?.set(path, `it cannot be read as ignore rules: ${problem}`);
|
|
1722
|
-
break;
|
|
1723
|
-
}
|
|
1724
|
-
await writeAt(target, bytes, listing);
|
|
1725
|
-
next[path] = sha(bytes);
|
|
1726
|
-
if (rules) pass.landed?.add(path);
|
|
1727
|
-
budget.files++;
|
|
1728
|
-
budget.bytes += bytes.length;
|
|
1729
|
-
info(`${SIGN.pull} ${path}`);
|
|
1730
|
-
break;
|
|
1731
|
-
}
|
|
1732
2146
|
case "delete-remote":
|
|
1733
2147
|
// Deletes do NOT propagate unless asked for.
|
|
1734
2148
|
//
|
|
@@ -1795,11 +2209,17 @@ async function applyPlan({ vfs, dir, root, state, plan, pass, opts, sess, signal
|
|
|
1795
2209
|
}
|
|
1796
2210
|
}
|
|
1797
2211
|
} catch (err) {
|
|
1798
|
-
|
|
1799
|
-
warnOnce(sess, `error:${path}:${message}`, `${path}: ${message}`);
|
|
1800
|
-
if (action === "pull" && isIgnoreFile(path.slice(1))) pass.notLanded?.set(path, message);
|
|
2212
|
+
failed(action, path, err);
|
|
1801
2213
|
}
|
|
1802
2214
|
}
|
|
2215
|
+
// Land what is still fetching — or, stopped, drop it: a fetch is not a
|
|
2216
|
+
// step, and nothing is recorded for a pull that did not land. Its fetch is
|
|
2217
|
+
// still waited for, so none runs on after the pass has returned.
|
|
2218
|
+
while (window.length && !signal?.aborted) await landOldest();
|
|
2219
|
+
if (window.length) {
|
|
2220
|
+
await Promise.allSettled(window.map((p) => p.bytes));
|
|
2221
|
+
window.length = 0;
|
|
2222
|
+
}
|
|
1803
2223
|
if (budget.full) {
|
|
1804
2224
|
sess.more = true;
|
|
1805
2225
|
warnOnce(
|
|
@@ -1914,6 +2334,24 @@ function unreadRules(pass, state) {
|
|
|
1914
2334
|
export async function reconcile({ vfs, dir, root, state, opts = {}, session: s, signal }) {
|
|
1915
2335
|
const sess = session(s);
|
|
1916
2336
|
if (signal?.aborted) return state;
|
|
2337
|
+
// A restart begins with what earlier runs hashed (see `loadHashCache`).
|
|
2338
|
+
if (!sess.hashCacheLoaded) {
|
|
2339
|
+
sess.hashCacheLoaded = true;
|
|
2340
|
+
if (sess.hashes.size === 0) sess.hashes = await loadHashCache(dir);
|
|
2341
|
+
sess.hashCacheSaved = new Map(sess.hashes);
|
|
2342
|
+
}
|
|
2343
|
+
try {
|
|
2344
|
+
return await reconcilePass({ vfs, dir, root, state, opts, sess, signal });
|
|
2345
|
+
} finally {
|
|
2346
|
+
// Written only when it changed: most passes hash nothing new.
|
|
2347
|
+
if (!sameHashes(sess.hashes, sess.hashCacheSaved)) {
|
|
2348
|
+
await saveHashCache(dir, sess.hashes).catch((err) => warn(`could not save the content hash cache: ${err?.message ?? err}`));
|
|
2349
|
+
sess.hashCacheSaved = new Map(sess.hashes);
|
|
2350
|
+
}
|
|
2351
|
+
}
|
|
2352
|
+
}
|
|
2353
|
+
|
|
2354
|
+
async function reconcilePass({ vfs, dir, root, state, opts, sess, signal }) {
|
|
1917
2355
|
const budget = { bytes: 0, files: 0 }; // what the whole pass may pull, both rounds
|
|
1918
2356
|
const read = { bytes: 0, files: 0 }; // and read to compare
|
|
1919
2357
|
let pass = await preparePass({ vfs, dir, root, state, opts, sess, signal, read });
|
|
@@ -1958,7 +2396,10 @@ export async function reconcile({ vfs, dir, root, state, opts = {}, session: s,
|
|
|
1958
2396
|
);
|
|
1959
2397
|
}
|
|
1960
2398
|
const agreed = withAgreed(state, pass);
|
|
1961
|
-
|
|
2399
|
+
let next = plan.length ? await applyPlan({ vfs, dir, root, state: agreed, plan, pass, opts, sess, signal, budget }) : agreed;
|
|
2400
|
+
// While uploads wait on ignore rules, so do the folders going up.
|
|
2401
|
+
const dirPlan = waiting.size ? pass.dirPlan.filter(({ action }) => !action.endsWith("-remote")) : pass.dirPlan;
|
|
2402
|
+
if (dirPlan.length && !signal?.aborted) next = await applyDirPlan({ vfs, dir, root, state: next, plan: dirPlan, opts, sess, signal });
|
|
1962
2403
|
await persist(dir, root, state, next);
|
|
1963
2404
|
const short = pass.outstanding?.space ?? 0;
|
|
1964
2405
|
if (short) {
|
package/src/lib/syncPlan.js
CHANGED
|
@@ -67,6 +67,64 @@ export function planSync({ last, local, remote, unknown = [] }) {
|
|
|
67
67
|
return plan;
|
|
68
68
|
}
|
|
69
69
|
|
|
70
|
+
/** What a folder is in the state file, in place of a content hash. */
|
|
71
|
+
export const DIR = "dir";
|
|
72
|
+
|
|
73
|
+
/** A folder's key in the state file: its path with a trailing slash, so it can never be a file's. */
|
|
74
|
+
export const dirKey = (path) => `${path}/`;
|
|
75
|
+
|
|
76
|
+
/** The state without its folders: what the file planner, and its breaker, count. */
|
|
77
|
+
export function filesOf(state) {
|
|
78
|
+
const out = {};
|
|
79
|
+
for (const [key, hash] of Object.entries(state)) if (!key.endsWith("/")) out[key] = hash;
|
|
80
|
+
return out;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/**
|
|
84
|
+
* Plan the folders, by the same three-way rule as the files ({@link planPath}),
|
|
85
|
+
* with {@link DIR} standing in for a hash. `local` and `remote` are the folder
|
|
86
|
+
* paths each side holds (`remote` null when the folders could not be listed:
|
|
87
|
+
* then nothing is planned, since a folder not listed is not a folder gone).
|
|
88
|
+
*
|
|
89
|
+
* Returns `{ path, action }` with action one of: "record" (on both sides, not
|
|
90
|
+
* yet in the state), "mkdir-local", "mkdir-remote", "rmdir-local",
|
|
91
|
+
* "rmdir-remote", "forget". Paths in `unknown` are left alone, and so, when
|
|
92
|
+
* `fold` is given (a volume that folds names), is every folder whose folded
|
|
93
|
+
* name more than one spelling shares: which of them this disk holds cannot be
|
|
94
|
+
* told apart, and reading one as missing would plan its removal.
|
|
95
|
+
*/
|
|
96
|
+
export function planDirs({ last, local, remote, unknown = [], fold = null }) {
|
|
97
|
+
if (!remote) return [];
|
|
98
|
+
const isUnknown = coveredBy(unknown);
|
|
99
|
+
const before = new Set(Object.keys(last).filter((k) => k.endsWith("/")).map((k) => k.slice(0, -1)));
|
|
100
|
+
const here = new Set(local);
|
|
101
|
+
const there = new Set(remote);
|
|
102
|
+
const paths = new Set([...before, ...here, ...there]);
|
|
103
|
+
const clash = new Set();
|
|
104
|
+
if (fold) {
|
|
105
|
+
const spellings = new Map();
|
|
106
|
+
for (const path of paths) {
|
|
107
|
+
const folded = fold(path);
|
|
108
|
+
if (!spellings.has(folded)) spellings.set(folded, []);
|
|
109
|
+
spellings.get(folded).push(path);
|
|
110
|
+
}
|
|
111
|
+
for (const group of spellings.values()) if (group.length > 1) for (const path of group) clash.add(path);
|
|
112
|
+
}
|
|
113
|
+
const ACTIONS = { push: "mkdir-remote", pull: "mkdir-local", "delete-remote": "rmdir-remote", "delete-local": "rmdir-local", forget: "forget" };
|
|
114
|
+
const plan = [];
|
|
115
|
+
for (const path of [...paths].sort()) {
|
|
116
|
+
if (isUnknown(path) || clash.has(path)) continue;
|
|
117
|
+
const was = before.has(path) ? DIR : undefined;
|
|
118
|
+
const action = planPath({ last: was, local: here.has(path) ? DIR : undefined, remote: there.has(path) ? DIR : undefined });
|
|
119
|
+
if (action === "none") {
|
|
120
|
+
if (!was) plan.push({ path, action: "record" });
|
|
121
|
+
} else if (ACTIONS[action]) {
|
|
122
|
+
plan.push({ path, action: ACTIONS[action] });
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
return plan;
|
|
126
|
+
}
|
|
127
|
+
|
|
70
128
|
/**
|
|
71
129
|
* `(path) => boolean`: is `path`, or a directory above it, in `prefixes`?
|
|
72
130
|
*
|