@muhkoo/cli 0.15.0-alpha.5 → 0.15.0-alpha.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +2 -2
- package/src/lib/mount.js +318 -88
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@muhkoo/cli",
|
|
3
|
-
"version": "0.15.0-alpha.
|
|
3
|
+
"version": "0.15.0-alpha.6",
|
|
4
4
|
"description": "Command-line interface for the Muhkoo platform — provision backends, deploy apps, and manage agents, functions, databases, and domains.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|
|
@@ -40,7 +40,7 @@
|
|
|
40
40
|
"release": "npm publish --tag alpha && npm dist-tag add @muhkoo/cli@$npm_package_version latest"
|
|
41
41
|
},
|
|
42
42
|
"dependencies": {
|
|
43
|
-
"@muhkoo/connect": "0.15.0-alpha.
|
|
43
|
+
"@muhkoo/connect": "0.15.0-alpha.6",
|
|
44
44
|
"snarkjs": "^0.7.0"
|
|
45
45
|
},
|
|
46
46
|
"packageManager": "yarn@1.22.22+sha512.a6b2f7906b721bba3d67d4aff083df04dad64c399707841b7acf00f6b133b7ac24255f2652fa22ae3534329dc6180534e98d17432037ff6fd140556e2bb3137e",
|
package/src/lib/mount.js
CHANGED
|
@@ -77,12 +77,40 @@ const KEEP_HIDDEN = new Set([".vcsignore", ".gitignore"]);
|
|
|
77
77
|
const MAX_BYTES = 100 * 1024 * 1024;
|
|
78
78
|
|
|
79
79
|
/**
|
|
80
|
-
* How many files
|
|
80
|
+
* How many files the scan reads at once.
|
|
81
81
|
*
|
|
82
82
|
* Enough to hide per-request latency; low enough not to open a connection per
|
|
83
|
-
* file on a large project.
|
|
83
|
+
* file on a large project. A read costs 0.3–0.5 s against production, nearly
|
|
84
|
+
* all of it fixed, so throughput comes from overlapping them: 96 small files
|
|
85
|
+
* took 6.35 s eight at a time, 4.50 s at sixteen, and 4.12 s at thirty-two.
|
|
84
86
|
*/
|
|
85
|
-
const
|
|
87
|
+
export const SCAN_CONCURRENCY = 16;
|
|
88
|
+
|
|
89
|
+
/**
|
|
90
|
+
* And the most the files being read may claim between them — by their
|
|
91
|
+
* records' sizes, and their manifests' — before another read waits for one
|
|
92
|
+
* to finish. So raising the count does not raise the memory a run of large
|
|
93
|
+
* files holds at once. A file that claims more than this on its own is read
|
|
94
|
+
* alone: nothing else starts until it is done.
|
|
95
|
+
*/
|
|
96
|
+
export const SCAN_WINDOW_BYTES = 64 * 1024 * 1024;
|
|
97
|
+
|
|
98
|
+
/**
|
|
99
|
+
* How many pulls to fetch at once while the oldest lands (see `applyPlan`).
|
|
100
|
+
*
|
|
101
|
+
* A pull the scan did not keep the bytes of — past BLOB_CACHE_BYTES, or a
|
|
102
|
+
* follow-up pass — is a round trip of its own, and one at a time they cost
|
|
103
|
+
* the SUM of the latencies. Only the fetches overlap; every pull still lands
|
|
104
|
+
* one at a time, in plan order.
|
|
105
|
+
*/
|
|
106
|
+
const PULL_CONCURRENCY = 8;
|
|
107
|
+
|
|
108
|
+
/**
|
|
109
|
+
* And the most the records of the pulls being fetched may claim between
|
|
110
|
+
* them before the window waits for one to land — so a run of large files is
|
|
111
|
+
* not held in memory eight at a time. One is always let through.
|
|
112
|
+
*/
|
|
113
|
+
const PULL_WINDOW_BYTES = 64 * 1024 * 1024;
|
|
86
114
|
|
|
87
115
|
/**
|
|
88
116
|
* The most one pass pulls: bytes and files. Past it, the rest wait for a
|
|
@@ -1110,12 +1138,18 @@ export async function scanRemote(vfs, root, includeHidden = false, ignore = () =
|
|
|
1110
1138
|
const readPace = pacer(); // reads from a cache resolve at once, and never yield on their own
|
|
1111
1139
|
const seen = new Map(); // content key → { hash, size }, for the next pass
|
|
1112
1140
|
const reading = new Map(); // content key → the read under way
|
|
1141
|
+
const oversize = new Map(); // content key → its length, for a read that came back over MAX_BYTES
|
|
1113
1142
|
const blobs = ctx.blobs ?? blobCache(BLOB_CACHE_BYTES);
|
|
1114
1143
|
const budget = sess.readBudget;
|
|
1115
1144
|
const spent = ctx.spent ?? { bytes: 0, files: 0 };
|
|
1116
1145
|
const deferred = new Set();
|
|
1146
|
+
// Bytes in flight are bounded as well as reads (SCAN_WINDOW_BYTES), by the
|
|
1147
|
+
// same claim the budget is charged. A read waiting for room is not
|
|
1148
|
+
// started once the scan is stopped.
|
|
1149
|
+
const window = byteWindow(SCAN_WINDOW_BYTES);
|
|
1150
|
+
const STOPPED = Symbol("stopped");
|
|
1117
1151
|
await Promise.all(
|
|
1118
|
-
Array.from({ length: Math.min(
|
|
1152
|
+
Array.from({ length: Math.min(SCAN_CONCURRENCY, queue.length) }, async () => {
|
|
1119
1153
|
for (let item = queue.shift(); item !== undefined && !ctx.signal?.aborted; item = queue.shift()) {
|
|
1120
1154
|
const breather = readPace();
|
|
1121
1155
|
if (breather) await breather;
|
|
@@ -1140,6 +1174,12 @@ export async function scanRemote(vfs, root, includeHidden = false, ignore = () =
|
|
|
1140
1174
|
sizes[key] = known.size;
|
|
1141
1175
|
continue;
|
|
1142
1176
|
}
|
|
1177
|
+
const over = content.key && oversize.get(content.key);
|
|
1178
|
+
if (over) {
|
|
1179
|
+
tooBig(sess, key, over);
|
|
1180
|
+
unknown.add(key);
|
|
1181
|
+
continue;
|
|
1182
|
+
}
|
|
1143
1183
|
let read = content.key && reading.get(content.key);
|
|
1144
1184
|
if (!read) {
|
|
1145
1185
|
// Ignore files included: a 256 KB file of comments is
|
|
@@ -1154,16 +1194,37 @@ export async function scanRemote(vfs, root, includeHidden = false, ignore = () =
|
|
|
1154
1194
|
spent.files++;
|
|
1155
1195
|
spent.bytes += claim;
|
|
1156
1196
|
}
|
|
1157
|
-
read =
|
|
1158
|
-
|
|
1159
|
-
|
|
1160
|
-
|
|
1197
|
+
read = (async () => {
|
|
1198
|
+
const admitted = await window.acquire(claim, ctx.signal);
|
|
1199
|
+
if (admitted && ctx.signal?.aborted) window.release(claim); // let in, but stopped since
|
|
1200
|
+
if (!admitted || ctx.signal?.aborted) {
|
|
1201
|
+
if (budget) { // never read: give back what it was charged
|
|
1202
|
+
spent.files--;
|
|
1203
|
+
spent.bytes -= claim;
|
|
1204
|
+
}
|
|
1205
|
+
throw STOPPED;
|
|
1206
|
+
}
|
|
1207
|
+
try {
|
|
1208
|
+
const bytes = await content.read();
|
|
1209
|
+
spent.bytes += bytes.length - claim;
|
|
1210
|
+
return bytes;
|
|
1211
|
+
} finally {
|
|
1212
|
+
window.release(claim);
|
|
1213
|
+
}
|
|
1214
|
+
})();
|
|
1161
1215
|
if (content.key) reading.set(content.key, read);
|
|
1162
1216
|
}
|
|
1163
1217
|
const bytes = await read;
|
|
1164
1218
|
if (bytes.length > MAX_BYTES) {
|
|
1165
1219
|
tooBig(sess, key, bytes.length);
|
|
1166
1220
|
unknown.add(key);
|
|
1221
|
+
if (content.key) {
|
|
1222
|
+
// Only its length is kept: entries sharing this
|
|
1223
|
+
// content get the same answer without the read
|
|
1224
|
+
// (up to 100 MB) staying referenced.
|
|
1225
|
+
oversize.set(content.key, bytes.length);
|
|
1226
|
+
reading.delete(content.key);
|
|
1227
|
+
}
|
|
1167
1228
|
continue;
|
|
1168
1229
|
}
|
|
1169
1230
|
const hash = sha(bytes);
|
|
@@ -1172,8 +1233,18 @@ export async function scanRemote(vfs, root, includeHidden = false, ignore = () =
|
|
|
1172
1233
|
if (content.key) {
|
|
1173
1234
|
seen.set(content.key, { hash, size: bytes.length });
|
|
1174
1235
|
blobs.put(content.key, bytes);
|
|
1236
|
+
// `seen` answers for this content from here on. Left
|
|
1237
|
+
// in `reading`, the settled read held every byte the
|
|
1238
|
+
// pass read, past the blob cache's limit and the window.
|
|
1239
|
+
reading.delete(content.key);
|
|
1175
1240
|
}
|
|
1176
1241
|
} catch (err) {
|
|
1242
|
+
// Stopped before its read began: not read, and nothing to
|
|
1243
|
+
// warn of. The caller does not plan from a stopped scan.
|
|
1244
|
+
if (err === STOPPED) {
|
|
1245
|
+
unknown.add(key);
|
|
1246
|
+
continue;
|
|
1247
|
+
}
|
|
1177
1248
|
// One unreadable entry — a lost shard, a planted manifest, a
|
|
1178
1249
|
// dropped request — must not stop the whole sync, and must
|
|
1179
1250
|
// not read as a deletion either.
|
|
@@ -1213,10 +1284,11 @@ export async function scanRemote(vfs, root, includeHidden = false, ignore = () =
|
|
|
1213
1284
|
*/
|
|
1214
1285
|
async function remoteContent(vfs, abs) {
|
|
1215
1286
|
if (typeof vfs.statManifest !== "function") {
|
|
1216
|
-
const
|
|
1287
|
+
const size = recordSize((await vfs.stat(abs)).size);
|
|
1217
1288
|
return { size, key: null, bound: null, read: () => vfs.readFile(abs) };
|
|
1218
1289
|
}
|
|
1219
|
-
const { manifest, size } = await vfs.statManifest(abs);
|
|
1290
|
+
const { manifest, size: claimed } = await vfs.statManifest(abs);
|
|
1291
|
+
const size = recordSize(claimed);
|
|
1220
1292
|
const exact = typeof vfs.readByManifest === "function";
|
|
1221
1293
|
return {
|
|
1222
1294
|
size,
|
|
@@ -1226,6 +1298,15 @@ async function remoteContent(vfs, abs) {
|
|
|
1226
1298
|
};
|
|
1227
1299
|
}
|
|
1228
1300
|
|
|
1301
|
+
/**
|
|
1302
|
+
* A record's size, as a number the budgets can add. The record is written by
|
|
1303
|
+
* the remote party, so the size may be absent, a string, negative or NaN; any
|
|
1304
|
+
* of those counts as 0 (the bytes that come back are what is held to the limit).
|
|
1305
|
+
*/
|
|
1306
|
+
function recordSize(size) {
|
|
1307
|
+
return Number.isSafeInteger(size) && size >= 0 ? size : 0;
|
|
1308
|
+
}
|
|
1309
|
+
|
|
1229
1310
|
/**
|
|
1230
1311
|
* What a manifest lets a read bring back (see `remoteContent`), or null when
|
|
1231
1312
|
* it is not in a shape this knows. Each chunk decrypts to its ciphertext less
|
|
@@ -1246,6 +1327,63 @@ function manifestBound(manifest) {
|
|
|
1246
1327
|
return { plain, fetched };
|
|
1247
1328
|
}
|
|
1248
1329
|
|
|
1330
|
+
/**
|
|
1331
|
+
* A first-come, first-served allowance of `limit` bytes for reads in flight.
|
|
1332
|
+
*
|
|
1333
|
+
* `acquire(n, signal)` resolves true once `n` more bytes fit — or at once
|
|
1334
|
+
* when nothing is held, so one claim larger than the whole window is still
|
|
1335
|
+
* let through, alone — and false, without taking anything, when `signal`
|
|
1336
|
+
* has fired. Waiters are served in order, so a large claim is not starved by
|
|
1337
|
+
* small ones arriving behind it. `release(n)` gives back what an acquire took.
|
|
1338
|
+
*/
|
|
1339
|
+
function byteWindow(limit) {
|
|
1340
|
+
let held = 0;
|
|
1341
|
+
// A claim that is not a finite, non-negative number is 0: NaN in `held`
|
|
1342
|
+
// would make nothing fit again, and the scan would never finish.
|
|
1343
|
+
const sane = (n) => (Number.isFinite(n) && n > 0 ? n : 0);
|
|
1344
|
+
const waiting = [];
|
|
1345
|
+
const fits = (n) => held === 0 || held + n <= limit;
|
|
1346
|
+
const wake = () => {
|
|
1347
|
+
while (waiting.length && fits(waiting[0].n)) {
|
|
1348
|
+
const w = waiting.shift();
|
|
1349
|
+
held += w.n;
|
|
1350
|
+
w.done(true);
|
|
1351
|
+
}
|
|
1352
|
+
};
|
|
1353
|
+
return {
|
|
1354
|
+
acquire(claim, signal) {
|
|
1355
|
+
const n = sane(claim);
|
|
1356
|
+
if (signal?.aborted) return Promise.resolve(false);
|
|
1357
|
+
if (!waiting.length && fits(n)) {
|
|
1358
|
+
held += n;
|
|
1359
|
+
return Promise.resolve(true);
|
|
1360
|
+
}
|
|
1361
|
+
return new Promise((resolve) => {
|
|
1362
|
+
const onAbort = () => {
|
|
1363
|
+
const i = waiting.indexOf(w);
|
|
1364
|
+
if (i < 0) return;
|
|
1365
|
+
waiting.splice(i, 1);
|
|
1366
|
+
resolve(false);
|
|
1367
|
+
wake(); // the head may have been what held the rest
|
|
1368
|
+
};
|
|
1369
|
+
const w = {
|
|
1370
|
+
n,
|
|
1371
|
+
done: (ok) => {
|
|
1372
|
+
signal?.removeEventListener("abort", onAbort);
|
|
1373
|
+
resolve(ok);
|
|
1374
|
+
},
|
|
1375
|
+
};
|
|
1376
|
+
waiting.push(w);
|
|
1377
|
+
signal?.addEventListener("abort", onAbort, { once: true });
|
|
1378
|
+
});
|
|
1379
|
+
},
|
|
1380
|
+
release(claim) {
|
|
1381
|
+
held -= sane(claim);
|
|
1382
|
+
wake();
|
|
1383
|
+
},
|
|
1384
|
+
};
|
|
1385
|
+
}
|
|
1386
|
+
|
|
1249
1387
|
/**
|
|
1250
1388
|
* The bytes a pass has read, by content key, up to `limit` bytes: the oldest
|
|
1251
1389
|
* go first. What lets a pull of content the scan already read — two hundred
|
|
@@ -1617,10 +1755,22 @@ function breakerMessage({ action, count, tracked, empty }, dir, root) {
|
|
|
1617
1755
|
/**
|
|
1618
1756
|
* Apply a plan, and return the new last-synced view.
|
|
1619
1757
|
*
|
|
1620
|
-
*
|
|
1621
|
-
*
|
|
1622
|
-
*
|
|
1623
|
-
*
|
|
1758
|
+
* Every step that changes something — a write to the filesystem, a delete, a
|
|
1759
|
+
* file landing on this disk — is made one at a time, in plan order: writes
|
|
1760
|
+
* touch one directory record each, and two concurrent writes to the same
|
|
1761
|
+
* record would have one silently lose ([[VfsNamespace]] consistency note).
|
|
1762
|
+
* Pushes, deletes and conflicts are applied exactly as before, in sequence.
|
|
1763
|
+
*
|
|
1764
|
+
* Only the FETCH of a pull overlaps: up to PULL_CONCURRENCY pulls, in plan
|
|
1765
|
+
* order, are fetched while the oldest lands. A pull reads nothing but the
|
|
1766
|
+
* network and the pass's blob cache, so overlapping them changes no state;
|
|
1767
|
+
* landing stays in plan order, on the same checks, so what lands — and which
|
|
1768
|
+
* of two spellings a folding volume refuses — is what a sequential pass would
|
|
1769
|
+
* land. A step that is not a pull first lands every pull before it.
|
|
1770
|
+
*
|
|
1771
|
+
* `signal` stops it between steps — never inside one — and what was done by
|
|
1772
|
+
* then is what it returns. Pulls fetched but not yet landed when it fires are
|
|
1773
|
+
* dropped, unrecorded, and the next pass plans them again.
|
|
1624
1774
|
*
|
|
1625
1775
|
* A step this disk declines (a pull onto a symlink, a remote file where a
|
|
1626
1776
|
* directory stands) is warned about once and left out of the result, so the
|
|
@@ -1635,9 +1785,139 @@ async function applyPlan({ vfs, dir, root, state, plan, pass, opts, sess, signal
|
|
|
1635
1785
|
const listing = new Map();
|
|
1636
1786
|
const localKeys = new Set(Object.keys(pass.local));
|
|
1637
1787
|
const decline = (action, path, reason) => warnOnce(sess, `declined:${action}:${path}`, `skipping ${path} — ${reason}`);
|
|
1788
|
+
const failed = (action, path, err) => {
|
|
1789
|
+
const message = err?.message ?? String(err);
|
|
1790
|
+
warnOnce(sess, `error:${path}:${message}`, `${path}: ${message}`);
|
|
1791
|
+
if (action === "pull" && isIgnoreFile(path.slice(1))) pass.notLanded?.set(path, message);
|
|
1792
|
+
};
|
|
1638
1793
|
let short = 0; // pulls held for want of disk space
|
|
1639
1794
|
let over = 0; // and past the pass's budget
|
|
1640
1795
|
|
|
1796
|
+
// ---- pulls: admitted and fetched ahead, landed in order ----
|
|
1797
|
+
//
|
|
1798
|
+
// Held pulls change nothing, so the next pass plans them again. Each limit
|
|
1799
|
+
// is checked twice: on the size the scan read, so a pull that cannot fit is
|
|
1800
|
+
// not fetched at all, and then on the bytes actually fetched — the file may
|
|
1801
|
+
// have changed since, and they are what land on this disk.
|
|
1802
|
+
//
|
|
1803
|
+
// A pull is RESERVED against both limits when it is admitted, before it is
|
|
1804
|
+
// fetched: `inFlight` is what has been admitted and not yet landed. Without
|
|
1805
|
+
// it, pulls checked together, before any had landed, would each see the
|
|
1806
|
+
// same free space and the same budget, and together overshoot both.
|
|
1807
|
+
//
|
|
1808
|
+
// Ignore files are held to the same limits. They were exempt as "small",
|
|
1809
|
+
// and a 256 KB file of comments, copied to thousands of directories, put
|
|
1810
|
+
// 7 GiB on a disk already under its reserve in one pass. One held here
|
|
1811
|
+
// holds this pass's uploads instead (see `reconcile`).
|
|
1812
|
+
const limit = sess.pullBudget;
|
|
1813
|
+
const inFlight = { files: 0, bytes: 0 };
|
|
1814
|
+
const window = []; // pulls being fetched, in plan order
|
|
1815
|
+
const fetching = new Map(); // content key → its fetch, so two entries sharing it fetch it once
|
|
1816
|
+
/**
|
|
1817
|
+
* Why a pull of `size` more bytes must wait, or null. `admitted`: the pull
|
|
1818
|
+
* was let in already, so a budget that has filled behind it — with pulls
|
|
1819
|
+
* admitted after it, which it lands ahead of — does not stop it; only its
|
|
1820
|
+
* own bytes, with what is still in flight, going over does.
|
|
1821
|
+
*/
|
|
1822
|
+
const holdFor = async (size, admitted = false) => {
|
|
1823
|
+
const files = budget.files + inFlight.files;
|
|
1824
|
+
if ((budget.full && !admitted) || (files && (files + 1 > limit.files || budget.bytes + inFlight.bytes + size > limit.bytes))) {
|
|
1825
|
+
budget.full = true;
|
|
1826
|
+
over++;
|
|
1827
|
+
return "this pass has pulled all it may; the next pass carries on";
|
|
1828
|
+
}
|
|
1829
|
+
if (await roomFor(dir, inFlight.bytes + size, sess)) return null;
|
|
1830
|
+
short++;
|
|
1831
|
+
return "the disk would be left with less than its reserve free";
|
|
1832
|
+
};
|
|
1833
|
+
/** The bytes of `remoteAbs`. Content the scan read — or an earlier pull, of an entry sharing it — is not fetched again. */
|
|
1834
|
+
const fetchBytes = async (remoteAbs) => {
|
|
1835
|
+
const content = await remoteContent(vfs, remoteAbs);
|
|
1836
|
+
const kept = pass.blobs?.get(content.key);
|
|
1837
|
+
if (kept) return kept;
|
|
1838
|
+
let read = content.key ? fetching.get(content.key) : undefined;
|
|
1839
|
+
if (!read) {
|
|
1840
|
+
read = content.read();
|
|
1841
|
+
if (content.key) {
|
|
1842
|
+
fetching.set(content.key, read);
|
|
1843
|
+
read.then(() => fetching.delete(content.key), () => fetching.delete(content.key));
|
|
1844
|
+
}
|
|
1845
|
+
}
|
|
1846
|
+
const bytes = await read;
|
|
1847
|
+
pass.blobs?.put(content.key, bytes);
|
|
1848
|
+
return bytes;
|
|
1849
|
+
};
|
|
1850
|
+
/** Admit a pull and start fetching it; null when it is held or declined here. */
|
|
1851
|
+
const startPull = async (path, remoteAbs) => {
|
|
1852
|
+
const claim = pass.sizes?.[path] ?? 0;
|
|
1853
|
+
const hold = await holdFor(claim);
|
|
1854
|
+
if (hold) {
|
|
1855
|
+
pass.notLanded?.set(path, hold);
|
|
1856
|
+
return null;
|
|
1857
|
+
}
|
|
1858
|
+
// A target this disk refuses is not fetched at all. Checked again when
|
|
1859
|
+
// it lands: the pulls landing ahead of it may have changed the answer.
|
|
1860
|
+
const target = await writeTarget(dir, path, { localKeys, listing });
|
|
1861
|
+
if (!target.ok) {
|
|
1862
|
+
decline("pull", path, target.reason);
|
|
1863
|
+
pass.notLanded?.set(path, target.reason);
|
|
1864
|
+
return null;
|
|
1865
|
+
}
|
|
1866
|
+
inFlight.files++;
|
|
1867
|
+
inFlight.bytes += claim;
|
|
1868
|
+
const bytes = fetchBytes(remoteAbs);
|
|
1869
|
+
bytes.catch(() => {}); // its failure is reported when it lands
|
|
1870
|
+
return { path, remoteAbs, claim, bytes };
|
|
1871
|
+
};
|
|
1872
|
+
/** Land a fetched pull: the same checks a sequential pass made, then the write. */
|
|
1873
|
+
const land = async ({ path, remoteAbs, claim, bytes: fetched }) => {
|
|
1874
|
+
inFlight.files--;
|
|
1875
|
+
inFlight.bytes -= claim;
|
|
1876
|
+
const bytes = await fetched;
|
|
1877
|
+
if (bytes.length > MAX_BYTES) {
|
|
1878
|
+
tooBig(sess, path, bytes.length);
|
|
1879
|
+
pass.notLanded?.set(path, "it is over the mount's size limit");
|
|
1880
|
+
return;
|
|
1881
|
+
}
|
|
1882
|
+
const hold = await holdFor(bytes.length, true);
|
|
1883
|
+
if (hold) {
|
|
1884
|
+
pass.notLanded?.set(path, hold);
|
|
1885
|
+
return;
|
|
1886
|
+
}
|
|
1887
|
+
// An ignore file that cannot be read as rules is not pulled: landed, it
|
|
1888
|
+
// would stop every pass (see `see`), and uploads wait for it instead
|
|
1889
|
+
// (see `reconcile`).
|
|
1890
|
+
const rules = isIgnoreFile(path.slice(1));
|
|
1891
|
+
const problem = rules && ignoreFileProblem(bytes);
|
|
1892
|
+
if (problem) {
|
|
1893
|
+
decline("pull", path, `it cannot be read as ignore rules: ${problem}. If it is not yours, remove it: muhkoo vfs rm ${remoteAbs}`);
|
|
1894
|
+
pass.notLanded?.set(path, `it cannot be read as ignore rules: ${problem}`);
|
|
1895
|
+
return;
|
|
1896
|
+
}
|
|
1897
|
+
const target = await writeTarget(dir, path, { localKeys, listing });
|
|
1898
|
+
if (!target.ok) {
|
|
1899
|
+
decline("pull", path, target.reason);
|
|
1900
|
+
pass.notLanded?.set(path, target.reason);
|
|
1901
|
+
return;
|
|
1902
|
+
}
|
|
1903
|
+
await writeAt(target, bytes, listing);
|
|
1904
|
+
next[path] = sha(bytes);
|
|
1905
|
+
if (rules) pass.landed?.add(path);
|
|
1906
|
+
budget.files++;
|
|
1907
|
+
budget.bytes += bytes.length;
|
|
1908
|
+
info(`${SIGN.pull} ${path}`);
|
|
1909
|
+
};
|
|
1910
|
+
const landOldest = async () => {
|
|
1911
|
+
const pull = window.shift();
|
|
1912
|
+
try {
|
|
1913
|
+
await land(pull);
|
|
1914
|
+
} catch (err) {
|
|
1915
|
+
failed("pull", pull.path, err);
|
|
1916
|
+
}
|
|
1917
|
+
};
|
|
1918
|
+
/** Room for another pull in the window? Bounded in files, and — by what the records claim — in bytes held. */
|
|
1919
|
+
const windowFull = () => window.length >= PULL_CONCURRENCY || (window.length > 0 && inFlight.bytes >= PULL_WINDOW_BYTES);
|
|
1920
|
+
|
|
1641
1921
|
for (const { path, action } of plan) {
|
|
1642
1922
|
if (signal?.aborted) break;
|
|
1643
1923
|
const localAbs = localPathFor(dir, path);
|
|
@@ -1649,6 +1929,21 @@ async function applyPlan({ vfs, dir, root, state, plan, pass, opts, sess, signal
|
|
|
1649
1929
|
continue;
|
|
1650
1930
|
}
|
|
1651
1931
|
const remoteAbs = root === "/" ? path : root + path;
|
|
1932
|
+
if (action === "pull") {
|
|
1933
|
+
while (windowFull() && !signal?.aborted) await landOldest();
|
|
1934
|
+
if (signal?.aborted) break;
|
|
1935
|
+
try {
|
|
1936
|
+
const pull = await startPull(path, remoteAbs);
|
|
1937
|
+
if (pull) window.push(pull);
|
|
1938
|
+
} catch (err) {
|
|
1939
|
+
failed(action, path, err);
|
|
1940
|
+
}
|
|
1941
|
+
continue;
|
|
1942
|
+
}
|
|
1943
|
+
// Anything else waits for the pulls before it, so steps still take
|
|
1944
|
+
// effect in plan order.
|
|
1945
|
+
while (window.length && !signal?.aborted) await landOldest();
|
|
1946
|
+
if (signal?.aborted) break;
|
|
1652
1947
|
try {
|
|
1653
1948
|
switch (action) {
|
|
1654
1949
|
case "push": {
|
|
@@ -1658,77 +1953,6 @@ async function applyPlan({ vfs, dir, root, state, plan, pass, opts, sess, signal
|
|
|
1658
1953
|
info(`${SIGN.push} ${path}`);
|
|
1659
1954
|
break;
|
|
1660
1955
|
}
|
|
1661
|
-
case "pull": {
|
|
1662
|
-
// Held pulls change nothing, so the next pass plans them
|
|
1663
|
-
// again. Each limit is checked twice: on the size the scan
|
|
1664
|
-
// read, so a pull that cannot fit is not fetched at all, and
|
|
1665
|
-
// then on the bytes actually fetched — the file may have
|
|
1666
|
-
// changed since, and they are what land on this disk.
|
|
1667
|
-
//
|
|
1668
|
-
// Ignore files are held to the same limits. They were
|
|
1669
|
-
// exempt as "small", and a 256 KB file of comments, copied
|
|
1670
|
-
// to thousands of directories, put 7 GiB on a disk already
|
|
1671
|
-
// under its reserve in one pass. One held here holds this
|
|
1672
|
-
// pass's uploads instead (see `reconcile`).
|
|
1673
|
-
const limit = sess.pullBudget;
|
|
1674
|
-
/** Why a pull of `size` bytes must wait, or null. */
|
|
1675
|
-
const holdFor = async (size) => {
|
|
1676
|
-
if (budget.full || (budget.files && (budget.files + 1 > limit.files || budget.bytes + size > limit.bytes))) {
|
|
1677
|
-
budget.full = true;
|
|
1678
|
-
over++;
|
|
1679
|
-
return "this pass has pulled all it may; the next pass carries on";
|
|
1680
|
-
}
|
|
1681
|
-
if (await roomFor(dir, size, sess)) return null;
|
|
1682
|
-
short++;
|
|
1683
|
-
return "the disk would be left with less than its reserve free";
|
|
1684
|
-
};
|
|
1685
|
-
let hold = await holdFor(pass.sizes?.[path] ?? 0);
|
|
1686
|
-
if (hold) {
|
|
1687
|
-
pass.notLanded?.set(path, hold);
|
|
1688
|
-
break;
|
|
1689
|
-
}
|
|
1690
|
-
const target = await writeTarget(dir, path, { localKeys, listing });
|
|
1691
|
-
if (!target.ok) {
|
|
1692
|
-
decline(action, path, target.reason);
|
|
1693
|
-
pass.notLanded?.set(path, target.reason);
|
|
1694
|
-
break;
|
|
1695
|
-
}
|
|
1696
|
-
// Content the scan read — or an earlier pull, of an entry
|
|
1697
|
-
// sharing it — is not fetched again.
|
|
1698
|
-
const content = await remoteContent(vfs, remoteAbs);
|
|
1699
|
-
let bytes = pass.blobs?.get(content.key);
|
|
1700
|
-
if (!bytes) {
|
|
1701
|
-
bytes = await content.read();
|
|
1702
|
-
pass.blobs?.put(content.key, bytes);
|
|
1703
|
-
}
|
|
1704
|
-
if (bytes.length > MAX_BYTES) {
|
|
1705
|
-
tooBig(sess, path, bytes.length);
|
|
1706
|
-
pass.notLanded?.set(path, "it is over the mount's size limit");
|
|
1707
|
-
break;
|
|
1708
|
-
}
|
|
1709
|
-
hold = await holdFor(bytes.length);
|
|
1710
|
-
if (hold) {
|
|
1711
|
-
pass.notLanded?.set(path, hold);
|
|
1712
|
-
break;
|
|
1713
|
-
}
|
|
1714
|
-
// An ignore file that cannot be read as rules is not
|
|
1715
|
-
// pulled: landed, it would stop every pass (see `see`), and
|
|
1716
|
-
// uploads wait for it instead (see `reconcile`).
|
|
1717
|
-
const rules = isIgnoreFile(path.slice(1));
|
|
1718
|
-
const problem = rules && ignoreFileProblem(bytes);
|
|
1719
|
-
if (problem) {
|
|
1720
|
-
decline(action, path, `it cannot be read as ignore rules: ${problem}. If it is not yours, remove it: muhkoo vfs rm ${remoteAbs}`);
|
|
1721
|
-
pass.notLanded?.set(path, `it cannot be read as ignore rules: ${problem}`);
|
|
1722
|
-
break;
|
|
1723
|
-
}
|
|
1724
|
-
await writeAt(target, bytes, listing);
|
|
1725
|
-
next[path] = sha(bytes);
|
|
1726
|
-
if (rules) pass.landed?.add(path);
|
|
1727
|
-
budget.files++;
|
|
1728
|
-
budget.bytes += bytes.length;
|
|
1729
|
-
info(`${SIGN.pull} ${path}`);
|
|
1730
|
-
break;
|
|
1731
|
-
}
|
|
1732
1956
|
case "delete-remote":
|
|
1733
1957
|
// Deletes do NOT propagate unless asked for.
|
|
1734
1958
|
//
|
|
@@ -1795,11 +2019,17 @@ async function applyPlan({ vfs, dir, root, state, plan, pass, opts, sess, signal
|
|
|
1795
2019
|
}
|
|
1796
2020
|
}
|
|
1797
2021
|
} catch (err) {
|
|
1798
|
-
|
|
1799
|
-
warnOnce(sess, `error:${path}:${message}`, `${path}: ${message}`);
|
|
1800
|
-
if (action === "pull" && isIgnoreFile(path.slice(1))) pass.notLanded?.set(path, message);
|
|
2022
|
+
failed(action, path, err);
|
|
1801
2023
|
}
|
|
1802
2024
|
}
|
|
2025
|
+
// Land what is still fetching — or, stopped, drop it: a fetch is not a
|
|
2026
|
+
// step, and nothing is recorded for a pull that did not land. Its fetch is
|
|
2027
|
+
// still waited for, so none runs on after the pass has returned.
|
|
2028
|
+
while (window.length && !signal?.aborted) await landOldest();
|
|
2029
|
+
if (window.length) {
|
|
2030
|
+
await Promise.allSettled(window.map((p) => p.bytes));
|
|
2031
|
+
window.length = 0;
|
|
2032
|
+
}
|
|
1803
2033
|
if (budget.full) {
|
|
1804
2034
|
sess.more = true;
|
|
1805
2035
|
warnOnce(
|