mainflow 0.43.0-alpha.4 → 0.44.0-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +7 -5
- package/dist/{dist-D5rZ0MQH.js → dist-CXrUfRQp.js} +111 -10
- package/dist/facts.d.ts +22 -5
- package/dist/facts.js +28 -7
- package/dist/grant-check.d.ts +45 -0
- package/dist/grant-check.js +572 -0
- package/dist/grant-commands.d.ts +31 -3
- package/dist/grant-commands.js +14 -4
- package/dist/hooks.d.ts +2 -18
- package/dist/hooks.js +103 -15
- package/dist/{host-BJvqZy-i.js → host-C74j2Qox.js} +174 -57
- package/dist/host.d.ts +1 -1
- package/dist/host.js +1 -1
- package/dist/{index-DRpamPge.d.ts → index-jq7lN8V8.d.ts} +9 -3
- package/dist/{index-BqPQp6TF.d.ts → index-l8QUG80k.d.ts} +11 -0
- package/dist/launch.d.ts +1 -1
- package/dist/launch.js +2 -2
- package/dist/native/mainflow-swap-darwin-arm64 +0 -0
- package/dist/native/mainflow-swap-darwin-x64 +0 -0
- package/dist/{operation-BhBbGauo.js → operation-BmCeGKHw.js} +344 -132
- package/dist/operation-BroV98VB.d.ts +19 -0
- package/dist/{ui-xxzmhAqF.js → ui-8NTeAfC7.js} +3 -3
- package/package.json +1 -1
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { At as hubRequirements, B as applyOffline, Ct as comparePaths, Dt as textBytes, Et as requestTime, G as scopeOfPath, H as renumberFiles, J as syncedDir, Jt as nodeStorage, K as hubDigest, St as VERSION_HEADER, Tt as readsAsText, U as renumberPath, Ut as registeredName, V as renumberRequest, W as renumberText, X as applyOperation, Y as syncedPath, _t as MAX_BLOB_BYTES, an as splitFrontMatter, bt as ROUTES, ft as nowMinute, gt as BLOB_HASH, ht as ACCESS_HEADERS, in as contentHash, jt as versionMismatch, q as mayHoldSynced, rn as WriteError, tn as ConflictError, wt as exceedsFileLimit, xt as SOCKET_MAX_AGE_CLOSE, yt as MAX_FILE_BYTES } from "./host-C74j2Qox.js";
|
|
2
2
|
import { t as CLI_VERSION } from "./version-Dvv4jwF1.js";
|
|
3
3
|
import { createHash, randomBytes } from "node:crypto";
|
|
4
4
|
import { accessSync, appendFileSync, chmodSync, closeSync, constants, copyFileSync, createReadStream, createWriteStream, existsSync, linkSync, lstatSync, mkdirSync, openSync, readFileSync, readSync, readdirSync, renameSync, rmSync, rmdirSync, statSync, watch, writeFileSync } from "node:fs";
|
|
@@ -8,8 +8,6 @@ import { execFileSync, spawn, spawnSync } from "node:child_process";
|
|
|
8
8
|
import { parse as parse$1 } from "yaml";
|
|
9
9
|
import { homedir } from "node:os";
|
|
10
10
|
import { createBrotliDecompress, createGunzip, createInflate, createZstdDecompress } from "node:zlib";
|
|
11
|
-
import { Agent, createServer, request } from "node:http";
|
|
12
|
-
import { Agent as Agent$1, request as request$1 } from "node:https";
|
|
13
11
|
import { Readable, Transform } from "node:stream";
|
|
14
12
|
import { pipeline } from "node:stream/promises";
|
|
15
13
|
import { EventEmitter } from "node:events";
|
|
@@ -253,6 +251,12 @@ function decoded(res) {
|
|
|
253
251
|
}
|
|
254
252
|
return decode ? res.pipe(decode()) : res;
|
|
255
253
|
}
|
|
254
|
+
/**
|
|
255
|
+
* node:http and node:https load node:tls, which reads the system certificates when NODE_USE_SYSTEM_CA is set and
|
|
256
|
+
* costs about 220 ms. Load them on the first request so commands that never reach the server do not pay for it.
|
|
257
|
+
*/
|
|
258
|
+
const http = () => process.getBuiltinModule("node:http");
|
|
259
|
+
const https = () => process.getBuiltinModule("node:https");
|
|
256
260
|
var ServerApi = class {
|
|
257
261
|
settings;
|
|
258
262
|
timeoutMs;
|
|
@@ -260,14 +264,14 @@ var ServerApi = class {
|
|
|
260
264
|
* The connection writes go on, kept open between them: opening one costs two round trips, for TCP and TLS. One
|
|
261
265
|
* connection at a time, since the client sends one write at a time; a request waits for the one before it.
|
|
262
266
|
*/
|
|
263
|
-
|
|
267
|
+
writeAgent;
|
|
264
268
|
/** The connections snapshots go on, apart from the writes', so a snapshot never waits behind a write or holds one up */
|
|
265
|
-
|
|
269
|
+
readAgent;
|
|
266
270
|
/**
|
|
267
271
|
* The connections binary files' contents go up and down on, apart from the writes' and the snapshots', so a large file
|
|
268
272
|
* holds neither up; a few at a time, as the files of a snapshot are downloaded side by side.
|
|
269
273
|
*/
|
|
270
|
-
|
|
274
|
+
transferAgent;
|
|
271
275
|
/** Requests on `agent` not answered yet */
|
|
272
276
|
busy = 0;
|
|
273
277
|
transfer = {
|
|
@@ -277,28 +281,30 @@ var ServerApi = class {
|
|
|
277
281
|
body: 0,
|
|
278
282
|
sent: 0,
|
|
279
283
|
listed: 0,
|
|
280
|
-
unchanged: 0
|
|
284
|
+
unchanged: 0,
|
|
285
|
+
changes: 0
|
|
281
286
|
};
|
|
282
287
|
constructor(settings, timeoutMs = 3e4) {
|
|
283
288
|
this.settings = settings;
|
|
284
289
|
this.timeoutMs = timeoutMs;
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
maxFreeSockets: 1
|
|
290
|
-
});
|
|
291
|
-
this.reads = new Agent$2({
|
|
292
|
-
keepAlive: true,
|
|
293
|
-
maxSockets: 2,
|
|
294
|
-
maxFreeSockets: 1
|
|
295
|
-
});
|
|
296
|
-
this.transfers = new Agent$2({
|
|
290
|
+
}
|
|
291
|
+
makeAgent(maxSockets) {
|
|
292
|
+
const { Agent } = this.settings.url.startsWith("https:") ? https() : http();
|
|
293
|
+
return new Agent({
|
|
297
294
|
keepAlive: true,
|
|
298
|
-
maxSockets
|
|
295
|
+
maxSockets,
|
|
299
296
|
maxFreeSockets: 1
|
|
300
297
|
});
|
|
301
298
|
}
|
|
299
|
+
get agent() {
|
|
300
|
+
return this.writeAgent ??= this.makeAgent(1);
|
|
301
|
+
}
|
|
302
|
+
get reads() {
|
|
303
|
+
return this.readAgent ??= this.makeAgent(2);
|
|
304
|
+
}
|
|
305
|
+
get transfers() {
|
|
306
|
+
return this.transferAgent ??= this.makeAgent(4);
|
|
307
|
+
}
|
|
302
308
|
/**
|
|
303
309
|
* The service token's headers, and the CLI's version when the settings name it (VERSION_HEADER), which every request
|
|
304
310
|
* and the WebSocket's upgrade carry. Never logged.
|
|
@@ -380,7 +386,7 @@ var ServerApi = class {
|
|
|
380
386
|
* the raw bytes of a stream, or nothing; `headers` add to the token's and say what the body is.
|
|
381
387
|
*/
|
|
382
388
|
open(method, url, body, headers, agent, timeoutMs) {
|
|
383
|
-
const send = url.startsWith("https:") ? request
|
|
389
|
+
const send = url.startsWith("https:") ? https().request : http().request;
|
|
384
390
|
return new Promise((resolve, reject) => {
|
|
385
391
|
const req = send(url, {
|
|
386
392
|
method,
|
|
@@ -406,7 +412,7 @@ var ServerApi = class {
|
|
|
406
412
|
"content-type": "application/json",
|
|
407
413
|
"content-length": String(Buffer.byteLength(body))
|
|
408
414
|
} : {};
|
|
409
|
-
if (agent === this.
|
|
415
|
+
if (agent === this.writeAgent) this.busy++;
|
|
410
416
|
try {
|
|
411
417
|
const res = await this.open(method, url, body, {
|
|
412
418
|
...kind,
|
|
@@ -434,7 +440,7 @@ var ServerApi = class {
|
|
|
434
440
|
stream.on("error", reject);
|
|
435
441
|
});
|
|
436
442
|
} finally {
|
|
437
|
-
if (agent === this.
|
|
443
|
+
if (agent === this.writeAgent) this.busy--;
|
|
438
444
|
}
|
|
439
445
|
}
|
|
440
446
|
/**
|
|
@@ -553,9 +559,9 @@ var ServerApi = class {
|
|
|
553
559
|
}
|
|
554
560
|
/** Close the kept connections; a later request opens others. */
|
|
555
561
|
close() {
|
|
556
|
-
this.
|
|
557
|
-
this.
|
|
558
|
-
this.
|
|
562
|
+
this.writeAgent?.destroy();
|
|
563
|
+
this.readAgent?.destroy();
|
|
564
|
+
this.transferAgent?.destroy();
|
|
559
565
|
}
|
|
560
566
|
/** Send one request; the response is the hub's answer, applied or refused. */
|
|
561
567
|
async send(request, options = {}) {
|
|
@@ -568,26 +574,13 @@ var ServerApi = class {
|
|
|
568
574
|
this.transfer.batches++;
|
|
569
575
|
this.transfer.files += body.files.filter((entry) => entry.kind === "file").length;
|
|
570
576
|
this.transfer.listed += body.files.filter((entry) => entry.kind === "listed").length;
|
|
571
|
-
if (body.unchanged === true) this.transfer.unchanged++;
|
|
577
|
+
if ("unchanged" in body && body.unchanged === true) this.transfer.unchanged++;
|
|
578
|
+
if ("changes" in body && body.changes === true) this.transfer.changes++;
|
|
572
579
|
this.transfer.wire += wire;
|
|
573
580
|
this.transfer.body += size;
|
|
574
581
|
this.transfer.sent += sent;
|
|
575
582
|
return body;
|
|
576
583
|
}
|
|
577
|
-
/**
|
|
578
|
-
* The hub's snapshot, batch by batch (docs/sync-protocol.md "快照與整份上傳"). `have` gives the content hash of the
|
|
579
|
-
* text the client holds of each file, so the server sends only the files that differ, and names the ones of `have` it
|
|
580
|
-
* does not hold as removed; each batch is sent only the part of `have` in its own range. A path of `have` that no
|
|
581
|
-
* batch mentions is a file the server holds with that text. The batches are in order, each with its seq; one whose seq
|
|
582
|
-
* is below the batch before, as when the server's storage was replaced meanwhile, starts the snapshot over. A path of
|
|
583
|
-
* `have` the server never held a file at comes as `unknown` rather than `removed`, from a server that tells them apart.
|
|
584
|
-
* `listOnly` asks for the files under those prefixes as `listed`, without their text. A part of the hub alone is taken
|
|
585
|
-
* from `after` on, and ends with the batch that covers `through`; only the paths of that range are told of. `blobs` takes
|
|
586
|
-
* the binary files too, each that differs from `have` as `blob`; `have` names a binary file by its sha256 then.
|
|
587
|
-
* `digest` names the digest of `have` and the directories the client holds (hubDigest), asked on the first batch of a
|
|
588
|
-
* whole hub's snapshot with `blobs`: a server whose hub it is the digest of answers that nothing differs, and the
|
|
589
|
-
* snapshot is that one batch, with the client's directories as the entries, as the server would have listed them.
|
|
590
|
-
*/
|
|
591
584
|
async snapshot(have = /* @__PURE__ */ new Map(), options = {}) {
|
|
592
585
|
const paths = [...have.keys()].sort(comparePaths);
|
|
593
586
|
const { through } = options;
|
|
@@ -599,15 +592,26 @@ var ServerApi = class {
|
|
|
599
592
|
while (after !== void 0 && start < paths.length && comparePaths(paths[start], after) <= 0) start++;
|
|
600
593
|
const range = Object.fromEntries(paths.slice(start).map((path) => [path, have.get(path)]));
|
|
601
594
|
const asking = options.digest !== void 0 && options.blobs === true && after === void 0;
|
|
602
|
-
const
|
|
595
|
+
const since = options.since !== void 0 && options.blobs === true && after === void 0 && attempt === 1 ? options.since : void 0;
|
|
596
|
+
const answer = await this.batch({
|
|
603
597
|
protocol: 3,
|
|
604
598
|
have: range,
|
|
605
599
|
unknown: true,
|
|
606
600
|
...after === void 0 ? {} : { after },
|
|
607
601
|
...asking ? { digest: options.digest.value } : {},
|
|
602
|
+
...since === void 0 ? {} : { since },
|
|
608
603
|
...options.listOnly?.length ? { listOnly: options.listOnly } : {},
|
|
609
604
|
...options.blobs ? { blobs: true } : {}
|
|
610
605
|
});
|
|
606
|
+
if ("changes" in answer && answer.changes === true) {
|
|
607
|
+
if (since === void 0) throw new Unreachable("the sync server answered a snapshot that named no since with what changed since", true);
|
|
608
|
+
return {
|
|
609
|
+
seq: answer.seq,
|
|
610
|
+
files: answer.files,
|
|
611
|
+
digest: answer.digest
|
|
612
|
+
};
|
|
613
|
+
}
|
|
614
|
+
const { seq, files, next, unchanged } = answer;
|
|
611
615
|
if (unchanged === true && asking) return [{
|
|
612
616
|
seq,
|
|
613
617
|
files: options.digest.dirs.map((path) => ({
|
|
@@ -1281,6 +1285,8 @@ function exists(file) {
|
|
|
1281
1285
|
const isBlob = (file) => typeof file === "object" && file !== null && file.kind === "blob";
|
|
1282
1286
|
/** The hash the server names a file of its by: a text's content hash, a binary file's sha256. */
|
|
1283
1287
|
const serverHash = (file) => typeof file === "string" ? contentHash(file) : file.hash;
|
|
1288
|
+
/** Whether the client holds `held` as the server's version of a path, the same file as `file`. */
|
|
1289
|
+
const sameServerFile = (held, file) => held !== void 0 && held !== null && (typeof held === "string" ? held === file : isBlob(file) && held.hash === file.hash);
|
|
1284
1290
|
/** Whether two tool calls are of one session, the same subagent in it or both of the session itself. */
|
|
1285
1291
|
const sameCaller = (a, b) => a !== void 0 && b !== void 0 && a.session === b.session && a.agent === b.agent;
|
|
1286
1292
|
/** Whether `edit` makes `to` of `from`, as the Edit tool makes it (applyEdit). */
|
|
@@ -1329,6 +1335,12 @@ const BINARY_RECHECK_MS = 150;
|
|
|
1329
1335
|
* program that rewrites a file as `cat >` does empties it first and fills it a moment later
|
|
1330
1336
|
*/
|
|
1331
1337
|
const EMPTY_RECHECK_MS = 300;
|
|
1338
|
+
/**
|
|
1339
|
+
* How long taking in what changed since a seq (takeChanges) waits for one step to be done on disk before the next, when
|
|
1340
|
+
* a path changes kind: the files removed before the directories, those before the directories made, and those before
|
|
1341
|
+
* the files written. A step that takes longer, as a file whose removal waits for the write lock, is not waited out.
|
|
1342
|
+
*/
|
|
1343
|
+
const CHANGES_STEP_MS = 5e3;
|
|
1332
1344
|
/** How long after the exchange the displaced file is read again, for a program that still held it open */
|
|
1333
1345
|
const LATE_WRITE_MS = 150;
|
|
1334
1346
|
/**
|
|
@@ -1639,6 +1651,19 @@ var SyncClient = class extends EventEmitter {
|
|
|
1639
1651
|
dirCheck = null;
|
|
1640
1652
|
/** The hub's seq up to which the client has taken in every change: the last snapshot and the broadcasts after it */
|
|
1641
1653
|
hubSeq = -1;
|
|
1654
|
+
/**
|
|
1655
|
+
* Whether `cloud` and `remote` hold every file the server holds as of hubSeq, binary files included, and syncedDirs
|
|
1656
|
+
* every directory, but for the paths changed after it: a whole snapshot with binary files was taken in this run, and
|
|
1657
|
+
* nothing since made the client unsure of what it holds, as a server that stopped keeping binary files, or a hub
|
|
1658
|
+
* reloaded. Only then is a snapshot asked for what changed since hubSeq (`since`), as a path the answer does not name
|
|
1659
|
+
* holds what the client takes the server to hold. Not kept across runs: known.json names no seq.
|
|
1660
|
+
*/
|
|
1661
|
+
holdsHub = false;
|
|
1662
|
+
/**
|
|
1663
|
+
* How many reload broadcasts came (the hub replaced, migrated or restored): a snapshot under way when one comes does
|
|
1664
|
+
* not leave the client holding the hub (holdsHub), whatever it took in, so the next one is whole.
|
|
1665
|
+
*/
|
|
1666
|
+
reloads = 0;
|
|
1642
1667
|
/** The seqs of the broadcasts on the current connection above hubSeq with a change between that never arrived */
|
|
1643
1668
|
ahead = /* @__PURE__ */ new Set();
|
|
1644
1669
|
waiters = /* @__PURE__ */ new Set();
|
|
@@ -2091,6 +2116,8 @@ var SyncClient = class extends EventEmitter {
|
|
|
2091
2116
|
if (seq === this.hubSeq + 1) this.takenUpTo(seq);
|
|
2092
2117
|
else if (seq > this.hubSeq + 1) this.ahead.add(seq);
|
|
2093
2118
|
if ("reload" in message && message.reload) {
|
|
2119
|
+
this.reloads++;
|
|
2120
|
+
this.holdsHub = false;
|
|
2094
2121
|
this.track(this.resync());
|
|
2095
2122
|
return;
|
|
2096
2123
|
}
|
|
@@ -2175,7 +2202,7 @@ var SyncClient = class extends EventEmitter {
|
|
|
2175
2202
|
return;
|
|
2176
2203
|
}
|
|
2177
2204
|
this.knowDir(path);
|
|
2178
|
-
this.track(mkdir(dir, { recursive: true }));
|
|
2205
|
+
return this.track(mkdir(dir, { recursive: true }));
|
|
2179
2206
|
}
|
|
2180
2207
|
/**
|
|
2181
2208
|
* The server removed the directory at `path` as of the hub's `seq`, as a broadcast tells, or a snapshot that no longer
|
|
@@ -2188,7 +2215,7 @@ var SyncClient = class extends EventEmitter {
|
|
|
2188
2215
|
this.dirSeq.set(path, seq);
|
|
2189
2216
|
this.forgetDir(path);
|
|
2190
2217
|
this.trace(`dir ${path} removed seq ${seq}`);
|
|
2191
|
-
this.track(this.whenFree(() => this.locked(() => this.removeEmptyDir(path))).catch((error) => this.log(`${path}/: ${error.message}`)));
|
|
2218
|
+
return this.track(this.whenFree(() => this.locked(() => this.removeEmptyDir(path))).catch((error) => this.log(`${path}/: ${error.message}`)));
|
|
2192
2219
|
}
|
|
2193
2220
|
/** Remove the directory at `path` when nothing is in it; one gone already, or holding anything, stays as it is. True when it went. */
|
|
2194
2221
|
removeEmptyDir(path) {
|
|
@@ -2248,12 +2275,13 @@ var SyncClient = class extends EventEmitter {
|
|
|
2248
2275
|
clearTimeout(timer);
|
|
2249
2276
|
}
|
|
2250
2277
|
/**
|
|
2251
|
-
* Take the server's
|
|
2252
|
-
* does a change still missing among the broadcasts once the snapshot
|
|
2253
|
-
* reconciled: the server broadcasts a change after storing it, so a
|
|
2278
|
+
* Take the server's hub, or only what changed in it since hubSeq (takeSnapshot), and reconcile every path; a request
|
|
2279
|
+
* made while one runs runs once more after it. So does a change still missing among the broadcasts once the snapshot
|
|
2280
|
+
* is in, including ones that came while it was reconciled: the server broadcasts a change after storing it, so a
|
|
2281
|
+
* snapshot asked for later holds it. One that
|
|
2254
2282
|
* still does not, and brought nothing newer, is asked for again after a pause; after STALLED_SNAPSHOTS of them the
|
|
2255
2283
|
* server is taken to have lost those changes, as when its storage was replaced: the versions the client holds from
|
|
2256
|
-
* after the gap are forgotten and the snapshot is taken once more, as it is. The connection has its
|
|
2284
|
+
* after the gap are forgotten and the snapshot is taken once more, whole, as it is. The connection has its
|
|
2257
2285
|
* snapshot once one ends while it is open (I1); one that fails leaves it without, and the client reconnects to take a
|
|
2258
2286
|
* fresh one, rather than send on the snapshot of an earlier connection.
|
|
2259
2287
|
*/
|
|
@@ -2276,6 +2304,7 @@ var SyncClient = class extends EventEmitter {
|
|
|
2276
2304
|
if (stalled >= STALLED_SNAPSHOTS) {
|
|
2277
2305
|
this.log(`the sync server's snapshot stays without the hub's changes ${missing}; taking it as complete`);
|
|
2278
2306
|
this.ahead.clear();
|
|
2307
|
+
this.holdsHub = false;
|
|
2279
2308
|
for (const [path, seq] of this.fileSeq) if (seq > this.hubSeq) this.fileSeq.delete(path);
|
|
2280
2309
|
for (const [path, seq] of this.dirSeq) if (seq > this.hubSeq) this.dirSeq.delete(path);
|
|
2281
2310
|
stalled = 0;
|
|
@@ -2346,12 +2375,24 @@ var SyncClient = class extends EventEmitter {
|
|
|
2346
2375
|
* upload (forgetBinaries), so they are files never synced here, sent as new even when this snapshot fails or the
|
|
2347
2376
|
* client is killed; and in the snapshot after the upload, a binary file of the client's the server names removed, as
|
|
2348
2377
|
* when it once held one there, is taken as one it never held (unheld), also in a run that started without known.json.
|
|
2378
|
+
*
|
|
2379
|
+
* Once a whole snapshot with binary files was taken in this run (holdsHub), the client holds every file and directory
|
|
2380
|
+
* the server holds as of hubSeq, and the next snapshot, after a reconnect or a missed broadcast, asks for what changed
|
|
2381
|
+
* since hubSeq alone (`since`). A server whose change log tells answers with the paths changed (SnapshotDelta): taken
|
|
2382
|
+
* in over what the client holds once they make the hub's digest (changesHold, takeChanges), and else dropped for the
|
|
2383
|
+
* whole snapshot. A server that cannot tell, or one from before it, answers the first batch, and the snapshot goes on
|
|
2384
|
+
* whole as ever.
|
|
2349
2385
|
*/
|
|
2350
2386
|
takeSnapshot() {
|
|
2351
2387
|
return this.exclusive(() => this.snapshotNow());
|
|
2352
2388
|
}
|
|
2353
2389
|
async snapshotNow() {
|
|
2354
|
-
let parts;
|
|
2390
|
+
let parts = [];
|
|
2391
|
+
/** What changed since `since`, when the server answered with it and it makes the hub's digest */
|
|
2392
|
+
let delta = null;
|
|
2393
|
+
let since;
|
|
2394
|
+
/** What the client took the server to hold as of `since`, which `delta` is taken in over */
|
|
2395
|
+
let deltaBase = null;
|
|
2355
2396
|
let held;
|
|
2356
2397
|
let heldBlobs;
|
|
2357
2398
|
let have;
|
|
@@ -2362,10 +2403,23 @@ var SyncClient = class extends EventEmitter {
|
|
|
2362
2403
|
/** Whether the snapshot is asked for with the binary files */
|
|
2363
2404
|
let withBlobs = false;
|
|
2364
2405
|
if (!this.probed) await this.probeBlobs();
|
|
2406
|
+
/** The reload broadcasts that came before this snapshot: one that comes while it runs leaves the next one whole */
|
|
2407
|
+
const reloads = this.reloads;
|
|
2408
|
+
/**
|
|
2409
|
+
* Whether the snapshot may ask for what changed since hubSeq, read once the probe told whether the server keeps
|
|
2410
|
+
* binary files; not once an answer of it failed the digest
|
|
2411
|
+
*/
|
|
2412
|
+
let askChanges = this.holdsHub;
|
|
2365
2413
|
for (;;) {
|
|
2366
2414
|
before = this.hubSeq;
|
|
2415
|
+
const base = askChanges && this.blobSupport === true && before >= 0 ? this.serverCopy() : null;
|
|
2367
2416
|
({texts: held, blobs: heldBlobs} = await this.holdings());
|
|
2368
|
-
|
|
2417
|
+
let dropped = false;
|
|
2418
|
+
for (const path of this.remote.keys()) {
|
|
2419
|
+
if (!(held.has(path) || heldBlobs.has(path) || this.expected.get(path) !== void 0)) continue;
|
|
2420
|
+
this.remote.delete(path);
|
|
2421
|
+
dropped = true;
|
|
2422
|
+
}
|
|
2369
2423
|
have = new Map([...held].map(([path, text]) => [path, this.hashOf(path, text)]));
|
|
2370
2424
|
for (const [path, blob] of heldBlobs) have.set(path, blob.hash);
|
|
2371
2425
|
for (const [path, hash] of this.remote) have.set(path, hash);
|
|
@@ -2379,11 +2433,25 @@ var SyncClient = class extends EventEmitter {
|
|
|
2379
2433
|
value: hubDigest(have, dirs),
|
|
2380
2434
|
dirs
|
|
2381
2435
|
} : void 0;
|
|
2382
|
-
|
|
2436
|
+
since = base !== null && withBlobs && !dropped ? before : void 0;
|
|
2437
|
+
const taken = await this.api.snapshot(have, {
|
|
2383
2438
|
listOnly: this.listOnly(this.copied && !this.synced),
|
|
2384
|
-
blobs:
|
|
2439
|
+
blobs: withBlobs,
|
|
2440
|
+
since,
|
|
2385
2441
|
...digest ? { digest } : {}
|
|
2386
2442
|
});
|
|
2443
|
+
if (!Array.isArray(taken)) {
|
|
2444
|
+
const verdict = this.changesHold(taken, base);
|
|
2445
|
+
if (verdict === "held") {
|
|
2446
|
+
delta = taken;
|
|
2447
|
+
deltaBase = base;
|
|
2448
|
+
break;
|
|
2449
|
+
}
|
|
2450
|
+
this.log(verdict === "differs" ? `what changed since seq ${since} does not make the hub's digest; taking the whole snapshot` : `what changed since seq ${since} leaves the hub without files; taking the whole snapshot`);
|
|
2451
|
+
askChanges = false;
|
|
2452
|
+
continue;
|
|
2453
|
+
}
|
|
2454
|
+
parts = taken;
|
|
2387
2455
|
if (uploaded || holdsFiles(parts, have)) break;
|
|
2388
2456
|
if (this.opts.settings.onDemand === true) {
|
|
2389
2457
|
if (!this.uploadWanted) {
|
|
@@ -2411,7 +2479,7 @@ var SyncClient = class extends EventEmitter {
|
|
|
2411
2479
|
this.backoff = 500;
|
|
2412
2480
|
}
|
|
2413
2481
|
if (this.closing) return;
|
|
2414
|
-
const seq = parts[0].seq;
|
|
2482
|
+
const seq = delta !== null ? delta.seq : parts[0].seq;
|
|
2415
2483
|
if (seq < before) {
|
|
2416
2484
|
this.log(`the sync server's seq went back from ${before} to ${seq}; taking its snapshot as it is`);
|
|
2417
2485
|
this.fileSeq.clear();
|
|
@@ -2419,94 +2487,102 @@ var SyncClient = class extends EventEmitter {
|
|
|
2419
2487
|
this.hubSeq = -1;
|
|
2420
2488
|
this.ahead.clear();
|
|
2421
2489
|
}
|
|
2422
|
-
|
|
2423
|
-
|
|
2424
|
-
|
|
2425
|
-
|
|
2426
|
-
|
|
2427
|
-
|
|
2428
|
-
|
|
2429
|
-
|
|
2430
|
-
|
|
2431
|
-
|
|
2432
|
-
|
|
2433
|
-
|
|
2434
|
-
|
|
2435
|
-
|
|
2436
|
-
|
|
2437
|
-
|
|
2438
|
-
|
|
2439
|
-
|
|
2490
|
+
let wanted = /* @__PURE__ */ new Set();
|
|
2491
|
+
if (delta !== null) {
|
|
2492
|
+
const taken = await this.takeChanges(delta, deltaBase, held);
|
|
2493
|
+
if (this.closing) return;
|
|
2494
|
+
wanted = taken.wanted;
|
|
2495
|
+
this.log(`took what changed since seq ${since} at seq ${seq}: ${delta.files.length} path${delta.files.length === 1 ? "" : "s"}, ${taken.sent} files sent${this.remote.size > 0 ? `, ${this.remote.size} only on the server` : ""}`);
|
|
2496
|
+
} else {
|
|
2497
|
+
const named = /* @__PURE__ */ new Set();
|
|
2498
|
+
const present = /* @__PURE__ */ new Set();
|
|
2499
|
+
let sent = 0;
|
|
2500
|
+
this.snapshotAnswers.clear();
|
|
2501
|
+
const fromCopy = this.copied && !this.synced;
|
|
2502
|
+
const blobsFromCopy = withBlobs && !this.blobsCompared;
|
|
2503
|
+
/** The directories the snapshot names: every one the server holds */
|
|
2504
|
+
const dirs = /* @__PURE__ */ new Set();
|
|
2505
|
+
for (const part of parts) for (const event of part.files) {
|
|
2506
|
+
if (event.kind === "dir") {
|
|
2507
|
+
named.add(event.path);
|
|
2508
|
+
dirs.add(event.path);
|
|
2509
|
+
this.receiveDir(event.path, part.seq, false);
|
|
2510
|
+
continue;
|
|
2511
|
+
}
|
|
2512
|
+
if (!this.syncs(event.path)) continue;
|
|
2513
|
+
if (event.kind === "listed") {
|
|
2514
|
+
named.add(event.path);
|
|
2515
|
+
if (this.mayStay(event.path, held)) this.keepRemote(event.path, event.hash, part.seq);
|
|
2516
|
+
else {
|
|
2517
|
+
wanted.add(event.path);
|
|
2518
|
+
this.awaiting.add(event.path);
|
|
2519
|
+
if (part.seq > (this.fileSeq.get(event.path) ?? -1)) this.fileSeq.set(event.path, part.seq);
|
|
2520
|
+
}
|
|
2521
|
+
continue;
|
|
2522
|
+
}
|
|
2440
2523
|
named.add(event.path);
|
|
2441
|
-
if (
|
|
2442
|
-
|
|
2443
|
-
|
|
2444
|
-
this.awaiting.add(event.path);
|
|
2445
|
-
if (part.seq > (this.fileSeq.get(event.path) ?? -1)) this.fileSeq.set(event.path, part.seq);
|
|
2524
|
+
if (event.kind === "file" || event.kind === "blob") {
|
|
2525
|
+
present.add(event.path);
|
|
2526
|
+
if (event.kind === "file") sent++;
|
|
2446
2527
|
}
|
|
2447
|
-
|
|
2448
|
-
|
|
2449
|
-
|
|
2450
|
-
|
|
2451
|
-
|
|
2452
|
-
|
|
2528
|
+
if (uploaded && event.kind === "removed" && heldBlobs.has(event.path)) {
|
|
2529
|
+
this.receive({
|
|
2530
|
+
kind: "unknown",
|
|
2531
|
+
path: event.path
|
|
2532
|
+
}, part.seq);
|
|
2533
|
+
continue;
|
|
2534
|
+
}
|
|
2535
|
+
if (event.kind === "removed" && (held.has(event.path) ? fromCopy : blobsFromCopy && heldBlobs.has(event.path))) {
|
|
2536
|
+
const text = held.get(event.path);
|
|
2537
|
+
this.snapshotAnswers.set(event.path, {
|
|
2538
|
+
hash: text === void 0 ? heldBlobs.get(event.path).hash : this.hashOf(event.path, text),
|
|
2539
|
+
removed: true
|
|
2540
|
+
});
|
|
2541
|
+
}
|
|
2542
|
+
this.receive(event, part.seq);
|
|
2453
2543
|
}
|
|
2454
|
-
|
|
2544
|
+
for (const [path, text] of held) {
|
|
2545
|
+
if (named.has(path)) continue;
|
|
2546
|
+
present.add(path);
|
|
2547
|
+
this.snapshotAnswers.set(path, {
|
|
2548
|
+
hash: this.hashOf(path, text),
|
|
2549
|
+
removed: false
|
|
2550
|
+
});
|
|
2455
2551
|
this.receive({
|
|
2456
|
-
kind: "
|
|
2457
|
-
path
|
|
2458
|
-
|
|
2459
|
-
|
|
2552
|
+
kind: "file",
|
|
2553
|
+
path,
|
|
2554
|
+
text
|
|
2555
|
+
}, seq);
|
|
2460
2556
|
}
|
|
2461
|
-
|
|
2462
|
-
|
|
2463
|
-
|
|
2464
|
-
|
|
2465
|
-
|
|
2557
|
+
for (const [path, blob] of heldBlobs) {
|
|
2558
|
+
if (named.has(path)) continue;
|
|
2559
|
+
present.add(path);
|
|
2560
|
+
this.snapshotAnswers.set(path, {
|
|
2561
|
+
hash: blob.hash,
|
|
2562
|
+
removed: false
|
|
2466
2563
|
});
|
|
2564
|
+
this.receive({
|
|
2565
|
+
kind: "blob",
|
|
2566
|
+
path,
|
|
2567
|
+
hash: blob.hash,
|
|
2568
|
+
size: blob.size ?? 0
|
|
2569
|
+
}, seq);
|
|
2467
2570
|
}
|
|
2468
|
-
this.
|
|
2469
|
-
|
|
2470
|
-
|
|
2471
|
-
|
|
2472
|
-
|
|
2473
|
-
|
|
2474
|
-
|
|
2475
|
-
|
|
2476
|
-
}
|
|
2477
|
-
this.
|
|
2478
|
-
|
|
2479
|
-
path,
|
|
2480
|
-
text
|
|
2481
|
-
}, seq);
|
|
2482
|
-
}
|
|
2483
|
-
for (const [path, blob] of heldBlobs) {
|
|
2484
|
-
if (named.has(path)) continue;
|
|
2485
|
-
present.add(path);
|
|
2486
|
-
this.snapshotAnswers.set(path, {
|
|
2487
|
-
hash: blob.hash,
|
|
2488
|
-
removed: false
|
|
2489
|
-
});
|
|
2490
|
-
this.receive({
|
|
2491
|
-
kind: "blob",
|
|
2492
|
-
path,
|
|
2493
|
-
hash: blob.hash,
|
|
2494
|
-
size: blob.size ?? 0
|
|
2495
|
-
}, seq);
|
|
2496
|
-
}
|
|
2497
|
-
const blind = this.blobSupport !== true;
|
|
2498
|
-
for (const path of /* @__PURE__ */ new Set([...this.cloud.keys(), ...this.expected.keys()])) {
|
|
2499
|
-
if (present.has(path) || named.has(path) || !this.syncs(path)) continue;
|
|
2500
|
-
if (blind && this.knownBinary(path)) continue;
|
|
2501
|
-
this.receive({
|
|
2502
|
-
kind: "removed",
|
|
2503
|
-
path
|
|
2504
|
-
}, seq);
|
|
2571
|
+
const blind = this.blobSupport !== true;
|
|
2572
|
+
for (const path of /* @__PURE__ */ new Set([...this.cloud.keys(), ...this.expected.keys()])) {
|
|
2573
|
+
if (present.has(path) || named.has(path) || !this.syncs(path)) continue;
|
|
2574
|
+
if (blind && this.knownBinary(path)) continue;
|
|
2575
|
+
this.receive({
|
|
2576
|
+
kind: "removed",
|
|
2577
|
+
path
|
|
2578
|
+
}, seq);
|
|
2579
|
+
}
|
|
2580
|
+
for (const path of this.syncedDirs) if (!dirs.has(path)) this.receiveDirRemoved(path, seq);
|
|
2581
|
+
this.log(`took the snapshot at seq ${seq} in ${parts.length} batch${parts.length === 1 ? "" : "es"}: ${sent} files sent, ${present.size - sent} already held here${this.remote.size > 0 ? `, ${this.remote.size} only on the server` : ""}`);
|
|
2505
2582
|
}
|
|
2506
|
-
for (const path of this.syncedDirs) if (!dirs.has(path)) this.receiveDirRemoved(path, seq);
|
|
2507
|
-
this.log(`took the snapshot at seq ${seq} in ${parts.length} batch${parts.length === 1 ? "" : "es"}: ${sent} files sent, ${present.size - sent} already held here${this.remote.size > 0 ? `, ${this.remote.size} only on the server` : ""}`);
|
|
2508
2583
|
this.takenUpTo(seq);
|
|
2509
2584
|
this.synced = true;
|
|
2585
|
+
this.holdsHub = withBlobs && this.reloads === reloads;
|
|
2510
2586
|
if (withBlobs && !this.blobsCompared) {
|
|
2511
2587
|
this.blobsCompared = true;
|
|
2512
2588
|
rmSync(this.paths.copyBlobs, { force: true });
|
|
@@ -2521,6 +2597,141 @@ var SyncClient = class extends EventEmitter {
|
|
|
2521
2597
|
await this.queueRemovedDirs();
|
|
2522
2598
|
}
|
|
2523
2599
|
/**
|
|
2600
|
+
* What the client takes the server to hold, while it holds the hub (holdsHub): the server's version of each file it
|
|
2601
|
+
* took in, a binary file's included, the hash of each file only on the server, and the directories the server holds.
|
|
2602
|
+
* A file here the server never had is not among them, though a snapshot request names it.
|
|
2603
|
+
*/
|
|
2604
|
+
serverCopy() {
|
|
2605
|
+
const files = /* @__PURE__ */ new Map();
|
|
2606
|
+
for (const [path, file] of this.cloud) if (file !== null && this.syncs(path)) files.set(path, file);
|
|
2607
|
+
return {
|
|
2608
|
+
files,
|
|
2609
|
+
remote: new Map(this.remote),
|
|
2610
|
+
dirs: new Set(this.syncedDirs)
|
|
2611
|
+
};
|
|
2612
|
+
}
|
|
2613
|
+
/**
|
|
2614
|
+
* Whether the changes the server answered with (SnapshotDelta), taken in over what the client took it to hold as of
|
|
2615
|
+
* `since` (`base`), make the hub's digest as of their seq (`held`), so they are the hub's whole difference; `differs`
|
|
2616
|
+
* when they do not, as when the server's storage was put back to an earlier point; `empty` when they leave no file,
|
|
2617
|
+
* which a whole snapshot tells apart from a server without the hub. A `dir`, `file` or `blob` replaces whatever kind
|
|
2618
|
+
* the path held; `removed`, `unknown` and `dirRemoved` leave nothing there.
|
|
2619
|
+
*/
|
|
2620
|
+
changesHold(delta, base) {
|
|
2621
|
+
const files = /* @__PURE__ */ new Map();
|
|
2622
|
+
for (const [path, file] of base.files) files.set(path, isBlob(file) ? file.hash : this.hashOf(path, file));
|
|
2623
|
+
for (const [path, hash] of base.remote) files.set(path, hash);
|
|
2624
|
+
const dirs = new Set(base.dirs);
|
|
2625
|
+
for (const entry of delta.files) {
|
|
2626
|
+
const { path } = entry;
|
|
2627
|
+
if (entry.kind === "file") files.set(path, contentHash(entry.text));
|
|
2628
|
+
else if (entry.kind === "blob" || entry.kind === "listed") files.set(path, entry.hash);
|
|
2629
|
+
else files.delete(path);
|
|
2630
|
+
if (entry.kind === "dir") dirs.add(path);
|
|
2631
|
+
else dirs.delete(path);
|
|
2632
|
+
}
|
|
2633
|
+
if (hubDigest(files, dirs) !== delta.digest) return "differs";
|
|
2634
|
+
return [...files.keys()].some((path) => this.syncs(path)) ? "held" : "empty";
|
|
2635
|
+
}
|
|
2636
|
+
/**
|
|
2637
|
+
* Take in what changed since the `since` a snapshot named (SnapshotDelta), once its digest showed that the client's
|
|
2638
|
+
* copy with these changes is the hub as of their seq (changesHold). Only what the answer names changes: a path it does
|
|
2639
|
+
* not name holds what the client took the server to hold (`base`), so nothing is made of a file here the answer does
|
|
2640
|
+
* not name, as a whole snapshot makes of one it does not name. The changes are taken in in the order the server puts
|
|
2641
|
+
* a hub back in (docs/sync-protocol.md "增量快照"): files removed, a file that turned into a directory among them;
|
|
2642
|
+
* then directories removed, the deepest first, a directory that turned into a file among them; then directories made,
|
|
2643
|
+
* the shallowest first; then files written, and binary files downloaded. When a directory comes or goes, each step
|
|
2644
|
+
* waits for the one before to be done on disk, at most CHANGES_STEP_MS. An `unknown` path is not removed: a file here
|
|
2645
|
+
* there is new to the server (receive). A path the answer does not name keeps the version `base` holds as of the
|
|
2646
|
+
* answer's seq, as a whole snapshot's file left out as held does: a broadcast that changed it meanwhile and is no
|
|
2647
|
+
* newer is undone, as it was undone on the server too, and an older one arriving later is not taken in. Returns the
|
|
2648
|
+
* listed files to download, and how many texts came.
|
|
2649
|
+
*/
|
|
2650
|
+
async takeChanges(delta, base, held) {
|
|
2651
|
+
const { seq } = delta;
|
|
2652
|
+
const wanted = /* @__PURE__ */ new Set();
|
|
2653
|
+
let sent = 0;
|
|
2654
|
+
const depth = (path) => path.split("/").length;
|
|
2655
|
+
const named = new Set(delta.files.map((entry) => entry.path));
|
|
2656
|
+
for (const [path, file] of base.files) {
|
|
2657
|
+
if (named.has(path) || (this.fileSeq.get(path) ?? -1) > seq) continue;
|
|
2658
|
+
if (sameServerFile(this.cloud.get(path), file)) this.fileSeq.set(path, seq);
|
|
2659
|
+
else this.receive(isBlob(file) ? {
|
|
2660
|
+
kind: "blob",
|
|
2661
|
+
path,
|
|
2662
|
+
hash: file.hash,
|
|
2663
|
+
size: file.size ?? 0
|
|
2664
|
+
} : {
|
|
2665
|
+
kind: "file",
|
|
2666
|
+
path,
|
|
2667
|
+
text: file
|
|
2668
|
+
}, seq);
|
|
2669
|
+
}
|
|
2670
|
+
for (const [path, hash] of base.remote) {
|
|
2671
|
+
if (named.has(path) || (this.fileSeq.get(path) ?? -1) > seq) continue;
|
|
2672
|
+
if (this.remote.get(path) === hash) this.fileSeq.set(path, seq);
|
|
2673
|
+
else this.keepRemote(path, hash, seq);
|
|
2674
|
+
}
|
|
2675
|
+
for (const path of base.dirs) {
|
|
2676
|
+
if (named.has(path) || (this.dirSeq.get(path) ?? -1) > seq) continue;
|
|
2677
|
+
if (this.syncedDirs.has(path)) this.dirSeq.set(path, seq);
|
|
2678
|
+
else this.receiveDir(path, seq, false);
|
|
2679
|
+
}
|
|
2680
|
+
const gone = delta.files.flatMap((entry) => entry.kind === "dirRemoved" || (entry.kind === "file" || entry.kind === "blob" || entry.kind === "listed") && this.syncedDirs.has(entry.path) ? [entry.path] : []);
|
|
2681
|
+
const made = delta.files.flatMap((entry) => entry.kind === "dir" ? [entry.path] : []);
|
|
2682
|
+
const steps = gone.length > 0 || made.length > 0;
|
|
2683
|
+
const done = (work) => steps && work.length > 0 ? Promise.race([Promise.allSettled(work), sleep$1(CHANGES_STEP_MS)]) : Promise.resolve();
|
|
2684
|
+
const removed = [];
|
|
2685
|
+
const isFile = (path) => (this.cloud.get(path) ?? null) !== null || this.remote.has(path);
|
|
2686
|
+
for (const entry of delta.files) {
|
|
2687
|
+
if (!this.syncs(entry.path)) continue;
|
|
2688
|
+
if (entry.kind === "removed" || entry.kind === "unknown") this.receive(entry, seq);
|
|
2689
|
+
else if (entry.kind === "dir" && isFile(entry.path)) this.receive({
|
|
2690
|
+
kind: "removed",
|
|
2691
|
+
path: entry.path
|
|
2692
|
+
}, seq);
|
|
2693
|
+
else continue;
|
|
2694
|
+
removed.push(entry.path);
|
|
2695
|
+
}
|
|
2696
|
+
await done(removed.map((path) => this.chains.get(path)));
|
|
2697
|
+
if (this.closing) return {
|
|
2698
|
+
wanted,
|
|
2699
|
+
sent
|
|
2700
|
+
};
|
|
2701
|
+
const rmdirs = [];
|
|
2702
|
+
for (const path of gone.sort((a, b) => depth(b) - depth(a) || comparePaths(a, b))) rmdirs.push(this.receiveDirRemoved(path, seq));
|
|
2703
|
+
await done(rmdirs);
|
|
2704
|
+
if (this.closing) return {
|
|
2705
|
+
wanted,
|
|
2706
|
+
sent
|
|
2707
|
+
};
|
|
2708
|
+
const mkdirs = [];
|
|
2709
|
+
for (const path of made.sort((a, b) => depth(a) - depth(b) || comparePaths(a, b))) mkdirs.push(this.receiveDir(path, seq, false));
|
|
2710
|
+
await done(mkdirs);
|
|
2711
|
+
if (this.closing) return {
|
|
2712
|
+
wanted,
|
|
2713
|
+
sent
|
|
2714
|
+
};
|
|
2715
|
+
for (const entry of delta.files) {
|
|
2716
|
+
if (!this.syncs(entry.path)) continue;
|
|
2717
|
+
if (entry.kind === "listed") {
|
|
2718
|
+
if (this.mayStay(entry.path, held)) this.keepRemote(entry.path, entry.hash, seq);
|
|
2719
|
+
else {
|
|
2720
|
+
wanted.add(entry.path);
|
|
2721
|
+
this.awaiting.add(entry.path);
|
|
2722
|
+
if (seq > (this.fileSeq.get(entry.path) ?? -1)) this.fileSeq.set(entry.path, seq);
|
|
2723
|
+
}
|
|
2724
|
+
} else if (entry.kind === "file" || entry.kind === "blob") {
|
|
2725
|
+
if (entry.kind === "file") sent++;
|
|
2726
|
+
this.receive(entry, seq);
|
|
2727
|
+
}
|
|
2728
|
+
}
|
|
2729
|
+
return {
|
|
2730
|
+
wanted,
|
|
2731
|
+
sent
|
|
2732
|
+
};
|
|
2733
|
+
}
|
|
2734
|
+
/**
|
|
2524
2735
|
* Ask the server whether it keeps binary files (blobSupport), once per connection, before its snapshot. A probe that
|
|
2525
2736
|
* fails leaves what the last one found, null on a first connection, so the texts sync all the same and binary files
|
|
2526
2737
|
* wait as they are here: the snapshot is taken again after PROBE_RETRY_MS, doubling up to PROBE_RETRY_MAX_MS while it
|
|
@@ -2532,6 +2743,7 @@ var SyncClient = class extends EventEmitter {
|
|
|
2532
2743
|
await this.askBlobs();
|
|
2533
2744
|
} finally {
|
|
2534
2745
|
if (had !== true && this.blobSupport === true) this.fileSeq.clear();
|
|
2746
|
+
if (had !== this.blobSupport) this.holdsHub = false;
|
|
2535
2747
|
}
|
|
2536
2748
|
}
|
|
2537
2749
|
async askBlobs() {
|
|
@@ -5596,7 +5808,7 @@ function callIn(body) {
|
|
|
5596
5808
|
/** Serve `client` to this machine's mainflow commands until closed. */
|
|
5597
5809
|
async function serveIpc(dirs, client) {
|
|
5598
5810
|
const secret = randomBytes(16).toString("hex");
|
|
5599
|
-
const server = createServer(async (req, res) => {
|
|
5811
|
+
const server = process.getBuiltinModule("node:http").createServer(async (req, res) => {
|
|
5600
5812
|
const reply = (status, value) => res.writeHead(status, { "content-type": "application/json" }).end(JSON.stringify(value));
|
|
5601
5813
|
if (req.headers["x-mainflow-sync"] !== secret) return reply(403, { error: "wrong secret" });
|
|
5602
5814
|
const chunks = [];
|