mainflow 0.43.0 → 0.44.0-alpha.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +7 -5
- package/dist/{dist-D5rZ0MQH.js → dist-CXrUfRQp.js} +111 -10
- package/dist/facts.d.ts +22 -5
- package/dist/facts.js +28 -7
- package/dist/grant-check.d.ts +45 -0
- package/dist/grant-check.js +572 -0
- package/dist/grant-commands.d.ts +31 -3
- package/dist/grant-commands.js +14 -4
- package/dist/hooks.d.ts +2 -18
- package/dist/hooks.js +103 -15
- package/dist/{host-BJvqZy-i.js → host-C74j2Qox.js} +174 -57
- package/dist/host.d.ts +1 -1
- package/dist/host.js +1 -1
- package/dist/{index-DRpamPge.d.ts → index-jq7lN8V8.d.ts} +9 -3
- package/dist/{index-BqPQp6TF.d.ts → index-l8QUG80k.d.ts} +11 -0
- package/dist/launch.d.ts +1 -1
- package/dist/launch.js +2 -2
- package/dist/native/mainflow-swap-darwin-arm64 +0 -0
- package/dist/native/mainflow-swap-darwin-x64 +0 -0
- package/dist/operation-BroV98VB.d.ts +19 -0
- package/dist/{operation-BhBbGauo.js → operation-CUeOFEaf.js} +504 -151
- package/dist/{ui-xxzmhAqF.js → ui-qqVCewFz.js} +3 -3
- package/package.json +1 -1
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { At as hubRequirements, B as applyOffline, Ct as comparePaths, Dt as textBytes, Et as requestTime, G as scopeOfPath, H as renumberFiles, J as syncedDir, Jt as nodeStorage, K as hubDigest, St as VERSION_HEADER, Tt as readsAsText, U as renumberPath, Ut as registeredName, V as renumberRequest, W as renumberText, X as applyOperation, Y as syncedPath, _t as MAX_BLOB_BYTES, an as splitFrontMatter, bt as ROUTES, ft as nowMinute, gt as BLOB_HASH, ht as ACCESS_HEADERS, in as contentHash, jt as versionMismatch, q as mayHoldSynced, rn as WriteError, tn as ConflictError, wt as exceedsFileLimit, xt as SOCKET_MAX_AGE_CLOSE, yt as MAX_FILE_BYTES } from "./host-C74j2Qox.js";
|
|
2
2
|
import { t as CLI_VERSION } from "./version-Dvv4jwF1.js";
|
|
3
3
|
import { createHash, randomBytes } from "node:crypto";
|
|
4
4
|
import { accessSync, appendFileSync, chmodSync, closeSync, constants, copyFileSync, createReadStream, createWriteStream, existsSync, linkSync, lstatSync, mkdirSync, openSync, readFileSync, readSync, readdirSync, renameSync, rmSync, rmdirSync, statSync, watch, writeFileSync } from "node:fs";
|
|
@@ -8,8 +8,6 @@ import { execFileSync, spawn, spawnSync } from "node:child_process";
|
|
|
8
8
|
import { parse as parse$1 } from "yaml";
|
|
9
9
|
import { homedir } from "node:os";
|
|
10
10
|
import { createBrotliDecompress, createGunzip, createInflate, createZstdDecompress } from "node:zlib";
|
|
11
|
-
import { Agent, createServer, request } from "node:http";
|
|
12
|
-
import { Agent as Agent$1, request as request$1 } from "node:https";
|
|
13
11
|
import { Readable, Transform } from "node:stream";
|
|
14
12
|
import { pipeline } from "node:stream/promises";
|
|
15
13
|
import { EventEmitter } from "node:events";
|
|
@@ -253,6 +251,12 @@ function decoded(res) {
|
|
|
253
251
|
}
|
|
254
252
|
return decode ? res.pipe(decode()) : res;
|
|
255
253
|
}
|
|
254
|
+
/**
|
|
255
|
+
* node:http and node:https load node:tls, which reads the system certificates when NODE_USE_SYSTEM_CA is set and
|
|
256
|
+
* costs about 220 ms. Load them on the first request so commands that never reach the server do not pay for it.
|
|
257
|
+
*/
|
|
258
|
+
const http = () => process.getBuiltinModule("node:http");
|
|
259
|
+
const https = () => process.getBuiltinModule("node:https");
|
|
256
260
|
var ServerApi = class {
|
|
257
261
|
settings;
|
|
258
262
|
timeoutMs;
|
|
@@ -260,14 +264,14 @@ var ServerApi = class {
|
|
|
260
264
|
* The connection writes go on, kept open between them: opening one costs two round trips, for TCP and TLS. One
|
|
261
265
|
* connection at a time, since the client sends one write at a time; a request waits for the one before it.
|
|
262
266
|
*/
|
|
263
|
-
|
|
267
|
+
writeAgent;
|
|
264
268
|
/** The connections snapshots go on, apart from the writes', so a snapshot never waits behind a write or holds one up */
|
|
265
|
-
|
|
269
|
+
readAgent;
|
|
266
270
|
/**
|
|
267
271
|
* The connections binary files' contents go up and down on, apart from the writes' and the snapshots', so a large file
|
|
268
272
|
* holds neither up; a few at a time, as the files of a snapshot are downloaded side by side.
|
|
269
273
|
*/
|
|
270
|
-
|
|
274
|
+
transferAgent;
|
|
271
275
|
/** Requests on `agent` not answered yet */
|
|
272
276
|
busy = 0;
|
|
273
277
|
transfer = {
|
|
@@ -277,28 +281,30 @@ var ServerApi = class {
|
|
|
277
281
|
body: 0,
|
|
278
282
|
sent: 0,
|
|
279
283
|
listed: 0,
|
|
280
|
-
unchanged: 0
|
|
284
|
+
unchanged: 0,
|
|
285
|
+
changes: 0
|
|
281
286
|
};
|
|
282
287
|
constructor(settings, timeoutMs = 3e4) {
|
|
283
288
|
this.settings = settings;
|
|
284
289
|
this.timeoutMs = timeoutMs;
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
maxFreeSockets: 1
|
|
290
|
-
});
|
|
291
|
-
this.reads = new Agent$2({
|
|
292
|
-
keepAlive: true,
|
|
293
|
-
maxSockets: 2,
|
|
294
|
-
maxFreeSockets: 1
|
|
295
|
-
});
|
|
296
|
-
this.transfers = new Agent$2({
|
|
290
|
+
}
|
|
291
|
+
makeAgent(maxSockets) {
|
|
292
|
+
const { Agent } = this.settings.url.startsWith("https:") ? https() : http();
|
|
293
|
+
return new Agent({
|
|
297
294
|
keepAlive: true,
|
|
298
|
-
maxSockets
|
|
295
|
+
maxSockets,
|
|
299
296
|
maxFreeSockets: 1
|
|
300
297
|
});
|
|
301
298
|
}
|
|
299
|
+
get agent() {
|
|
300
|
+
return this.writeAgent ??= this.makeAgent(1);
|
|
301
|
+
}
|
|
302
|
+
get reads() {
|
|
303
|
+
return this.readAgent ??= this.makeAgent(2);
|
|
304
|
+
}
|
|
305
|
+
get transfers() {
|
|
306
|
+
return this.transferAgent ??= this.makeAgent(4);
|
|
307
|
+
}
|
|
302
308
|
/**
|
|
303
309
|
* The service token's headers, and the CLI's version when the settings name it (VERSION_HEADER), which every request
|
|
304
310
|
* and the WebSocket's upgrade carry. Never logged.
|
|
@@ -380,7 +386,7 @@ var ServerApi = class {
|
|
|
380
386
|
* the raw bytes of a stream, or nothing; `headers` add to the token's and say what the body is.
|
|
381
387
|
*/
|
|
382
388
|
open(method, url, body, headers, agent, timeoutMs) {
|
|
383
|
-
const send = url.startsWith("https:") ? request
|
|
389
|
+
const send = url.startsWith("https:") ? https().request : http().request;
|
|
384
390
|
return new Promise((resolve, reject) => {
|
|
385
391
|
const req = send(url, {
|
|
386
392
|
method,
|
|
@@ -406,7 +412,7 @@ var ServerApi = class {
|
|
|
406
412
|
"content-type": "application/json",
|
|
407
413
|
"content-length": String(Buffer.byteLength(body))
|
|
408
414
|
} : {};
|
|
409
|
-
if (agent === this.
|
|
415
|
+
if (agent === this.writeAgent) this.busy++;
|
|
410
416
|
try {
|
|
411
417
|
const res = await this.open(method, url, body, {
|
|
412
418
|
...kind,
|
|
@@ -434,7 +440,7 @@ var ServerApi = class {
|
|
|
434
440
|
stream.on("error", reject);
|
|
435
441
|
});
|
|
436
442
|
} finally {
|
|
437
|
-
if (agent === this.
|
|
443
|
+
if (agent === this.writeAgent) this.busy--;
|
|
438
444
|
}
|
|
439
445
|
}
|
|
440
446
|
/**
|
|
@@ -553,9 +559,9 @@ var ServerApi = class {
|
|
|
553
559
|
}
|
|
554
560
|
/** Close the kept connections; a later request opens others. */
|
|
555
561
|
close() {
|
|
556
|
-
this.
|
|
557
|
-
this.
|
|
558
|
-
this.
|
|
562
|
+
this.writeAgent?.destroy();
|
|
563
|
+
this.readAgent?.destroy();
|
|
564
|
+
this.transferAgent?.destroy();
|
|
559
565
|
}
|
|
560
566
|
/** Send one request; the response is the hub's answer, applied or refused. */
|
|
561
567
|
async send(request, options = {}) {
|
|
@@ -568,26 +574,13 @@ var ServerApi = class {
|
|
|
568
574
|
this.transfer.batches++;
|
|
569
575
|
this.transfer.files += body.files.filter((entry) => entry.kind === "file").length;
|
|
570
576
|
this.transfer.listed += body.files.filter((entry) => entry.kind === "listed").length;
|
|
571
|
-
if (body.unchanged === true) this.transfer.unchanged++;
|
|
577
|
+
if ("unchanged" in body && body.unchanged === true) this.transfer.unchanged++;
|
|
578
|
+
if ("changes" in body && body.changes === true) this.transfer.changes++;
|
|
572
579
|
this.transfer.wire += wire;
|
|
573
580
|
this.transfer.body += size;
|
|
574
581
|
this.transfer.sent += sent;
|
|
575
582
|
return body;
|
|
576
583
|
}
|
|
577
|
-
/**
|
|
578
|
-
* The hub's snapshot, batch by batch (docs/sync-protocol.md "快照與整份上傳"). `have` gives the content hash of the
|
|
579
|
-
* text the client holds of each file, so the server sends only the files that differ, and names the ones of `have` it
|
|
580
|
-
* does not hold as removed; each batch is sent only the part of `have` in its own range. A path of `have` that no
|
|
581
|
-
* batch mentions is a file the server holds with that text. The batches are in order, each with its seq; one whose seq
|
|
582
|
-
* is below the batch before, as when the server's storage was replaced meanwhile, starts the snapshot over. A path of
|
|
583
|
-
* `have` the server never held a file at comes as `unknown` rather than `removed`, from a server that tells them apart.
|
|
584
|
-
* `listOnly` asks for the files under those prefixes as `listed`, without their text. A part of the hub alone is taken
|
|
585
|
-
* from `after` on, and ends with the batch that covers `through`; only the paths of that range are told of. `blobs` takes
|
|
586
|
-
* the binary files too, each that differs from `have` as `blob`; `have` names a binary file by its sha256 then.
|
|
587
|
-
* `digest` names the digest of `have` and the directories the client holds (hubDigest), asked on the first batch of a
|
|
588
|
-
* whole hub's snapshot with `blobs`: a server whose hub it is the digest of answers that nothing differs, and the
|
|
589
|
-
* snapshot is that one batch, with the client's directories as the entries, as the server would have listed them.
|
|
590
|
-
*/
|
|
591
584
|
async snapshot(have = /* @__PURE__ */ new Map(), options = {}) {
|
|
592
585
|
const paths = [...have.keys()].sort(comparePaths);
|
|
593
586
|
const { through } = options;
|
|
@@ -599,15 +592,26 @@ var ServerApi = class {
|
|
|
599
592
|
while (after !== void 0 && start < paths.length && comparePaths(paths[start], after) <= 0) start++;
|
|
600
593
|
const range = Object.fromEntries(paths.slice(start).map((path) => [path, have.get(path)]));
|
|
601
594
|
const asking = options.digest !== void 0 && options.blobs === true && after === void 0;
|
|
602
|
-
const
|
|
595
|
+
const since = options.since !== void 0 && options.blobs === true && after === void 0 && attempt === 1 ? options.since : void 0;
|
|
596
|
+
const answer = await this.batch({
|
|
603
597
|
protocol: 3,
|
|
604
598
|
have: range,
|
|
605
599
|
unknown: true,
|
|
606
600
|
...after === void 0 ? {} : { after },
|
|
607
601
|
...asking ? { digest: options.digest.value } : {},
|
|
602
|
+
...since === void 0 ? {} : { since },
|
|
608
603
|
...options.listOnly?.length ? { listOnly: options.listOnly } : {},
|
|
609
604
|
...options.blobs ? { blobs: true } : {}
|
|
610
605
|
});
|
|
606
|
+
if ("changes" in answer && answer.changes === true) {
|
|
607
|
+
if (since === void 0) throw new Unreachable("the sync server answered a snapshot that named no since with what changed since", true);
|
|
608
|
+
return {
|
|
609
|
+
seq: answer.seq,
|
|
610
|
+
files: answer.files,
|
|
611
|
+
digest: answer.digest
|
|
612
|
+
};
|
|
613
|
+
}
|
|
614
|
+
const { seq, files, next, unchanged } = answer;
|
|
611
615
|
if (unchanged === true && asking) return [{
|
|
612
616
|
seq,
|
|
613
617
|
files: options.digest.dirs.map((path) => ({
|
|
@@ -916,6 +920,11 @@ const readKnownDirs = (hub) => {
|
|
|
916
920
|
const dirs = readJson(syncPaths(hub).known, {}).dirs;
|
|
917
921
|
return Array.isArray(dirs) ? dirs.filter((dir) => typeof dir === "string") : [];
|
|
918
922
|
};
|
|
923
|
+
/** The hub's seq known.json names (KnownFile.seq); undefined when it names none, or none that is a seq. */
|
|
924
|
+
const readKnownSeq = (hub) => {
|
|
925
|
+
const seq = readJson(syncPaths(hub).known, {}).seq;
|
|
926
|
+
return Number.isInteger(seq) && seq >= 0 ? seq : void 0;
|
|
927
|
+
};
|
|
919
928
|
/** What known.json writes before a binary file's sha256, apart from a text's content hash. */
|
|
920
929
|
const BLOB_PREFIX = "blob:";
|
|
921
930
|
/** The text of the version known.json names by `hash`; undefined when it was not kept, as by a sync client before known/. */
|
|
@@ -932,9 +941,9 @@ function readKnownText(dirs, hash) {
|
|
|
932
941
|
* there yet; the texts of versions it no longer names are removed. A version whose text the client does not have keeps
|
|
933
942
|
* the text known/ holds, if any. The new texts go first, then known.json, then the removals, so a client killed at any
|
|
934
943
|
* step leaves a known.json whose every text is in known/. A binary file is given as its sha256 with BLOB_PREFIX, and no text.
|
|
935
|
-
* `dirs` are the directories it names.
|
|
944
|
+
* `dirs` are the directories it names, and `seq` the hub's seq they are as of, written with them (KnownFile.seq).
|
|
936
945
|
*/
|
|
937
|
-
function writeKnown(hub, files, dirs = []) {
|
|
946
|
+
function writeKnown(hub, files, dirs = [], seq) {
|
|
938
947
|
const dir = syncPaths(hub).knownTexts;
|
|
939
948
|
mkdirSync(dir, { recursive: true });
|
|
940
949
|
const named = new Set(Object.values(files).map((file) => file.hash));
|
|
@@ -947,7 +956,8 @@ function writeKnown(hub, files, dirs = []) {
|
|
|
947
956
|
}
|
|
948
957
|
const known = {
|
|
949
958
|
files: Object.fromEntries(Object.entries(files).map(([path, file]) => [path, file.hash])),
|
|
950
|
-
...dirs.length > 0 ? { dirs } : {}
|
|
959
|
+
...dirs.length > 0 ? { dirs } : {},
|
|
960
|
+
...seq === void 0 ? {} : { seq }
|
|
951
961
|
};
|
|
952
962
|
writeJson(syncPaths(hub).known, known);
|
|
953
963
|
for (const name of held) if (!named.has(name)) rmSync(join(dir, name), { force: true });
|
|
@@ -1281,6 +1291,8 @@ function exists(file) {
|
|
|
1281
1291
|
const isBlob = (file) => typeof file === "object" && file !== null && file.kind === "blob";
|
|
1282
1292
|
/** The hash the server names a file of its by: a text's content hash, a binary file's sha256. */
|
|
1283
1293
|
const serverHash = (file) => typeof file === "string" ? contentHash(file) : file.hash;
|
|
1294
|
+
/** Whether the client holds `held` as the server's version of a path, the same file as `file`. */
|
|
1295
|
+
const sameServerFile = (held, file) => held !== void 0 && held !== null && (typeof held === "string" ? held === file : isBlob(file) && held.hash === file.hash);
|
|
1284
1296
|
/** Whether two tool calls are of one session, the same subagent in it or both of the session itself. */
|
|
1285
1297
|
const sameCaller = (a, b) => a !== void 0 && b !== void 0 && a.session === b.session && a.agent === b.agent;
|
|
1286
1298
|
/** Whether `edit` makes `to` of `from`, as the Edit tool makes it (applyEdit). */
|
|
@@ -1329,6 +1341,12 @@ const BINARY_RECHECK_MS = 150;
|
|
|
1329
1341
|
* program that rewrites a file as `cat >` does empties it first and fills it a moment later
|
|
1330
1342
|
*/
|
|
1331
1343
|
const EMPTY_RECHECK_MS = 300;
|
|
1344
|
+
/**
|
|
1345
|
+
* How long taking in what changed since a seq (takeChanges) waits for one step to be done on disk before the next, when
|
|
1346
|
+
* a path changes kind: the files removed before the directories, those before the directories made, and those before
|
|
1347
|
+
* the files written. A step that takes longer, as a file whose removal waits for the write lock, is not waited out.
|
|
1348
|
+
*/
|
|
1349
|
+
const CHANGES_STEP_MS = 5e3;
|
|
1332
1350
|
/** How long after the exchange the displaced file is read again, for a program that still held it open */
|
|
1333
1351
|
const LATE_WRITE_MS = 150;
|
|
1334
1352
|
/**
|
|
@@ -1639,6 +1657,34 @@ var SyncClient = class extends EventEmitter {
|
|
|
1639
1657
|
dirCheck = null;
|
|
1640
1658
|
/** The hub's seq up to which the client has taken in every change: the last snapshot and the broadcasts after it */
|
|
1641
1659
|
hubSeq = -1;
|
|
1660
|
+
/**
|
|
1661
|
+
* Whether `cloud` and `remote` hold every file the server holds as of hubSeq, binary files included, and syncedDirs
|
|
1662
|
+
* every directory, but for the paths changed after it: a whole snapshot with binary files was taken in this run, and
|
|
1663
|
+
* nothing since made the client unsure of what it holds, as a server that stopped keeping binary files, or a hub
|
|
1664
|
+
* reloaded. Only then is a snapshot asked for what changed since hubSeq (`since`), as a path the answer does not name
|
|
1665
|
+
* holds what the client takes the server to hold. A run after it starts from known.json instead, which names the seq
|
|
1666
|
+
* (knownSeq, resumeSeq).
|
|
1667
|
+
*/
|
|
1668
|
+
holdsHub = false;
|
|
1669
|
+
/**
|
|
1670
|
+
* How many reload broadcasts came (the hub replaced, migrated or restored): a snapshot under way when one comes does
|
|
1671
|
+
* not leave the client holding the hub (holdsHub), whatever it took in, so the next one is whole.
|
|
1672
|
+
*/
|
|
1673
|
+
reloads = 0;
|
|
1674
|
+
/**
|
|
1675
|
+
* The hub's seq known.json named when this run started (readKnownSeq): the files it names, its directories and
|
|
1676
|
+
* remote.json are the hub as of it, so the run's first snapshot asks for what changed since, as one after a reconnect
|
|
1677
|
+
* does (holdsHub). Undefined once a snapshot was taken, or when known.json names none or the run cannot take it, as
|
|
1678
|
+
* when a text it names is not in known/.
|
|
1679
|
+
*/
|
|
1680
|
+
resumeSeq;
|
|
1681
|
+
/**
|
|
1682
|
+
* For each path, a seq at or before which the hub held another version than the one the client knows of now (in
|
|
1683
|
+
* `cloud`, or `remote`): a broadcast's own seq, or, for a snapshot's, one after the seq of the version before it. A
|
|
1684
|
+
* path whose known.json version lags behind is changed on the server after the seq known.json is written with
|
|
1685
|
+
* (knownSeq), so what changed since names it.
|
|
1686
|
+
*/
|
|
1687
|
+
changedAt = /* @__PURE__ */ new Map();
|
|
1642
1688
|
/** The seqs of the broadcasts on the current connection above hubSeq with a change between that never arrived */
|
|
1643
1689
|
ahead = /* @__PURE__ */ new Set();
|
|
1644
1690
|
waiters = /* @__PURE__ */ new Set();
|
|
@@ -1836,6 +1882,7 @@ var SyncClient = class extends EventEmitter {
|
|
|
1836
1882
|
async restoreKnown() {
|
|
1837
1883
|
this.copied = !existsSync(this.paths.known);
|
|
1838
1884
|
this.blobsCompared = !this.copied && !existsSync(this.paths.copyBlobs);
|
|
1885
|
+
this.resumeSeq = this.blobsCompared ? readKnownSeq(this.dirs) : void 0;
|
|
1839
1886
|
for (const dir of readKnownDirs(this.dirs)) this.syncedDirs.add(dir);
|
|
1840
1887
|
for (const [path, hash] of Object.entries(readKnown(this.dirs))) {
|
|
1841
1888
|
if (hash.startsWith("blob:")) {
|
|
@@ -1859,7 +1906,9 @@ var SyncClient = class extends EventEmitter {
|
|
|
1859
1906
|
text,
|
|
1860
1907
|
server: text
|
|
1861
1908
|
});
|
|
1909
|
+
else if (this.syncs(path)) this.resumeSeq = void 0;
|
|
1862
1910
|
}
|
|
1911
|
+
if (this.resumeSeq !== void 0) this.log(`known.json is of the hub's seq ${this.resumeSeq}: the first snapshot asks for what changed since`);
|
|
1863
1912
|
}
|
|
1864
1913
|
/**
|
|
1865
1914
|
* The refusals a run killed before it settled them left in the outbox (src/state.ts): each text not saved yet is saved,
|
|
@@ -2091,12 +2140,15 @@ var SyncClient = class extends EventEmitter {
|
|
|
2091
2140
|
if (seq === this.hubSeq + 1) this.takenUpTo(seq);
|
|
2092
2141
|
else if (seq > this.hubSeq + 1) this.ahead.add(seq);
|
|
2093
2142
|
if ("reload" in message && message.reload) {
|
|
2143
|
+
this.reloads++;
|
|
2144
|
+
this.holdsHub = false;
|
|
2145
|
+
this.resumeSeq = void 0;
|
|
2094
2146
|
this.track(this.resync());
|
|
2095
2147
|
return;
|
|
2096
2148
|
}
|
|
2097
2149
|
for (const event of message.files) if (event.kind === "dir") this.receiveDir(event.path, seq, true);
|
|
2098
2150
|
else if (event.kind === "dirRemoved") this.receiveDirRemoved(event.path, seq);
|
|
2099
|
-
else this.receive(event, seq);
|
|
2151
|
+
else this.receive(event, seq, true);
|
|
2100
2152
|
if (this.ahead.size > 0 && this.synced && this.resyncing === null) {
|
|
2101
2153
|
this.log(`missed the hub's changes ${this.hubSeq + 1} to ${Math.min(...this.ahead) - 1}; reloading the snapshot`);
|
|
2102
2154
|
this.track(this.resync());
|
|
@@ -2117,7 +2169,7 @@ var SyncClient = class extends EventEmitter {
|
|
|
2117
2169
|
* that may stay on the server, as a new screenshot of a closed mainline, goes into `remote` instead of being downloaded,
|
|
2118
2170
|
* as a listed text does in a snapshot.
|
|
2119
2171
|
*/
|
|
2120
|
-
receive(event, seq) {
|
|
2172
|
+
receive(event, seq, live = false) {
|
|
2121
2173
|
if (!this.syncs(event.path)) return;
|
|
2122
2174
|
const file = event.kind === "file" ? event.text : event.kind === "blob" ? {
|
|
2123
2175
|
kind: "blob",
|
|
@@ -2130,6 +2182,7 @@ var SyncClient = class extends EventEmitter {
|
|
|
2130
2182
|
this.trace(`older ${event.path} ${label} seq ${seq}, holding seq ${held}`);
|
|
2131
2183
|
return;
|
|
2132
2184
|
}
|
|
2185
|
+
this.noteChange(event.path, file === null ? null : serverHash(file), live ? seq : held + 1);
|
|
2133
2186
|
this.fileSeq.set(event.path, seq);
|
|
2134
2187
|
if (this.downloadErrors.has(event.path)) {
|
|
2135
2188
|
const before = this.cloud.get(event.path);
|
|
@@ -2175,7 +2228,7 @@ var SyncClient = class extends EventEmitter {
|
|
|
2175
2228
|
return;
|
|
2176
2229
|
}
|
|
2177
2230
|
this.knowDir(path);
|
|
2178
|
-
this.track(mkdir(dir, { recursive: true }));
|
|
2231
|
+
return this.track(mkdir(dir, { recursive: true }));
|
|
2179
2232
|
}
|
|
2180
2233
|
/**
|
|
2181
2234
|
* The server removed the directory at `path` as of the hub's `seq`, as a broadcast tells, or a snapshot that no longer
|
|
@@ -2188,7 +2241,7 @@ var SyncClient = class extends EventEmitter {
|
|
|
2188
2241
|
this.dirSeq.set(path, seq);
|
|
2189
2242
|
this.forgetDir(path);
|
|
2190
2243
|
this.trace(`dir ${path} removed seq ${seq}`);
|
|
2191
|
-
this.track(this.whenFree(() => this.locked(() => this.removeEmptyDir(path))).catch((error) => this.log(`${path}/: ${error.message}`)));
|
|
2244
|
+
return this.track(this.whenFree(() => this.locked(() => this.removeEmptyDir(path))).catch((error) => this.log(`${path}/: ${error.message}`)));
|
|
2192
2245
|
}
|
|
2193
2246
|
/** Remove the directory at `path` when nothing is in it; one gone already, or holding anything, stays as it is. True when it went. */
|
|
2194
2247
|
removeEmptyDir(path) {
|
|
@@ -2211,6 +2264,16 @@ var SyncClient = class extends EventEmitter {
|
|
|
2211
2264
|
forgetDir(path) {
|
|
2212
2265
|
if (this.syncedDirs.delete(path)) this.saveKnown();
|
|
2213
2266
|
}
|
|
2267
|
+
/** The hash the client knows the server's file at `path` by: of its version in `cloud`, or `remote`; null for none. */
|
|
2268
|
+
serverView(path) {
|
|
2269
|
+
const file = this.cloud.get(path);
|
|
2270
|
+
if (file !== void 0 && file !== null) return serverHash(file);
|
|
2271
|
+
return this.remote.get(path) ?? null;
|
|
2272
|
+
}
|
|
2273
|
+
/** The server's file at `path` is to be known by `hash` (null for none), a change of the hub at seq `at` or later (changedAt). */
|
|
2274
|
+
noteChange(path, hash, at) {
|
|
2275
|
+
if (this.serverView(path) !== hash) this.changedAt.set(path, at);
|
|
2276
|
+
}
|
|
2214
2277
|
/** Whether the client holds `path` as of the hub's `seq` or later. */
|
|
2215
2278
|
reached(path, seq) {
|
|
2216
2279
|
return this.hubSeq >= seq || (this.fileSeq.get(path) ?? -1) >= seq;
|
|
@@ -2248,12 +2311,13 @@ var SyncClient = class extends EventEmitter {
|
|
|
2248
2311
|
clearTimeout(timer);
|
|
2249
2312
|
}
|
|
2250
2313
|
/**
|
|
2251
|
-
* Take the server's
|
|
2252
|
-
* does a change still missing among the broadcasts once the snapshot
|
|
2253
|
-
* reconciled: the server broadcasts a change after storing it, so a
|
|
2314
|
+
* Take the server's hub, or only what changed in it since hubSeq (takeSnapshot), and reconcile every path; a request
|
|
2315
|
+
* made while one runs runs once more after it. So does a change still missing among the broadcasts once the snapshot
|
|
2316
|
+
* is in, including ones that came while it was reconciled: the server broadcasts a change after storing it, so a
|
|
2317
|
+
* snapshot asked for later holds it. One that
|
|
2254
2318
|
* still does not, and brought nothing newer, is asked for again after a pause; after STALLED_SNAPSHOTS of them the
|
|
2255
2319
|
* server is taken to have lost those changes, as when its storage was replaced: the versions the client holds from
|
|
2256
|
-
* after the gap are forgotten and the snapshot is taken once more, as it is. The connection has its
|
|
2320
|
+
* after the gap are forgotten and the snapshot is taken once more, whole, as it is. The connection has its
|
|
2257
2321
|
* snapshot once one ends while it is open (I1); one that fails leaves it without, and the client reconnects to take a
|
|
2258
2322
|
* fresh one, rather than send on the snapshot of an earlier connection.
|
|
2259
2323
|
*/
|
|
@@ -2276,6 +2340,7 @@ var SyncClient = class extends EventEmitter {
|
|
|
2276
2340
|
if (stalled >= STALLED_SNAPSHOTS) {
|
|
2277
2341
|
this.log(`the sync server's snapshot stays without the hub's changes ${missing}; taking it as complete`);
|
|
2278
2342
|
this.ahead.clear();
|
|
2343
|
+
this.holdsHub = false;
|
|
2279
2344
|
for (const [path, seq] of this.fileSeq) if (seq > this.hubSeq) this.fileSeq.delete(path);
|
|
2280
2345
|
for (const [path, seq] of this.dirSeq) if (seq > this.hubSeq) this.dirSeq.delete(path);
|
|
2281
2346
|
stalled = 0;
|
|
@@ -2346,12 +2411,25 @@ var SyncClient = class extends EventEmitter {
|
|
|
2346
2411
|
* upload (forgetBinaries), so they are files never synced here, sent as new even when this snapshot fails or the
|
|
2347
2412
|
* client is killed; and in the snapshot after the upload, a binary file of the client's the server names removed, as
|
|
2348
2413
|
* when it once held one there, is taken as one it never held (unheld), also in a run that started without known.json.
|
|
2414
|
+
*
|
|
2415
|
+
* Once a whole snapshot with binary files was taken in this run (holdsHub), the client holds every file and directory
|
|
2416
|
+
* the server holds as of hubSeq, and the next snapshot, after a reconnect or a missed broadcast, asks for what changed
|
|
2417
|
+
* since hubSeq alone (`since`). A server whose change log tells answers with the paths changed (SnapshotDelta): taken
|
|
2418
|
+
* in over what the client holds once they make the hub's digest (changesHold, takeChanges), and else dropped for the
|
|
2419
|
+
* whole snapshot. A server that cannot tell, or one from before it, answers the first batch, and the snapshot goes on
|
|
2420
|
+
* whole as ever. A run's first snapshot asks the same way from the seq known.json names (resumeSeq), taking known.json,
|
|
2421
|
+
* remote.json and the directories known.json names for what the server held as of it (knownCopy).
|
|
2349
2422
|
*/
|
|
2350
2423
|
takeSnapshot() {
|
|
2351
2424
|
return this.exclusive(() => this.snapshotNow());
|
|
2352
2425
|
}
|
|
2353
2426
|
async snapshotNow() {
|
|
2354
|
-
let parts;
|
|
2427
|
+
let parts = [];
|
|
2428
|
+
/** What changed since `since`, when the server answered with it and it makes the hub's digest */
|
|
2429
|
+
let delta = null;
|
|
2430
|
+
let since;
|
|
2431
|
+
/** What the client took the server to hold as of `since`, which `delta` is taken in over */
|
|
2432
|
+
let deltaBase = null;
|
|
2355
2433
|
let held;
|
|
2356
2434
|
let heldBlobs;
|
|
2357
2435
|
let have;
|
|
@@ -2362,10 +2440,25 @@ var SyncClient = class extends EventEmitter {
|
|
|
2362
2440
|
/** Whether the snapshot is asked for with the binary files */
|
|
2363
2441
|
let withBlobs = false;
|
|
2364
2442
|
if (!this.probed) await this.probeBlobs();
|
|
2443
|
+
/** The reload broadcasts that came before this snapshot: one that comes while it runs leaves the next one whole */
|
|
2444
|
+
const reloads = this.reloads;
|
|
2445
|
+
/**
|
|
2446
|
+
* Whether the snapshot may ask for what changed since a seq: hubSeq while the client holds the hub, or the seq
|
|
2447
|
+
* known.json names before the run's first snapshot (resumeSeq). Read once the probe told whether the server keeps
|
|
2448
|
+
* binary files; not once an answer of it failed the digest
|
|
2449
|
+
*/
|
|
2450
|
+
let askChanges = this.holdsHub || this.resumeSeq !== void 0;
|
|
2365
2451
|
for (;;) {
|
|
2366
2452
|
before = this.hubSeq;
|
|
2453
|
+
const from = !askChanges || this.blobSupport !== true ? null : this.holdsHub ? before >= 0 ? before : null : this.resumeSeq ?? null;
|
|
2454
|
+
const base = from === null ? null : this.holdsHub ? this.serverCopy() : this.knownCopy();
|
|
2367
2455
|
({texts: held, blobs: heldBlobs} = await this.holdings());
|
|
2368
|
-
|
|
2456
|
+
let dropped = false;
|
|
2457
|
+
for (const path of this.remote.keys()) {
|
|
2458
|
+
if (!(held.has(path) || heldBlobs.has(path) || this.expected.get(path) !== void 0)) continue;
|
|
2459
|
+
this.remote.delete(path);
|
|
2460
|
+
dropped = true;
|
|
2461
|
+
}
|
|
2369
2462
|
have = new Map([...held].map(([path, text]) => [path, this.hashOf(path, text)]));
|
|
2370
2463
|
for (const [path, blob] of heldBlobs) have.set(path, blob.hash);
|
|
2371
2464
|
for (const [path, hash] of this.remote) have.set(path, hash);
|
|
@@ -2379,11 +2472,25 @@ var SyncClient = class extends EventEmitter {
|
|
|
2379
2472
|
value: hubDigest(have, dirs),
|
|
2380
2473
|
dirs
|
|
2381
2474
|
} : void 0;
|
|
2382
|
-
|
|
2475
|
+
since = base !== null && from !== null && withBlobs && !dropped ? from : void 0;
|
|
2476
|
+
const taken = await this.api.snapshot(have, {
|
|
2383
2477
|
listOnly: this.listOnly(this.copied && !this.synced),
|
|
2384
|
-
blobs:
|
|
2478
|
+
blobs: withBlobs,
|
|
2479
|
+
since,
|
|
2385
2480
|
...digest ? { digest } : {}
|
|
2386
2481
|
});
|
|
2482
|
+
if (!Array.isArray(taken)) {
|
|
2483
|
+
const verdict = this.changesHold(taken, base);
|
|
2484
|
+
if (verdict === "held") {
|
|
2485
|
+
delta = taken;
|
|
2486
|
+
deltaBase = base;
|
|
2487
|
+
break;
|
|
2488
|
+
}
|
|
2489
|
+
this.log(verdict === "differs" ? `what changed since seq ${since} does not make the hub's digest; taking the whole snapshot` : `what changed since seq ${since} leaves the hub without files; taking the whole snapshot`);
|
|
2490
|
+
askChanges = false;
|
|
2491
|
+
continue;
|
|
2492
|
+
}
|
|
2493
|
+
parts = taken;
|
|
2387
2494
|
if (uploaded || holdsFiles(parts, have)) break;
|
|
2388
2495
|
if (this.opts.settings.onDemand === true) {
|
|
2389
2496
|
if (!this.uploadWanted) {
|
|
@@ -2411,7 +2518,7 @@ var SyncClient = class extends EventEmitter {
|
|
|
2411
2518
|
this.backoff = 500;
|
|
2412
2519
|
}
|
|
2413
2520
|
if (this.closing) return;
|
|
2414
|
-
const seq = parts[0].seq;
|
|
2521
|
+
const seq = delta !== null ? delta.seq : parts[0].seq;
|
|
2415
2522
|
if (seq < before) {
|
|
2416
2523
|
this.log(`the sync server's seq went back from ${before} to ${seq}; taking its snapshot as it is`);
|
|
2417
2524
|
this.fileSeq.clear();
|
|
@@ -2419,94 +2526,104 @@ var SyncClient = class extends EventEmitter {
|
|
|
2419
2526
|
this.hubSeq = -1;
|
|
2420
2527
|
this.ahead.clear();
|
|
2421
2528
|
}
|
|
2422
|
-
|
|
2423
|
-
|
|
2424
|
-
|
|
2425
|
-
|
|
2426
|
-
|
|
2427
|
-
|
|
2428
|
-
|
|
2429
|
-
|
|
2430
|
-
|
|
2431
|
-
|
|
2432
|
-
|
|
2433
|
-
|
|
2434
|
-
|
|
2435
|
-
|
|
2436
|
-
|
|
2437
|
-
|
|
2438
|
-
|
|
2439
|
-
|
|
2529
|
+
let wanted = /* @__PURE__ */ new Set();
|
|
2530
|
+
if (delta !== null) {
|
|
2531
|
+
const taken = await this.takeChanges(delta, deltaBase, held);
|
|
2532
|
+
if (this.closing) return;
|
|
2533
|
+
wanted = taken.wanted;
|
|
2534
|
+
this.log(`took what changed since seq ${since} at seq ${seq}: ${delta.files.length} path${delta.files.length === 1 ? "" : "s"}, ${taken.sent} files sent${this.remote.size > 0 ? `, ${this.remote.size} only on the server` : ""}`);
|
|
2535
|
+
} else {
|
|
2536
|
+
const named = /* @__PURE__ */ new Set();
|
|
2537
|
+
const present = /* @__PURE__ */ new Set();
|
|
2538
|
+
let sent = 0;
|
|
2539
|
+
this.snapshotAnswers.clear();
|
|
2540
|
+
const fromCopy = this.copied && !this.synced;
|
|
2541
|
+
const blobsFromCopy = withBlobs && !this.blobsCompared;
|
|
2542
|
+
/** The directories the snapshot names: every one the server holds */
|
|
2543
|
+
const dirs = /* @__PURE__ */ new Set();
|
|
2544
|
+
for (const part of parts) for (const event of part.files) {
|
|
2545
|
+
if (event.kind === "dir") {
|
|
2546
|
+
named.add(event.path);
|
|
2547
|
+
dirs.add(event.path);
|
|
2548
|
+
this.receiveDir(event.path, part.seq, false);
|
|
2549
|
+
continue;
|
|
2550
|
+
}
|
|
2551
|
+
if (!this.syncs(event.path)) continue;
|
|
2552
|
+
if (event.kind === "listed") {
|
|
2553
|
+
named.add(event.path);
|
|
2554
|
+
if (this.mayStay(event.path, held)) this.keepRemote(event.path, event.hash, part.seq);
|
|
2555
|
+
else {
|
|
2556
|
+
wanted.add(event.path);
|
|
2557
|
+
this.awaiting.add(event.path);
|
|
2558
|
+
if (part.seq > (this.fileSeq.get(event.path) ?? -1)) this.fileSeq.set(event.path, part.seq);
|
|
2559
|
+
}
|
|
2560
|
+
continue;
|
|
2561
|
+
}
|
|
2440
2562
|
named.add(event.path);
|
|
2441
|
-
if (
|
|
2442
|
-
|
|
2443
|
-
|
|
2444
|
-
this.awaiting.add(event.path);
|
|
2445
|
-
if (part.seq > (this.fileSeq.get(event.path) ?? -1)) this.fileSeq.set(event.path, part.seq);
|
|
2563
|
+
if (event.kind === "file" || event.kind === "blob") {
|
|
2564
|
+
present.add(event.path);
|
|
2565
|
+
if (event.kind === "file") sent++;
|
|
2446
2566
|
}
|
|
2447
|
-
|
|
2448
|
-
|
|
2449
|
-
|
|
2450
|
-
|
|
2451
|
-
|
|
2452
|
-
|
|
2567
|
+
if (uploaded && event.kind === "removed" && heldBlobs.has(event.path)) {
|
|
2568
|
+
this.receive({
|
|
2569
|
+
kind: "unknown",
|
|
2570
|
+
path: event.path
|
|
2571
|
+
}, part.seq);
|
|
2572
|
+
continue;
|
|
2573
|
+
}
|
|
2574
|
+
if (event.kind === "removed" && (held.has(event.path) ? fromCopy : blobsFromCopy && heldBlobs.has(event.path))) {
|
|
2575
|
+
const text = held.get(event.path);
|
|
2576
|
+
this.snapshotAnswers.set(event.path, {
|
|
2577
|
+
hash: text === void 0 ? heldBlobs.get(event.path).hash : this.hashOf(event.path, text),
|
|
2578
|
+
removed: true
|
|
2579
|
+
});
|
|
2580
|
+
}
|
|
2581
|
+
this.receive(event, part.seq);
|
|
2453
2582
|
}
|
|
2454
|
-
|
|
2583
|
+
for (const [path, text] of held) {
|
|
2584
|
+
if (named.has(path)) continue;
|
|
2585
|
+
present.add(path);
|
|
2586
|
+
this.snapshotAnswers.set(path, {
|
|
2587
|
+
hash: this.hashOf(path, text),
|
|
2588
|
+
removed: false
|
|
2589
|
+
});
|
|
2455
2590
|
this.receive({
|
|
2456
|
-
kind: "
|
|
2457
|
-
path
|
|
2458
|
-
|
|
2459
|
-
|
|
2591
|
+
kind: "file",
|
|
2592
|
+
path,
|
|
2593
|
+
text
|
|
2594
|
+
}, seq);
|
|
2460
2595
|
}
|
|
2461
|
-
|
|
2462
|
-
|
|
2463
|
-
|
|
2464
|
-
|
|
2465
|
-
|
|
2596
|
+
for (const [path, blob] of heldBlobs) {
|
|
2597
|
+
if (named.has(path)) continue;
|
|
2598
|
+
present.add(path);
|
|
2599
|
+
this.snapshotAnswers.set(path, {
|
|
2600
|
+
hash: blob.hash,
|
|
2601
|
+
removed: false
|
|
2466
2602
|
});
|
|
2603
|
+
this.receive({
|
|
2604
|
+
kind: "blob",
|
|
2605
|
+
path,
|
|
2606
|
+
hash: blob.hash,
|
|
2607
|
+
size: blob.size ?? 0
|
|
2608
|
+
}, seq);
|
|
2467
2609
|
}
|
|
2468
|
-
this.
|
|
2469
|
-
|
|
2470
|
-
|
|
2471
|
-
|
|
2472
|
-
|
|
2473
|
-
|
|
2474
|
-
|
|
2475
|
-
|
|
2476
|
-
}
|
|
2477
|
-
this.
|
|
2478
|
-
|
|
2479
|
-
path,
|
|
2480
|
-
text
|
|
2481
|
-
}, seq);
|
|
2482
|
-
}
|
|
2483
|
-
for (const [path, blob] of heldBlobs) {
|
|
2484
|
-
if (named.has(path)) continue;
|
|
2485
|
-
present.add(path);
|
|
2486
|
-
this.snapshotAnswers.set(path, {
|
|
2487
|
-
hash: blob.hash,
|
|
2488
|
-
removed: false
|
|
2489
|
-
});
|
|
2490
|
-
this.receive({
|
|
2491
|
-
kind: "blob",
|
|
2492
|
-
path,
|
|
2493
|
-
hash: blob.hash,
|
|
2494
|
-
size: blob.size ?? 0
|
|
2495
|
-
}, seq);
|
|
2496
|
-
}
|
|
2497
|
-
const blind = this.blobSupport !== true;
|
|
2498
|
-
for (const path of /* @__PURE__ */ new Set([...this.cloud.keys(), ...this.expected.keys()])) {
|
|
2499
|
-
if (present.has(path) || named.has(path) || !this.syncs(path)) continue;
|
|
2500
|
-
if (blind && this.knownBinary(path)) continue;
|
|
2501
|
-
this.receive({
|
|
2502
|
-
kind: "removed",
|
|
2503
|
-
path
|
|
2504
|
-
}, seq);
|
|
2610
|
+
const blind = this.blobSupport !== true;
|
|
2611
|
+
for (const path of /* @__PURE__ */ new Set([...this.cloud.keys(), ...this.expected.keys()])) {
|
|
2612
|
+
if (present.has(path) || named.has(path) || !this.syncs(path)) continue;
|
|
2613
|
+
if (blind && this.knownBinary(path)) continue;
|
|
2614
|
+
this.receive({
|
|
2615
|
+
kind: "removed",
|
|
2616
|
+
path
|
|
2617
|
+
}, seq);
|
|
2618
|
+
}
|
|
2619
|
+
for (const path of this.syncedDirs) if (!dirs.has(path)) this.receiveDirRemoved(path, seq);
|
|
2620
|
+
this.log(`took the snapshot at seq ${seq} in ${parts.length} batch${parts.length === 1 ? "" : "es"}: ${sent} files sent, ${present.size - sent} already held here${this.remote.size > 0 ? `, ${this.remote.size} only on the server` : ""}`);
|
|
2505
2621
|
}
|
|
2506
|
-
for (const path of this.syncedDirs) if (!dirs.has(path)) this.receiveDirRemoved(path, seq);
|
|
2507
|
-
this.log(`took the snapshot at seq ${seq} in ${parts.length} batch${parts.length === 1 ? "" : "es"}: ${sent} files sent, ${present.size - sent} already held here${this.remote.size > 0 ? `, ${this.remote.size} only on the server` : ""}`);
|
|
2508
2622
|
this.takenUpTo(seq);
|
|
2509
2623
|
this.synced = true;
|
|
2624
|
+
this.holdsHub = withBlobs && this.reloads === reloads;
|
|
2625
|
+
this.resumeSeq = void 0;
|
|
2626
|
+
this.saveKnown();
|
|
2510
2627
|
if (withBlobs && !this.blobsCompared) {
|
|
2511
2628
|
this.blobsCompared = true;
|
|
2512
2629
|
rmSync(this.paths.copyBlobs, { force: true });
|
|
@@ -2521,6 +2638,167 @@ var SyncClient = class extends EventEmitter {
|
|
|
2521
2638
|
await this.queueRemovedDirs();
|
|
2522
2639
|
}
|
|
2523
2640
|
/**
|
|
2641
|
+
* What the client takes the server to hold as of the seq known.json names, before the run's first snapshot
|
|
2642
|
+
* (resumeSeq): the version known.json names of each file, but one only on the server (remote.json), and the
|
|
2643
|
+
* directories it names; with what a broadcast taken in since the run started holds, which is of a later seq.
|
|
2644
|
+
*/
|
|
2645
|
+
knownCopy() {
|
|
2646
|
+
const files = /* @__PURE__ */ new Map();
|
|
2647
|
+
for (const [path, disk] of this.confirmed) {
|
|
2648
|
+
if (!disk || !this.syncs(path) || this.remote.has(path)) continue;
|
|
2649
|
+
if (disk.blob) files.set(path, {
|
|
2650
|
+
kind: "blob",
|
|
2651
|
+
hash: disk.hash
|
|
2652
|
+
});
|
|
2653
|
+
else if (disk.text !== void 0) files.set(path, disk.text);
|
|
2654
|
+
}
|
|
2655
|
+
for (const [path, file] of this.cloud) {
|
|
2656
|
+
if (!this.syncs(path)) continue;
|
|
2657
|
+
if (file === null) files.delete(path);
|
|
2658
|
+
else files.set(path, file);
|
|
2659
|
+
}
|
|
2660
|
+
return {
|
|
2661
|
+
files,
|
|
2662
|
+
remote: new Map(this.remote),
|
|
2663
|
+
dirs: new Set(this.syncedDirs)
|
|
2664
|
+
};
|
|
2665
|
+
}
|
|
2666
|
+
/**
|
|
2667
|
+
* What the client takes the server to hold, while it holds the hub (holdsHub): the server's version of each file it
|
|
2668
|
+
* took in, a binary file's included, the hash of each file only on the server, and the directories the server holds.
|
|
2669
|
+
* A file here the server never had is not among them, though a snapshot request names it.
|
|
2670
|
+
*/
|
|
2671
|
+
serverCopy() {
|
|
2672
|
+
const files = /* @__PURE__ */ new Map();
|
|
2673
|
+
for (const [path, file] of this.cloud) if (file !== null && this.syncs(path)) files.set(path, file);
|
|
2674
|
+
return {
|
|
2675
|
+
files,
|
|
2676
|
+
remote: new Map(this.remote),
|
|
2677
|
+
dirs: new Set(this.syncedDirs)
|
|
2678
|
+
};
|
|
2679
|
+
}
|
|
2680
|
+
/**
|
|
2681
|
+
* Whether the changes the server answered with (SnapshotDelta), taken in over what the client took it to hold as of
|
|
2682
|
+
* `since` (`base`), make the hub's digest as of their seq (`held`), so they are the hub's whole difference; `differs`
|
|
2683
|
+
* when they do not, as when the server's storage was put back to an earlier point; `empty` when they leave no file,
|
|
2684
|
+
* which a whole snapshot tells apart from a server without the hub. A `dir`, `file` or `blob` replaces whatever kind
|
|
2685
|
+
* the path held; `removed`, `unknown` and `dirRemoved` leave nothing there.
|
|
2686
|
+
*/
|
|
2687
|
+
changesHold(delta, base) {
|
|
2688
|
+
const files = /* @__PURE__ */ new Map();
|
|
2689
|
+
for (const [path, file] of base.files) files.set(path, isBlob(file) ? file.hash : this.hashOf(path, file));
|
|
2690
|
+
for (const [path, hash] of base.remote) files.set(path, hash);
|
|
2691
|
+
const dirs = new Set(base.dirs);
|
|
2692
|
+
for (const entry of delta.files) {
|
|
2693
|
+
const { path } = entry;
|
|
2694
|
+
if (entry.kind === "file") files.set(path, contentHash(entry.text));
|
|
2695
|
+
else if (entry.kind === "blob" || entry.kind === "listed") files.set(path, entry.hash);
|
|
2696
|
+
else files.delete(path);
|
|
2697
|
+
if (entry.kind === "dir") dirs.add(path);
|
|
2698
|
+
else dirs.delete(path);
|
|
2699
|
+
}
|
|
2700
|
+
if (hubDigest(files, dirs) !== delta.digest) return "differs";
|
|
2701
|
+
return [...files.keys()].some((path) => this.syncs(path)) ? "held" : "empty";
|
|
2702
|
+
}
|
|
2703
|
+
/**
|
|
2704
|
+
* Take in what changed since the `since` a snapshot named (SnapshotDelta), once its digest showed that the client's
|
|
2705
|
+
* copy with these changes is the hub as of their seq (changesHold). Only what the answer names changes: a path it does
|
|
2706
|
+
* not name holds what the client took the server to hold (`base`), so nothing is made of a file here the answer does
|
|
2707
|
+
* not name, as a whole snapshot makes of one it does not name. The changes are taken in in the order the server puts
|
|
2708
|
+
* a hub back in (docs/sync-protocol.md "增量快照"): files removed, a file that turned into a directory among them;
|
|
2709
|
+
* then directories removed, the deepest first, a directory that turned into a file among them; then directories made,
|
|
2710
|
+
* the shallowest first; then files written, and binary files downloaded. When a directory comes or goes, each step
|
|
2711
|
+
* waits for the one before to be done on disk, at most CHANGES_STEP_MS. An `unknown` path is not removed: a file here
|
|
2712
|
+
* there is new to the server (receive). A path the answer does not name keeps the version `base` holds as of the
|
|
2713
|
+
* answer's seq, as a whole snapshot's file left out as held does: a broadcast that changed it meanwhile and is no
|
|
2714
|
+
* newer is undone, as it was undone on the server too, and an older one arriving later is not taken in. Returns the
|
|
2715
|
+
* listed files to download, and how many texts came.
|
|
2716
|
+
*/
|
|
2717
|
+
async takeChanges(delta, base, held) {
|
|
2718
|
+
const { seq } = delta;
|
|
2719
|
+
const wanted = /* @__PURE__ */ new Set();
|
|
2720
|
+
let sent = 0;
|
|
2721
|
+
const depth = (path) => path.split("/").length;
|
|
2722
|
+
const named = new Set(delta.files.map((entry) => entry.path));
|
|
2723
|
+
for (const [path, file] of base.files) {
|
|
2724
|
+
if (named.has(path) || (this.fileSeq.get(path) ?? -1) > seq) continue;
|
|
2725
|
+
if (sameServerFile(this.cloud.get(path), file)) this.fileSeq.set(path, seq);
|
|
2726
|
+
else this.receive(isBlob(file) ? {
|
|
2727
|
+
kind: "blob",
|
|
2728
|
+
path,
|
|
2729
|
+
hash: file.hash,
|
|
2730
|
+
size: file.size ?? 0
|
|
2731
|
+
} : {
|
|
2732
|
+
kind: "file",
|
|
2733
|
+
path,
|
|
2734
|
+
text: file
|
|
2735
|
+
}, seq);
|
|
2736
|
+
}
|
|
2737
|
+
for (const [path, hash] of base.remote) {
|
|
2738
|
+
if (named.has(path) || (this.fileSeq.get(path) ?? -1) > seq) continue;
|
|
2739
|
+
if (this.remote.get(path) === hash) this.fileSeq.set(path, seq);
|
|
2740
|
+
else this.keepRemote(path, hash, seq);
|
|
2741
|
+
}
|
|
2742
|
+
for (const path of base.dirs) {
|
|
2743
|
+
if (named.has(path) || (this.dirSeq.get(path) ?? -1) > seq) continue;
|
|
2744
|
+
if (this.syncedDirs.has(path)) this.dirSeq.set(path, seq);
|
|
2745
|
+
else this.receiveDir(path, seq, false);
|
|
2746
|
+
}
|
|
2747
|
+
const gone = delta.files.flatMap((entry) => entry.kind === "dirRemoved" || (entry.kind === "file" || entry.kind === "blob" || entry.kind === "listed") && this.syncedDirs.has(entry.path) ? [entry.path] : []);
|
|
2748
|
+
const made = delta.files.flatMap((entry) => entry.kind === "dir" ? [entry.path] : []);
|
|
2749
|
+
const steps = gone.length > 0 || made.length > 0;
|
|
2750
|
+
const done = (work) => steps && work.length > 0 ? Promise.race([Promise.allSettled(work), sleep$1(CHANGES_STEP_MS)]) : Promise.resolve();
|
|
2751
|
+
const removed = [];
|
|
2752
|
+
const isFile = (path) => (this.cloud.get(path) ?? null) !== null || this.remote.has(path);
|
|
2753
|
+
for (const entry of delta.files) {
|
|
2754
|
+
if (!this.syncs(entry.path)) continue;
|
|
2755
|
+
if (entry.kind === "removed" || entry.kind === "unknown") this.receive(entry, seq);
|
|
2756
|
+
else if (entry.kind === "dir" && isFile(entry.path)) this.receive({
|
|
2757
|
+
kind: "removed",
|
|
2758
|
+
path: entry.path
|
|
2759
|
+
}, seq);
|
|
2760
|
+
else continue;
|
|
2761
|
+
removed.push(entry.path);
|
|
2762
|
+
}
|
|
2763
|
+
await done(removed.map((path) => this.chains.get(path)));
|
|
2764
|
+
if (this.closing) return {
|
|
2765
|
+
wanted,
|
|
2766
|
+
sent
|
|
2767
|
+
};
|
|
2768
|
+
const rmdirs = [];
|
|
2769
|
+
for (const path of gone.sort((a, b) => depth(b) - depth(a) || comparePaths(a, b))) rmdirs.push(this.receiveDirRemoved(path, seq));
|
|
2770
|
+
await done(rmdirs);
|
|
2771
|
+
if (this.closing) return {
|
|
2772
|
+
wanted,
|
|
2773
|
+
sent
|
|
2774
|
+
};
|
|
2775
|
+
const mkdirs = [];
|
|
2776
|
+
for (const path of made.sort((a, b) => depth(a) - depth(b) || comparePaths(a, b))) mkdirs.push(this.receiveDir(path, seq, false));
|
|
2777
|
+
await done(mkdirs);
|
|
2778
|
+
if (this.closing) return {
|
|
2779
|
+
wanted,
|
|
2780
|
+
sent
|
|
2781
|
+
};
|
|
2782
|
+
for (const entry of delta.files) {
|
|
2783
|
+
if (!this.syncs(entry.path)) continue;
|
|
2784
|
+
if (entry.kind === "listed") {
|
|
2785
|
+
if (this.mayStay(entry.path, held)) this.keepRemote(entry.path, entry.hash, seq);
|
|
2786
|
+
else {
|
|
2787
|
+
wanted.add(entry.path);
|
|
2788
|
+
this.awaiting.add(entry.path);
|
|
2789
|
+
if (seq > (this.fileSeq.get(entry.path) ?? -1)) this.fileSeq.set(entry.path, seq);
|
|
2790
|
+
}
|
|
2791
|
+
} else if (entry.kind === "file" || entry.kind === "blob") {
|
|
2792
|
+
if (entry.kind === "file") sent++;
|
|
2793
|
+
this.receive(entry, seq);
|
|
2794
|
+
}
|
|
2795
|
+
}
|
|
2796
|
+
return {
|
|
2797
|
+
wanted,
|
|
2798
|
+
sent
|
|
2799
|
+
};
|
|
2800
|
+
}
|
|
2801
|
+
/**
|
|
2524
2802
|
* Ask the server whether it keeps binary files (blobSupport), once per connection, before its snapshot. A probe that
|
|
2525
2803
|
* fails leaves what the last one found, null on a first connection, so the texts sync all the same and binary files
|
|
2526
2804
|
* wait as they are here: the snapshot is taken again after PROBE_RETRY_MS, doubling up to PROBE_RETRY_MAX_MS while it
|
|
@@ -2532,6 +2810,7 @@ var SyncClient = class extends EventEmitter {
|
|
|
2532
2810
|
await this.askBlobs();
|
|
2533
2811
|
} finally {
|
|
2534
2812
|
if (had !== true && this.blobSupport === true) this.fileSeq.clear();
|
|
2813
|
+
if (had !== this.blobSupport) this.holdsHub = false;
|
|
2535
2814
|
}
|
|
2536
2815
|
}
|
|
2537
2816
|
async askBlobs() {
|
|
@@ -3025,7 +3304,9 @@ var SyncClient = class extends EventEmitter {
|
|
|
3025
3304
|
}
|
|
3026
3305
|
/** Note `path` as only on the server, with the hash of the server's text as of `seq`, unless a newer version came. */
|
|
3027
3306
|
keepRemote(path, hash, seq) {
|
|
3028
|
-
|
|
3307
|
+
const held = this.fileSeq.get(path) ?? -1;
|
|
3308
|
+
if (seq <= held) return;
|
|
3309
|
+
this.noteChange(path, hash, held + 1);
|
|
3029
3310
|
this.fileSeq.set(path, seq);
|
|
3030
3311
|
this.remote.set(path, hash);
|
|
3031
3312
|
this.cloud.delete(path);
|
|
@@ -5476,12 +5757,39 @@ var SyncClient = class extends EventEmitter {
|
|
|
5476
5757
|
this.knownTimer = void 0;
|
|
5477
5758
|
const files = this.knownFiles();
|
|
5478
5759
|
const dirs = this.knownDirs();
|
|
5760
|
+
const seq = this.knownSeq();
|
|
5761
|
+
if (seq !== void 0) this.saveRemoteNow(final);
|
|
5479
5762
|
try {
|
|
5480
|
-
underLock(this.dirs, () => writeKnown(this.dirs, files, dirs));
|
|
5763
|
+
underLock(this.dirs, () => writeKnown(this.dirs, files, dirs, seq));
|
|
5481
5764
|
} catch (error) {
|
|
5482
5765
|
this.log(`could not save local/sync/known.json: ${error.message}`);
|
|
5483
5766
|
}
|
|
5484
5767
|
}
|
|
5768
|
+
/**
|
|
5769
|
+
* The hub's seq known.json is written with (KnownFile.seq): one as of which the files it names, its directories and
|
|
5770
|
+
* remote.json are the hub's, but for paths changed on the server after it, so that what changed since names every
|
|
5771
|
+
* path whose version there differs. Only while the client holds the hub (holdsHub) and no snapshot is under way, as
|
|
5772
|
+
* the files it is taking in are of later seqs; and none while a listed file is being downloaded. It is hubSeq, or
|
|
5773
|
+
* lower when a file's version known.json names is not the server's the client knows of, as one the outbox holds a
|
|
5774
|
+
* write about while the server changed it: one before that change (changedAt). Undefined when there is none.
|
|
5775
|
+
*/
|
|
5776
|
+
knownSeq() {
|
|
5777
|
+
if (!this.holdsHub || this.hubSeq < 0 || this.resyncing !== null || this.awaiting.size > 0) return void 0;
|
|
5778
|
+
const listed = this.listedRemote();
|
|
5779
|
+
let seq = this.hubSeq;
|
|
5780
|
+
for (const path of /* @__PURE__ */ new Set([
|
|
5781
|
+
...this.cloud.keys(),
|
|
5782
|
+
...this.confirmed.keys(),
|
|
5783
|
+
...this.remote.keys()
|
|
5784
|
+
])) {
|
|
5785
|
+
if (!this.syncs(path)) continue;
|
|
5786
|
+
const disk = this.confirmed.get(path);
|
|
5787
|
+
if ((listed.get(path) ?? (disk ? disk.hash : null)) === this.serverView(path)) continue;
|
|
5788
|
+
seq = Math.min(seq, (this.changedAt.get(path) ?? 0) - 1);
|
|
5789
|
+
if (seq < 0) return void 0;
|
|
5790
|
+
}
|
|
5791
|
+
return seq;
|
|
5792
|
+
}
|
|
5485
5793
|
};
|
|
5486
5794
|
//#endregion
|
|
5487
5795
|
//#region ../sync/dist/ipc.js
|
|
@@ -5512,30 +5820,75 @@ var DaemonUnavailable = class extends Error {
|
|
|
5512
5820
|
};
|
|
5513
5821
|
/** Another sync client already runs for the hub. */
|
|
5514
5822
|
var DaemonRunning = class extends Error {};
|
|
5823
|
+
/** The answer's body could not be read to its end, after its status came. */
|
|
5824
|
+
var BodyUnread = class extends Error {};
|
|
5825
|
+
/**
|
|
5826
|
+
* One request to the sync client on the loopback interface, with the status and the whole body of its answer. It uses
|
|
5827
|
+
* node:http, taken when first called, rather than fetch: fetch loads undici and with it tls, and a process with
|
|
5828
|
+
* NODE_USE_SYSTEM_CA set reads the system's certificates when tls loads, which costs every hook that writes about 140 ms.
|
|
5829
|
+
* `agent: false` opens a connection for this request alone, so no kept-alive socket holds a short-lived command open.
|
|
5830
|
+
* No answer within `timeoutMs` rejects with an error named TimeoutError, which callDaemon takes as a silent sync client;
|
|
5831
|
+
* the time running out while the body is read rejects with BodyUnread, as does any failure reading it.
|
|
5832
|
+
*/
|
|
5833
|
+
function loopbackRequest(port, path, headers, body, timeoutMs) {
|
|
5834
|
+
return new Promise((resolve, reject) => {
|
|
5835
|
+
let answered = false;
|
|
5836
|
+
const request = process.getBuiltinModule("node:http").request({
|
|
5837
|
+
host: "127.0.0.1",
|
|
5838
|
+
port,
|
|
5839
|
+
path,
|
|
5840
|
+
method: body === void 0 ? "GET" : "POST",
|
|
5841
|
+
headers,
|
|
5842
|
+
agent: false
|
|
5843
|
+
}, (response) => {
|
|
5844
|
+
answered = true;
|
|
5845
|
+
const chunks = [];
|
|
5846
|
+
response.on("data", (chunk) => chunks.push(chunk));
|
|
5847
|
+
response.on("end", () => {
|
|
5848
|
+
clearTimeout(timer);
|
|
5849
|
+
resolve({
|
|
5850
|
+
status: response.statusCode ?? 0,
|
|
5851
|
+
text: Buffer.concat(chunks).toString("utf8")
|
|
5852
|
+
});
|
|
5853
|
+
});
|
|
5854
|
+
response.on("error", (error) => {
|
|
5855
|
+
clearTimeout(timer);
|
|
5856
|
+
reject(new BodyUnread(error.message));
|
|
5857
|
+
});
|
|
5858
|
+
});
|
|
5859
|
+
const timer = setTimeout(() => {
|
|
5860
|
+
const error = answered ? new BodyUnread("the answer did not end in time") : Object.assign(/* @__PURE__ */ new Error("The operation was aborted due to timeout"), { name: "TimeoutError" });
|
|
5861
|
+
request.destroy(error);
|
|
5862
|
+
reject(error);
|
|
5863
|
+
}, timeoutMs);
|
|
5864
|
+
request.on("error", (error) => {
|
|
5865
|
+
clearTimeout(timer);
|
|
5866
|
+
reject(answered ? new BodyUnread(error.message) : error);
|
|
5867
|
+
});
|
|
5868
|
+
request.end(body);
|
|
5869
|
+
});
|
|
5870
|
+
}
|
|
5515
5871
|
async function callDaemon(daemon, route, body, timeoutMs = 6e4) {
|
|
5872
|
+
const notSync = () => new DaemonUnavailable(`what answers on port ${daemon.port} is not mainflow sync (pid ${daemon.pid})`);
|
|
5516
5873
|
let response;
|
|
5517
5874
|
try {
|
|
5518
|
-
response = await
|
|
5519
|
-
|
|
5520
|
-
|
|
5521
|
-
|
|
5522
|
-
"content-type": "application/json"
|
|
5523
|
-
},
|
|
5524
|
-
body: body === void 0 ? void 0 : JSON.stringify(body),
|
|
5525
|
-
signal: AbortSignal.timeout(timeoutMs)
|
|
5526
|
-
});
|
|
5875
|
+
response = await loopbackRequest(daemon.port, route, {
|
|
5876
|
+
[SECRET_HEADER]: daemon.secret,
|
|
5877
|
+
"content-type": "application/json"
|
|
5878
|
+
}, body === void 0 ? void 0 : JSON.stringify(body), timeoutMs);
|
|
5527
5879
|
} catch (error) {
|
|
5880
|
+
if (error instanceof BodyUnread) throw notSync();
|
|
5528
5881
|
const silent = error.name === "TimeoutError";
|
|
5529
5882
|
throw new DaemonUnavailable(`mainflow sync (pid ${daemon.pid}) does not answer: ${error.message}`, silent);
|
|
5530
5883
|
}
|
|
5531
5884
|
let parsed;
|
|
5532
5885
|
try {
|
|
5533
|
-
parsed =
|
|
5886
|
+
parsed = JSON.parse(response.text);
|
|
5534
5887
|
} catch {
|
|
5535
|
-
throw
|
|
5888
|
+
throw notSync();
|
|
5536
5889
|
}
|
|
5537
5890
|
if (response.status === 403) throw new DaemonUnavailable(`what answers on port ${daemon.port} is not this hub's mainflow sync (pid ${daemon.pid})`);
|
|
5538
|
-
if (
|
|
5891
|
+
if (response.status < 200 || response.status > 299) throw new Error(`mainflow sync failed: ${parsed.error}`);
|
|
5539
5892
|
return parsed;
|
|
5540
5893
|
}
|
|
5541
5894
|
/** How long a probe waits for the sync client's answer; a client holding the write lock answers after it. */
|
|
@@ -5596,7 +5949,7 @@ function callIn(body) {
|
|
|
5596
5949
|
/** Serve `client` to this machine's mainflow commands until closed. */
|
|
5597
5950
|
async function serveIpc(dirs, client) {
|
|
5598
5951
|
const secret = randomBytes(16).toString("hex");
|
|
5599
|
-
const server = createServer(async (req, res) => {
|
|
5952
|
+
const server = process.getBuiltinModule("node:http").createServer(async (req, res) => {
|
|
5600
5953
|
const reply = (status, value) => res.writeHead(status, { "content-type": "application/json" }).end(JSON.stringify(value));
|
|
5601
5954
|
if (req.headers["x-mainflow-sync"] !== secret) return reply(403, { error: "wrong secret" });
|
|
5602
5955
|
const chunks = [];
|