@narumitw/pi-sync-daemon 0.9.1 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +11 -0
- package/README.md +11 -1
- package/dist/cli.js +2 -0
- package/dist/client-transport.d.ts +14 -0
- package/dist/client-transport.js +66 -0
- package/dist/client.d.ts +15 -8
- package/dist/client.js +116 -86
- package/dist/consent.js +33 -11
- package/dist/daemon-debug.d.ts +3 -0
- package/dist/daemon-debug.js +33 -0
- package/dist/daemon-recovery.d.ts +4 -0
- package/dist/daemon-recovery.js +75 -0
- package/dist/engine/backends/git/git-backend.d.ts +8 -0
- package/dist/engine/backends/git/git-backend.js +39 -12
- package/dist/engine/backends/git/git-runner.js +6 -1
- package/dist/engine/backends/git/git-storage.js +5 -9
- package/dist/engine/backends/publication-encoding.d.ts +3 -0
- package/dist/engine/backends/publication-encoding.js +19 -0
- package/dist/engine/backends/s3/s3-backend.js +10 -12
- package/dist/engine/backends/sync-backend.d.ts +4 -0
- package/dist/engine/backends/webdav/webdav-backend.d.ts +0 -2
- package/dist/engine/backends/webdav/webdav-backend.js +15 -26
- package/dist/engine/backends/webdav/webdav-client.js +10 -0
- package/dist/engine/commands/command-types.d.ts +1 -0
- package/dist/engine/debug-log.d.ts +27 -0
- package/dist/engine/debug-log.js +134 -0
- package/dist/engine/paths.d.ts +2 -0
- package/dist/engine/paths.js +7 -1
- package/dist/engine/settings/config-file.d.ts +10 -1
- package/dist/engine/settings/config-file.js +62 -17
- package/dist/engine/settings/config.d.ts +7 -0
- package/dist/engine/settings/config.js +49 -14
- package/dist/engine/snapshot/snapshot-apply.d.ts +2 -1
- package/dist/engine/snapshot/snapshot-apply.js +30 -8
- package/dist/engine/snapshot/snapshot-codec.js +3 -1
- package/dist/engine/snapshot/snapshot-payload.d.ts +4 -0
- package/dist/engine/snapshot/snapshot-payload.js +32 -0
- package/dist/engine/snapshot/snapshot-transaction-images.d.ts +2 -0
- package/dist/engine/snapshot/snapshot-transaction-images.js +34 -0
- package/dist/engine/snapshot/snapshot-transaction.d.ts +4 -1
- package/dist/engine/snapshot/snapshot-transaction.js +50 -7
- package/dist/engine/snapshot/snapshot-types.d.ts +10 -2
- package/dist/engine/snapshot/snapshot.d.ts +3 -1
- package/dist/engine/snapshot/snapshot.js +64 -25
- package/dist/engine/state/lock.d.ts +2 -1
- package/dist/engine/state/lock.js +157 -17
- package/dist/engine/sync/continuous-sync.d.ts +10 -1
- package/dist/engine/sync/continuous-sync.js +35 -11
- package/dist/engine/sync/initialization.d.ts +1 -1
- package/dist/engine/sync/initialization.js +2 -2
- package/dist/engine/sync/local-fields.d.ts +6 -1
- package/dist/engine/sync/local-fields.js +42 -19
- package/dist/engine/sync/merged-sync.js +13 -10
- package/dist/engine/sync/remote-snapshot.js +18 -3
- package/dist/engine/sync/session-push.js +13 -4
- package/dist/engine/sync/startup-bootstrap.d.ts +1 -0
- package/dist/engine/sync/startup-bootstrap.js +105 -0
- package/dist/engine/sync/startup-recovery-evidence.d.ts +1 -0
- package/dist/engine/sync/startup-recovery-evidence.js +37 -0
- package/dist/engine/sync/startup-timings.d.ts +9 -2
- package/dist/engine/sync/startup-timings.js +80 -8
- package/dist/engine/sync/sync-error-guidance.js +6 -1
- package/dist/engine/sync/sync-inspection.js +4 -7
- package/dist/engine/sync/sync-local.d.ts +2 -1
- package/dist/engine/sync/sync-local.js +3 -1
- package/dist/engine/sync/sync-mutations.js +9 -3
- package/dist/engine/sync/sync-policy.d.ts +2 -0
- package/dist/engine/sync/sync-policy.js +9 -1
- package/dist/engine/sync/sync-state.d.ts +8 -3
- package/dist/engine/sync/sync-state.js +12 -4
- package/dist/engine-service.d.ts +6 -0
- package/dist/engine-service.js +54 -10
- package/dist/identity.d.ts +5 -2
- package/dist/identity.js +36 -6
- package/dist/index.d.ts +2 -1
- package/dist/index.js +2 -1
- package/dist/monitor.js +20 -1
- package/dist/paged-review.d.ts +35 -0
- package/dist/paged-review.js +177 -0
- package/dist/participant-ledger.d.ts +16 -0
- package/dist/participant-ledger.js +149 -0
- package/dist/protocol.d.ts +15 -0
- package/dist/protocol.js +4 -0
- package/dist/server.d.ts +7 -0
- package/dist/server.js +268 -120
- package/dist/service-host.d.ts +2 -1
- package/dist/service-host.js +29 -9
- package/dist/state-wait-jobs.d.ts +8 -0
- package/dist/state-wait-jobs.js +23 -0
- package/dist/state-wait.d.ts +7 -0
- package/dist/state-wait.js +34 -0
- package/docs/architecture.md +22 -0
- package/package.json +1 -1
package/dist/server.js
CHANGED
|
@@ -1,16 +1,21 @@
|
|
|
1
1
|
import { createHash, randomBytes, randomUUID, timingSafeEqual } from "node:crypto";
|
|
2
2
|
import { addAbortListener } from "node:events";
|
|
3
|
+
import { lstatSync } from "node:fs";
|
|
3
4
|
import fs from "node:fs/promises";
|
|
4
5
|
import { createServer } from "node:http";
|
|
5
|
-
import path from "node:path";
|
|
6
6
|
import lockfile from "proper-lockfile";
|
|
7
7
|
import { DAEMON_BUILD_VERSION } from "./build-version.js";
|
|
8
8
|
import { endpointOwner } from "./endpoint.js";
|
|
9
|
-
import {
|
|
9
|
+
import { debugOperation } from "./engine/debug-log.js";
|
|
10
|
+
import { SyncDecisionRequiredError } from "./engine/sync/sync-errors.js";
|
|
11
|
+
import { canonicalRootGuard, daemonPaths, matchesPublishedIdentity, privateDirectory, processExists, publishIdentity, readIdentity, readJsonBody, readUpgradeOwner, runtimePaths, } from "./identity.js";
|
|
10
12
|
import { createIdleExit } from "./idle-exit.js";
|
|
11
13
|
import { withCleanup } from "./lifecycle.js";
|
|
14
|
+
import { createPagedReviewOffer, reviewScope } from "./paged-review.js";
|
|
12
15
|
import { ParticipantAdmission } from "./participant-admission.js";
|
|
13
|
-
import {
|
|
16
|
+
import { createParticipantLedger } from "./participant-ledger.js";
|
|
17
|
+
import { ClientContactExpiredError, DAEMON_JOB_CAPACITY, DaemonStoppingError, MAX_MESSAGE_BYTES, MAX_REVIEW_BYTES, operationPrefix, PROTOCOL_VERSION, REVIEW_PAGE_CHARS, record, remoteError, UNKNOWN_JOB_RESULT, validateFacts, validateHandoff, validId, } from "./protocol.js";
|
|
18
|
+
import { ParticipantStateWait } from "./state-wait.js";
|
|
14
19
|
export async function startServer(root, execute, hooks = {}) {
|
|
15
20
|
const base = await runtimePaths(root);
|
|
16
21
|
const identity = {
|
|
@@ -29,6 +34,7 @@ export async function startServer(root, execute, hooks = {}) {
|
|
|
29
34
|
throw new Error("A daemon owner is still alive; refusing replacement.");
|
|
30
35
|
let compromised = false;
|
|
31
36
|
let published = false;
|
|
37
|
+
let released = false;
|
|
32
38
|
const release = await lockfile.lock(paths.directory, {
|
|
33
39
|
realpath: false,
|
|
34
40
|
lockfilePath: paths.owner,
|
|
@@ -40,7 +46,28 @@ export async function startServer(root, execute, hooks = {}) {
|
|
|
40
46
|
void close().catch(() => { });
|
|
41
47
|
},
|
|
42
48
|
});
|
|
49
|
+
const diagnostic = hooks.diagnostic;
|
|
50
|
+
const ownsLog = () => {
|
|
51
|
+
if (!published || compromised || released)
|
|
52
|
+
return false;
|
|
53
|
+
assertRoot();
|
|
54
|
+
return (lstatSync(paths.owner, { throwIfNoEntry: false })?.isDirectory() === true &&
|
|
55
|
+
matchesPublishedIdentity(paths, identity));
|
|
56
|
+
};
|
|
57
|
+
const debug = diagnostic
|
|
58
|
+
? (event, current) => {
|
|
59
|
+
try {
|
|
60
|
+
diagnostic(event, () => ownsLog() && (current?.() ?? true));
|
|
61
|
+
}
|
|
62
|
+
catch { }
|
|
63
|
+
}
|
|
64
|
+
: undefined;
|
|
65
|
+
const failure = () => {
|
|
66
|
+
debug?.({ event: "lifecycle", outcome: "failed" });
|
|
67
|
+
hooks.failure?.();
|
|
68
|
+
};
|
|
43
69
|
const participants = new Map();
|
|
70
|
+
const waiters = new WeakMap();
|
|
44
71
|
const admission = new ParticipantAdmission();
|
|
45
72
|
const jobs = new Map();
|
|
46
73
|
const backgroundJobs = new Set();
|
|
@@ -49,22 +76,22 @@ export async function startServer(root, execute, hooks = {}) {
|
|
|
49
76
|
// Retain tombstones after result eviction; replay must never dispatch a second write.
|
|
50
77
|
const retired = new Set();
|
|
51
78
|
const pending = new Map();
|
|
52
|
-
const
|
|
79
|
+
const ledger = createParticipantLedger(paths, () => participants.values(), failure);
|
|
53
80
|
let sequence = 0;
|
|
54
81
|
let participantGeneration = 0;
|
|
55
82
|
let closed = false;
|
|
56
83
|
let stopRequested = false;
|
|
57
84
|
let automaticClosing = false;
|
|
58
|
-
let peerWriteFailed = false;
|
|
59
85
|
let idleExit;
|
|
60
86
|
let expiring;
|
|
61
87
|
const peerOperations = new Set();
|
|
62
88
|
let queue = Promise.resolve();
|
|
63
89
|
let closing;
|
|
64
|
-
let peerWrites = Promise.resolve();
|
|
65
90
|
let heartbeat;
|
|
66
91
|
function participantsChanged() {
|
|
67
92
|
participantGeneration++;
|
|
93
|
+
for (const peer of participants.values())
|
|
94
|
+
waiters.get(peer)?.changed();
|
|
68
95
|
for (const job of [...jobs.values(), ...backgroundJobs]) {
|
|
69
96
|
if (job.participantGeneration !== undefined &&
|
|
70
97
|
!job.committed &&
|
|
@@ -72,87 +99,26 @@ export async function startServer(root, execute, hooks = {}) {
|
|
|
72
99
|
job.controller.abort(new DOMException("Daemon participants changed before automatic work completed.", "AbortError"));
|
|
73
100
|
}
|
|
74
101
|
}
|
|
75
|
-
function savePeers(candidate) {
|
|
102
|
+
function savePeers(candidate, allowNoop = false) {
|
|
76
103
|
idleExit?.changed();
|
|
77
|
-
|
|
78
|
-
peerWrites = operation.catch(() => {
|
|
79
|
-
peerWriteFailed = true;
|
|
80
|
-
hooks.failure?.();
|
|
81
|
-
});
|
|
82
|
-
return operation;
|
|
83
|
-
}
|
|
84
|
-
async function writePeers(candidate) {
|
|
85
|
-
const temporary = path.join(paths.directory, `.${randomUUID()}.clients`);
|
|
86
|
-
const data = {
|
|
87
|
-
version: 1,
|
|
88
|
-
root: paths.root,
|
|
89
|
-
clients: [...participants.values()]
|
|
90
|
-
.filter((peer) => processExists(peer.facts.pid))
|
|
91
|
-
.map((peer) => ({ ...(candidate?.peer === peer ? candidate.facts : peer.facts), handoff: peer.handoff })),
|
|
92
|
-
};
|
|
93
|
-
const handle = await fs.open(temporary, "wx", 0o600);
|
|
94
|
-
try {
|
|
95
|
-
await handle.writeFile(JSON.stringify(data));
|
|
96
|
-
await handle.sync();
|
|
97
|
-
await handle.close();
|
|
98
|
-
await fs.rename(temporary, peersFile);
|
|
99
|
-
}
|
|
100
|
-
finally {
|
|
101
|
-
await handle.close().catch(() => { });
|
|
102
|
-
await fs.rm(temporary, { force: true });
|
|
103
|
-
}
|
|
104
|
+
return ledger.save(candidate, allowNoop);
|
|
104
105
|
}
|
|
105
106
|
try {
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
data.version !== 1 ||
|
|
122
|
-
data.root !== paths.root ||
|
|
123
|
-
!Array.isArray(data.clients) ||
|
|
124
|
-
data.clients.length > 64)
|
|
125
|
-
throw new Error("Invalid daemon client record; evidence retained.");
|
|
126
|
-
for (const stored of data.clients) {
|
|
127
|
-
if (!record(stored))
|
|
128
|
-
throw new Error("Invalid stored participant.");
|
|
129
|
-
const { handoff, ...value } = stored;
|
|
130
|
-
validateFacts(value);
|
|
131
|
-
if (handoff !== undefined)
|
|
132
|
-
validateHandoff(handoff);
|
|
133
|
-
if (processExists(value.pid)) {
|
|
134
|
-
if (handoff?.kind === "exit")
|
|
135
|
-
closedClients.add(value.id);
|
|
136
|
-
participants.set(value.id, {
|
|
137
|
-
facts: value,
|
|
138
|
-
handoff,
|
|
139
|
-
callbacks: new Map(),
|
|
140
|
-
events: [],
|
|
141
|
-
connected: false,
|
|
142
|
-
closing: false,
|
|
143
|
-
suspected: handoff?.kind !== "exit",
|
|
144
|
-
seen: 0,
|
|
145
|
-
});
|
|
146
|
-
}
|
|
147
|
-
}
|
|
148
|
-
}
|
|
149
|
-
finally {
|
|
150
|
-
await handle.close();
|
|
151
|
-
}
|
|
152
|
-
}
|
|
153
|
-
catch (error) {
|
|
154
|
-
if (error.code !== "ENOENT")
|
|
155
|
-
throw error;
|
|
107
|
+
for (const { facts, handoff } of await ledger.restore()) {
|
|
108
|
+
if (!processExists(facts.pid))
|
|
109
|
+
continue;
|
|
110
|
+
if (handoff?.kind === "exit")
|
|
111
|
+
closedClients.add(facts.id);
|
|
112
|
+
participants.set(facts.id, {
|
|
113
|
+
facts,
|
|
114
|
+
handoff,
|
|
115
|
+
callbacks: new Map(),
|
|
116
|
+
events: [],
|
|
117
|
+
connected: false,
|
|
118
|
+
closing: false,
|
|
119
|
+
suspected: handoff?.kind !== "exit",
|
|
120
|
+
seen: 0,
|
|
121
|
+
});
|
|
156
122
|
}
|
|
157
123
|
}
|
|
158
124
|
catch (error) {
|
|
@@ -173,6 +139,7 @@ export async function startServer(root, execute, hooks = {}) {
|
|
|
173
139
|
peer.events.push(event);
|
|
174
140
|
while (peer.events.length > 128 || Buffer.byteLength(JSON.stringify(peer.events)) > MAX_MESSAGE_BYTES / 8)
|
|
175
141
|
peer.events.shift();
|
|
142
|
+
waiters.get(peer)?.changed();
|
|
176
143
|
}
|
|
177
144
|
};
|
|
178
145
|
const assertOwner = () => {
|
|
@@ -247,20 +214,28 @@ export async function startServer(root, execute, hooks = {}) {
|
|
|
247
214
|
const peer = requireParticipant(target, cleanup);
|
|
248
215
|
const callback = { id: randomUUID(), job: job.view.id, kind, payload: data };
|
|
249
216
|
let review;
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
typeof data.content === "string" &&
|
|
253
|
-
Buffer.byteLength(data.content) > REVIEW_PAGE_CHARS) {
|
|
217
|
+
let pagedReview;
|
|
218
|
+
if (kind === "merge-review" && record(data) && typeof data.content === "string") {
|
|
254
219
|
const bytes = Buffer.byteLength(data.content);
|
|
255
220
|
if (bytes > MAX_REVIEW_BYTES)
|
|
256
221
|
throw new Error("Exact review exceeds its byte limit; no truncated review was approved.");
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
222
|
+
if (bytes > REVIEW_PAGE_CHARS) {
|
|
223
|
+
if (peer.reviewDetails)
|
|
224
|
+
pagedReview = createPagedReviewOffer(reviewScope(identity, peer.facts, callback), data, data.content, bytes);
|
|
225
|
+
if (pagedReview) {
|
|
226
|
+
callback.payload = pagedReview.summary;
|
|
227
|
+
callback.review = pagedReview.reference;
|
|
228
|
+
}
|
|
229
|
+
else {
|
|
230
|
+
review = data.content;
|
|
231
|
+
callback.payload = { ...data, content: undefined };
|
|
232
|
+
callback.review = {
|
|
233
|
+
pages: Math.ceil(review.length / REVIEW_PAGE_CHARS),
|
|
234
|
+
bytes,
|
|
235
|
+
sha256: createHash("sha256").update(review).digest("hex"),
|
|
236
|
+
};
|
|
237
|
+
}
|
|
238
|
+
}
|
|
264
239
|
}
|
|
265
240
|
if (peer.callbacks.size >= 64 ||
|
|
266
241
|
Buffer.byteLength(JSON.stringify([...peer.callbacks.values(), callback])) > MAX_MESSAGE_BYTES / 8)
|
|
@@ -292,20 +267,33 @@ export async function startServer(root, execute, hooks = {}) {
|
|
|
292
267
|
if (cleanup && reservation && error === undefined && value === true)
|
|
293
268
|
peer.reservations?.delete(reservation);
|
|
294
269
|
sequence++;
|
|
270
|
+
waiters.get(peer)?.changed();
|
|
295
271
|
if (error)
|
|
296
272
|
reject(error);
|
|
297
273
|
else
|
|
298
274
|
resolve(value);
|
|
299
275
|
};
|
|
276
|
+
let startupReviewLock;
|
|
277
|
+
if (!job.committed && ["confirm", "select", "merge-review"].includes(kind)) {
|
|
278
|
+
try {
|
|
279
|
+
startupReviewLock = hooks.startup?.captureReviewLock();
|
|
280
|
+
}
|
|
281
|
+
catch {
|
|
282
|
+
startupReviewLock = undefined;
|
|
283
|
+
}
|
|
284
|
+
}
|
|
300
285
|
pending.set(callback.id, {
|
|
301
286
|
callback,
|
|
287
|
+
startupReviewLock,
|
|
302
288
|
review,
|
|
289
|
+
pagedReview,
|
|
303
290
|
target,
|
|
304
291
|
resolve: (value) => finish(value),
|
|
305
292
|
reject: (error) => finish(undefined, error),
|
|
306
293
|
});
|
|
307
294
|
peer.callbacks.set(callback.id, callback);
|
|
308
295
|
sequence++;
|
|
296
|
+
waiters.get(peer)?.changed();
|
|
309
297
|
effectiveSignal?.addEventListener("abort", abort, { once: true });
|
|
310
298
|
if (kind.startsWith("queue-")) {
|
|
311
299
|
timer = setTimeout(() => finish(undefined, new Error("Client file-queue acknowledgement timed out; work was stopped.")), 30_000);
|
|
@@ -329,6 +317,10 @@ export async function startServer(root, execute, hooks = {}) {
|
|
|
329
317
|
validate,
|
|
330
318
|
request,
|
|
331
319
|
emit: (kind, data, target = job.client) => emit(kind, data, target),
|
|
320
|
+
debug,
|
|
321
|
+
noteDebugOutcome(outcome) {
|
|
322
|
+
job.debugOutcome = outcome;
|
|
323
|
+
},
|
|
332
324
|
committed: () => {
|
|
333
325
|
job.committed = true;
|
|
334
326
|
emit("commit", { job: job.view.id }, job.client);
|
|
@@ -350,14 +342,17 @@ export async function startServer(root, execute, hooks = {}) {
|
|
|
350
342
|
throw new Error("Operation result expired; reconcile the existing journal before starting new work.");
|
|
351
343
|
if (retired.size + jobs.size >= 4096)
|
|
352
344
|
throw new Error("Daemon operation ledger is full; drain and restart explicitly.");
|
|
353
|
-
if (jobs.size >=
|
|
345
|
+
if (jobs.size >= DAEMON_JOB_CAPACITY) {
|
|
354
346
|
for (const [key, job] of jobs)
|
|
355
347
|
if (job.view.phase === "done" || job.view.phase === "failed") {
|
|
356
348
|
retired.add(key);
|
|
357
349
|
jobs.delete(key);
|
|
350
|
+
const owner = job.client ? participants.get(job.client) : undefined;
|
|
351
|
+
if (owner)
|
|
352
|
+
waiters.get(owner)?.changed();
|
|
358
353
|
break;
|
|
359
354
|
}
|
|
360
|
-
if (jobs.size >=
|
|
355
|
+
if (jobs.size >= DAEMON_JOB_CAPACITY)
|
|
361
356
|
throw new Error("Daemon job capacity reached.");
|
|
362
357
|
}
|
|
363
358
|
const job = {
|
|
@@ -381,19 +376,32 @@ export async function startServer(root, execute, hooks = {}) {
|
|
|
381
376
|
if (cancelled.has(`${client}:${id}`))
|
|
382
377
|
job.controller.abort(new DOMException("Operation cancelled before dispatch", "AbortError"));
|
|
383
378
|
const execution = makeExecution(job);
|
|
379
|
+
const kind = debug && debugOperation(operation);
|
|
380
|
+
const note = (outcome) => {
|
|
381
|
+
if (kind)
|
|
382
|
+
debug?.({ event: "job", operation: kind, outcome, operationId: id });
|
|
383
|
+
};
|
|
384
384
|
job.promise = queue
|
|
385
385
|
.then(async () => {
|
|
386
386
|
execution.validate();
|
|
387
387
|
job.view.phase = "running";
|
|
388
|
+
note("started");
|
|
388
389
|
const result = await execute(operation, payload, execution);
|
|
389
390
|
if (Buffer.byteLength(JSON.stringify(result ?? null)) > MAX_MESSAGE_BYTES / 2)
|
|
390
391
|
throw new Error("Operation result exceeds its byte limit; use bounded inspection.");
|
|
391
392
|
job.view.result = result;
|
|
392
393
|
job.view.phase = "done";
|
|
394
|
+
note(result === "cancelled" ? "cancelled" : (job.debugOutcome ?? "completed"));
|
|
393
395
|
})
|
|
394
396
|
.catch((error) => {
|
|
395
397
|
job.view.phase = "failed";
|
|
396
398
|
job.view.error = remoteError(error);
|
|
399
|
+
note(!job.committed && (job.controller.signal.aborted || (error instanceof Error && error.name === "AbortError"))
|
|
400
|
+
? "cancelled"
|
|
401
|
+
: (job.debugOutcome ??
|
|
402
|
+
(error instanceof SyncDecisionRequiredError || error instanceof DaemonStoppingError
|
|
403
|
+
? "blocked"
|
|
404
|
+
: "failed")));
|
|
397
405
|
if (job.committed)
|
|
398
406
|
for (const id of new Set([client, ...job.peers])) {
|
|
399
407
|
const affected = participants.get(id);
|
|
@@ -405,6 +413,9 @@ export async function startServer(root, execute, hooks = {}) {
|
|
|
405
413
|
for (const waiting of [...pending.values()])
|
|
406
414
|
if (waiting.callback.job === id)
|
|
407
415
|
waiting.reject(new Error("Operation finished."));
|
|
416
|
+
const owner = participants.get(client);
|
|
417
|
+
if (owner)
|
|
418
|
+
waiters.get(owner)?.changed();
|
|
408
419
|
hooks.idle?.();
|
|
409
420
|
});
|
|
410
421
|
queue = job.promise;
|
|
@@ -416,6 +427,66 @@ export async function startServer(root, execute, hooks = {}) {
|
|
|
416
427
|
peer.expiry = undefined;
|
|
417
428
|
peer.resumeFacts = undefined;
|
|
418
429
|
}
|
|
430
|
+
async function bootstrapStartup(client, generation, fingerprint, debug, signal, startup) {
|
|
431
|
+
const peer = requireParticipant(client);
|
|
432
|
+
const before = peer.facts;
|
|
433
|
+
const currentParticipant = () => {
|
|
434
|
+
assertOwner();
|
|
435
|
+
signal.throwIfAborted();
|
|
436
|
+
return !ledger.failed && requireParticipant(client) === peer && peer.facts === before;
|
|
437
|
+
};
|
|
438
|
+
if (before.generation !== generation || !before.hasUI || !["tui", "rpc"].includes(before.mode)) {
|
|
439
|
+
throw new Error("Only the attached interactive session can finish startup.");
|
|
440
|
+
}
|
|
441
|
+
const reviewState = () => {
|
|
442
|
+
if (backgroundJobs.size ||
|
|
443
|
+
[...participants.values()].some((item) => item.resumeUnsafe || item.reservations?.size))
|
|
444
|
+
return;
|
|
445
|
+
const running = [...jobs.values()].filter((job) => job.view.phase === "running" || job.view.phase === "queued");
|
|
446
|
+
if (!running.length && !pending.size)
|
|
447
|
+
return null;
|
|
448
|
+
if (running.length !== 1 || pending.size !== 1)
|
|
449
|
+
return;
|
|
450
|
+
const [job] = running;
|
|
451
|
+
const review = pending.values().next().value;
|
|
452
|
+
if (job.view.phase === "running" &&
|
|
453
|
+
!job.committed &&
|
|
454
|
+
!job.controller.signal.aborted &&
|
|
455
|
+
job.participantGeneration === undefined &&
|
|
456
|
+
review?.startupReviewLock &&
|
|
457
|
+
review.callback.job === job.view.id &&
|
|
458
|
+
review.target !== client)
|
|
459
|
+
return review;
|
|
460
|
+
};
|
|
461
|
+
if (before.ready || !currentParticipant() || reviewState() === undefined)
|
|
462
|
+
return false;
|
|
463
|
+
await hooks.factsChanged?.();
|
|
464
|
+
return admission.run(async () => {
|
|
465
|
+
const review = reviewState();
|
|
466
|
+
if (review === undefined || !currentParticipant())
|
|
467
|
+
return false;
|
|
468
|
+
const validateEvidence = await startup.probe(fingerprint, signal, review?.startupReviewLock);
|
|
469
|
+
const recheck = () => currentParticipant() && reviewState() === review;
|
|
470
|
+
if (!validateEvidence || !recheck())
|
|
471
|
+
return false;
|
|
472
|
+
const ready = { ...before, ready: true, startupDebug: debug };
|
|
473
|
+
await savePeers({ peer, facts: ready });
|
|
474
|
+
let accepted = false;
|
|
475
|
+
try {
|
|
476
|
+
if (!recheck() || !validateEvidence())
|
|
477
|
+
return false;
|
|
478
|
+
peer.facts = ready;
|
|
479
|
+
accepted = true;
|
|
480
|
+
participantsChanged();
|
|
481
|
+
hooks.idle?.();
|
|
482
|
+
return true;
|
|
483
|
+
}
|
|
484
|
+
finally {
|
|
485
|
+
if (!accepted)
|
|
486
|
+
await savePeers();
|
|
487
|
+
}
|
|
488
|
+
}, signal);
|
|
489
|
+
}
|
|
419
490
|
async function resumeContact(facts, disconnect) {
|
|
420
491
|
const peer = requireParticipant(facts.id, true);
|
|
421
492
|
const expiry = peer.expiry;
|
|
@@ -434,7 +505,7 @@ export async function startServer(root, execute, hooks = {}) {
|
|
|
434
505
|
const recheck = () => {
|
|
435
506
|
assertOwner();
|
|
436
507
|
signal.throwIfAborted();
|
|
437
|
-
if (
|
|
508
|
+
if (ledger.failed)
|
|
438
509
|
throw new Error("Daemon client ledger persistence is uncertain; reconcile with the existing owner before reconnecting.");
|
|
439
510
|
if (participants.get(facts.id) !== peer ||
|
|
440
511
|
peer.expiry !== expiry ||
|
|
@@ -523,10 +594,10 @@ export async function startServer(root, execute, hooks = {}) {
|
|
|
523
594
|
hooks.idle?.();
|
|
524
595
|
return result;
|
|
525
596
|
}
|
|
526
|
-
async function handle(method, params, signal) {
|
|
597
|
+
async function handle(method, params, signal, reviewDetails = false) {
|
|
527
598
|
if (automaticClosing && method !== "stop")
|
|
528
599
|
throw new DaemonStoppingError();
|
|
529
|
-
const cleanup = ["answer", "poll", "cancel", "detach", "pause", "handoff", "stop"].includes(method);
|
|
600
|
+
const cleanup = ["answer", "poll", "state-wait", "cancel", "detach", "pause", "handoff", "stop"].includes(method);
|
|
530
601
|
if (!published && !cleanup)
|
|
531
602
|
throw new Error("Daemon readiness is not published.");
|
|
532
603
|
if (!cleanup)
|
|
@@ -564,6 +635,7 @@ export async function startServer(root, execute, hooks = {}) {
|
|
|
564
635
|
participantsChanged();
|
|
565
636
|
participants.set(params.id, {
|
|
566
637
|
facts: structuredClone(params),
|
|
638
|
+
reviewDetails,
|
|
567
639
|
reservations: old?.reservations,
|
|
568
640
|
resumeUnsafe: old?.resumeUnsafe,
|
|
569
641
|
callbacks: old?.callbacks ?? new Map(),
|
|
@@ -611,12 +683,26 @@ export async function startServer(root, execute, hooks = {}) {
|
|
|
611
683
|
// Monitoring can await a queued reservation; never call it while admission is held.
|
|
612
684
|
await hooks.factsChanged?.();
|
|
613
685
|
assertOwner();
|
|
614
|
-
return poll(attached, 0);
|
|
686
|
+
return { ...poll(attached, 0), stateWait: 1 };
|
|
615
687
|
}
|
|
616
688
|
if (method === "resume-contact") {
|
|
617
689
|
validateFacts(params);
|
|
618
690
|
return resumeContact(params, signal);
|
|
619
691
|
}
|
|
692
|
+
if (method === "startup.bootstrap" && hooks.startup) {
|
|
693
|
+
if (!record(params) ||
|
|
694
|
+
Object.keys(params).length !== 4 ||
|
|
695
|
+
!validId(params.client) ||
|
|
696
|
+
!validId(params.generation) ||
|
|
697
|
+
typeof params.fingerprint !== "string" ||
|
|
698
|
+
!/^[a-f0-9]{64}$/u.test(params.fingerprint) ||
|
|
699
|
+
typeof params.debug !== "boolean" ||
|
|
700
|
+
!signal)
|
|
701
|
+
throw new Error("Invalid startup bootstrap request.");
|
|
702
|
+
const peer = requireParticipant(params.client);
|
|
703
|
+
peer.seen = Date.now();
|
|
704
|
+
return bootstrapStartup(params.client, params.generation, params.fingerprint, params.debug, signal, hooks.startup);
|
|
705
|
+
}
|
|
620
706
|
if (!record(params))
|
|
621
707
|
throw new Error("Invalid request parameters.");
|
|
622
708
|
const contact = typeof params.client === "string" ? participants.get(params.client) : undefined;
|
|
@@ -638,10 +724,11 @@ export async function startServer(root, execute, hooks = {}) {
|
|
|
638
724
|
requireParticipant(params.facts.id);
|
|
639
725
|
peer.facts = structuredClone(params.facts);
|
|
640
726
|
const published = peer.facts;
|
|
641
|
-
|
|
727
|
+
const unchanged = JSON.stringify(old) === JSON.stringify(published);
|
|
728
|
+
if (!unchanged)
|
|
642
729
|
participantsChanged();
|
|
643
730
|
try {
|
|
644
|
-
await savePeers();
|
|
731
|
+
await savePeers(undefined, unchanged);
|
|
645
732
|
}
|
|
646
733
|
catch (error) {
|
|
647
734
|
if (peer.facts === published)
|
|
@@ -679,18 +766,52 @@ export async function startServer(root, execute, hooks = {}) {
|
|
|
679
766
|
throw new Error("Invalid operation envelope.");
|
|
680
767
|
return enqueue(params.id, params.client, params.operation, params.payload).view;
|
|
681
768
|
}
|
|
769
|
+
if (method === "state-wait") {
|
|
770
|
+
const peer = requireParticipant(params.client, true);
|
|
771
|
+
const client = peer.facts.id;
|
|
772
|
+
const generation = peer.facts.generation;
|
|
773
|
+
const requested = params.jobs;
|
|
774
|
+
if (params.generation !== generation ||
|
|
775
|
+
!Array.isArray(requested) ||
|
|
776
|
+
requested.length > DAEMON_JOB_CAPACITY ||
|
|
777
|
+
!requested.every(validId))
|
|
778
|
+
throw new Error("Invalid state wait.");
|
|
779
|
+
let mailbox = waiters.get(peer);
|
|
780
|
+
if (!mailbox) {
|
|
781
|
+
mailbox = new ParticipantStateWait();
|
|
782
|
+
waiters.set(peer, mailbox);
|
|
783
|
+
}
|
|
784
|
+
await mailbox.wait(params.revision, signal);
|
|
785
|
+
signal?.throwIfAborted();
|
|
786
|
+
if (participants.get(client) !== peer || peer.facts.generation !== generation)
|
|
787
|
+
throw new Error("Client changed during state wait.");
|
|
788
|
+
const finished = [...jobs.values()]
|
|
789
|
+
.filter((job) => job.client === client && (job.view.phase === "done" || job.view.phase === "failed"))
|
|
790
|
+
.map((job) => job.view.id);
|
|
791
|
+
return {
|
|
792
|
+
...poll(peer, params.cursor),
|
|
793
|
+
stateRevision: mailbox.version,
|
|
794
|
+
finished,
|
|
795
|
+
missing: requested.filter((id) => jobs.get(id)?.client !== client),
|
|
796
|
+
};
|
|
797
|
+
}
|
|
682
798
|
if (method === "poll") {
|
|
683
799
|
const peer = requireParticipant(params.client, true);
|
|
684
800
|
const job = typeof params.job === "string" ? jobs.get(params.job) : undefined;
|
|
685
801
|
if (params.job !== undefined && !job)
|
|
686
|
-
throw new Error(
|
|
802
|
+
throw new Error(UNKNOWN_JOB_RESULT);
|
|
687
803
|
if (job && job.client !== params.client)
|
|
688
804
|
throw new Error("Operation belongs to another client.");
|
|
689
805
|
return poll(peer, params.cursor, job);
|
|
690
806
|
}
|
|
691
807
|
if (method === "review-page") {
|
|
692
|
-
requireParticipant(params.client);
|
|
808
|
+
const peer = requireParticipant(params.client);
|
|
693
809
|
const waiting = typeof params.id === "string" ? pending.get(params.id) : undefined;
|
|
810
|
+
if (waiting?.pagedReview) {
|
|
811
|
+
if (waiting.pagedReview.generation !== peer.facts.generation)
|
|
812
|
+
throw new Error("Stale, foreign, or invalid review page.");
|
|
813
|
+
return waiting.pagedReview.page(params);
|
|
814
|
+
}
|
|
694
815
|
if (!waiting?.review ||
|
|
695
816
|
waiting.target !== params.client ||
|
|
696
817
|
!Number.isSafeInteger(params.index) ||
|
|
@@ -701,12 +822,20 @@ export async function startServer(root, execute, hooks = {}) {
|
|
|
701
822
|
return { id: params.id, index: params.index, content: waiting.review.slice(start, start + REVIEW_PAGE_CHARS) };
|
|
702
823
|
}
|
|
703
824
|
if (method === "answer") {
|
|
704
|
-
requireParticipant(params.client, true);
|
|
825
|
+
const peer = requireParticipant(params.client, true);
|
|
705
826
|
const waiting = typeof params.id === "string" ? pending.get(params.id) : undefined;
|
|
706
|
-
|
|
827
|
+
const offer = waiting?.pagedReview;
|
|
828
|
+
if (!waiting ||
|
|
829
|
+
waiting.target !== params.client ||
|
|
830
|
+
(offer && (offer.generation !== peer.facts.generation || params.generation !== offer.generation)))
|
|
707
831
|
throw new Error("Stale or foreign callback response.");
|
|
708
832
|
if (typeof params.error === "string")
|
|
709
833
|
waiting.reject(new Error(params.error.slice(0, 1024)));
|
|
834
|
+
else if (offer && params.value !== false) {
|
|
835
|
+
assertOwner();
|
|
836
|
+
requireParticipant(params.client);
|
|
837
|
+
waiting.resolve(offer.approved(params.value));
|
|
838
|
+
}
|
|
710
839
|
else
|
|
711
840
|
waiting.resolve(params.value);
|
|
712
841
|
return {};
|
|
@@ -731,6 +860,8 @@ export async function startServer(root, execute, hooks = {}) {
|
|
|
731
860
|
}
|
|
732
861
|
if (method === "stop") {
|
|
733
862
|
stopRequested = true;
|
|
863
|
+
for (const peer of participants.values())
|
|
864
|
+
waiters.get(peer)?.changed();
|
|
734
865
|
setImmediate(() => {
|
|
735
866
|
void close().catch(() => { });
|
|
736
867
|
});
|
|
@@ -767,14 +898,14 @@ export async function startServer(root, execute, hooks = {}) {
|
|
|
767
898
|
respond(response, 403, { error: { name: "Error", message: "Invalid daemon identity or protocol." } });
|
|
768
899
|
return;
|
|
769
900
|
}
|
|
770
|
-
if (
|
|
901
|
+
if (["attach", "resume-contact", "startup.bootstrap", "state-wait"].includes(body.method)) {
|
|
771
902
|
admissionController = new AbortController();
|
|
772
903
|
response.once("close", disconnect);
|
|
773
904
|
if (response.destroyed)
|
|
774
905
|
disconnect();
|
|
775
906
|
}
|
|
776
|
-
const operation = handle(body.method, body.params, admissionController?.signal);
|
|
777
|
-
if (["attach", "resume-contact", "update", "detach", "pause", "handoff"].includes(body.method)) {
|
|
907
|
+
const operation = handle(body.method, body.params, admissionController?.signal, body.reviewDetails === 1);
|
|
908
|
+
if (["attach", "resume-contact", "startup.bootstrap", "update", "detach", "pause", "handoff"].includes(body.method)) {
|
|
778
909
|
peerOperations.add(operation);
|
|
779
910
|
void operation.finally(() => peerOperations.delete(operation)).catch(() => { });
|
|
780
911
|
}
|
|
@@ -801,6 +932,7 @@ export async function startServer(root, execute, hooks = {}) {
|
|
|
801
932
|
await publishIdentity(paths, identity);
|
|
802
933
|
assertOwner();
|
|
803
934
|
published = true;
|
|
935
|
+
debug?.({ event: "lifecycle", outcome: "starting" });
|
|
804
936
|
}
|
|
805
937
|
catch (error) {
|
|
806
938
|
closed = true;
|
|
@@ -830,6 +962,7 @@ export async function startServer(root, execute, hooks = {}) {
|
|
|
830
962
|
invalidateExpiry(peer);
|
|
831
963
|
else if (!peer.closing && !peer.publishing)
|
|
832
964
|
peer.expiry = new AbortController();
|
|
965
|
+
waiters.get(peer)?.changed();
|
|
833
966
|
changed = true;
|
|
834
967
|
for (const job of [...jobs.values(), ...backgroundJobs])
|
|
835
968
|
if (!job.committed && (job.client === id || job.peers.has(id)))
|
|
@@ -853,7 +986,7 @@ export async function startServer(root, execute, hooks = {}) {
|
|
|
853
986
|
if (participationChanged)
|
|
854
987
|
await hooks.factsChanged?.();
|
|
855
988
|
}
|
|
856
|
-
const lifetime = async () => closed || compromised ||
|
|
989
|
+
const lifetime = async () => closed || compromised || ledger.failed ? "pause" : ((await hooks.lifetime?.()) ?? "pause");
|
|
857
990
|
if (hooks.lifetime) {
|
|
858
991
|
idleExit = createIdleExit({
|
|
859
992
|
live: () => liveClientFacts().length > 0,
|
|
@@ -866,12 +999,12 @@ export async function startServer(root, execute, hooks = {}) {
|
|
|
866
999
|
await Promise.allSettled([...jobs.values()].map((job) => job.promise));
|
|
867
1000
|
await Promise.allSettled([...peerOperations]);
|
|
868
1001
|
await expiring;
|
|
869
|
-
await
|
|
1002
|
+
await ledger.drain();
|
|
870
1003
|
},
|
|
871
1004
|
close: (current) => admission.run(async () => {
|
|
872
|
-
if (!current() || closed ||
|
|
1005
|
+
if (!current() || closed || ledger.failed || peerOperations.size)
|
|
873
1006
|
return false;
|
|
874
|
-
await
|
|
1007
|
+
await ledger.drain();
|
|
875
1008
|
const owned = await known();
|
|
876
1009
|
const policy = await lifetime();
|
|
877
1010
|
if (!owned ||
|
|
@@ -879,7 +1012,7 @@ export async function startServer(root, execute, hooks = {}) {
|
|
|
879
1012
|
!current() ||
|
|
880
1013
|
closed ||
|
|
881
1014
|
compromised ||
|
|
882
|
-
|
|
1015
|
+
ledger.failed ||
|
|
883
1016
|
peerOperations.size ||
|
|
884
1017
|
backgroundJobs.size ||
|
|
885
1018
|
pending.size ||
|
|
@@ -895,7 +1028,7 @@ export async function startServer(root, execute, hooks = {}) {
|
|
|
895
1028
|
return true;
|
|
896
1029
|
}),
|
|
897
1030
|
resume: () => hooks.idle?.(),
|
|
898
|
-
failed:
|
|
1031
|
+
failed: failure,
|
|
899
1032
|
});
|
|
900
1033
|
void idleExit.tick();
|
|
901
1034
|
}
|
|
@@ -910,8 +1043,10 @@ export async function startServer(root, execute, hooks = {}) {
|
|
|
910
1043
|
return closing;
|
|
911
1044
|
closing = (async () => {
|
|
912
1045
|
closed = true;
|
|
913
|
-
for (const peer of participants.values())
|
|
1046
|
+
for (const peer of participants.values()) {
|
|
914
1047
|
invalidateExpiry(peer);
|
|
1048
|
+
waiters.get(peer)?.changed();
|
|
1049
|
+
}
|
|
915
1050
|
clearInterval(heartbeat);
|
|
916
1051
|
for (const waiting of [...pending.values()])
|
|
917
1052
|
if (!waiting.callback.kind.startsWith("queue-"))
|
|
@@ -931,19 +1066,31 @@ export async function startServer(root, execute, hooks = {}) {
|
|
|
931
1066
|
await admission.run(async () => { });
|
|
932
1067
|
await Promise.allSettled([...peerOperations]);
|
|
933
1068
|
await expiring;
|
|
934
|
-
await
|
|
1069
|
+
await ledger.drain();
|
|
935
1070
|
for (const waiting of [...pending.values()])
|
|
936
1071
|
waiting.reject(new Error("Daemon stopping."));
|
|
937
1072
|
// A failed engine drain must retain the live owner, not permit an implicit restart.
|
|
938
1073
|
if (closeError)
|
|
939
1074
|
throw closeError;
|
|
940
1075
|
await endpoint.close();
|
|
941
|
-
|
|
1076
|
+
const owned = await known();
|
|
1077
|
+
if (owned)
|
|
942
1078
|
await fs.rm(paths.identity, { force: true });
|
|
943
1079
|
await release();
|
|
1080
|
+
released = true;
|
|
1081
|
+
if (owned && !compromised) {
|
|
1082
|
+
try {
|
|
1083
|
+
diagnostic?.({ event: "lifecycle", outcome: "drained" }, () => {
|
|
1084
|
+
assertRoot();
|
|
1085
|
+
return (!lstatSync(paths.owner, { throwIfNoEntry: false }) &&
|
|
1086
|
+
!lstatSync(paths.identity, { throwIfNoEntry: false }));
|
|
1087
|
+
});
|
|
1088
|
+
}
|
|
1089
|
+
catch { }
|
|
1090
|
+
}
|
|
944
1091
|
hooks.stopped?.();
|
|
945
1092
|
})().catch((error) => {
|
|
946
|
-
|
|
1093
|
+
failure();
|
|
947
1094
|
closing = undefined;
|
|
948
1095
|
throw error;
|
|
949
1096
|
});
|
|
@@ -974,6 +1121,7 @@ export async function startServer(root, execute, hooks = {}) {
|
|
|
974
1121
|
identity,
|
|
975
1122
|
paths,
|
|
976
1123
|
close,
|
|
1124
|
+
recordBootOutcome: (outcome) => debug?.({ event: "lifecycle", outcome }),
|
|
977
1125
|
backgroundExecution,
|
|
978
1126
|
get retainedEndpoint() {
|
|
979
1127
|
return endpoint.retainedDirectory;
|