@cotal-ai/manager 0.11.6 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +42 -0
- package/dist/commands.js +27 -4
- package/dist/commands.js.map +1 -1
- package/dist/index.d.ts +2 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -1
- package/dist/index.js.map +1 -1
- package/dist/launch.d.ts +6 -1
- package/dist/launch.d.ts.map +1 -1
- package/dist/launch.js +12 -2
- package/dist/launch.js.map +1 -1
- package/dist/manager.d.ts +257 -3
- package/dist/manager.d.ts.map +1 -1
- package/dist/manager.js +1443 -46
- package/dist/manager.js.map +1 -1
- package/dist/resume.d.ts +17 -0
- package/dist/resume.d.ts.map +1 -0
- package/dist/resume.js +142 -0
- package/dist/resume.js.map +1 -0
- package/dist/runtime/pty.d.ts.map +1 -1
- package/dist/runtime/pty.js +11 -0
- package/dist/runtime/pty.js.map +1 -1
- package/package.json +5 -5
package/dist/manager.js
CHANGED
|
@@ -1,13 +1,16 @@
|
|
|
1
1
|
import { execFile } from "node:child_process";
|
|
2
|
-
import {
|
|
2
|
+
import { createHash, randomUUID, randomBytes } from "node:crypto";
|
|
3
|
+
import { connect, credsAuthenticator } from "@nats-io/transport-node";
|
|
4
|
+
import { existsSync, lstatSync, readFileSync, rmSync } from "node:fs";
|
|
3
5
|
import { join, dirname, resolve } from "node:path";
|
|
4
|
-
import { CotalEndpoint, DEFAULT_SERVER, DEV_OWNER, MANAGER_LEASE_TTL_MS, STANDING_RENEWABLE_TTL_SEC, agentFilePath, clearSpaceHistory, connectorServers, deprovisionAgent, firstFreeName, loadAgentFile, loadCotalConfig, mintCreds, mkSecretDir, newIdentity, parsePrincipalKey, parseShareSelection, principalKey, provisionAgent, provisionAgentDurables, registry, resolveAuthProvider, saveAgentFile,
|
|
5
|
-
import { agentAuthState, authDir, connectorInstallHint, DEFAULT_CONNECTOR, defaultAgentType, findCotalRoot, loadMeshes, loadSpaceAuth, manifestExtensionNames, materializeFromManifest, mergeLaunchOptions, remintDaemonCreds, resolveOnPath, userAuthStateDir, writeRenewalRecord } from "@cotal-ai/workspace";
|
|
6
|
+
import { CotalEndpoint, DEFAULT_SERVER, DEV_OWNER, MANAGER_LEASE_TTL_MS, STANDING_RENEWABLE_TTL_SEC, agentFilePath, clearSpaceHistory, connectorServers, deprovisionAgent, firstFreeName, idFromCreds, loadAgentFile, loadCotalConfig, mintCreds, mintLifecycleUid, mkSecretDir, newIdentity, parsePrincipalKey, parseShareSelection, principalKey, probeConnect, provisionAgent, provisionAgentDurables, registry, resolveAuthProvider, saveAgentFile, subjectMatches, CONTROL_PRIVILEGED, CONTROL_SELF_SERVICE, CONTROL_ADMIN, CONTROL_AUTH_ADMIN, controlServiceSubject, } from "@cotal-ai/core";
|
|
7
|
+
import { agentActorTokenKey, agentAuthState, agentCredsDir, agentCredsKey, agentSecretFilePaths, agentSentinelCredsKey, authDir, connectorInstallHint, DEFAULT_CONNECTOR, defaultAgentType, findCotalRoot, loadMeshes, loadSpaceAuth, manifestExtensionNames, materializeFromManifest, materializeSecretToFile, mergeLaunchOptions, remintDaemonCreds, resolveOnPath, userAuthStateDir, workspaceSecretStore, writeRenewalRecord } from "@cotal-ai/workspace";
|
|
6
8
|
import { createRuntime, } from "./runtime/index.js";
|
|
7
9
|
import { AttachEndpoint } from "./attach-endpoint.js";
|
|
8
10
|
import { launchSpecForRun, materializePersona, launchAgentToStartOpts } from "./launch.js";
|
|
9
11
|
import { authorizeLaunch, authorizeNamedControl } from "./authorize.js";
|
|
10
12
|
import { controlShutdown } from "./control-shutdown.js";
|
|
13
|
+
import { parseResumeCommitArgs, parseResumeControlArgs, parseResumeFinalizeArgs } from "./resume.js";
|
|
11
14
|
/** Concurrency ceiling — the manager refuses to hold more than this many live + in-flight +
|
|
12
15
|
* cooling slots at once (P4a). Bounds a fork-bomb: spawn is a full agent process per call. */
|
|
13
16
|
const MAX_AGENTS = 50;
|
|
@@ -29,6 +32,16 @@ export const READINESS_TIMEOUT_MS = 30_000;
|
|
|
29
32
|
* `.catch`. Generous over the helper's 5s connect timeout to allow the two consumer-deletes + ACL purge
|
|
30
33
|
* + drain on a healthy-but-slow broker. */
|
|
31
34
|
const DEPROVISION_TIMEOUT_MS = 15_000;
|
|
35
|
+
/** A hard preservation stop should settle quickly. The manager still waits and reports a partial
|
|
36
|
+
* cut rather than pretending a child is gone. Held in ManagerOptions so fake runtimes can shorten it. */
|
|
37
|
+
const PRESERVE_STOP_TIMEOUT_MS = 10_000;
|
|
38
|
+
/** The STABLE retirement opId for one lifecycle (#29 piece 3): deterministic from the uid, so a
|
|
39
|
+
* despawn retry, a same-name-spawn nudge, and the auth service's boot resume all drive the SAME
|
|
40
|
+
* operation (the rail's idempotence table needs exactly one op per retiring incarnation). 26 hex
|
|
41
|
+
* chars = in the lifecycle-token grammar `[a-z0-9]{26,32}`, collision-resistant. */
|
|
42
|
+
function retireOpId(lifecycleUid) {
|
|
43
|
+
return createHash("sha256").update(`retire:${lifecycleUid}`).digest("hex").slice(0, 26);
|
|
44
|
+
}
|
|
32
45
|
/** Sentinel owner-filter value that matches NO agent's `userOwner` (owner tokens never contain a
|
|
33
46
|
* dash) — what {@link Manager.psOwnerFilter} returns for an unparseable caller so a malformed
|
|
34
47
|
* principal fail-closes to an empty `ps` instead of an unbounded one. */
|
|
@@ -54,6 +67,9 @@ function withTimeout(p, ms, msg) {
|
|
|
54
67
|
});
|
|
55
68
|
return Promise.race([p.finally(() => clearTimeout(timer)), timeout]);
|
|
56
69
|
}
|
|
70
|
+
function sameStrings(a, b) {
|
|
71
|
+
return JSON.stringify([...(a ?? [])].sort()) === JSON.stringify([...(b ?? [])].sort());
|
|
72
|
+
}
|
|
57
73
|
/**
|
|
58
74
|
* The agent supervisor: a long-lived mesh node that owns agent process lifecycle.
|
|
59
75
|
* It serves control requests on the "manager" service and spawns/kills agents
|
|
@@ -68,6 +84,7 @@ export class Manager {
|
|
|
68
84
|
/** See {@link ManagerOptions.installedExtensions}. */
|
|
69
85
|
installedExtensions;
|
|
70
86
|
runtime;
|
|
87
|
+
preserveStopTimeoutMs;
|
|
71
88
|
agents = new Map();
|
|
72
89
|
/** Names whose spawn is in flight (reserved synchronously before the provision await) — counted
|
|
73
90
|
* toward the ceiling so two concurrent same-name spawns can't both pass the gate (P4a). */
|
|
@@ -75,6 +92,29 @@ export class Manager {
|
|
|
75
92
|
/** Expiry stamps (`startedAt + MIN_LIFETIME`) for slots that freed while still young — a
|
|
76
93
|
* count-only, lazily-pruned recycle floor (P4c). Pruned + summed into the ceiling gate. */
|
|
77
94
|
cooling = [];
|
|
95
|
+
/** Names RESERVED PENDING RETIREMENT (#29 piece 3): a despawned agent's name stays held until
|
|
96
|
+
* the auth plane confirms its lifecycle's retirement TERMINAL over the auth-admin rail — the
|
|
97
|
+
* alias-reuse gate that closes the same-name despawn→respawn race at its root. An UNCERTAIN
|
|
98
|
+
* outcome (rail down, timeout) keeps the hold with the last attempt's copy; a same-name spawn
|
|
99
|
+
* refuses legibly AND re-fires the request. In-memory: across a manager restart the durable
|
|
100
|
+
* truth is the auth-side lifecycle head itself (an unretired head refuses issuance — the
|
|
101
|
+
* named residual this belt narrows, not replaces). */
|
|
102
|
+
retiring = new Map();
|
|
103
|
+
/** SINGLE-FLIGHT guard for {@link requestRetirement} (audit #1): one in-flight rail round-trip per
|
|
104
|
+
* (name, lifecycleUid). The detached `deprovision` call and every same-name-spawn nudge for THAT
|
|
105
|
+
* lifecycle JOIN the same promise instead of stacking independent requests that dual-enter the
|
|
106
|
+
* barrier; a fresh trigger after it settles re-drives. Keyed by (name, uid) — NOT name alone — so a
|
|
107
|
+
* same-name SUCCESSOR (which can spawn after the hold clears but before this flight's `nc.close`
|
|
108
|
+
* yield settles) never joins the predecessor's rail request and skips its own retirement. */
|
|
109
|
+
retiringFlight = new Map();
|
|
110
|
+
/** SINGLE-FLIGHT guard for {@link deprovision} (INT-2/C): one in-flight teardown per
|
|
111
|
+
* (name, lifecycleUid). The detached freeSlot teardown and every same-name-spawn nudge that
|
|
112
|
+
* re-drives it JOIN one promise instead of launching a SECOND, concurrent teardown. Without it,
|
|
113
|
+
* two teardowns race the NAME-KEYED ledger revoke: once the first frees the alias and a successor
|
|
114
|
+
* mints its own row, the second's delayed revoke (which carries no lifecycle coordinate) would
|
|
115
|
+
* delete the SUCCESSOR's standing authority. Keyed by (name, uid) so a later same-name lifecycle
|
|
116
|
+
* gets its own flight; a fresh trigger after settle re-drives only if the hold still stands. */
|
|
117
|
+
deprovisioningFlight = new Map();
|
|
78
118
|
attach;
|
|
79
119
|
ep;
|
|
80
120
|
/** Space trust material when the mesh runs in auth mode (`.cotal/auth` present);
|
|
@@ -92,6 +132,29 @@ export class Manager {
|
|
|
92
132
|
leaseTimer;
|
|
93
133
|
/** The class-2 renewal owner's half-TTL schedule (D5 slice 5); armed only on auth meshes. */
|
|
94
134
|
credRenewTimer;
|
|
135
|
+
maintenanceState = "active";
|
|
136
|
+
lifecycleInFlight = 0;
|
|
137
|
+
lifecycleDrainWaiters = [];
|
|
138
|
+
preservationTask;
|
|
139
|
+
preparationTask;
|
|
140
|
+
preservationGeneration = 0;
|
|
141
|
+
preservationAttemptId;
|
|
142
|
+
preservationStarted = false;
|
|
143
|
+
preservationFailures = [];
|
|
144
|
+
unverifiedStops = [];
|
|
145
|
+
preservationInventory;
|
|
146
|
+
resumeAttemptId;
|
|
147
|
+
resumeInventoryDigest;
|
|
148
|
+
resumeInventory;
|
|
149
|
+
resumeTask;
|
|
150
|
+
resumeResult;
|
|
151
|
+
resumeRequired = false;
|
|
152
|
+
resumeAwaitingCommit = false;
|
|
153
|
+
resumeCommitted = false;
|
|
154
|
+
resumeCommitTask;
|
|
155
|
+
resumeFinalized = false;
|
|
156
|
+
resumeDurableCommitToken;
|
|
157
|
+
resumedAgentNames = new Set();
|
|
95
158
|
constructor(opts) {
|
|
96
159
|
this.space = opts.space;
|
|
97
160
|
this.servers = opts.servers;
|
|
@@ -99,7 +162,17 @@ export class Manager {
|
|
|
99
162
|
this.workspaceRoot = opts.workspaceRoot ?? findCotalRoot();
|
|
100
163
|
this.installedExtensions = opts.installedExtensions ?? false;
|
|
101
164
|
this.runtime = createRuntime(opts.runtime ?? "auto", `cotal-${this.space}`);
|
|
102
|
-
this.
|
|
165
|
+
this.preserveStopTimeoutMs = opts.preserveStopTimeoutMs ?? PRESERVE_STOP_TIMEOUT_MS;
|
|
166
|
+
if (opts.resumeAttemptId && !/^[A-Za-z0-9_-]{1,128}$/.test(opts.resumeAttemptId))
|
|
167
|
+
throw new Error("resumeAttemptId must be a safe token (letters, digits, _, -; max 128)");
|
|
168
|
+
if (opts.resumeDurableCommitToken && !/^[a-f0-9]{64}$/.test(opts.resumeDurableCommitToken))
|
|
169
|
+
throw new Error("resumeDurableCommitToken must be a lowercase 32-byte token");
|
|
170
|
+
if (opts.resumeDurableCommitToken && !opts.resumeAttemptId)
|
|
171
|
+
throw new Error("resumeDurableCommitToken requires resumeAttemptId");
|
|
172
|
+
this.resumeAttemptId = opts.resumeAttemptId;
|
|
173
|
+
this.resumeRequired = opts.resumeAttemptId !== undefined;
|
|
174
|
+
this.resumeDurableCommitToken = opts.resumeDurableCommitToken;
|
|
175
|
+
this.attach = new AttachEndpoint((name) => this.maintenanceState === "active" && !this.resumeRequired ? this.agents.get(name)?.handle : undefined, () => this.list(),
|
|
103
176
|
// Initial /feed replay for a connecting console: the current peer roster.
|
|
104
177
|
() => [{ event: "roster", data: this.ep?.getRoster() ?? [] }], opts.consolePort ?? 0);
|
|
105
178
|
}
|
|
@@ -149,6 +222,11 @@ export class Manager {
|
|
|
149
222
|
servers: this.servers,
|
|
150
223
|
channels: [],
|
|
151
224
|
creds,
|
|
225
|
+
// The supervisor registers on the roster, and an authed presence-registering endpoint is
|
|
226
|
+
// lifecycle-keyed (SPEC 13.1, fail-before-presence). The manager process is the top of its
|
|
227
|
+
// own launch chain (the operator command IS its launcher), so it mints its incarnation's
|
|
228
|
+
// uid here - one per supervisor process, never reused across restarts.
|
|
229
|
+
lifecycleUid: mintLifecycleUid(),
|
|
152
230
|
// The supervisor serves control + watches presence; it never consumes chat/dm/task
|
|
153
231
|
// (no message handler). consume:false avoids binding consumers it doesn't use — and
|
|
154
232
|
// under auth avoids trying to bind its own DM/task durables that nothing pre-created.
|
|
@@ -221,6 +299,9 @@ export class Manager {
|
|
|
221
299
|
* (no responder) is recorded honestly: the daemon's 75% source re-read remains the adoption backstop.
|
|
222
300
|
* Never throws — renewal failure must be LOUD (log + record), not fatal to the supervisor. */
|
|
223
301
|
async renewDaemonCreds() {
|
|
302
|
+
const release = this.beginLifecycle();
|
|
303
|
+
if (!release)
|
|
304
|
+
return;
|
|
224
305
|
try {
|
|
225
306
|
const results = await remintDaemonCreds(this.workspaceRoot);
|
|
226
307
|
const resigned = results.filter((r) => r.ok);
|
|
@@ -243,6 +324,360 @@ export class Manager {
|
|
|
243
324
|
catch (e) {
|
|
244
325
|
console.error(`! credential renewal pass failed: ${e.message}`);
|
|
245
326
|
}
|
|
327
|
+
finally {
|
|
328
|
+
release();
|
|
329
|
+
}
|
|
330
|
+
}
|
|
331
|
+
/** Admit one lifecycle/control operation while active. The synchronous increment is the fence:
|
|
332
|
+
* preserveState flips state before its first await, so work is either counted or rejected. */
|
|
333
|
+
beginLifecycle(resumeOperation = false) {
|
|
334
|
+
if (this.maintenanceState !== "active" || (this.resumeRequired && !resumeOperation))
|
|
335
|
+
return undefined;
|
|
336
|
+
this.lifecycleInFlight++;
|
|
337
|
+
let released = false;
|
|
338
|
+
return () => {
|
|
339
|
+
if (released)
|
|
340
|
+
return;
|
|
341
|
+
released = true;
|
|
342
|
+
this.releaseLifecycle();
|
|
343
|
+
};
|
|
344
|
+
}
|
|
345
|
+
releaseLifecycle() {
|
|
346
|
+
this.lifecycleInFlight--;
|
|
347
|
+
if (this.lifecycleInFlight !== 0)
|
|
348
|
+
return;
|
|
349
|
+
const waiters = this.lifecycleDrainWaiters;
|
|
350
|
+
this.lifecycleDrainWaiters = [];
|
|
351
|
+
for (const wake of waiters)
|
|
352
|
+
wake();
|
|
353
|
+
}
|
|
354
|
+
/** A cleanup spawned by accepted active-mode work is part of that work for maintenance draining,
|
|
355
|
+
* even where the ordinary control reply remains fire-and-forget. */
|
|
356
|
+
trackDeprovision(a, context = "") {
|
|
357
|
+
this.lifecycleInFlight++;
|
|
358
|
+
void this.deprovision(a)
|
|
359
|
+
.catch((e) => console.error(`deprovision${context ? ` ${context}` : ""} ${a.name} (${a.id}): ${e.message}`))
|
|
360
|
+
.finally(() => this.releaseLifecycle());
|
|
361
|
+
}
|
|
362
|
+
async awaitLifecycleDrain() {
|
|
363
|
+
if (this.lifecycleInFlight === 0)
|
|
364
|
+
return;
|
|
365
|
+
await new Promise((resolve) => this.lifecycleDrainWaiters.push(resolve));
|
|
366
|
+
}
|
|
367
|
+
maintenanceError() {
|
|
368
|
+
if (this.resumeRequired)
|
|
369
|
+
return `manager is waiting for resume attempt ${this.resumeAttemptId}; ordinary lifecycle/control work is fenced`;
|
|
370
|
+
return `manager is in ${this.maintenanceState} mode; new lifecycle/control work is fenced`;
|
|
371
|
+
}
|
|
372
|
+
/** Fence and build the inventory without stopping a child. The coordinator must durably persist
|
|
373
|
+
* this exact plan before calling commitPreservation with the same attempt id. */
|
|
374
|
+
preparePreservation(attemptId) {
|
|
375
|
+
if (!attemptId.trim())
|
|
376
|
+
return Promise.reject(new Error("preservation attemptId is required"));
|
|
377
|
+
if (this.preservationAttemptId && this.preservationAttemptId !== attemptId)
|
|
378
|
+
return Promise.reject(new Error(`manager is fenced for preservation attempt ${this.preservationAttemptId}; refusing different attempt ${attemptId}`));
|
|
379
|
+
if (this.maintenanceState === "preserved" && this.preservationInventory)
|
|
380
|
+
return Promise.resolve({ ok: true, attemptId, state: "preserved", inventory: this.preservationInventory, failures: [] });
|
|
381
|
+
if (this.preparationTask)
|
|
382
|
+
return this.preparationTask;
|
|
383
|
+
if (this.maintenanceState === "active") {
|
|
384
|
+
// The fence lands before any await. Accepted work has already incremented lifecycleInFlight.
|
|
385
|
+
this.maintenanceState = "preserving";
|
|
386
|
+
this.preservationAttemptId = attemptId;
|
|
387
|
+
this.preservationGeneration++;
|
|
388
|
+
if (this.credRenewTimer) {
|
|
389
|
+
clearInterval(this.credRenewTimer);
|
|
390
|
+
this.credRenewTimer = undefined;
|
|
391
|
+
}
|
|
392
|
+
}
|
|
393
|
+
const generation = this.preservationGeneration;
|
|
394
|
+
const task = this.runPreparation(attemptId, generation);
|
|
395
|
+
let wrapped;
|
|
396
|
+
wrapped = task.finally(() => {
|
|
397
|
+
if (this.preservationGeneration === generation && this.preparationTask === wrapped)
|
|
398
|
+
this.preparationTask = undefined;
|
|
399
|
+
});
|
|
400
|
+
this.preparationTask = wrapped;
|
|
401
|
+
return wrapped;
|
|
402
|
+
}
|
|
403
|
+
assertPreservationGeneration(attemptId, generation) {
|
|
404
|
+
if (this.preservationAttemptId !== attemptId || this.preservationGeneration !== generation)
|
|
405
|
+
throw new Error(`preservation attempt ${attemptId} was abandoned before preparation completed`);
|
|
406
|
+
}
|
|
407
|
+
async runPreparation(attemptId, generation) {
|
|
408
|
+
await this.awaitLifecycleDrain();
|
|
409
|
+
this.assertPreservationGeneration(attemptId, generation);
|
|
410
|
+
const inventory = this.preservationInventory ?? {
|
|
411
|
+
version: "cotal-manager-resume/v1",
|
|
412
|
+
space: this.space,
|
|
413
|
+
createdAt: new Date().toISOString(),
|
|
414
|
+
agents: [...this.agents.values()].map((a) => this.resumeEntry(a)),
|
|
415
|
+
};
|
|
416
|
+
const failures = [];
|
|
417
|
+
for (const entry of inventory.agents) {
|
|
418
|
+
const error = this.inventoryReferenceError(entry);
|
|
419
|
+
if (error)
|
|
420
|
+
failures.push({
|
|
421
|
+
name: entry.name,
|
|
422
|
+
id: entry.identity.mode === "user" ? principalKey(entry.identity.owner, entry.identity.actor).key : entry.identity.id,
|
|
423
|
+
error,
|
|
424
|
+
});
|
|
425
|
+
}
|
|
426
|
+
const unverifiedStops = this.unverifiedStops.filter((stopped) => {
|
|
427
|
+
try {
|
|
428
|
+
if (!stopped.authoritative && stopped.handle.status() === "exited")
|
|
429
|
+
return false;
|
|
430
|
+
}
|
|
431
|
+
catch { /* fail closed below */ }
|
|
432
|
+
failures.push({
|
|
433
|
+
name: stopped.name,
|
|
434
|
+
id: stopped.id,
|
|
435
|
+
error: stopped.error ?? `an earlier stop on runtime "${stopped.handle.kind}" cannot prove the child is gone`,
|
|
436
|
+
});
|
|
437
|
+
return true;
|
|
438
|
+
});
|
|
439
|
+
this.assertPreservationGeneration(attemptId, generation);
|
|
440
|
+
// The prepared inventory must round-trip through the EXACT resume control parser (schema and
|
|
441
|
+
// byte cap) NOW, before any child stops: a cut that cannot resume must fail at prepare time,
|
|
442
|
+
// never after listener exposure.
|
|
443
|
+
try {
|
|
444
|
+
parseResumeControlArgs({ attemptId, inventory });
|
|
445
|
+
}
|
|
446
|
+
catch (e) {
|
|
447
|
+
failures.push({ name: "<inventory>", id: attemptId, error: `prepared inventory would be rejected at resume: ${e.message}` });
|
|
448
|
+
}
|
|
449
|
+
this.preservationInventory = inventory;
|
|
450
|
+
this.preservationFailures = failures;
|
|
451
|
+
this.unverifiedStops = unverifiedStops;
|
|
452
|
+
return {
|
|
453
|
+
ok: failures.length === 0,
|
|
454
|
+
attemptId,
|
|
455
|
+
state: "prepared",
|
|
456
|
+
inventory,
|
|
457
|
+
failures: [...failures],
|
|
458
|
+
};
|
|
459
|
+
}
|
|
460
|
+
/** Stop children only after the coordinator has persisted the prepared inventory. Same-attempt
|
|
461
|
+
* retries are idempotent; a different attempt is refused. */
|
|
462
|
+
commitPreservation(attemptId) {
|
|
463
|
+
if (!this.preservationAttemptId || this.preservationAttemptId !== attemptId)
|
|
464
|
+
return Promise.reject(new Error(`preservation attempt ${attemptId} was not prepared by this manager`));
|
|
465
|
+
if (!this.preservationInventory)
|
|
466
|
+
return Promise.reject(new Error(`preservation attempt ${attemptId} has no prepared inventory`));
|
|
467
|
+
if (this.preservationFailures.length)
|
|
468
|
+
return Promise.resolve({
|
|
469
|
+
ok: false,
|
|
470
|
+
attemptId,
|
|
471
|
+
state: "preserving",
|
|
472
|
+
inventory: this.preservationInventory,
|
|
473
|
+
failures: [...this.preservationFailures],
|
|
474
|
+
});
|
|
475
|
+
if (this.maintenanceState === "preserved")
|
|
476
|
+
return Promise.resolve({ ok: true, attemptId, state: "preserved", inventory: this.preservationInventory, failures: [] });
|
|
477
|
+
if (this.preservationTask)
|
|
478
|
+
return this.preservationTask;
|
|
479
|
+
this.preservationStarted = true;
|
|
480
|
+
this.preservationTask = this.runPreservation(attemptId).finally(() => {
|
|
481
|
+
this.preservationTask = undefined;
|
|
482
|
+
});
|
|
483
|
+
return this.preservationTask;
|
|
484
|
+
}
|
|
485
|
+
/** Recover an abandoned prepare before any child stop. Once commit begins, preservation is
|
|
486
|
+
* irreversible and remains fenced until the coordinator records failure/recourse. */
|
|
487
|
+
abortPreservation(attemptId) {
|
|
488
|
+
if (this.preservationAttemptId !== attemptId)
|
|
489
|
+
throw new Error(`preservation attempt ${attemptId} is not the active manager attempt`);
|
|
490
|
+
if (this.preparationTask || this.lifecycleInFlight > 0)
|
|
491
|
+
throw new Error(`preservation attempt ${attemptId} is still preparing or draining accepted lifecycle work and cannot be aborted`);
|
|
492
|
+
if (this.preservationStarted || this.preservationTask || this.maintenanceState === "preserved")
|
|
493
|
+
throw new Error(`preservation attempt ${attemptId} has begun stopping children and cannot return to active mode`);
|
|
494
|
+
this.preservationGeneration++;
|
|
495
|
+
this.maintenanceState = "active";
|
|
496
|
+
this.preservationAttemptId = undefined;
|
|
497
|
+
this.preservationInventory = undefined;
|
|
498
|
+
this.preservationFailures = [];
|
|
499
|
+
if (this.auth && !this.credRenewTimer) {
|
|
500
|
+
this.credRenewTimer = setInterval(() => { void this.renewDaemonCreds(); }, (STANDING_RENEWABLE_TTL_SEC / 2) * 1000);
|
|
501
|
+
this.credRenewTimer.unref?.();
|
|
502
|
+
}
|
|
503
|
+
// Exit watchers were suppressed while the fence stood: reconcile every child that died during
|
|
504
|
+
// preparation now, or its slot/credential footprint would linger unreaped after the abort.
|
|
505
|
+
for (const agent of [...this.agents.values()]) {
|
|
506
|
+
try {
|
|
507
|
+
if (agent.handle.status() === "exited")
|
|
508
|
+
this.onAgentExit(agent);
|
|
509
|
+
}
|
|
510
|
+
catch { /* status unavailable - the exit watcher fires again on real exit */ }
|
|
511
|
+
}
|
|
512
|
+
}
|
|
513
|
+
/** In-process convenience that preserves the crash barrier by awaiting durable persistence between
|
|
514
|
+
* prepare and commit. Wire callers use the explicit two-phase admin operations. */
|
|
515
|
+
async preserveState(opts) {
|
|
516
|
+
const plan = await this.preparePreservation(opts.attemptId);
|
|
517
|
+
if (!plan.ok)
|
|
518
|
+
return { ok: false, attemptId: opts.attemptId, state: "preserving", inventory: plan.inventory, failures: plan.failures };
|
|
519
|
+
await opts.persistInventory(plan.inventory);
|
|
520
|
+
return this.commitPreservation(opts.attemptId);
|
|
521
|
+
}
|
|
522
|
+
async runPreservation(attemptId) {
|
|
523
|
+
const failures = [];
|
|
524
|
+
for (const a of [...this.agents.values()])
|
|
525
|
+
a.suppressCleanup = true;
|
|
526
|
+
await Promise.all([...this.agents.values()].map(async (a) => {
|
|
527
|
+
try {
|
|
528
|
+
// A preservation cut must not run the connector's logical leave/cleanup hooks.
|
|
529
|
+
a.handle.stop({ graceful: false });
|
|
530
|
+
}
|
|
531
|
+
catch (e) {
|
|
532
|
+
failures.push({ name: a.name, id: a.id, error: `stop failed: ${e.message}` });
|
|
533
|
+
return;
|
|
534
|
+
}
|
|
535
|
+
try {
|
|
536
|
+
await this.awaitHandleExit(a.handle);
|
|
537
|
+
if (this.agents.get(a.name) === a)
|
|
538
|
+
this.agents.delete(a.name);
|
|
539
|
+
}
|
|
540
|
+
catch (e) {
|
|
541
|
+
failures.push({ name: a.name, id: a.id, error: e.message });
|
|
542
|
+
}
|
|
543
|
+
}));
|
|
544
|
+
if (failures.length === 0)
|
|
545
|
+
this.maintenanceState = "preserved";
|
|
546
|
+
return {
|
|
547
|
+
ok: failures.length === 0,
|
|
548
|
+
attemptId,
|
|
549
|
+
state: this.maintenanceState === "preserved" ? "preserved" : "preserving",
|
|
550
|
+
inventory: this.preservationInventory,
|
|
551
|
+
failures,
|
|
552
|
+
};
|
|
553
|
+
}
|
|
554
|
+
async awaitHandleExit(handle) {
|
|
555
|
+
if (!handle.waitForExit)
|
|
556
|
+
throw new Error(`runtime "${handle.kind}" cannot prove child exit (AgentHandle.waitForExit is not implemented)`);
|
|
557
|
+
if (handle.status() === "exited")
|
|
558
|
+
return;
|
|
559
|
+
await withTimeout(handle.waitForExit(), this.preserveStopTimeoutMs, `child did not exit within ${this.preserveStopTimeoutMs}ms`);
|
|
560
|
+
if (handle.status() !== "exited")
|
|
561
|
+
throw new Error(`runtime "${handle.kind}" reported exit completion but status is still running`);
|
|
562
|
+
}
|
|
563
|
+
inventoryReferenceError(entry) {
|
|
564
|
+
if (entry.launch.source.kind === "manifest" && !entry.launch.source.runId)
|
|
565
|
+
return "resolved manifest launch has no retained runId";
|
|
566
|
+
if (entry.launch.unresolvedLaunchOptionKeys?.length)
|
|
567
|
+
return `imperative launch options have no non-secret durable source (${entry.launch.unresolvedLaunchOptionKeys.join(", ")})`;
|
|
568
|
+
if (!entry.dependencies.some((path) => resolve(path) === resolve(entry.launch.source.configPath)))
|
|
569
|
+
return `launch config is not declared as a retained dependency: ${entry.launch.source.configPath}`;
|
|
570
|
+
if (entry.launch.source.kind === "manifest" && entry.launch.source.runId) {
|
|
571
|
+
const specPath = join(this.workspaceRoot, ".cotal", "run", `${entry.launch.source.runId}.json`);
|
|
572
|
+
if (!entry.dependencies.some((path) => resolve(path) === resolve(specPath)))
|
|
573
|
+
return `manifest source is not declared as a retained dependency: ${specPath}`;
|
|
574
|
+
}
|
|
575
|
+
const required = [...entry.dependencies];
|
|
576
|
+
if (entry.identity.mode === "static")
|
|
577
|
+
required.push(entry.identity.credential.path);
|
|
578
|
+
if (entry.identity.mode === "user") {
|
|
579
|
+
required.push(entry.identity.actorToken.path, entry.identity.sentinelCredential.path);
|
|
580
|
+
}
|
|
581
|
+
for (const path of required) {
|
|
582
|
+
try {
|
|
583
|
+
const st = lstatSync(path);
|
|
584
|
+
if (!st.isFile() || st.isSymbolicLink())
|
|
585
|
+
return `retained reference is not a regular non-symlink file: ${path}`;
|
|
586
|
+
}
|
|
587
|
+
catch (e) {
|
|
588
|
+
return `retained reference unavailable: ${path} (${e.message})`;
|
|
589
|
+
}
|
|
590
|
+
}
|
|
591
|
+
if (process.platform !== "win32") {
|
|
592
|
+
const secrets = entry.identity.mode === "static"
|
|
593
|
+
? [entry.identity.credential.path]
|
|
594
|
+
: entry.identity.mode === "user"
|
|
595
|
+
? [entry.identity.actorToken.path, entry.identity.sentinelCredential.path]
|
|
596
|
+
: [];
|
|
597
|
+
for (const path of secrets)
|
|
598
|
+
if ((lstatSync(path).mode & 0o077) !== 0)
|
|
599
|
+
return `retained identity file is not private (expected 0600): ${path}`;
|
|
600
|
+
}
|
|
601
|
+
try {
|
|
602
|
+
if (this.fileDigest(entry.launch.source.configPath) !== entry.launch.source.configSha256)
|
|
603
|
+
return `launch config changed since it became effective: ${entry.launch.source.configPath}`;
|
|
604
|
+
if (entry.identity.mode === "static" && this.fileDigest(entry.identity.credential.path) !== entry.identity.credential.sha256)
|
|
605
|
+
return `retained credential changed after the cut: ${entry.identity.credential.path}`;
|
|
606
|
+
if (entry.identity.mode === "user" &&
|
|
607
|
+
(this.fileDigest(entry.identity.actorToken.path) !== entry.identity.actorToken.sha256 ||
|
|
608
|
+
this.fileDigest(entry.identity.sentinelCredential.path) !== entry.identity.sentinelCredential.sha256))
|
|
609
|
+
return `retained user identity files changed after the cut for ${entry.name}`;
|
|
610
|
+
if (entry.launch.source.kind === "manifest" && entry.launch.source.runId) {
|
|
611
|
+
const specPath = join(this.workspaceRoot, ".cotal", "run", `${entry.launch.source.runId}.json`);
|
|
612
|
+
if (!entry.launch.source.manifestSha256 || this.fileDigest(specPath) !== entry.launch.source.manifestSha256)
|
|
613
|
+
return `manifest source changed since it became effective: ${specPath}`;
|
|
614
|
+
}
|
|
615
|
+
}
|
|
616
|
+
catch (e) {
|
|
617
|
+
return `retained reference cannot be hashed: ${e.message}`;
|
|
618
|
+
}
|
|
619
|
+
return undefined;
|
|
620
|
+
}
|
|
621
|
+
fileDigest(path) {
|
|
622
|
+
return createHash("sha256").update(readFileSync(path)).digest("hex");
|
|
623
|
+
}
|
|
624
|
+
fileDigestOrEmpty(path) {
|
|
625
|
+
try {
|
|
626
|
+
return this.fileDigest(path);
|
|
627
|
+
}
|
|
628
|
+
catch {
|
|
629
|
+
return "";
|
|
630
|
+
}
|
|
631
|
+
}
|
|
632
|
+
resumeEntry(a) {
|
|
633
|
+
const principal = a.userOwner
|
|
634
|
+
? parsePrincipalKey(a.id)
|
|
635
|
+
: { owner: DEV_OWNER, actor: a.id };
|
|
636
|
+
if (!principal)
|
|
637
|
+
throw new Error(`managed agent ${a.name} has an invalid principal ${a.id}`);
|
|
638
|
+
const files = agentSecretFilePaths(this.workspaceRoot, a.name);
|
|
639
|
+
const identity = a.userOwner
|
|
640
|
+
? {
|
|
641
|
+
mode: "user",
|
|
642
|
+
owner: principal.owner,
|
|
643
|
+
actor: principal.actor,
|
|
644
|
+
lifecycleUid: a.lifecycleUid,
|
|
645
|
+
actorToken: { kind: "file", path: files.actorToken, sha256: this.fileDigestOrEmpty(files.actorToken) },
|
|
646
|
+
sentinelCredential: { kind: "file", path: files.sentinelCreds, sha256: this.fileDigestOrEmpty(files.sentinelCreds) },
|
|
647
|
+
health: { kind: "file", path: files.health },
|
|
648
|
+
}
|
|
649
|
+
: this.auth
|
|
650
|
+
? { mode: "static", id: principal.actor, lifecycleUid: a.lifecycleUid, credential: { kind: "file", path: files.creds, sha256: this.fileDigestOrEmpty(files.creds) } }
|
|
651
|
+
: { mode: "open", id: principal.actor, lifecycleUid: a.lifecycleUid };
|
|
652
|
+
const dependencies = [a.launch.source.configPath];
|
|
653
|
+
if (a.launch.source.kind === "manifest" && a.launch.source.runId)
|
|
654
|
+
dependencies.unshift(join(this.workspaceRoot, ".cotal", "run", `${a.launch.source.runId}.json`));
|
|
655
|
+
return {
|
|
656
|
+
space: this.space,
|
|
657
|
+
name: a.name,
|
|
658
|
+
role: a.role,
|
|
659
|
+
identity,
|
|
660
|
+
launch: {
|
|
661
|
+
connector: a.agent,
|
|
662
|
+
runtime: a.handle.kind,
|
|
663
|
+
cwd: a.launch.cwd,
|
|
664
|
+
source: a.launch.source,
|
|
665
|
+
model: a.launch.model,
|
|
666
|
+
variant: a.launch.variant,
|
|
667
|
+
subscribe: a.launch.subscribe,
|
|
668
|
+
allowSubscribe: a.launch.allowSubscribe,
|
|
669
|
+
allowPublish: a.launch.allowPublish,
|
|
670
|
+
capabilities: a.launch.capabilities,
|
|
671
|
+
transcript: a.launch.transcript,
|
|
672
|
+
shareTools: a.launch.shareTools,
|
|
673
|
+
forkSource: a.launch.forkSource,
|
|
674
|
+
unresolvedLaunchOptionKeys: a.launch.unresolvedLaunchOptionKeys,
|
|
675
|
+
},
|
|
676
|
+
dependencies,
|
|
677
|
+
spawner: a.spawner,
|
|
678
|
+
authorityParent: a.authorityParent,
|
|
679
|
+
startedAt: new Date(a.startedAt).toISOString(),
|
|
680
|
+
};
|
|
246
681
|
}
|
|
247
682
|
/** Tear down every managed agent's footprint — the shared teardown for EVERY manager-exit path (#159
|
|
248
683
|
* B2): graceful {@link stop} AND the fail-closed lease-loss exit ({@link renewLease}). A manager exit is
|
|
@@ -262,14 +697,44 @@ export class Manager {
|
|
|
262
697
|
this.stopHandle(a, false);
|
|
263
698
|
}
|
|
264
699
|
// Deprovision EVERY snapshot entry regardless of whether its stop failed (allSettled + a loud log).
|
|
265
|
-
await Promise.allSettled(managed.map((a) => this.deprovision(a).catch((e) => console.error(`deprovision ${a.name} (${a.id}) on shutdown: ${e.message}`))));
|
|
700
|
+
await Promise.allSettled(managed.filter((a) => !a.suppressCleanup).map((a) => this.deprovision(a).catch((e) => console.error(`deprovision ${a.name} (${a.id}) on shutdown: ${e.message}`))));
|
|
701
|
+
}
|
|
702
|
+
async stopRetainedAgentsOnExit() {
|
|
703
|
+
const managed = [...this.agents.values()];
|
|
704
|
+
for (const a of managed)
|
|
705
|
+
a.suppressCleanup = true;
|
|
706
|
+
const failures = [];
|
|
707
|
+
await Promise.all(managed.map(async (a) => {
|
|
708
|
+
try {
|
|
709
|
+
a.handle.stop({ graceful: false });
|
|
710
|
+
}
|
|
711
|
+
catch (e) {
|
|
712
|
+
failures.push(`${a.name}: stop failed: ${e.message}`);
|
|
713
|
+
}
|
|
714
|
+
try {
|
|
715
|
+
await this.awaitHandleExit(a.handle);
|
|
716
|
+
if (this.agents.get(a.name) === a)
|
|
717
|
+
this.agents.delete(a.name);
|
|
718
|
+
}
|
|
719
|
+
catch (e) {
|
|
720
|
+
failures.push(`${a.name}: ${e.message}`);
|
|
721
|
+
}
|
|
722
|
+
}));
|
|
723
|
+
if (failures.length)
|
|
724
|
+
throw new Error(`manager preservation shutdown incomplete: ${failures.join("; ")}`);
|
|
266
725
|
}
|
|
267
726
|
async stop() {
|
|
268
727
|
if (this.leaseTimer)
|
|
269
728
|
clearInterval(this.leaseTimer);
|
|
270
729
|
if (this.credRenewTimer)
|
|
271
730
|
clearInterval(this.credRenewTimer);
|
|
272
|
-
|
|
731
|
+
if (this.maintenanceState === "active" && !this.resumeRequired) {
|
|
732
|
+
await this.teardownManagedAgents(); // normal shutdown stays destructive (#159 B2)
|
|
733
|
+
}
|
|
734
|
+
else {
|
|
735
|
+
// A signal after a partial preservation must never fall back into destructive teardown.
|
|
736
|
+
await this.stopRetainedAgentsOnExit();
|
|
737
|
+
}
|
|
273
738
|
await this.ep.releaseManagerLease(this.leaseRevision);
|
|
274
739
|
await this.ep.stop();
|
|
275
740
|
await this.attach.stop();
|
|
@@ -279,9 +744,9 @@ export class Manager {
|
|
|
279
744
|
* with the new holder, and exit. We deliberately do NOT re-acquire (a replacement may already be live
|
|
280
745
|
* while we'd still be serving) and do NOT release the key — it now belongs to that replacement. */
|
|
281
746
|
async renewLease() {
|
|
282
|
-
if (!this.leaseInfo || this.leaseRevision === undefined)
|
|
283
|
-
return;
|
|
284
747
|
try {
|
|
748
|
+
if (!this.leaseInfo || this.leaseRevision === undefined)
|
|
749
|
+
return;
|
|
285
750
|
this.leaseRevision = await this.ep.renewManagerLease(this.leaseInfo, this.leaseRevision);
|
|
286
751
|
}
|
|
287
752
|
catch (e) {
|
|
@@ -291,7 +756,10 @@ export class Manager {
|
|
|
291
756
|
// Tear down our managed agents' footprints too (#159 B2) — this exit path leaks them otherwise. Do
|
|
292
757
|
// NOT release the lease key (it may belong to the replacement holder). Best-effort, like ep/attach.
|
|
293
758
|
try {
|
|
294
|
-
|
|
759
|
+
if (this.maintenanceState === "active" && !this.resumeRequired)
|
|
760
|
+
await this.teardownManagedAgents();
|
|
761
|
+
else
|
|
762
|
+
await this.stopRetainedAgentsOnExit();
|
|
295
763
|
}
|
|
296
764
|
catch { /* best effort */ }
|
|
297
765
|
try {
|
|
@@ -306,6 +774,187 @@ export class Manager {
|
|
|
306
774
|
}
|
|
307
775
|
}
|
|
308
776
|
async handle(req, tier) {
|
|
777
|
+
if (req.op === "finalizeResume") {
|
|
778
|
+
if (tier !== CONTROL_ADMIN)
|
|
779
|
+
return { ok: false, error: "finalizeResume is admin-only; not allowed on this control subject" };
|
|
780
|
+
let args;
|
|
781
|
+
try {
|
|
782
|
+
args = parseResumeFinalizeArgs(req.args);
|
|
783
|
+
}
|
|
784
|
+
catch (e) {
|
|
785
|
+
return { ok: false, error: e.message };
|
|
786
|
+
}
|
|
787
|
+
if (!this.resumeAttemptId || this.resumeAttemptId !== args.attemptId)
|
|
788
|
+
return { ok: false, error: `manager expects resume attempt ${this.resumeAttemptId ?? "<none>"}, not ${args.attemptId}` };
|
|
789
|
+
if (!this.resumeCommitted || !this.resumeDurableCommitToken)
|
|
790
|
+
return { ok: false, error: `resume attempt ${args.attemptId} has no successful commit to finalize` };
|
|
791
|
+
if (this.resumeDurableCommitToken !== args.durableCommitToken)
|
|
792
|
+
return { ok: false, error: `resume attempt ${args.attemptId} durable commit token does not match` };
|
|
793
|
+
if (this.resumeFinalized)
|
|
794
|
+
return { ok: true, data: { attemptId: args.attemptId, state: "active" } };
|
|
795
|
+
const inventory = this.resumeInventory;
|
|
796
|
+
if (!inventory)
|
|
797
|
+
return { ok: false, error: `resume attempt ${args.attemptId} has no bound inventory` };
|
|
798
|
+
let inactive;
|
|
799
|
+
try {
|
|
800
|
+
inactive = this.resumeLivenessErrors(inventory, this.ep.getRoster());
|
|
801
|
+
}
|
|
802
|
+
catch (e) {
|
|
803
|
+
return { ok: false, error: `resume attempt ${args.attemptId} cannot verify live principals at finalize: ${e.message}` };
|
|
804
|
+
}
|
|
805
|
+
if (inactive.length)
|
|
806
|
+
return { ok: false, error: `resume attempt ${args.attemptId} is not live at finalize: ${inactive.join("; ")}` };
|
|
807
|
+
for (const entry of this.resumeInventory?.agents ?? []) {
|
|
808
|
+
const managed = this.agents.get(entry.name);
|
|
809
|
+
if (managed)
|
|
810
|
+
managed.suppressCleanup = false;
|
|
811
|
+
}
|
|
812
|
+
this.resumeFinalized = true;
|
|
813
|
+
this.resumeRequired = false;
|
|
814
|
+
return { ok: true, data: { attemptId: args.attemptId, state: "active" } };
|
|
815
|
+
}
|
|
816
|
+
if (req.op === "commitResume") {
|
|
817
|
+
if (tier !== CONTROL_ADMIN)
|
|
818
|
+
return { ok: false, error: "commitResume is admin-only; not allowed on this control subject" };
|
|
819
|
+
let attemptId;
|
|
820
|
+
try {
|
|
821
|
+
attemptId = parseResumeCommitArgs(req.args).attemptId;
|
|
822
|
+
}
|
|
823
|
+
catch (e) {
|
|
824
|
+
return { ok: false, error: e.message };
|
|
825
|
+
}
|
|
826
|
+
if (!this.resumeAttemptId || this.resumeAttemptId !== attemptId)
|
|
827
|
+
return { ok: false, error: `manager expects resume attempt ${this.resumeAttemptId ?? "<none>"}, not ${attemptId}` };
|
|
828
|
+
if (this.resumeCommitted)
|
|
829
|
+
return {
|
|
830
|
+
ok: true,
|
|
831
|
+
data: {
|
|
832
|
+
attemptId,
|
|
833
|
+
state: this.resumeFinalized ? "active" : "awaitingFinalize",
|
|
834
|
+
durableCommitToken: this.resumeDurableCommitToken,
|
|
835
|
+
},
|
|
836
|
+
};
|
|
837
|
+
if (this.resumeCommitTask)
|
|
838
|
+
return this.resumeCommitTask;
|
|
839
|
+
const task = this.commitResumeActivation(attemptId);
|
|
840
|
+
this.resumeCommitTask = task;
|
|
841
|
+
try {
|
|
842
|
+
return await task;
|
|
843
|
+
}
|
|
844
|
+
finally {
|
|
845
|
+
if (this.resumeCommitTask === task)
|
|
846
|
+
this.resumeCommitTask = undefined;
|
|
847
|
+
}
|
|
848
|
+
}
|
|
849
|
+
if (req.op === "resumePreserved") {
|
|
850
|
+
if (tier !== CONTROL_ADMIN)
|
|
851
|
+
return { ok: false, error: "resumePreserved is admin-only; not allowed on this control subject" };
|
|
852
|
+
try {
|
|
853
|
+
const args = parseResumeControlArgs(req.args);
|
|
854
|
+
const inventoryDigest = createHash("sha256").update(JSON.stringify(args.inventory)).digest("hex");
|
|
855
|
+
if (!this.resumeAttemptId)
|
|
856
|
+
return { ok: false, error: "resumePreserved requires a manager started with --resume-attempt" };
|
|
857
|
+
if (this.resumeAttemptId !== args.attemptId)
|
|
858
|
+
return { ok: false, error: `manager expects resume attempt ${this.resumeAttemptId}, not ${args.attemptId}` };
|
|
859
|
+
if (this.resumeInventoryDigest && this.resumeInventoryDigest !== inventoryDigest)
|
|
860
|
+
return { ok: false, error: `resume attempt ${args.attemptId} is already bound to a different inventory` };
|
|
861
|
+
if (!this.resumeInventoryDigest) {
|
|
862
|
+
this.resumeInventoryDigest = inventoryDigest;
|
|
863
|
+
this.resumeInventory = args.inventory;
|
|
864
|
+
}
|
|
865
|
+
if (!this.resumeTask && !this.resumeResult) {
|
|
866
|
+
this.resumeTask = this.resumePreserved(args.inventory).then((result) => {
|
|
867
|
+
if (result.ok || this.resumedAgentNames.size > 0)
|
|
868
|
+
this.resumeResult = result;
|
|
869
|
+
return result;
|
|
870
|
+
}).finally(() => {
|
|
871
|
+
this.resumeTask = undefined;
|
|
872
|
+
});
|
|
873
|
+
}
|
|
874
|
+
const result = this.resumeResult ?? await this.resumeTask;
|
|
875
|
+
const data = { attemptId: args.attemptId, state: result.ok ? "awaitingCommit" : "degraded", ...result };
|
|
876
|
+
return result.ok
|
|
877
|
+
? { ok: true, data }
|
|
878
|
+
: { ok: false, data, error: result.error ?? "retained-agent resume failed" };
|
|
879
|
+
}
|
|
880
|
+
catch (e) {
|
|
881
|
+
return { ok: false, error: e.message };
|
|
882
|
+
}
|
|
883
|
+
}
|
|
884
|
+
if (req.op === "preparePreservation" || req.op === "commitPreservation" || req.op === "abortPreservation") {
|
|
885
|
+
if (tier !== CONTROL_ADMIN)
|
|
886
|
+
return { ok: false, error: `${req.op} is admin-only; not allowed on this control subject` };
|
|
887
|
+
if (this.resumeRequired)
|
|
888
|
+
return { ok: false, error: this.maintenanceError() };
|
|
889
|
+
const attemptId = String(req.args?.attemptId ?? "").trim();
|
|
890
|
+
if (!attemptId)
|
|
891
|
+
return { ok: false, error: `${req.op} requires attemptId` };
|
|
892
|
+
try {
|
|
893
|
+
if (req.op === "abortPreservation") {
|
|
894
|
+
this.abortPreservation(attemptId);
|
|
895
|
+
return { ok: true, data: { attemptId, state: "active" } };
|
|
896
|
+
}
|
|
897
|
+
const result = req.op === "preparePreservation"
|
|
898
|
+
? await this.preparePreservation(attemptId)
|
|
899
|
+
: await this.commitPreservation(attemptId);
|
|
900
|
+
return result.ok
|
|
901
|
+
? { ok: true, data: result }
|
|
902
|
+
: {
|
|
903
|
+
ok: false,
|
|
904
|
+
data: result,
|
|
905
|
+
error: `preservation incomplete: ${result.failures.map((f) => `${f.name}: ${f.error}`).join("; ")}`,
|
|
906
|
+
};
|
|
907
|
+
}
|
|
908
|
+
catch (e) {
|
|
909
|
+
return { ok: false, error: e.message };
|
|
910
|
+
}
|
|
911
|
+
}
|
|
912
|
+
const release = this.beginLifecycle();
|
|
913
|
+
if (!release)
|
|
914
|
+
return { ok: false, error: this.maintenanceError() };
|
|
915
|
+
try {
|
|
916
|
+
return await this.handleActive(req, tier);
|
|
917
|
+
}
|
|
918
|
+
finally {
|
|
919
|
+
release();
|
|
920
|
+
}
|
|
921
|
+
}
|
|
922
|
+
async commitResumeActivation(attemptId) {
|
|
923
|
+
if (!this.resumeAwaitingCommit || !this.resumeResult?.ok)
|
|
924
|
+
return { ok: false, error: `resume attempt ${attemptId} has no successful activation to commit` };
|
|
925
|
+
const inventory = this.resumeInventory;
|
|
926
|
+
if (!inventory)
|
|
927
|
+
return { ok: false, error: `resume attempt ${attemptId} has no bound inventory` };
|
|
928
|
+
const authority = await Promise.all(inventory.agents.map(async (entry) => {
|
|
929
|
+
try {
|
|
930
|
+
await this.validateRetainedAuthority(entry);
|
|
931
|
+
return undefined;
|
|
932
|
+
}
|
|
933
|
+
catch (e) {
|
|
934
|
+
return `${entry.name}: ${e.message}`;
|
|
935
|
+
}
|
|
936
|
+
}));
|
|
937
|
+
const drift = authority.filter((error) => error !== undefined);
|
|
938
|
+
if (drift.length)
|
|
939
|
+
return { ok: false, error: `resume attempt ${attemptId} retained authority changed before commit: ${drift.join("; ")}` };
|
|
940
|
+
let inactive;
|
|
941
|
+
try {
|
|
942
|
+
inactive = this.resumeLivenessErrors(inventory, this.ep.getRoster());
|
|
943
|
+
}
|
|
944
|
+
catch (e) {
|
|
945
|
+
return { ok: false, error: `resume attempt ${attemptId} cannot verify live principals: ${e.message}` };
|
|
946
|
+
}
|
|
947
|
+
if (inactive.length)
|
|
948
|
+
return { ok: false, error: `resume attempt ${attemptId} is not live at commit: ${inactive.join("; ")}` };
|
|
949
|
+
this.resumeAwaitingCommit = false;
|
|
950
|
+
this.resumeCommitted = true;
|
|
951
|
+
this.resumeDurableCommitToken ??= randomBytes(32).toString("hex");
|
|
952
|
+
return {
|
|
953
|
+
ok: true,
|
|
954
|
+
data: { attemptId, state: "awaitingFinalize", durableCommitToken: this.resumeDurableCommitToken },
|
|
955
|
+
};
|
|
956
|
+
}
|
|
957
|
+
async handleActive(req, tier) {
|
|
309
958
|
const args = req.args ?? {};
|
|
310
959
|
// `req.from.id` is non-forgeable in auth mode: serveControl rejects any request whose payload
|
|
311
960
|
// `from.id` doesn't match the subject sender (endpoint.ts). In open mode there are no creds, so
|
|
@@ -401,6 +1050,55 @@ export class Manager {
|
|
|
401
1050
|
managedPrincipal(a) {
|
|
402
1051
|
return a.userOwner ? a.id : principalKey(DEV_OWNER, a.id).key;
|
|
403
1052
|
}
|
|
1053
|
+
resumeLivenessErrors(inventory, roster) {
|
|
1054
|
+
const inactive = [];
|
|
1055
|
+
const expectedNames = new Set(inventory.agents.map((entry) => entry.name));
|
|
1056
|
+
for (const name of this.resumedAgentNames)
|
|
1057
|
+
if (!expectedNames.has(name))
|
|
1058
|
+
inactive.push(`${name} is not part of the bound inventory`);
|
|
1059
|
+
for (const entry of inventory.agents) {
|
|
1060
|
+
const managed = this.agents.get(entry.name);
|
|
1061
|
+
if (!managed) {
|
|
1062
|
+
inactive.push(`${entry.name} is no longer managed`);
|
|
1063
|
+
continue;
|
|
1064
|
+
}
|
|
1065
|
+
const expectedId = entry.identity.mode === "user"
|
|
1066
|
+
? principalKey(entry.identity.owner, entry.identity.actor).key
|
|
1067
|
+
: entry.identity.id;
|
|
1068
|
+
const expectedPrincipal = entry.identity.mode === "user"
|
|
1069
|
+
? expectedId
|
|
1070
|
+
: principalKey(DEV_OWNER, entry.identity.id).key;
|
|
1071
|
+
if (managed.id !== expectedId || this.managedPrincipal(managed) !== expectedPrincipal) {
|
|
1072
|
+
inactive.push(`${entry.name} no longer holds retained principal ${expectedPrincipal}`);
|
|
1073
|
+
continue;
|
|
1074
|
+
}
|
|
1075
|
+
// The late paths (commit/finalize) must prove the SAME incarnation the incarnation-exact
|
|
1076
|
+
// readiness fence proved (§13.1): a principal-only match lets a wrong/absent-uid presence under
|
|
1077
|
+
// the reused alias satisfy commit/finalize after a readiness timeout, undoing the fence.
|
|
1078
|
+
if (managed.lifecycleUid !== entry.identity.lifecycleUid) {
|
|
1079
|
+
inactive.push(`${entry.name} manager metadata incarnation ${managed.lifecycleUid} drifted from the inventory's ${entry.identity.lifecycleUid}`);
|
|
1080
|
+
continue;
|
|
1081
|
+
}
|
|
1082
|
+
if (managed.handle.name !== entry.name || managed.handle.kind !== entry.launch.runtime) {
|
|
1083
|
+
inactive.push(`${entry.name} is not attached to its exact retained ${entry.launch.runtime} handle`);
|
|
1084
|
+
continue;
|
|
1085
|
+
}
|
|
1086
|
+
try {
|
|
1087
|
+
if (managed.handle.status() !== "running") {
|
|
1088
|
+
inactive.push(`${entry.name} runtime is not running`);
|
|
1089
|
+
continue;
|
|
1090
|
+
}
|
|
1091
|
+
}
|
|
1092
|
+
catch (e) {
|
|
1093
|
+
inactive.push(`${entry.name} runtime status failed: ${e.message}`);
|
|
1094
|
+
continue;
|
|
1095
|
+
}
|
|
1096
|
+
if (!roster.some((presence) => presence.card.id === expectedPrincipal && presence.card.name === entry.name && presence.status !== "offline" &&
|
|
1097
|
+
presence.lifecycleUid === entry.identity.lifecycleUid))
|
|
1098
|
+
inactive.push(`${entry.name} incarnation ${entry.identity.lifecycleUid} (principal ${expectedPrincipal}) is not exactly present`);
|
|
1099
|
+
}
|
|
1100
|
+
return inactive;
|
|
1101
|
+
}
|
|
404
1102
|
/** Self-despawn (P2b): stop the managed agent whose id == the authenticated caller. The
|
|
405
1103
|
* no-name self-op can only ever resolve to the caller's OWN managed entry (ids are unique
|
|
406
1104
|
* per spawn + non-forgeable in auth mode), never a peer — so it's structurally incapable of
|
|
@@ -412,7 +1110,7 @@ export class Manager {
|
|
|
412
1110
|
return { ok: false, error: `self-stop: caller ${callerId} is not a managed agent` };
|
|
413
1111
|
const graceful = args.graceful !== false;
|
|
414
1112
|
this.stopHandle(target, graceful);
|
|
415
|
-
this.
|
|
1113
|
+
this.trackStoppedHandle(target, true);
|
|
416
1114
|
return { ok: true, data: { name: target.name, stopped: true, graceful } };
|
|
417
1115
|
}
|
|
418
1116
|
// Plane-3 durable join/leave/list ops moved OFF the manager onto the server-side delivery daemon's
|
|
@@ -441,6 +1139,49 @@ export class Manager {
|
|
|
441
1139
|
console.error(`stop ${a.name} (${a.id}): ${e.message}`);
|
|
442
1140
|
}
|
|
443
1141
|
}
|
|
1142
|
+
/** Keep an accepted stop inside the lifecycle drain until the runtime proves the child is gone,
|
|
1143
|
+
* so a maintenance prepare can never fence ahead of a child that is still dying.
|
|
1144
|
+
*
|
|
1145
|
+
* An operator-accepted stop frees its slot at once: `stop` replying ✓ means `ps` no longer lists
|
|
1146
|
+
* the agent. That cannot omit a still-live child from a cut, because runPreparation drains the
|
|
1147
|
+
* lifecycle BEFORE it reads the roster — the exit proof below is what closes the race, not the
|
|
1148
|
+
* slot lingering. A recursive reap (`requireAuthoritativeExit`) instead keeps the slot until the
|
|
1149
|
+
* wait proves exit: nobody asked for those children to be gone, so they stay managed until the
|
|
1150
|
+
* runtime says otherwise, and a runtime that cannot prove exit records an unverified stop. */
|
|
1151
|
+
trackStoppedHandle(a, floor, requireAuthoritativeExit = false) {
|
|
1152
|
+
if (!a.handle.waitForExit) {
|
|
1153
|
+
// Preserve ordinary external-runtime stop behavior, but retain enough evidence for a later
|
|
1154
|
+
// maintenance prepare to fail if that runtime still cannot prove the surface disappeared.
|
|
1155
|
+
this.unverifiedStops.push({
|
|
1156
|
+
name: a.name,
|
|
1157
|
+
id: a.id,
|
|
1158
|
+
handle: a.handle,
|
|
1159
|
+
authoritative: requireAuthoritativeExit,
|
|
1160
|
+
error: requireAuthoritativeExit
|
|
1161
|
+
? `recursive reap cannot prove exit on runtime "${a.handle.kind}" (AgentHandle.waitForExit is not implemented)`
|
|
1162
|
+
: undefined,
|
|
1163
|
+
});
|
|
1164
|
+
if (requireAuthoritativeExit)
|
|
1165
|
+
return;
|
|
1166
|
+
this.freeSlot(a, floor, true);
|
|
1167
|
+
return;
|
|
1168
|
+
}
|
|
1169
|
+
if (!requireAuthoritativeExit)
|
|
1170
|
+
this.freeSlot(a, floor, true);
|
|
1171
|
+
this.lifecycleInFlight++;
|
|
1172
|
+
void this.awaitHandleExit(a.handle)
|
|
1173
|
+
.then(() => this.freeSlot(a, floor, true)) // no-op once an accepted stop already freed it
|
|
1174
|
+
.catch((e) => {
|
|
1175
|
+
this.unverifiedStops.push({
|
|
1176
|
+
name: a.name,
|
|
1177
|
+
id: a.id,
|
|
1178
|
+
handle: a.handle,
|
|
1179
|
+
authoritative: true,
|
|
1180
|
+
error: `accepted stop could not prove exit: ${e.message}`,
|
|
1181
|
+
});
|
|
1182
|
+
})
|
|
1183
|
+
.finally(() => this.releaseLifecycle());
|
|
1184
|
+
}
|
|
444
1185
|
/** USER-MODE spawn provisioning (the gate-1 counterpart to the static mint block): resolve the
|
|
445
1186
|
* OWNER (ctl caller's principal, or the manifest's stamped owner — never a payload field),
|
|
446
1187
|
* pre-create the principal-keyed durables + ACL row on the ephemeral provisioner, author the
|
|
@@ -469,15 +1210,19 @@ export class Manager {
|
|
|
469
1210
|
// pass through too (a persona may hold delegable roles) — the ledger's envelope walk still
|
|
470
1211
|
// attenuates every one of these against the spawner chain.
|
|
471
1212
|
const scope = (opts.capabilities ?? []).filter((c) => c === "spawn" || c === "admin" || /^role:[A-Za-z0-9_-]+$/.test(c));
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
1213
|
+
// LOCAL composition, hardcoded: until the manager's entry is store-threaded (the same
|
|
1214
|
+
// later slice as its renewal-owner store, with/after the membership-rw reader migration),
|
|
1215
|
+
// a pure-KMS hosted manager CANNOT read the callout material this grant needs — hosted
|
|
1216
|
+
// user-mode spawn via the manager is UNAVAILABLE, not silently degraded, until then.
|
|
1217
|
+
const secrets = workspaceSecretStore(this.workspaceRoot);
|
|
1218
|
+
const files = agentSecretFilePaths(this.workspaceRoot, name);
|
|
1219
|
+
const { actorToken: tokenPath, sentinelCreds: sentinelPath, health: healthPath } = files;
|
|
476
1220
|
try {
|
|
477
1221
|
// The GRANT first — it is the envelope-rule enforcement point (a delegation must sit within
|
|
478
1222
|
// the spawner's own grant), so a refused delegation exits here having touched nothing beyond
|
|
479
1223
|
// the ledger: no durables, no broker footprint, nothing for a corrected respawn to race.
|
|
480
1224
|
const grant = await provider.grantAgent({
|
|
1225
|
+
store: secrets,
|
|
481
1226
|
dir,
|
|
482
1227
|
space: this.space,
|
|
483
1228
|
owner,
|
|
@@ -488,17 +1233,23 @@ export class Manager {
|
|
|
488
1233
|
role: opts.role,
|
|
489
1234
|
parent: spawnerPr ? opts.spawner : undefined,
|
|
490
1235
|
label: opts.label,
|
|
1236
|
+
lifecycleUid: opts.lifecycleUid,
|
|
491
1237
|
});
|
|
492
|
-
// Durables + ACL row,
|
|
493
|
-
// (a user agent's credential is its bearer, minted by the callout per connect
|
|
494
|
-
|
|
1238
|
+
// Durables + ACL row, LIFECYCLE-keyed (SPEC 13.1) — the same onboarding as static agents minus
|
|
1239
|
+
// the mint (a user agent's credential is its bearer, minted by the callout per connect from the
|
|
1240
|
+
// ledger row's recorded lifecycleUid — the same value provisioned here).
|
|
1241
|
+
await this.withProvisioner((prov) => provisionAgentDurables(prov, { owner, actor: name, lifecycleUid: opts.lifecycleUid }, {
|
|
495
1242
|
subscribe: opts.subscribe,
|
|
496
1243
|
allowSubscribe: opts.allowSubscribe,
|
|
497
1244
|
role: opts.role,
|
|
498
1245
|
}));
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
1246
|
+
// The store holds the source of truth; the bearer re-exec (`--token-file`) and the launch's
|
|
1247
|
+
// sentinel handoff read FILES, so materialize both at the canonical paths (under the local
|
|
1248
|
+
// FS composition, a byte-identical rewrite of the keys' own locations).
|
|
1249
|
+
await secrets.put(agentActorTokenKey(name), grant.actorToken);
|
|
1250
|
+
await secrets.put(agentSentinelCredsKey(name), grant.sentinelCreds);
|
|
1251
|
+
await materializeSecretToFile(secrets, agentActorTokenKey(name), tokenPath);
|
|
1252
|
+
await materializeSecretToFile(secrets, agentSentinelCredsKey(name), sentinelPath);
|
|
502
1253
|
rmSync(healthPath, { force: true }); // a fresh start opens a fresh health window
|
|
503
1254
|
const bearerCmd = [
|
|
504
1255
|
// The manager's own invocation prefix (node + loader flags + the cotal entry) — the agent
|
|
@@ -525,10 +1276,12 @@ export class Manager {
|
|
|
525
1276
|
// may respawn the moment it reads the refusal, and a detached teardown would race (and
|
|
526
1277
|
// delete) that fresh spawn's just-provisioned durables.
|
|
527
1278
|
await provider.revokeAgent({ dir, owner, actor: name }).catch(() => { });
|
|
1279
|
+
await secrets.delete(agentActorTokenKey(name)).catch(() => { });
|
|
1280
|
+
await secrets.delete(agentSentinelCredsKey(name)).catch(() => { });
|
|
528
1281
|
rmSync(tokenPath, { force: true });
|
|
529
1282
|
rmSync(sentinelPath, { force: true });
|
|
530
1283
|
rmSync(healthPath, { force: true });
|
|
531
|
-
await this.deprovision({ id: principalKey(owner, name).key, name, userOwner: owner }).catch((err) => console.error(`rollback deprovision ${name}: ${err.message}`));
|
|
1284
|
+
await this.deprovision({ id: principalKey(owner, name).key, name, lifecycleUid: opts.lifecycleUid, userOwner: owner }).catch((err) => console.error(`rollback deprovision ${name}: ${err.message}`));
|
|
532
1285
|
return { error: `agent auth preflight failed for "${name}": ${e.message}` };
|
|
533
1286
|
}
|
|
534
1287
|
}
|
|
@@ -537,17 +1290,31 @@ export class Manager {
|
|
|
537
1290
|
* expires — flooring the RECYCLE, not the call, so both free paths (despawn + exit/reap) are
|
|
538
1291
|
* covered (P4c). Floor self + own-child despawn and natural exit; NEVER admin despawn (operator
|
|
539
1292
|
* emergency-kill stays unthrottled) and NEVER the reserved-rollback path (no cold-start paid). */
|
|
540
|
-
freeSlot(a, floor) {
|
|
1293
|
+
freeSlot(a, floor, acceptedBeforeFence = false) {
|
|
541
1294
|
if (this.agents.get(a.name) !== a)
|
|
542
1295
|
return; // already freed (exit raced despawn, etc.)
|
|
543
1296
|
this.agents.delete(a.name);
|
|
544
1297
|
if (floor && Date.now() - a.startedAt < MIN_LIFETIME)
|
|
545
1298
|
this.cooling.push(a.startedAt + MIN_LIFETIME);
|
|
1299
|
+
// #29 piece 3: on a USER mesh the name is RESERVED PENDING RETIREMENT — despawn started this
|
|
1300
|
+
// lifecycle's FULL teardown (footprint + standing-authority revoke + the auth-side retirement),
|
|
1301
|
+
// and the alias frees only when all of it completes (not the retirement alone). The detached
|
|
1302
|
+
// deprovision below drives it; a failed revoke or an unreachable rail keeps the name held,
|
|
1303
|
+
// re-driven by a retry. Gate on userMode BY
|
|
1304
|
+
// CONSTRUCTION (NEW-1): a static-auth mint has no user-mode lifecycle head to retire, so the
|
|
1305
|
+
// reservation + rail request simply don't apply there (the incidental nkey-parse used to mask
|
|
1306
|
+
// this, but the intent is "user mode only", not "any principal-shaped id").
|
|
1307
|
+
if (this.userMode) {
|
|
1308
|
+
const p = parsePrincipalKey(a.id);
|
|
1309
|
+
if (p)
|
|
1310
|
+
this.retiring.set(a.name, { opId: retireOpId(a.lifecycleUid), lifecycleUid: a.lifecycleUid, owner: p.owner, actor: p.actor, agentId: a.id, userOwner: a.userOwner, startedAt: Date.now() });
|
|
1311
|
+
}
|
|
546
1312
|
// Auth mode: tear down the departed agent's minted broker footprint + creds file (#159 B2). The
|
|
547
1313
|
// process is already gone, so this must never block the slot free or throw into the caller — it runs
|
|
548
1314
|
// detached, and a failure is logged loudly (never swallowed), not retried. The `agents` guard above
|
|
549
1315
|
// makes this fire exactly once per agent across every free path (despawn / self-stop / reap / exit).
|
|
550
|
-
|
|
1316
|
+
if (!a.suppressCleanup && (this.maintenanceState === "active" || acceptedBeforeFence))
|
|
1317
|
+
this.trackDeprovision(a);
|
|
551
1318
|
}
|
|
552
1319
|
/** Tear down a departed agent's minted footprint (#159 B2, auth mode): its local-principal durables
|
|
553
1320
|
* (`dm_local-<id>`, `dlv_local-<id>`), its read-ACL row, and its creds file — everything the spawn's
|
|
@@ -564,53 +1331,208 @@ export class Manager {
|
|
|
564
1331
|
async deprovision(a) {
|
|
565
1332
|
if (!this.auth)
|
|
566
1333
|
return; // open mesh mints no creds/durables — nothing to tear down
|
|
1334
|
+
// SINGLE-FLIGHT per (name, lifecycleUid) (INT-2/C): join an in-flight teardown for this exact
|
|
1335
|
+
// lifecycle rather than launching a second concurrent one whose delayed name-keyed revoke could
|
|
1336
|
+
// outlive the hold-clear and delete a successor's row. A fresh trigger after settle re-drives.
|
|
1337
|
+
const key = JSON.stringify([a.name, a.lifecycleUid]); // ASCII-safe, delimiter-collision-free
|
|
1338
|
+
const inflight = this.deprovisioningFlight.get(key);
|
|
1339
|
+
if (inflight)
|
|
1340
|
+
return inflight;
|
|
1341
|
+
const flight = this.driveDeprovision(a).finally(() => {
|
|
1342
|
+
if (this.deprovisioningFlight.get(key) === flight)
|
|
1343
|
+
this.deprovisioningFlight.delete(key);
|
|
1344
|
+
});
|
|
1345
|
+
this.deprovisioningFlight.set(key, flight);
|
|
1346
|
+
return flight;
|
|
1347
|
+
}
|
|
1348
|
+
/** The actual footprint teardown (wrapped by {@link deprovision}'s single-flight). */
|
|
1349
|
+
async driveDeprovision(a) {
|
|
1350
|
+
if (!this.auth)
|
|
1351
|
+
return; // guaranteed by deprovision; re-checked for the deprovisionBroker narrowing
|
|
567
1352
|
// Drop the local creds file FIRST + unconditionally — it is a usable identity on disk, useless for a
|
|
568
1353
|
// departed agent, so it must not survive even if the broker teardown below fails or times out. The
|
|
569
1354
|
// teardown mints its OWN deprovisioner cred (not this file), so removing it early is independent.
|
|
570
|
-
|
|
1355
|
+
// Migrated kinds: the store delete is the authoritative removal; the rmSync clears the FS
|
|
1356
|
+
// materialization (a byte-identical no-op under the local composition, real once the manager
|
|
1357
|
+
// is store-threaded onto a non-FS store).
|
|
1358
|
+
const secrets = workspaceSecretStore(this.workspaceRoot);
|
|
1359
|
+
const files = agentSecretFilePaths(this.workspaceRoot, a.name);
|
|
1360
|
+
await secrets.delete(agentCredsKey(a.name));
|
|
1361
|
+
rmSync(files.creds, { force: true });
|
|
571
1362
|
if (a.userOwner) {
|
|
572
1363
|
// USER MODE: this teardown IS revocation, not just footprint reduction — the ledger row is
|
|
573
1364
|
// the agent's standing mint authority, so delete it (next exchange refused, next connect
|
|
574
1365
|
// denied) and shred the secret/sentinel/health files. A copied actor token dies here; a
|
|
575
1366
|
// still-LIVE connection ends at its bearer-bound JWT expiry (≤ the agent TTL).
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
1367
|
+
await secrets.delete(agentActorTokenKey(a.name));
|
|
1368
|
+
await secrets.delete(agentSentinelCredsKey(a.name));
|
|
1369
|
+
for (const f of [files.actorToken, files.sentinelCreds, files.health])
|
|
1370
|
+
rmSync(f, { force: true });
|
|
1371
|
+
// The ledger row IS the agent's STANDING mint authority (a different store from the auth-plane
|
|
1372
|
+
// cred ledger the rail retirement covers): while it lives, a copied actor token can still mint a
|
|
1373
|
+
// fresh connect credential. So a FAILED revoke must NOT be swallowed into a clean terminal (INT-2):
|
|
1374
|
+
// mark the standing authority live on the hold so the retirement can never free the name (a freed
|
|
1375
|
+
// name says "this lifecycle is fully gone" - false while the mint authority stands), and carry a
|
|
1376
|
+
// legible operator copy. A retry re-drives this whole teardown (the same-name-spawn nudge routes
|
|
1377
|
+
// through deprovision, not the rail alone), so the revoke is re-attempted, not stranded.
|
|
1378
|
+
const holdRevoke = this.retiring.get(a.name);
|
|
1379
|
+
if (holdRevoke && holdRevoke.lifecycleUid === a.lifecycleUid)
|
|
1380
|
+
holdRevoke.standingAuthorityLive = true;
|
|
579
1381
|
try {
|
|
580
1382
|
await resolveAuthProvider().revokeAgent({
|
|
581
1383
|
dir: userAuthStateDir(this.workspaceRoot, this.space),
|
|
582
1384
|
owner: a.userOwner,
|
|
583
1385
|
actor: a.name,
|
|
584
1386
|
});
|
|
1387
|
+
const done = this.retiring.get(a.name);
|
|
1388
|
+
if (done && done.lifecycleUid === a.lifecycleUid)
|
|
1389
|
+
done.standingAuthorityLive = false;
|
|
585
1390
|
}
|
|
586
1391
|
catch (e) {
|
|
1392
|
+
const h = this.retiring.get(a.name);
|
|
1393
|
+
if (h && h.lifecycleUid === a.lifecycleUid)
|
|
1394
|
+
h.lastError = `the agent's standing mint authority could not be revoked (${e.message}); the name stays held so a copied actor token cannot mint fresh credentials. NEXT: a same-name spawn re-drives the full teardown (including the revoke), or recover the auth state.`;
|
|
587
1395
|
console.error(`revoke agent grant ${a.name}: ${e.message}`);
|
|
588
1396
|
}
|
|
589
1397
|
}
|
|
590
|
-
|
|
1398
|
+
await this.deprovisionBroker(a);
|
|
1399
|
+
// #29 piece 3: after the footprint teardown, ask the AUTH plane to RETIRE the lifecycle over
|
|
1400
|
+
// the auth-admin rail. The rail re-checks the space-manager lease at serve time; the terminal
|
|
1401
|
+
// (or an already-retired answer) clears the name reservation. Failures keep the hold with
|
|
1402
|
+
// their operator copy — legible, retryable, never a silent half-state.
|
|
1403
|
+
await this.requestRetirement(a);
|
|
1404
|
+
}
|
|
1405
|
+
/** Request the auth-side retirement of a departed agent's lifecycle (#29 piece 3): an ephemeral
|
|
1406
|
+
* `retirement-requester` credential (request + reply only), the generic `retireLifecycle` op,
|
|
1407
|
+
* a STABLE opId (derived from the lifecycleUid, so every retry re-drives the SAME operation),
|
|
1408
|
+
* and the four-outcome handling in operator vocabulary. */
|
|
1409
|
+
async requestRetirement(a) {
|
|
1410
|
+
if (!this.userMode)
|
|
1411
|
+
return; // NEW-1: lifecycle retirement is a user-mesh concept; a static mint has no head to retire
|
|
1412
|
+
// SINGLE-FLIGHT per (name, lifecycleUid) (audit #1): the detached deprovision call and every
|
|
1413
|
+
// same-name-spawn nudge for THIS lifecycle share ONE in-flight retirement, so concurrent triggers
|
|
1414
|
+
// never stack independent rail requests that dual-enter runAgentRetirementBarrier. Keyed by
|
|
1415
|
+
// (name, uid), NOT name alone: driveRetirement clears the hold on rail ok but then yields at
|
|
1416
|
+
// `nc.close()` with the flight still stored, so the alias can free and a SUCCESSOR (new uid) spawn.
|
|
1417
|
+
// A name-only key would let that successor's own teardown JOIN the predecessor's still-pending
|
|
1418
|
+
// flight and never send its OWN retirement — leaving the successor's lifecycle unretired. The uid in
|
|
1419
|
+
// the key gives the successor a disjoint flight (mirrors {@link deprovisioningFlight}). A fresh
|
|
1420
|
+
// trigger after settle re-drives (a still-present hold => retirement not yet confirmed).
|
|
1421
|
+
const key = JSON.stringify([a.name, a.lifecycleUid]);
|
|
1422
|
+
const inflight = this.retiringFlight.get(key);
|
|
1423
|
+
if (inflight)
|
|
1424
|
+
return inflight;
|
|
1425
|
+
const flight = this.driveRetirement(a).finally(() => {
|
|
1426
|
+
if (this.retiringFlight.get(key) === flight)
|
|
1427
|
+
this.retiringFlight.delete(key);
|
|
1428
|
+
});
|
|
1429
|
+
this.retiringFlight.set(key, flight);
|
|
1430
|
+
return flight;
|
|
1431
|
+
}
|
|
1432
|
+
/** The rail round-trip for one retirement (wrapped by {@link requestRetirement}'s single-flight). */
|
|
1433
|
+
async driveRetirement(a) {
|
|
1434
|
+
if (!this.auth)
|
|
1435
|
+
return; // guaranteed by requestRetirement; re-checked for the type narrowing below
|
|
1436
|
+
const held = this.retiring.get(a.name);
|
|
1437
|
+
const me = parsePrincipalKey(this.ep.ref().id);
|
|
1438
|
+
const target = parsePrincipalKey(a.id);
|
|
1439
|
+
if (!me || !target) {
|
|
1440
|
+
if (held)
|
|
1441
|
+
held.lastError = "the manager or target principal could not be derived; the retirement was not requested";
|
|
1442
|
+
return;
|
|
1443
|
+
}
|
|
1444
|
+
const uncertain = (why) => `the despawn stopped "${a.name}", but the retirement's completion could NOT be confirmed (${why}). The name stays held - not failed, not done - and a same-name spawn re-drives the same teardown; the auth service also finishes any started retirement on its next boot. NEXT: if the auth rail stays unreachable, recover the stack (\`cotal supervise\`), then re-attempt the same-name spawn.`;
|
|
1445
|
+
try {
|
|
1446
|
+
const creds = await mintCreds(this.auth, newIdentity(), "retirement-requester", { retirementRequester: { owner: me.owner, actor: me.actor } });
|
|
1447
|
+
const nc = await connect({ servers: this.servers ?? DEFAULT_SERVER, authenticator: credsAuthenticator(new TextEncoder().encode(creds)), maxReconnectAttempts: 0 });
|
|
1448
|
+
try {
|
|
1449
|
+
const subject = controlServiceSubject(this.space, CONTROL_AUTH_ADMIN, me.owner, me.actor);
|
|
1450
|
+
const m = await nc.request(subject, JSON.stringify({ op: "retireLifecycle", args: { owner: target.owner, actor: target.actor, lifecycleUid: a.lifecycleUid, opId: retireOpId(a.lifecycleUid) } }), { timeout: 20_000, noMux: true, reply: `${subject}.reply.${randomUUID()}` });
|
|
1451
|
+
const r = m.json();
|
|
1452
|
+
if (r.ok) {
|
|
1453
|
+
// CAS the hold clear (audit #1 ABA): free the alias ONLY if the current hold is still THIS
|
|
1454
|
+
// lifecycle's - a late reply for a retired predecessor must never clear a successor's newer hold.
|
|
1455
|
+
const cur = this.retiring.get(a.name);
|
|
1456
|
+
if (cur && cur.lifecycleUid === a.lifecycleUid) {
|
|
1457
|
+
if (cur.standingAuthorityLive) {
|
|
1458
|
+
// INT-2: the auth-plane lifecycle retired, but the manager-side STANDING mint authority is
|
|
1459
|
+
// not yet revoked (a failed revoke). Freeing the name here would be a false terminal (a
|
|
1460
|
+
// copied token could still mint), so keep the hold with its revoke-failure copy; a retry
|
|
1461
|
+
// re-drives the full teardown (revoke included).
|
|
1462
|
+
console.error(`despawn ${a.name}: the auth-plane lifecycle retired, but the standing mint authority is not yet revoked; the name stays held. ${cur.lastError ?? ""}`);
|
|
1463
|
+
}
|
|
1464
|
+
else {
|
|
1465
|
+
this.retiring.delete(a.name);
|
|
1466
|
+
console.error(`despawn ${a.name}: the agent's retirement completed; the name is free for reuse`);
|
|
1467
|
+
}
|
|
1468
|
+
}
|
|
1469
|
+
else {
|
|
1470
|
+
console.error(`despawn ${a.name}: retirement confirmed for a prior lifecycle of "${a.name}"; the current hold is left intact`);
|
|
1471
|
+
}
|
|
1472
|
+
}
|
|
1473
|
+
else {
|
|
1474
|
+
// The rail's refusal is already the operator copy (lease-loss/stale/foreign-op faces,
|
|
1475
|
+
// full-no-op statements included) - surface it INTACT, never flattened.
|
|
1476
|
+
if (held)
|
|
1477
|
+
held.lastError = r.error ?? "the auth service refused the retirement without a reason";
|
|
1478
|
+
console.error(`despawn ${a.name}: ${r.error ?? "the auth service refused the retirement without a reason"}`);
|
|
1479
|
+
}
|
|
1480
|
+
}
|
|
1481
|
+
finally {
|
|
1482
|
+
await nc.close().catch(() => { });
|
|
1483
|
+
}
|
|
1484
|
+
}
|
|
1485
|
+
catch (e) {
|
|
1486
|
+
const copy = uncertain(e.message);
|
|
1487
|
+
if (held)
|
|
1488
|
+
held.lastError = copy;
|
|
1489
|
+
console.error(`despawn ${a.name}: ${copy}`);
|
|
1490
|
+
}
|
|
1491
|
+
}
|
|
1492
|
+
/** The teardown's ASYNC BROKER PHASE: mint the ephemeral target-pinned deprovisioner cred and
|
|
1493
|
+
* delete the agent's broker footprint (dm_/dlv_ durables + read-ACL row). Split from
|
|
1494
|
+
* {@link deprovision} because it runs LAST in the ordered teardown chain — after the creds/secret
|
|
1495
|
+
* shred and the awaited ledger revoke, which precede it in that same single-flighted chain (the
|
|
1496
|
+
* revoke is awaited and can be deliberately slow, so it is not merely a synchronous prefix). The name
|
|
1497
|
+
* is NOT freed while any teardown phase is still in flight — the hold clears only after the
|
|
1498
|
+
* standing-authority revoke AND the lifecycle retirement both confirm (see {@link driveRetirement}) —
|
|
1499
|
+
* but the deletes here are still lifecycle-uid-pinned so even a replayed/stale teardown can never
|
|
1500
|
+
* reach a same-name successor's footprint (its names embed a different uid). */
|
|
1501
|
+
async deprovisionBroker(a) {
|
|
1502
|
+
// LIFECYCLE-PINNED (SPEC 13.1): both the credential's exact-name grants and the delete names
|
|
1503
|
+
// carry a.lifecycleUid, so a stale/replayed teardown for this retired incarnation is broker-denied
|
|
1504
|
+
// against a same-name successor's footprint (its names embed a different uid).
|
|
1505
|
+
const creds = await mintCreds(this.auth, newIdentity(), "deprovisioner", {
|
|
1506
|
+
deprovisionTarget: { principal: a.id, lifecycleUid: a.lifecycleUid },
|
|
1507
|
+
});
|
|
591
1508
|
// Bound the detached broker teardown so a wedged broker can't leave the deprovision promise pending
|
|
592
1509
|
// forever with no log — the timeout rejects into freeSlot's fail-loud `.catch` (paired with the
|
|
593
1510
|
// helper's own fail-fast connect). The durables/ACL row still fall to space teardown as a backstop.
|
|
594
|
-
await withTimeout(deprovisionAgent({ servers: this.servers ?? DEFAULT_SERVER, space: this.space, targetId: a.id, creds }), DEPROVISION_TIMEOUT_MS, `deprovision ${a.name} (${a.id}): broker teardown timed out`);
|
|
1511
|
+
await withTimeout(deprovisionAgent({ servers: this.servers ?? DEFAULT_SERVER, space: this.space, targetId: a.id, lifecycleUid: a.lifecycleUid, creds }), DEPROVISION_TIMEOUT_MS, `deprovision ${a.name} (${a.id}): broker teardown timed out`);
|
|
595
1512
|
}
|
|
596
|
-
/** Reap a parent's children on its exit (P4b)
|
|
597
|
-
*
|
|
598
|
-
*
|
|
1513
|
+
/** Reap a parent's children on its exit (P4b). Every descendant remains managed until the runtime's
|
|
1514
|
+
* authoritative wait proves exit; the wait participates in the lifecycle drain, so preservation can
|
|
1515
|
+
* never omit a child that may still be alive. Recursive descendants are scheduled before their parent
|
|
1516
|
+
* slot can disappear. */
|
|
599
1517
|
reapChildrenOf(parentId) {
|
|
600
1518
|
for (const child of [...this.agents.values()]) {
|
|
601
1519
|
if (child.spawner !== parentId)
|
|
602
1520
|
continue;
|
|
1521
|
+
this.reapChildrenOf(this.managedPrincipal(child));
|
|
603
1522
|
this.stopHandle(child, false);
|
|
604
|
-
this.
|
|
605
|
-
this.reapChildrenOf(child.id);
|
|
1523
|
+
this.trackStoppedHandle(child, true, true);
|
|
606
1524
|
}
|
|
607
1525
|
}
|
|
608
1526
|
/** A managed agent's process exited on its own (crash, /exit, finished). Free its slot
|
|
609
1527
|
* (rate-floored — exit-driven churn counts) and reap any children it spawned. Idempotent via
|
|
610
1528
|
* freeSlot's identity guard, so a later graceful-stop SIGKILL firing exit again is a no-op. */
|
|
611
1529
|
onAgentExit(a) {
|
|
1530
|
+
// Preservation owns the child-stop snapshot. Exit watchers must neither delete that snapshot nor
|
|
1531
|
+
// trigger normal deprovision/reap while the cut is being formed.
|
|
1532
|
+
if (this.maintenanceState !== "active")
|
|
1533
|
+
return;
|
|
612
1534
|
this.freeSlot(a, true);
|
|
613
|
-
this.reapChildrenOf(a
|
|
1535
|
+
this.reapChildrenOf(this.managedPrincipal(a));
|
|
614
1536
|
}
|
|
615
1537
|
/** Agent names become `.cotal/agents/<name>.md` paths and mesh identities, so they must be bare
|
|
616
1538
|
* tokens, never a path — blocks traversal / arbitrary writes from a model-supplied name. */
|
|
@@ -623,7 +1545,7 @@ export class Manager {
|
|
|
623
1545
|
* in-flight (reserved) slots. Lets a colliding spawn auto-number instead of being rejected, so
|
|
624
1546
|
* callers never have to invent a unique name. */
|
|
625
1547
|
uniqueName(base) {
|
|
626
|
-
return firstFreeName(base, (n) => this.agents.has(n) || this.reserved.has(n));
|
|
1548
|
+
return firstFreeName(base, (n) => this.agents.has(n) || this.reserved.has(n) || this.retiring.has(n));
|
|
627
1549
|
}
|
|
628
1550
|
/** Spawn a teammate by persona ref (`name` loads `.cotal/agents/<name>.md`; the peer presents
|
|
629
1551
|
* under that file's own `name:`), as if a peer asked via the control plane. Used to pre-spawn the
|
|
@@ -835,7 +1757,7 @@ export class Manager {
|
|
|
835
1757
|
catch (e) {
|
|
836
1758
|
return { ok: false, error: e.message };
|
|
837
1759
|
}
|
|
838
|
-
const reply = await this.startAgent(launchAgentToStartOpts(la, configPath, spec.owner), caller);
|
|
1760
|
+
const reply = await this.startAgent(launchAgentToStartOpts(la, configPath, spec.owner, runId), caller);
|
|
839
1761
|
if (reply.ok)
|
|
840
1762
|
// `data.name` stays the spawned (numbered) identity — what creds are filed under and the ledger
|
|
841
1763
|
// keys on; `requested`/`runId`/`hash` give the CLI the manifest name + drift hash for the ledger.
|
|
@@ -849,6 +1771,17 @@ export class Manager {
|
|
|
849
1771
|
* defaulting to the manager's own id for roster/pre-spawn — recorded for the spawner
|
|
850
1772
|
* ledger (own-children despawn + reap-on-parent-exit). */
|
|
851
1773
|
async startAgent(opts, spawner) {
|
|
1774
|
+
const release = this.beginLifecycle();
|
|
1775
|
+
if (!release)
|
|
1776
|
+
return { ok: false, error: this.maintenanceError() };
|
|
1777
|
+
try {
|
|
1778
|
+
return await this.startAgentActive(opts, spawner);
|
|
1779
|
+
}
|
|
1780
|
+
finally {
|
|
1781
|
+
release();
|
|
1782
|
+
}
|
|
1783
|
+
}
|
|
1784
|
+
async startAgentActive(opts, spawner) {
|
|
852
1785
|
// The spawn argument is a persona REF — a filename in `.cotal/agents` (the unique spawn KEY), or
|
|
853
1786
|
// a path via `--config`. It is NOT the mesh identity: the identity comes from inside the file
|
|
854
1787
|
// (`name:`), so a persona can be filed descriptively (review-critic.md) yet present under a
|
|
@@ -968,6 +1901,20 @@ export class Manager {
|
|
|
968
1901
|
const idErr = this.nameError(identityName);
|
|
969
1902
|
if (idErr)
|
|
970
1903
|
return { ok: false, error: opts.resolved ? `launch agent: ${idErr}` : `persona ${configPath}: ${idErr}` };
|
|
1904
|
+
// The alias-reuse gate (#29 piece 3): a name whose previous agent is still retiring REFUSES
|
|
1905
|
+
// legibly (never a silent suffix), and the refusal re-drives the FULL durable teardown so
|
|
1906
|
+
// "retry the spawn" is also the nudge. It routes through `deprovision` (not `requestRetirement`
|
|
1907
|
+
// alone) so a retry re-drives the standing-authority revoke (INT-2) AND the broker cleanup before
|
|
1908
|
+
// any hold-clear (C): the alias must not free while the durable teardown or the revoke is still
|
|
1909
|
+
// outstanding. All the teardown ops are idempotent, and the rail request is single-flighted.
|
|
1910
|
+
const held = this.retiring.get(identityName);
|
|
1911
|
+
if (held !== undefined) {
|
|
1912
|
+
void this.deprovision({ id: held.agentId, name: identityName, lifecycleUid: held.lifecycleUid, userOwner: held.userOwner }).catch(() => { });
|
|
1913
|
+
return {
|
|
1914
|
+
ok: false,
|
|
1915
|
+
error: `the name "${identityName}" is reserved pending retirement: its previous agent's despawn started that lifecycle's teardown (footprint + standing-authority revoke + auth-side retirement), and the name frees only when all of it completes${held.lastError !== undefined ? ` (last attempt: ${held.lastError})` : ""}. NEXT: wait a moment and retry this spawn (retrying re-drives the whole teardown), or pick another name.`,
|
|
1916
|
+
};
|
|
1917
|
+
}
|
|
971
1918
|
if (variant && !connector.supportsModelVariant)
|
|
972
1919
|
return { ok: false, error: `${agent} connector does not support model variants (variant)` };
|
|
973
1920
|
const name = this.uniqueName(identityName);
|
|
@@ -1000,6 +1947,10 @@ export class Manager {
|
|
|
1000
1947
|
// A stable nkey identity assigned at spawn: the public key is the agent's card.id (threaded via
|
|
1001
1948
|
// COTAL_ID); the seed is retained to mint matching creds later.
|
|
1002
1949
|
const identity = newIdentity();
|
|
1950
|
+
// The incarnation's lifecycle UID (SPEC 13.1), minted ONCE per spawn: every lifecycle-keyed
|
|
1951
|
+
// broker resource (dm_/dlv_/chathist_ durables, ACL row, memberships) and the teardown
|
|
1952
|
+
// credential carry it, so a same-name successor's footprint is name-disjoint by construction.
|
|
1953
|
+
const lifecycleUid = mintLifecycleUid();
|
|
1003
1954
|
// In auth mode, mint the agent's creds from the space signing key and write them where the
|
|
1004
1955
|
// spawned session reads them (COTAL_CREDS path). Open mesh → no creds. Scope = the resolved
|
|
1005
1956
|
// subscribe/allowSubscribe (read) + allowPublish (post, default-deny).
|
|
@@ -1016,6 +1967,7 @@ export class Manager {
|
|
|
1016
1967
|
role,
|
|
1017
1968
|
capabilities,
|
|
1018
1969
|
label: ref,
|
|
1970
|
+
lifecycleUid,
|
|
1019
1971
|
});
|
|
1020
1972
|
if ("error" in prep) {
|
|
1021
1973
|
this.reserved.delete(name);
|
|
@@ -1023,7 +1975,7 @@ export class Manager {
|
|
|
1023
1975
|
}
|
|
1024
1976
|
userLaunch = prep.launch;
|
|
1025
1977
|
userOwner = prep.owner;
|
|
1026
|
-
provisioned = { id: principalKey(prep.owner, name).key, name, userOwner: prep.owner };
|
|
1978
|
+
provisioned = { id: principalKey(prep.owner, name).key, name, lifecycleUid, userOwner: prep.owner };
|
|
1027
1979
|
}
|
|
1028
1980
|
else if (this.auth) {
|
|
1029
1981
|
// Pre-create the agent's bind-only chat (+ DM + role TASK) durables and mint its scoped creds
|
|
@@ -1036,11 +1988,16 @@ export class Manager {
|
|
|
1036
1988
|
allowPublish,
|
|
1037
1989
|
role,
|
|
1038
1990
|
capabilities,
|
|
1991
|
+
lifecycleUid,
|
|
1039
1992
|
}));
|
|
1040
|
-
|
|
1041
|
-
|
|
1042
|
-
|
|
1043
|
-
|
|
1993
|
+
// Store first (the source of truth), then materialize: `buildLaunch` hands the CHILD this
|
|
1994
|
+
// file path, so the cred must exist as a file regardless of the store behind the seam.
|
|
1995
|
+
// LOCAL composition, hardcoded, same posture as provisionUserAgent's grant store above.
|
|
1996
|
+
const secrets = workspaceSecretStore(this.workspaceRoot);
|
|
1997
|
+
credsPath = agentSecretFilePaths(this.workspaceRoot, name).creds;
|
|
1998
|
+
await secrets.put(agentCredsKey(name), creds);
|
|
1999
|
+
await materializeSecretToFile(secrets, agentCredsKey(name), credsPath);
|
|
2000
|
+
provisioned = { id: identity.id, name, lifecycleUid }; // footprint now exists — the finally rolls it back if the spawn throws
|
|
1044
2001
|
}
|
|
1045
2002
|
// Personal MCP servers the operator opted to share with manager-spawned agents of this type
|
|
1046
2003
|
// (cotal config; default none → isolated, the memory-safe default this guards), narrowed by
|
|
@@ -1050,6 +2007,11 @@ export class Manager {
|
|
|
1050
2007
|
// arbitrary folders/repos. A relative path resolves against the workspace root; omitted → the
|
|
1051
2008
|
// agent shares the workspace root (the prior, unchanged behavior).
|
|
1052
2009
|
const cwd = opts.cwd ? resolve(this.workspaceRoot, opts.cwd) : this.workspaceRoot;
|
|
2010
|
+
const configSha256 = this.fileDigest(configPath);
|
|
2011
|
+
const manifestPath = opts.launchRef
|
|
2012
|
+
? join(this.workspaceRoot, ".cotal", "run", `${opts.launchRef.runId}.json`)
|
|
2013
|
+
: undefined;
|
|
2014
|
+
const manifestSha256 = manifestPath ? this.fileDigest(manifestPath) : undefined;
|
|
1053
2015
|
const spec = connector.buildLaunch({
|
|
1054
2016
|
space: this.space,
|
|
1055
2017
|
name,
|
|
@@ -1059,6 +2021,10 @@ export class Manager {
|
|
|
1059
2021
|
id: userLaunch ? undefined : identity.id,
|
|
1060
2022
|
creds: credsPath,
|
|
1061
2023
|
userAuth: userLaunch,
|
|
2024
|
+
// The incarnation's lifecycle UID: the agent endpoint binds its lifecycle-keyed dm/dlv/
|
|
2025
|
+
// chathist durables by this exact value (its creds pin the same names, so a mismatch fails
|
|
2026
|
+
// at the broker, never silently).
|
|
2027
|
+
lifecycleUid,
|
|
1062
2028
|
servers: this.servers,
|
|
1063
2029
|
configPath,
|
|
1064
2030
|
model,
|
|
@@ -1090,11 +2056,41 @@ export class Manager {
|
|
|
1090
2056
|
role,
|
|
1091
2057
|
agent,
|
|
1092
2058
|
id: userLaunch ? principalKey(userLaunch.owner, name).key : identity.id,
|
|
2059
|
+
lifecycleUid,
|
|
1093
2060
|
...(userLaunch ? { userOwner } : { seed: identity.seed }),
|
|
1094
2061
|
spawner: spawner ?? this.ep.ref().id,
|
|
2062
|
+
authorityParent: userLaunch && spawner && parsePrincipalKey(spawner) ? spawner : undefined,
|
|
1095
2063
|
startedAt: Date.now(),
|
|
1096
2064
|
handle,
|
|
1097
2065
|
control: spec.control,
|
|
2066
|
+
launch: {
|
|
2067
|
+
source: opts.resolved
|
|
2068
|
+
? {
|
|
2069
|
+
kind: "manifest",
|
|
2070
|
+
runId: opts.launchRef?.runId,
|
|
2071
|
+
requested: opts.launchRef?.requested ?? opts.resolved.name,
|
|
2072
|
+
hash: opts.launchRef?.hash ?? opts.resolved.hash,
|
|
2073
|
+
configPath,
|
|
2074
|
+
configSha256,
|
|
2075
|
+
manifestSha256,
|
|
2076
|
+
}
|
|
2077
|
+
: { kind: "persona", ref, configPath, configSha256 },
|
|
2078
|
+
cwd,
|
|
2079
|
+
model,
|
|
2080
|
+
variant,
|
|
2081
|
+
subscribe,
|
|
2082
|
+
allowSubscribe,
|
|
2083
|
+
allowPublish,
|
|
2084
|
+
capabilities,
|
|
2085
|
+
transcript,
|
|
2086
|
+
shareTools: opts.shareTools,
|
|
2087
|
+
forkSource: opts.resume,
|
|
2088
|
+
// Opaque values may contain secrets. Preserve only their keys and require the referenced
|
|
2089
|
+
// persona/manifest to resolve the values again; imperative overrides have no safe payload.
|
|
2090
|
+
unresolvedLaunchOptionKeys: opts.launchOptions && Object.keys(opts.launchOptions).length
|
|
2091
|
+
? Object.keys(opts.launchOptions).sort()
|
|
2092
|
+
: undefined,
|
|
2093
|
+
},
|
|
1098
2094
|
};
|
|
1099
2095
|
this.agents.set(name, managed);
|
|
1100
2096
|
// The live slot now owns teardown — freeSlot deprovisions this identity on exit — so the
|
|
@@ -1128,10 +2124,402 @@ export class Manager {
|
|
|
1128
2124
|
// orphan down (detached, fail-loud) so a failed spawn leaves no creds/durables behind (#159 B).
|
|
1129
2125
|
if (provisioned) {
|
|
1130
2126
|
const orphan = provisioned;
|
|
1131
|
-
|
|
2127
|
+
this.trackDeprovision(orphan, "(orphaned spawn)");
|
|
2128
|
+
}
|
|
2129
|
+
}
|
|
2130
|
+
}
|
|
2131
|
+
/** Preflight the whole inventory before launching its first process, then adopt each exact retained
|
|
2132
|
+
* principal without provisioning. A later runtime launch failure is reported per-agent, but malformed
|
|
2133
|
+
* or missing inventory material can never produce a partially resumed set. */
|
|
2134
|
+
async resumePreserved(inventory) {
|
|
2135
|
+
const release = this.beginLifecycle(true);
|
|
2136
|
+
if (!release)
|
|
2137
|
+
return { ok: false, agents: [], error: this.maintenanceError() };
|
|
2138
|
+
const batchReservations = [];
|
|
2139
|
+
try {
|
|
2140
|
+
if (inventory.version !== "cotal-manager-resume/v1")
|
|
2141
|
+
return { ok: false, agents: [], error: `unsupported manager resume inventory version ${String(inventory.version)}` };
|
|
2142
|
+
if (inventory.space !== this.space)
|
|
2143
|
+
return { ok: false, agents: [], error: `resume inventory belongs to space "${inventory.space}", not "${this.space}"` };
|
|
2144
|
+
const seen = new Set();
|
|
2145
|
+
const principals = new Set();
|
|
2146
|
+
await this.ep.waitForPresenceSnapshot();
|
|
2147
|
+
const livePrincipals = new Set(this.ep.getRoster()
|
|
2148
|
+
.filter((presence) => presence.status !== "offline")
|
|
2149
|
+
.map((presence) => presence.card.id));
|
|
2150
|
+
if (this.agents.size + this.reserved.size + this.coolingCount() + inventory.agents.length > MAX_AGENTS)
|
|
2151
|
+
return { ok: false, agents: [], error: `resume inventory would exceed manager capacity (${MAX_AGENTS})` };
|
|
2152
|
+
for (const entry of inventory.agents) {
|
|
2153
|
+
if (seen.has(entry.name))
|
|
2154
|
+
return { ok: false, agents: [], error: `resume inventory contains duplicate agent name "${entry.name}"` };
|
|
2155
|
+
seen.add(entry.name);
|
|
2156
|
+
let principal;
|
|
2157
|
+
try {
|
|
2158
|
+
principal = entry.identity.mode === "user"
|
|
2159
|
+
? principalKey(entry.identity.owner, entry.identity.actor).key
|
|
2160
|
+
: principalKey(DEV_OWNER, entry.identity.id).key;
|
|
2161
|
+
}
|
|
2162
|
+
catch (e) {
|
|
2163
|
+
return { ok: false, agents: [], error: `invalid retained principal for ${entry.name}: ${e.message}` };
|
|
2164
|
+
}
|
|
2165
|
+
if (principals.has(principal))
|
|
2166
|
+
return { ok: false, agents: [], error: `resume inventory contains duplicate principal "${principal}"` };
|
|
2167
|
+
principals.add(principal);
|
|
2168
|
+
if (livePrincipals.has(principal))
|
|
2169
|
+
return { ok: false, agents: [], error: `retained principal "${principal}" is already live and this runtime cannot authoritatively adopt it` };
|
|
2170
|
+
if (this.agents.has(entry.name) || this.reserved.has(entry.name))
|
|
2171
|
+
return { ok: false, agents: [], error: `retained agent "${entry.name}" is already managed or reserved` };
|
|
2172
|
+
}
|
|
2173
|
+
for (const entry of inventory.agents) {
|
|
2174
|
+
this.reserved.add(entry.name);
|
|
2175
|
+
batchReservations.push(entry.name);
|
|
2176
|
+
}
|
|
2177
|
+
const prepared = new Map();
|
|
2178
|
+
const preflight = [];
|
|
2179
|
+
for (const entry of inventory.agents) {
|
|
2180
|
+
const reply = await this.resumePreservedAgent(entry, true, true, prepared);
|
|
2181
|
+
preflight.push({ name: entry.name, reply });
|
|
2182
|
+
}
|
|
2183
|
+
const preflightFailures = preflight.filter(({ reply }) => !reply.ok);
|
|
2184
|
+
if (preflightFailures.length)
|
|
2185
|
+
return {
|
|
2186
|
+
ok: false,
|
|
2187
|
+
agents: preflight,
|
|
2188
|
+
error: `${preflightFailures.length} retained agent${preflightFailures.length === 1 ? "" : "s"} failed preflight`,
|
|
2189
|
+
};
|
|
2190
|
+
const agents = [];
|
|
2191
|
+
for (let i = 0; i < inventory.agents.length; i++) {
|
|
2192
|
+
const entry = inventory.agents[i];
|
|
2193
|
+
const reply = await this.resumePreservedAgent(entry, false, true, prepared);
|
|
2194
|
+
agents.push({ name: entry.name, reply });
|
|
2195
|
+
if (!reply.ok) {
|
|
2196
|
+
for (const skipped of inventory.agents.slice(i + 1))
|
|
2197
|
+
agents.push({ name: skipped.name, reply: { ok: false, error: `not launched because ${entry.name} failed` } });
|
|
2198
|
+
return { ok: false, agents, error: reply.error };
|
|
2199
|
+
}
|
|
2200
|
+
}
|
|
2201
|
+
if (this.resumeAttemptId)
|
|
2202
|
+
this.resumeAwaitingCommit = true;
|
|
2203
|
+
return { ok: true, agents };
|
|
2204
|
+
}
|
|
2205
|
+
finally {
|
|
2206
|
+
for (const name of batchReservations)
|
|
2207
|
+
this.reserved.delete(name);
|
|
2208
|
+
release();
|
|
2209
|
+
}
|
|
2210
|
+
}
|
|
2211
|
+
/** Re-read every retained identity input and its current authority without provisioning. This runs
|
|
2212
|
+
* during whole-inventory preflight, immediately before each individual spawn, and at commit. */
|
|
2213
|
+
async validateRetainedAuthority(entry) {
|
|
2214
|
+
const referenceError = this.inventoryReferenceError(entry);
|
|
2215
|
+
if (referenceError)
|
|
2216
|
+
throw new Error(`retained agent ${entry.name}: ${referenceError}`);
|
|
2217
|
+
if (entry.identity.mode === "open") {
|
|
2218
|
+
if (this.auth || this.userMode)
|
|
2219
|
+
throw new Error(`retained agent ${entry.name} is open-mode but the current manager is authenticated`);
|
|
2220
|
+
return { id: entry.identity.id };
|
|
2221
|
+
}
|
|
2222
|
+
if (entry.identity.mode === "static") {
|
|
2223
|
+
if (!this.auth || this.userMode)
|
|
2224
|
+
throw new Error(`retained agent ${entry.name} is static-auth but the current manager is not`);
|
|
2225
|
+
const expected = resolve(agentSecretFilePaths(this.workspaceRoot, entry.name).creds);
|
|
2226
|
+
if (resolve(entry.identity.credential.path) !== expected)
|
|
2227
|
+
throw new Error(`retained credential reference for ${entry.name} is not the manager-owned path ${expected}`);
|
|
2228
|
+
let credentialText;
|
|
2229
|
+
try {
|
|
2230
|
+
// The lstat guards the FS MATERIALIZATION the child will read at launch; the identity check
|
|
2231
|
+
// runs on the store's value — the source of truth (byte-identical here, the local FS
|
|
2232
|
+
// composition resolves the key to this same path).
|
|
2233
|
+
const st = lstatSync(expected);
|
|
2234
|
+
if (!st.isFile() || st.isSymbolicLink())
|
|
2235
|
+
throw new Error("not a regular non-symlink file");
|
|
2236
|
+
const stored = await workspaceSecretStore(this.workspaceRoot).get(agentCredsKey(entry.name));
|
|
2237
|
+
if (stored === undefined)
|
|
2238
|
+
throw new Error("the credential is not in the secret store");
|
|
2239
|
+
credentialText = stored;
|
|
2240
|
+
const actual = idFromCreds(credentialText);
|
|
2241
|
+
if (actual !== entry.identity.id)
|
|
2242
|
+
throw new Error(`retained credential identity ${actual} does not match inventory principal ${entry.identity.id}`);
|
|
1132
2243
|
}
|
|
2244
|
+
catch (e) {
|
|
2245
|
+
throw new Error(`retained credential for ${entry.name} is unusable: ${e.message}`);
|
|
2246
|
+
}
|
|
2247
|
+
const accepted = await this.probeStaticCredential(credentialText);
|
|
2248
|
+
if (!accepted.ok)
|
|
2249
|
+
throw new Error(`retained credential for ${entry.name} is not accepted by the current broker (${accepted.reason})`);
|
|
2250
|
+
return { id: entry.identity.id, creds: expected };
|
|
2251
|
+
}
|
|
2252
|
+
if (!this.userMode)
|
|
2253
|
+
throw new Error(`retained agent ${entry.name} is user-auth but the current manager is not`);
|
|
2254
|
+
try {
|
|
2255
|
+
const provider = resolveAuthProvider();
|
|
2256
|
+
// Mirror the static branch's expected-path equality: the store reads below are keyed by
|
|
2257
|
+
// NAME, so a retained record aimed at a foreign path would otherwise pass its digest checks
|
|
2258
|
+
// there while a different secret gets validated here. Canonical paths only.
|
|
2259
|
+
const files = agentSecretFilePaths(this.workspaceRoot, entry.name);
|
|
2260
|
+
if (resolve(entry.identity.actorToken.path) !== resolve(files.actorToken) ||
|
|
2261
|
+
resolve(entry.identity.sentinelCredential.path) !== resolve(files.sentinelCreds))
|
|
2262
|
+
throw new Error(`retained identity references are not the manager-owned paths under ${agentCredsDir(this.workspaceRoot)}`);
|
|
2263
|
+
const secrets = workspaceSecretStore(this.workspaceRoot);
|
|
2264
|
+
const actorToken = await secrets.get(agentActorTokenKey(entry.name));
|
|
2265
|
+
const sentinelCreds = await secrets.get(agentSentinelCredsKey(entry.name));
|
|
2266
|
+
if (actorToken === undefined || sentinelCreds === undefined)
|
|
2267
|
+
throw new Error("the retained actor token / sentinel credential is not in the secret store");
|
|
2268
|
+
const adopted = await provider.validateRetainedAgent({
|
|
2269
|
+
store: secrets,
|
|
2270
|
+
dir: userAuthStateDir(this.workspaceRoot, this.space),
|
|
2271
|
+
space: this.space,
|
|
2272
|
+
owner: entry.identity.owner,
|
|
2273
|
+
actor: entry.identity.actor,
|
|
2274
|
+
actorToken,
|
|
2275
|
+
sentinelCreds,
|
|
2276
|
+
});
|
|
2277
|
+
if (adopted.owner !== entry.identity.owner || adopted.actor !== entry.identity.actor)
|
|
2278
|
+
throw new Error(`auth provider returned a replacement principal; expected ${entry.identity.owner}.${entry.identity.actor}`);
|
|
2279
|
+
// Bind the inventory's uid to the CURRENT authority row BEFORE any spawn: a corrupt or
|
|
2280
|
+
// admin-supplied inventory naming a different incarnation is refused at pre-effect validation,
|
|
2281
|
+
// never left to broker-fail after the child is already running (SPEC §13.1).
|
|
2282
|
+
if (adopted.lifecycleUid !== entry.identity.lifecycleUid)
|
|
2283
|
+
throw new Error(`retained user authority for ${entry.identity.owner}.${entry.identity.actor} is incarnation ${adopted.lifecycleUid}, not the inventory's ${entry.identity.lifecycleUid}; a resume binds the exact recovered uid before any spawn (SPEC 13.1)`);
|
|
2284
|
+
if (!sameStrings(adopted.allowSubscribe, entry.launch.allowSubscribe) ||
|
|
2285
|
+
!sameStrings(adopted.allowPublish, entry.launch.allowPublish) ||
|
|
2286
|
+
!sameStrings(adopted.scope, entry.launch.capabilities) ||
|
|
2287
|
+
adopted.role !== entry.role || adopted.parent !== entry.authorityParent)
|
|
2288
|
+
throw new Error(`retained user authority for ${entry.identity.owner}.${entry.identity.actor} no longer matches the inventory`);
|
|
2289
|
+
return {
|
|
2290
|
+
userAuth: {
|
|
2291
|
+
owner: entry.identity.owner,
|
|
2292
|
+
actor: entry.identity.actor,
|
|
2293
|
+
sentinelCredsPath: entry.identity.sentinelCredential.path,
|
|
2294
|
+
bearerCmd: [
|
|
2295
|
+
process.execPath,
|
|
2296
|
+
...process.execArgv,
|
|
2297
|
+
process.argv[1],
|
|
2298
|
+
provider.agentBearerCommand,
|
|
2299
|
+
"--dir", userAuthStateDir(this.workspaceRoot, this.space),
|
|
2300
|
+
"--space", this.space,
|
|
2301
|
+
"--owner", entry.identity.owner,
|
|
2302
|
+
"--actor", entry.identity.actor,
|
|
2303
|
+
"--token-file", entry.identity.actorToken.path,
|
|
2304
|
+
"--health-file", entry.identity.health.path,
|
|
2305
|
+
],
|
|
2306
|
+
},
|
|
2307
|
+
};
|
|
2308
|
+
}
|
|
2309
|
+
catch (e) {
|
|
2310
|
+
throw new Error(`retained user principal ${entry.identity.owner}.${entry.identity.actor} could not be reused: ${e.message}`);
|
|
2311
|
+
}
|
|
2312
|
+
}
|
|
2313
|
+
/** Validate/relaunch one retained inventory entry. Called only through resumePreserved so all
|
|
2314
|
+
* records pass the same preflight before the first child is exposed. */
|
|
2315
|
+
async resumePreservedAgent(entry, preflightOnly = false, batchReserved = false, prepared) {
|
|
2316
|
+
const release = this.beginLifecycle(batchReserved);
|
|
2317
|
+
if (!release)
|
|
2318
|
+
return { ok: false, error: this.maintenanceError() };
|
|
2319
|
+
try {
|
|
2320
|
+
if (entry.space !== this.space)
|
|
2321
|
+
return { ok: false, error: `retained agent ${entry.name} belongs to space "${entry.space}", not "${this.space}"` };
|
|
2322
|
+
if (entry.launch.runtime !== this.runtime.kind)
|
|
2323
|
+
return { ok: false, error: `retained agent ${entry.name} requires runtime "${entry.launch.runtime}", current manager uses "${this.runtime.kind}"` };
|
|
2324
|
+
const nameErr = this.nameError(entry.name);
|
|
2325
|
+
if (nameErr)
|
|
2326
|
+
return { ok: false, error: nameErr };
|
|
2327
|
+
if (this.agents.has(entry.name) || (!batchReserved && this.reserved.has(entry.name)))
|
|
2328
|
+
return { ok: false, error: `retained agent "${entry.name}" is already managed or reserved; same-principal resume never auto-numbers` };
|
|
2329
|
+
if (!batchReserved && this.agents.size + this.reserved.size + this.coolingCount() >= MAX_AGENTS)
|
|
2330
|
+
return { ok: false, error: `at capacity (${MAX_AGENTS} agents incl. in-flight + cooling); same-principal resume refused` };
|
|
2331
|
+
const cached = prepared?.get(entry.name);
|
|
2332
|
+
if (!preflightOnly && cached) {
|
|
2333
|
+
try {
|
|
2334
|
+
// Do not trust the earlier batch preflight across another agent's sequential readiness wait.
|
|
2335
|
+
await this.validateRetainedAuthority(entry);
|
|
2336
|
+
}
|
|
2337
|
+
catch (e) {
|
|
2338
|
+
return { ok: false, error: e.message };
|
|
2339
|
+
}
|
|
2340
|
+
return this.launchPreparedResume(entry, cached, batchReserved);
|
|
2341
|
+
}
|
|
2342
|
+
try {
|
|
2343
|
+
const cwd = lstatSync(entry.launch.cwd);
|
|
2344
|
+
if (!cwd.isDirectory() || cwd.isSymbolicLink())
|
|
2345
|
+
return { ok: false, error: `retained cwd is not a real directory: ${entry.launch.cwd}` };
|
|
2346
|
+
}
|
|
2347
|
+
catch (e) {
|
|
2348
|
+
return { ok: false, error: `retained cwd unavailable: ${entry.launch.cwd} (${e.message})` };
|
|
2349
|
+
}
|
|
2350
|
+
let connector;
|
|
2351
|
+
try {
|
|
2352
|
+
connector = registry.resolve("connector", entry.launch.connector);
|
|
2353
|
+
}
|
|
2354
|
+
catch (e) {
|
|
2355
|
+
return { ok: false, error: e.message };
|
|
2356
|
+
}
|
|
2357
|
+
const missing = (connector.requires ?? []).filter((bin) => !resolveOnPath(bin));
|
|
2358
|
+
if (missing.length)
|
|
2359
|
+
return { ok: false, error: `${connector.name} harness needs ${missing.join(", ")} on PATH - not found` };
|
|
2360
|
+
if (entry.launch.variant && !connector.supportsModelVariant)
|
|
2361
|
+
return { ok: false, error: `${connector.name} connector does not support model variants (variant)` };
|
|
2362
|
+
let launchOptions;
|
|
2363
|
+
if (entry.launch.source.kind === "manifest") {
|
|
2364
|
+
const launchSource = entry.launch.source;
|
|
2365
|
+
if (!launchSource.runId)
|
|
2366
|
+
return { ok: false, error: `retained manifest launch for ${entry.name} has no runId; refusing to guess a .cotal/run source` };
|
|
2367
|
+
let spec;
|
|
2368
|
+
try {
|
|
2369
|
+
const source = launchSpecForRun(this.workspaceRoot, launchSource.runId);
|
|
2370
|
+
if (source.space !== this.space)
|
|
2371
|
+
return { ok: false, error: `retained launch spec space "${source.space}" does not match manager space "${this.space}"` };
|
|
2372
|
+
spec = source.agents.find((a) => a.name === launchSource.requested);
|
|
2373
|
+
}
|
|
2374
|
+
catch (e) {
|
|
2375
|
+
return { ok: false, error: e.message };
|
|
2376
|
+
}
|
|
2377
|
+
if (!spec || spec.hash !== launchSource.hash)
|
|
2378
|
+
return { ok: false, error: `retained manifest agent ${launchSource.requested} is missing or its hash changed; refusing same-principal resume` };
|
|
2379
|
+
launchOptions = spec.launchOptions;
|
|
2380
|
+
}
|
|
2381
|
+
else {
|
|
2382
|
+
try {
|
|
2383
|
+
launchOptions = loadAgentFile(entry.launch.source.configPath).launchOptions;
|
|
2384
|
+
}
|
|
2385
|
+
catch (e) {
|
|
2386
|
+
return { ok: false, error: e.message };
|
|
2387
|
+
}
|
|
2388
|
+
}
|
|
2389
|
+
let authority;
|
|
2390
|
+
try {
|
|
2391
|
+
authority = await this.validateRetainedAuthority(entry);
|
|
2392
|
+
}
|
|
2393
|
+
catch (e) {
|
|
2394
|
+
return { ok: false, error: e.message };
|
|
2395
|
+
}
|
|
2396
|
+
try {
|
|
2397
|
+
const mcpServers = connectorServers(loadCotalConfig(this.workspaceRoot), entry.launch.connector, parseShareSelection(entry.launch.shareTools));
|
|
2398
|
+
const spec = connector.buildLaunch({
|
|
2399
|
+
space: this.space,
|
|
2400
|
+
name: entry.name,
|
|
2401
|
+
role: entry.role,
|
|
2402
|
+
id: authority.id,
|
|
2403
|
+
creds: authority.creds,
|
|
2404
|
+
userAuth: authority.userAuth,
|
|
2405
|
+
// Recover the ORIGINAL incarnation uid (never a fresh mint on resume): the child endpoint
|
|
2406
|
+
// binds its lifecycle-keyed dm/dlv/chathist durables by this exact value, and its creds pin
|
|
2407
|
+
// the same names. Omitting it here (as the pre-fix resume path did) leaves the resumed child
|
|
2408
|
+
// with no COTAL_LIFECYCLE_UID: static/user fail the connector auth gate and open self-mints a
|
|
2409
|
+
// fresh uid that orphans the preserved durables and never matches the readiness fence.
|
|
2410
|
+
lifecycleUid: entry.identity.lifecycleUid,
|
|
2411
|
+
servers: this.servers,
|
|
2412
|
+
configPath: entry.launch.source.configPath,
|
|
2413
|
+
model: entry.launch.model,
|
|
2414
|
+
variant: entry.launch.variant,
|
|
2415
|
+
launchOptions,
|
|
2416
|
+
resume: entry.launch.forkSource,
|
|
2417
|
+
subscribe: entry.launch.subscribe,
|
|
2418
|
+
allowSubscribe: entry.launch.allowSubscribe,
|
|
2419
|
+
allowPublish: entry.launch.allowPublish,
|
|
2420
|
+
capabilities: entry.launch.capabilities,
|
|
2421
|
+
transcript: entry.launch.transcript,
|
|
2422
|
+
mcpServers,
|
|
2423
|
+
workspaceRoot: this.workspaceRoot,
|
|
2424
|
+
});
|
|
2425
|
+
const value = { spec, ...authority };
|
|
2426
|
+
prepared?.set(entry.name, value);
|
|
2427
|
+
if (preflightOnly)
|
|
2428
|
+
return { ok: true, data: { name: entry.name, preflight: true } };
|
|
2429
|
+
return this.launchPreparedResume(entry, value, batchReserved);
|
|
2430
|
+
}
|
|
2431
|
+
catch (e) {
|
|
2432
|
+
return { ok: false, error: e.message };
|
|
2433
|
+
}
|
|
2434
|
+
}
|
|
2435
|
+
finally {
|
|
2436
|
+
release();
|
|
1133
2437
|
}
|
|
1134
2438
|
}
|
|
2439
|
+
async launchPreparedResume(entry, prepared, batchReserved) {
|
|
2440
|
+
if (!batchReserved)
|
|
2441
|
+
this.reserved.add(entry.name);
|
|
2442
|
+
try {
|
|
2443
|
+
const handle = this.runtime.spawn(entry.name, prepared.spec, entry.launch.cwd);
|
|
2444
|
+
const managed = {
|
|
2445
|
+
name: entry.name,
|
|
2446
|
+
role: entry.role,
|
|
2447
|
+
agent: entry.launch.connector,
|
|
2448
|
+
id: entry.identity.mode === "user" ? principalKey(entry.identity.owner, entry.identity.actor).key : entry.identity.id,
|
|
2449
|
+
// Recover the ORIGINAL incarnation uid the durables are keyed by (never a fresh mint on resume).
|
|
2450
|
+
lifecycleUid: entry.identity.lifecycleUid,
|
|
2451
|
+
userOwner: entry.identity.mode === "user" ? entry.identity.owner : undefined,
|
|
2452
|
+
spawner: entry.spawner,
|
|
2453
|
+
authorityParent: entry.authorityParent,
|
|
2454
|
+
startedAt: Date.now(),
|
|
2455
|
+
handle,
|
|
2456
|
+
control: prepared.spec.control,
|
|
2457
|
+
launch: {
|
|
2458
|
+
source: entry.launch.source,
|
|
2459
|
+
cwd: entry.launch.cwd,
|
|
2460
|
+
model: entry.launch.model,
|
|
2461
|
+
variant: entry.launch.variant,
|
|
2462
|
+
subscribe: entry.launch.subscribe,
|
|
2463
|
+
allowSubscribe: entry.launch.allowSubscribe,
|
|
2464
|
+
allowPublish: entry.launch.allowPublish,
|
|
2465
|
+
capabilities: entry.launch.capabilities,
|
|
2466
|
+
transcript: entry.launch.transcript,
|
|
2467
|
+
shareTools: entry.launch.shareTools,
|
|
2468
|
+
forkSource: entry.launch.forkSource,
|
|
2469
|
+
},
|
|
2470
|
+
suppressCleanup: true,
|
|
2471
|
+
};
|
|
2472
|
+
this.agents.set(entry.name, managed);
|
|
2473
|
+
if (this.resumeAttemptId)
|
|
2474
|
+
this.resumedAgentNames.add(entry.name);
|
|
2475
|
+
const readiness = await this.awaitReadiness(managed);
|
|
2476
|
+
if (!readiness.ok && !readiness.uncertain)
|
|
2477
|
+
return { ok: false, error: readiness.detail };
|
|
2478
|
+
if (!readiness.ok) {
|
|
2479
|
+
this.watchExit(managed);
|
|
2480
|
+
this.watchResumeAdoption(managed);
|
|
2481
|
+
return { ok: false, error: readiness.detail };
|
|
2482
|
+
}
|
|
2483
|
+
if (!this.resumeAttemptId)
|
|
2484
|
+
managed.suppressCleanup = false;
|
|
2485
|
+
this.watchExit(managed);
|
|
2486
|
+
if (this.agents.get(managed.name) !== managed)
|
|
2487
|
+
return { ok: false, error: `${managed.name} exited immediately after same-principal readiness` };
|
|
2488
|
+
return {
|
|
2489
|
+
ok: true,
|
|
2490
|
+
data: { name: managed.name, role: managed.role, agent: managed.agent, id: managed.id, mode: handle.kind, resumed: true },
|
|
2491
|
+
};
|
|
2492
|
+
}
|
|
2493
|
+
catch (e) {
|
|
2494
|
+
return { ok: false, error: e.message };
|
|
2495
|
+
}
|
|
2496
|
+
finally {
|
|
2497
|
+
if (!batchReserved)
|
|
2498
|
+
this.reserved.delete(entry.name);
|
|
2499
|
+
}
|
|
2500
|
+
}
|
|
2501
|
+
probeStaticCredential(creds) {
|
|
2502
|
+
return probeConnect(this.servers ?? DEFAULT_SERVER, { creds, timeoutMs: 5_000 });
|
|
2503
|
+
}
|
|
2504
|
+
/** An uncertain resume remains non-destructive until exact-principal AND exact-incarnation presence
|
|
2505
|
+
* arrives later. Same predicate as the readiness fence: a principal-only match would let a
|
|
2506
|
+
* wrong/absent-uid presence under the reused alias clear cleanup suppression on another incarnation. */
|
|
2507
|
+
watchResumeAdoption(a) {
|
|
2508
|
+
const wanted = this.managedPrincipal(a);
|
|
2509
|
+
const onPresence = () => {
|
|
2510
|
+
if (this.agents.get(a.name) !== a) {
|
|
2511
|
+
this.ep.off("presence", onPresence);
|
|
2512
|
+
return;
|
|
2513
|
+
}
|
|
2514
|
+
if (!this.ep.getRoster().some((p) => p.card.id === wanted && p.status !== "offline" && p.lifecycleUid === a.lifecycleUid))
|
|
2515
|
+
return;
|
|
2516
|
+
if (!this.resumeRequired)
|
|
2517
|
+
a.suppressCleanup = false;
|
|
2518
|
+
this.ep.off("presence", onPresence);
|
|
2519
|
+
};
|
|
2520
|
+
this.ep.on("presence", onPresence);
|
|
2521
|
+
onPresence();
|
|
2522
|
+
}
|
|
1135
2523
|
/** #159 B1: wait for a detached launch to reach a REAL outcome before replying — never a liveness-
|
|
1136
2524
|
* inferring timer. Races three:
|
|
1137
2525
|
* • the assigned id joins presence (live) → **started** — the honest signal (the manager owns mesh
|
|
@@ -1158,7 +2546,16 @@ export class Manager {
|
|
|
1158
2546
|
// through managedPrincipal or a static launch can never be seen joining (every static spawn would
|
|
1159
2547
|
// resolve "uncertain"; caught by the lifecycle e2e).
|
|
1160
2548
|
const wanted = this.managedPrincipal(a);
|
|
1161
|
-
|
|
2549
|
+
// READINESS LIFECYCLE FENCE (SPEC 13.1): match the exact principal AND the exact lifecycle uid
|
|
2550
|
+
// the manager minted for THIS spawn (presence carries it, §6/:315). The endpoint's own
|
|
2551
|
+
// register-only broker proof is gated on the CLIENT-authored `card.kind`, which a managed child
|
|
2552
|
+
// holding a valid agent credential could set to "endpoint" to skip - so it is defense-in-depth,
|
|
2553
|
+
// NOT the authority boundary. This equality is: the manager (not the child) owns the expected
|
|
2554
|
+
// uid, so a ghost that advertises a wrong/absent uid never reports STARTED, whatever kind it
|
|
2555
|
+
// claims. The manager threads the uid into EVERY mode's launch (open included), so the child
|
|
2556
|
+
// adopts it over a self-mint and publishes it in presence; the uid is absent only from a peer
|
|
2557
|
+
// the manager never launched (a pure operator/daemon connection that never registers).
|
|
2558
|
+
const joined = () => this.ep.getRoster().some((p) => p.card.id === wanted && p.status !== "offline" && p.lifecycleUid === a.lifecycleUid);
|
|
1162
2559
|
return await new Promise((resolve) => {
|
|
1163
2560
|
let done = false;
|
|
1164
2561
|
let timer;
|
|
@@ -1255,7 +2652,7 @@ export class Manager {
|
|
|
1255
2652
|
return { ok: false, error: denied };
|
|
1256
2653
|
const graceful = args.graceful !== false;
|
|
1257
2654
|
this.stopHandle(a, graceful);
|
|
1258
|
-
this.
|
|
2655
|
+
this.trackStoppedHandle(a, !admin);
|
|
1259
2656
|
return { ok: true, data: { name, stopped: true, graceful } };
|
|
1260
2657
|
}
|
|
1261
2658
|
/** Open a short-lived PROVISIONER connection, run the onboarding ops on it, and drain it (closure (ii),
|
|
@@ -1392,7 +2789,7 @@ export class Manager {
|
|
|
1392
2789
|
// FAIL-CLOSED: a failed record is the failure + repair sentence; a missing/malformed or
|
|
1393
2790
|
// stale record on a live agent is auth-unknown/auth-stale, NEVER silently healthy.
|
|
1394
2791
|
const health = a.userOwner
|
|
1395
|
-
? agentAuthState(
|
|
2792
|
+
? agentAuthState(agentSecretFilePaths(this.workspaceRoot, a.name).health)
|
|
1396
2793
|
: undefined;
|
|
1397
2794
|
return {
|
|
1398
2795
|
name: a.name,
|