@cotal-ai/manager 0.11.6 → 0.13.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/manager.js CHANGED
@@ -1,13 +1,16 @@
1
1
  import { execFile } from "node:child_process";
2
- import { existsSync, rmSync } from "node:fs";
2
+ import { createHash, randomUUID, randomBytes } from "node:crypto";
3
+ import { connect, credsAuthenticator } from "@nats-io/transport-node";
4
+ import { existsSync, lstatSync, readFileSync, rmSync } from "node:fs";
3
5
  import { join, dirname, resolve } from "node:path";
4
- import { CotalEndpoint, DEFAULT_SERVER, DEV_OWNER, MANAGER_LEASE_TTL_MS, STANDING_RENEWABLE_TTL_SEC, agentFilePath, clearSpaceHistory, connectorServers, deprovisionAgent, firstFreeName, loadAgentFile, loadCotalConfig, mintCreds, mkSecretDir, newIdentity, parsePrincipalKey, parseShareSelection, principalKey, provisionAgent, provisionAgentDurables, registry, resolveAuthProvider, saveAgentFile, writeSecretFile, subjectMatches, CONTROL_PRIVILEGED, CONTROL_SELF_SERVICE, CONTROL_ADMIN, } from "@cotal-ai/core";
5
- import { agentAuthState, authDir, connectorInstallHint, DEFAULT_CONNECTOR, defaultAgentType, findCotalRoot, loadMeshes, loadSpaceAuth, manifestExtensionNames, materializeFromManifest, mergeLaunchOptions, remintDaemonCreds, resolveOnPath, userAuthStateDir, writeRenewalRecord } from "@cotal-ai/workspace";
6
+ import { CotalEndpoint, DEFAULT_SERVER, DEV_OWNER, MANAGER_LEASE_TTL_MS, STANDING_RENEWABLE_TTL_SEC, agentFilePath, clearSpaceHistory, connectorServers, deprovisionAgent, firstFreeName, idFromCreds, loadAgentFile, loadCotalConfig, mintCreds, mintLifecycleUid, mkSecretDir, newIdentity, parsePrincipalKey, parseShareSelection, principalKey, probeConnect, provisionAgent, provisionAgentDurables, registry, resolveAuthProvider, saveAgentFile, subjectMatches, CONTROL_PRIVILEGED, CONTROL_SELF_SERVICE, CONTROL_ADMIN, CONTROL_AUTH_ADMIN, controlServiceSubject, } from "@cotal-ai/core";
7
+ import { agentActorTokenKey, agentAuthState, agentCredsDir, agentCredsKey, agentSecretFilePaths, agentSentinelCredsKey, authDir, connectorInstallHint, DEFAULT_CONNECTOR, defaultAgentType, findCotalRoot, loadMeshes, loadSpaceAuth, manifestExtensionNames, materializeFromManifest, materializeSecretToFile, mergeLaunchOptions, remintDaemonCreds, resolveOnPath, userAuthStateDir, workspaceSecretStore, writeRenewalRecord } from "@cotal-ai/workspace";
6
8
  import { createRuntime, } from "./runtime/index.js";
7
9
  import { AttachEndpoint } from "./attach-endpoint.js";
8
10
  import { launchSpecForRun, materializePersona, launchAgentToStartOpts } from "./launch.js";
9
11
  import { authorizeLaunch, authorizeNamedControl } from "./authorize.js";
10
12
  import { controlShutdown } from "./control-shutdown.js";
13
+ import { parseResumeCommitArgs, parseResumeControlArgs, parseResumeFinalizeArgs } from "./resume.js";
11
14
  /** Concurrency ceiling — the manager refuses to hold more than this many live + in-flight +
12
15
  * cooling slots at once (P4a). Bounds a fork-bomb: spawn is a full agent process per call. */
13
16
  const MAX_AGENTS = 50;
@@ -29,6 +32,16 @@ export const READINESS_TIMEOUT_MS = 30_000;
29
32
  * `.catch`. Generous over the helper's 5s connect timeout to allow the two consumer-deletes + ACL purge
30
33
  * + drain on a healthy-but-slow broker. */
31
34
  const DEPROVISION_TIMEOUT_MS = 15_000;
35
+ /** A hard preservation stop should settle quickly. The manager still waits and reports a partial
36
+ * cut rather than pretending a child is gone. Held in ManagerOptions so fake runtimes can shorten it. */
37
+ const PRESERVE_STOP_TIMEOUT_MS = 10_000;
38
+ /** The STABLE retirement opId for one lifecycle (#29 piece 3): deterministic from the uid, so a
39
+ * despawn retry, a same-name-spawn nudge, and the auth service's boot resume all drive the SAME
40
+ * operation (the rail's idempotence table needs exactly one op per retiring incarnation). 26 hex
41
+ * chars = in the lifecycle-token grammar `[a-z0-9]{26,32}`, collision-resistant. */
42
+ function retireOpId(lifecycleUid) {
43
+ return createHash("sha256").update(`retire:${lifecycleUid}`).digest("hex").slice(0, 26);
44
+ }
32
45
  /** Sentinel owner-filter value that matches NO agent's `userOwner` (owner tokens never contain a
33
46
  * dash) — what {@link Manager.psOwnerFilter} returns for an unparseable caller so a malformed
34
47
  * principal fail-closes to an empty `ps` instead of an unbounded one. */
@@ -54,6 +67,9 @@ function withTimeout(p, ms, msg) {
54
67
  });
55
68
  return Promise.race([p.finally(() => clearTimeout(timer)), timeout]);
56
69
  }
70
+ function sameStrings(a, b) {
71
+ return JSON.stringify([...(a ?? [])].sort()) === JSON.stringify([...(b ?? [])].sort());
72
+ }
57
73
  /**
58
74
  * The agent supervisor: a long-lived mesh node that owns agent process lifecycle.
59
75
  * It serves control requests on the "manager" service and spawns/kills agents
@@ -68,6 +84,7 @@ export class Manager {
68
84
  /** See {@link ManagerOptions.installedExtensions}. */
69
85
  installedExtensions;
70
86
  runtime;
87
+ preserveStopTimeoutMs;
71
88
  agents = new Map();
72
89
  /** Names whose spawn is in flight (reserved synchronously before the provision await) — counted
73
90
  * toward the ceiling so two concurrent same-name spawns can't both pass the gate (P4a). */
@@ -75,6 +92,29 @@ export class Manager {
75
92
  /** Expiry stamps (`startedAt + MIN_LIFETIME`) for slots that freed while still young — a
76
93
  * count-only, lazily-pruned recycle floor (P4c). Pruned + summed into the ceiling gate. */
77
94
  cooling = [];
95
+ /** Names RESERVED PENDING RETIREMENT (#29 piece 3): a despawned agent's name stays held until
96
+ * the auth plane confirms its lifecycle's retirement TERMINAL over the auth-admin rail — the
97
+ * alias-reuse gate that closes the same-name despawn→respawn race at its root. An UNCERTAIN
98
+ * outcome (rail down, timeout) keeps the hold with the last attempt's copy; a same-name spawn
99
+ * refuses legibly AND re-fires the request. In-memory: across a manager restart the durable
100
+ * truth is the auth-side lifecycle head itself (an unretired head refuses issuance — the
101
+ * named residual this belt narrows, not replaces). */
102
+ retiring = new Map();
103
+ /** SINGLE-FLIGHT guard for {@link requestRetirement} (audit #1): one in-flight rail round-trip per
104
+ * (name, lifecycleUid). The detached `deprovision` call and every same-name-spawn nudge for THAT
105
+ * lifecycle JOIN the same promise instead of stacking independent requests that dual-enter the
106
+ * barrier; a fresh trigger after it settles re-drives. Keyed by (name, uid) — NOT name alone — so a
107
+ * same-name SUCCESSOR (which can spawn after the hold clears but before this flight's `nc.close`
108
+ * yield settles) never joins the predecessor's rail request and skips its own retirement. */
109
+ retiringFlight = new Map();
110
+ /** SINGLE-FLIGHT guard for {@link deprovision} (INT-2/C): one in-flight teardown per
111
+ * (name, lifecycleUid). The detached freeSlot teardown and every same-name-spawn nudge that
112
+ * re-drives it JOIN one promise instead of launching a SECOND, concurrent teardown. Without it,
113
+ * two teardowns race the NAME-KEYED ledger revoke: once the first frees the alias and a successor
114
+ * mints its own row, the second's delayed revoke (which carries no lifecycle coordinate) would
115
+ * delete the SUCCESSOR's standing authority. Keyed by (name, uid) so a later same-name lifecycle
116
+ * gets its own flight; a fresh trigger after settle re-drives only if the hold still stands. */
117
+ deprovisioningFlight = new Map();
78
118
  attach;
79
119
  ep;
80
120
  /** Space trust material when the mesh runs in auth mode (`.cotal/auth` present);
@@ -92,6 +132,29 @@ export class Manager {
92
132
  leaseTimer;
93
133
  /** The class-2 renewal owner's half-TTL schedule (D5 slice 5); armed only on auth meshes. */
94
134
  credRenewTimer;
135
+ maintenanceState = "active";
136
+ lifecycleInFlight = 0;
137
+ lifecycleDrainWaiters = [];
138
+ preservationTask;
139
+ preparationTask;
140
+ preservationGeneration = 0;
141
+ preservationAttemptId;
142
+ preservationStarted = false;
143
+ preservationFailures = [];
144
+ unverifiedStops = [];
145
+ preservationInventory;
146
+ resumeAttemptId;
147
+ resumeInventoryDigest;
148
+ resumeInventory;
149
+ resumeTask;
150
+ resumeResult;
151
+ resumeRequired = false;
152
+ resumeAwaitingCommit = false;
153
+ resumeCommitted = false;
154
+ resumeCommitTask;
155
+ resumeFinalized = false;
156
+ resumeDurableCommitToken;
157
+ resumedAgentNames = new Set();
95
158
  constructor(opts) {
96
159
  this.space = opts.space;
97
160
  this.servers = opts.servers;
@@ -99,7 +162,17 @@ export class Manager {
99
162
  this.workspaceRoot = opts.workspaceRoot ?? findCotalRoot();
100
163
  this.installedExtensions = opts.installedExtensions ?? false;
101
164
  this.runtime = createRuntime(opts.runtime ?? "auto", `cotal-${this.space}`);
102
- this.attach = new AttachEndpoint((name) => this.agents.get(name)?.handle, () => this.list(),
165
+ this.preserveStopTimeoutMs = opts.preserveStopTimeoutMs ?? PRESERVE_STOP_TIMEOUT_MS;
166
+ if (opts.resumeAttemptId && !/^[A-Za-z0-9_-]{1,128}$/.test(opts.resumeAttemptId))
167
+ throw new Error("resumeAttemptId must be a safe token (letters, digits, _, -; max 128)");
168
+ if (opts.resumeDurableCommitToken && !/^[a-f0-9]{64}$/.test(opts.resumeDurableCommitToken))
169
+ throw new Error("resumeDurableCommitToken must be a lowercase 32-byte token");
170
+ if (opts.resumeDurableCommitToken && !opts.resumeAttemptId)
171
+ throw new Error("resumeDurableCommitToken requires resumeAttemptId");
172
+ this.resumeAttemptId = opts.resumeAttemptId;
173
+ this.resumeRequired = opts.resumeAttemptId !== undefined;
174
+ this.resumeDurableCommitToken = opts.resumeDurableCommitToken;
175
+ this.attach = new AttachEndpoint((name) => this.maintenanceState === "active" && !this.resumeRequired ? this.agents.get(name)?.handle : undefined, () => this.list(),
103
176
  // Initial /feed replay for a connecting console: the current peer roster.
104
177
  () => [{ event: "roster", data: this.ep?.getRoster() ?? [] }], opts.consolePort ?? 0);
105
178
  }
@@ -149,6 +222,11 @@ export class Manager {
149
222
  servers: this.servers,
150
223
  channels: [],
151
224
  creds,
225
+ // The supervisor registers on the roster, and an authed presence-registering endpoint is
226
+ // lifecycle-keyed (SPEC 13.1, fail-before-presence). The manager process is the top of its
227
+ // own launch chain (the operator command IS its launcher), so it mints its incarnation's
228
+ // uid here - one per supervisor process, never reused across restarts.
229
+ lifecycleUid: mintLifecycleUid(),
152
230
  // The supervisor serves control + watches presence; it never consumes chat/dm/task
153
231
  // (no message handler). consume:false avoids binding consumers it doesn't use — and
154
232
  // under auth avoids trying to bind its own DM/task durables that nothing pre-created.
@@ -221,6 +299,9 @@ export class Manager {
221
299
  * (no responder) is recorded honestly: the daemon's 75% source re-read remains the adoption backstop.
222
300
  * Never throws — renewal failure must be LOUD (log + record), not fatal to the supervisor. */
223
301
  async renewDaemonCreds() {
302
+ const release = this.beginLifecycle();
303
+ if (!release)
304
+ return;
224
305
  try {
225
306
  const results = await remintDaemonCreds(this.workspaceRoot);
226
307
  const resigned = results.filter((r) => r.ok);
@@ -243,6 +324,360 @@ export class Manager {
243
324
  catch (e) {
244
325
  console.error(`! credential renewal pass failed: ${e.message}`);
245
326
  }
327
+ finally {
328
+ release();
329
+ }
330
+ }
331
+ /** Admit one lifecycle/control operation while active. The synchronous increment is the fence:
332
+ * preserveState flips state before its first await, so work is either counted or rejected. */
333
+ beginLifecycle(resumeOperation = false) {
334
+ if (this.maintenanceState !== "active" || (this.resumeRequired && !resumeOperation))
335
+ return undefined;
336
+ this.lifecycleInFlight++;
337
+ let released = false;
338
+ return () => {
339
+ if (released)
340
+ return;
341
+ released = true;
342
+ this.releaseLifecycle();
343
+ };
344
+ }
345
+ releaseLifecycle() {
346
+ this.lifecycleInFlight--;
347
+ if (this.lifecycleInFlight !== 0)
348
+ return;
349
+ const waiters = this.lifecycleDrainWaiters;
350
+ this.lifecycleDrainWaiters = [];
351
+ for (const wake of waiters)
352
+ wake();
353
+ }
354
+ /** A cleanup spawned by accepted active-mode work is part of that work for maintenance draining,
355
+ * even where the ordinary control reply remains fire-and-forget. */
356
+ trackDeprovision(a, context = "") {
357
+ this.lifecycleInFlight++;
358
+ void this.deprovision(a)
359
+ .catch((e) => console.error(`deprovision${context ? ` ${context}` : ""} ${a.name} (${a.id}): ${e.message}`))
360
+ .finally(() => this.releaseLifecycle());
361
+ }
362
+ async awaitLifecycleDrain() {
363
+ if (this.lifecycleInFlight === 0)
364
+ return;
365
+ await new Promise((resolve) => this.lifecycleDrainWaiters.push(resolve));
366
+ }
367
+ maintenanceError() {
368
+ if (this.resumeRequired)
369
+ return `manager is waiting for resume attempt ${this.resumeAttemptId}; ordinary lifecycle/control work is fenced`;
370
+ return `manager is in ${this.maintenanceState} mode; new lifecycle/control work is fenced`;
371
+ }
372
+ /** Fence and build the inventory without stopping a child. The coordinator must durably persist
373
+ * this exact plan before calling commitPreservation with the same attempt id. */
374
+ preparePreservation(attemptId) {
375
+ if (!attemptId.trim())
376
+ return Promise.reject(new Error("preservation attemptId is required"));
377
+ if (this.preservationAttemptId && this.preservationAttemptId !== attemptId)
378
+ return Promise.reject(new Error(`manager is fenced for preservation attempt ${this.preservationAttemptId}; refusing different attempt ${attemptId}`));
379
+ if (this.maintenanceState === "preserved" && this.preservationInventory)
380
+ return Promise.resolve({ ok: true, attemptId, state: "preserved", inventory: this.preservationInventory, failures: [] });
381
+ if (this.preparationTask)
382
+ return this.preparationTask;
383
+ if (this.maintenanceState === "active") {
384
+ // The fence lands before any await. Accepted work has already incremented lifecycleInFlight.
385
+ this.maintenanceState = "preserving";
386
+ this.preservationAttemptId = attemptId;
387
+ this.preservationGeneration++;
388
+ if (this.credRenewTimer) {
389
+ clearInterval(this.credRenewTimer);
390
+ this.credRenewTimer = undefined;
391
+ }
392
+ }
393
+ const generation = this.preservationGeneration;
394
+ const task = this.runPreparation(attemptId, generation);
395
+ let wrapped;
396
+ wrapped = task.finally(() => {
397
+ if (this.preservationGeneration === generation && this.preparationTask === wrapped)
398
+ this.preparationTask = undefined;
399
+ });
400
+ this.preparationTask = wrapped;
401
+ return wrapped;
402
+ }
403
+ assertPreservationGeneration(attemptId, generation) {
404
+ if (this.preservationAttemptId !== attemptId || this.preservationGeneration !== generation)
405
+ throw new Error(`preservation attempt ${attemptId} was abandoned before preparation completed`);
406
+ }
407
+ async runPreparation(attemptId, generation) {
408
+ await this.awaitLifecycleDrain();
409
+ this.assertPreservationGeneration(attemptId, generation);
410
+ const inventory = this.preservationInventory ?? {
411
+ version: "cotal-manager-resume/v1",
412
+ space: this.space,
413
+ createdAt: new Date().toISOString(),
414
+ agents: [...this.agents.values()].map((a) => this.resumeEntry(a)),
415
+ };
416
+ const failures = [];
417
+ for (const entry of inventory.agents) {
418
+ const error = this.inventoryReferenceError(entry);
419
+ if (error)
420
+ failures.push({
421
+ name: entry.name,
422
+ id: entry.identity.mode === "user" ? principalKey(entry.identity.owner, entry.identity.actor).key : entry.identity.id,
423
+ error,
424
+ });
425
+ }
426
+ const unverifiedStops = this.unverifiedStops.filter((stopped) => {
427
+ try {
428
+ if (!stopped.authoritative && stopped.handle.status() === "exited")
429
+ return false;
430
+ }
431
+ catch { /* fail closed below */ }
432
+ failures.push({
433
+ name: stopped.name,
434
+ id: stopped.id,
435
+ error: stopped.error ?? `an earlier stop on runtime "${stopped.handle.kind}" cannot prove the child is gone`,
436
+ });
437
+ return true;
438
+ });
439
+ this.assertPreservationGeneration(attemptId, generation);
440
+ // The prepared inventory must round-trip through the EXACT resume control parser (schema and
441
+ // byte cap) NOW, before any child stops: a cut that cannot resume must fail at prepare time,
442
+ // never after listener exposure.
443
+ try {
444
+ parseResumeControlArgs({ attemptId, inventory });
445
+ }
446
+ catch (e) {
447
+ failures.push({ name: "<inventory>", id: attemptId, error: `prepared inventory would be rejected at resume: ${e.message}` });
448
+ }
449
+ this.preservationInventory = inventory;
450
+ this.preservationFailures = failures;
451
+ this.unverifiedStops = unverifiedStops;
452
+ return {
453
+ ok: failures.length === 0,
454
+ attemptId,
455
+ state: "prepared",
456
+ inventory,
457
+ failures: [...failures],
458
+ };
459
+ }
460
+ /** Stop children only after the coordinator has persisted the prepared inventory. Same-attempt
461
+ * retries are idempotent; a different attempt is refused. */
462
+ commitPreservation(attemptId) {
463
+ if (!this.preservationAttemptId || this.preservationAttemptId !== attemptId)
464
+ return Promise.reject(new Error(`preservation attempt ${attemptId} was not prepared by this manager`));
465
+ if (!this.preservationInventory)
466
+ return Promise.reject(new Error(`preservation attempt ${attemptId} has no prepared inventory`));
467
+ if (this.preservationFailures.length)
468
+ return Promise.resolve({
469
+ ok: false,
470
+ attemptId,
471
+ state: "preserving",
472
+ inventory: this.preservationInventory,
473
+ failures: [...this.preservationFailures],
474
+ });
475
+ if (this.maintenanceState === "preserved")
476
+ return Promise.resolve({ ok: true, attemptId, state: "preserved", inventory: this.preservationInventory, failures: [] });
477
+ if (this.preservationTask)
478
+ return this.preservationTask;
479
+ this.preservationStarted = true;
480
+ this.preservationTask = this.runPreservation(attemptId).finally(() => {
481
+ this.preservationTask = undefined;
482
+ });
483
+ return this.preservationTask;
484
+ }
485
+ /** Recover an abandoned prepare before any child stop. Once commit begins, preservation is
486
+ * irreversible and remains fenced until the coordinator records failure/recourse. */
487
+ abortPreservation(attemptId) {
488
+ if (this.preservationAttemptId !== attemptId)
489
+ throw new Error(`preservation attempt ${attemptId} is not the active manager attempt`);
490
+ if (this.preparationTask || this.lifecycleInFlight > 0)
491
+ throw new Error(`preservation attempt ${attemptId} is still preparing or draining accepted lifecycle work and cannot be aborted`);
492
+ if (this.preservationStarted || this.preservationTask || this.maintenanceState === "preserved")
493
+ throw new Error(`preservation attempt ${attemptId} has begun stopping children and cannot return to active mode`);
494
+ this.preservationGeneration++;
495
+ this.maintenanceState = "active";
496
+ this.preservationAttemptId = undefined;
497
+ this.preservationInventory = undefined;
498
+ this.preservationFailures = [];
499
+ if (this.auth && !this.credRenewTimer) {
500
+ this.credRenewTimer = setInterval(() => { void this.renewDaemonCreds(); }, (STANDING_RENEWABLE_TTL_SEC / 2) * 1000);
501
+ this.credRenewTimer.unref?.();
502
+ }
503
+ // Exit watchers were suppressed while the fence stood: reconcile every child that died during
504
+ // preparation now, or its slot/credential footprint would linger unreaped after the abort.
505
+ for (const agent of [...this.agents.values()]) {
506
+ try {
507
+ if (agent.handle.status() === "exited")
508
+ this.onAgentExit(agent);
509
+ }
510
+ catch { /* status unavailable - the exit watcher fires again on real exit */ }
511
+ }
512
+ }
513
+ /** In-process convenience that preserves the crash barrier by awaiting durable persistence between
514
+ * prepare and commit. Wire callers use the explicit two-phase admin operations. */
515
+ async preserveState(opts) {
516
+ const plan = await this.preparePreservation(opts.attemptId);
517
+ if (!plan.ok)
518
+ return { ok: false, attemptId: opts.attemptId, state: "preserving", inventory: plan.inventory, failures: plan.failures };
519
+ await opts.persistInventory(plan.inventory);
520
+ return this.commitPreservation(opts.attemptId);
521
+ }
522
+ async runPreservation(attemptId) {
523
+ const failures = [];
524
+ for (const a of [...this.agents.values()])
525
+ a.suppressCleanup = true;
526
+ await Promise.all([...this.agents.values()].map(async (a) => {
527
+ try {
528
+ // A preservation cut must not run the connector's logical leave/cleanup hooks.
529
+ a.handle.stop({ graceful: false });
530
+ }
531
+ catch (e) {
532
+ failures.push({ name: a.name, id: a.id, error: `stop failed: ${e.message}` });
533
+ return;
534
+ }
535
+ try {
536
+ await this.awaitHandleExit(a.handle);
537
+ if (this.agents.get(a.name) === a)
538
+ this.agents.delete(a.name);
539
+ }
540
+ catch (e) {
541
+ failures.push({ name: a.name, id: a.id, error: e.message });
542
+ }
543
+ }));
544
+ if (failures.length === 0)
545
+ this.maintenanceState = "preserved";
546
+ return {
547
+ ok: failures.length === 0,
548
+ attemptId,
549
+ state: this.maintenanceState === "preserved" ? "preserved" : "preserving",
550
+ inventory: this.preservationInventory,
551
+ failures,
552
+ };
553
+ }
554
+ async awaitHandleExit(handle) {
555
+ if (!handle.waitForExit)
556
+ throw new Error(`runtime "${handle.kind}" cannot prove child exit (AgentHandle.waitForExit is not implemented)`);
557
+ if (handle.status() === "exited")
558
+ return;
559
+ await withTimeout(handle.waitForExit(), this.preserveStopTimeoutMs, `child did not exit within ${this.preserveStopTimeoutMs}ms`);
560
+ if (handle.status() !== "exited")
561
+ throw new Error(`runtime "${handle.kind}" reported exit completion but status is still running`);
562
+ }
563
+ inventoryReferenceError(entry) {
564
+ if (entry.launch.source.kind === "manifest" && !entry.launch.source.runId)
565
+ return "resolved manifest launch has no retained runId";
566
+ if (entry.launch.unresolvedLaunchOptionKeys?.length)
567
+ return `imperative launch options have no non-secret durable source (${entry.launch.unresolvedLaunchOptionKeys.join(", ")})`;
568
+ if (!entry.dependencies.some((path) => resolve(path) === resolve(entry.launch.source.configPath)))
569
+ return `launch config is not declared as a retained dependency: ${entry.launch.source.configPath}`;
570
+ if (entry.launch.source.kind === "manifest" && entry.launch.source.runId) {
571
+ const specPath = join(this.workspaceRoot, ".cotal", "run", `${entry.launch.source.runId}.json`);
572
+ if (!entry.dependencies.some((path) => resolve(path) === resolve(specPath)))
573
+ return `manifest source is not declared as a retained dependency: ${specPath}`;
574
+ }
575
+ const required = [...entry.dependencies];
576
+ if (entry.identity.mode === "static")
577
+ required.push(entry.identity.credential.path);
578
+ if (entry.identity.mode === "user") {
579
+ required.push(entry.identity.actorToken.path, entry.identity.sentinelCredential.path);
580
+ }
581
+ for (const path of required) {
582
+ try {
583
+ const st = lstatSync(path);
584
+ if (!st.isFile() || st.isSymbolicLink())
585
+ return `retained reference is not a regular non-symlink file: ${path}`;
586
+ }
587
+ catch (e) {
588
+ return `retained reference unavailable: ${path} (${e.message})`;
589
+ }
590
+ }
591
+ if (process.platform !== "win32") {
592
+ const secrets = entry.identity.mode === "static"
593
+ ? [entry.identity.credential.path]
594
+ : entry.identity.mode === "user"
595
+ ? [entry.identity.actorToken.path, entry.identity.sentinelCredential.path]
596
+ : [];
597
+ for (const path of secrets)
598
+ if ((lstatSync(path).mode & 0o077) !== 0)
599
+ return `retained identity file is not private (expected 0600): ${path}`;
600
+ }
601
+ try {
602
+ if (this.fileDigest(entry.launch.source.configPath) !== entry.launch.source.configSha256)
603
+ return `launch config changed since it became effective: ${entry.launch.source.configPath}`;
604
+ if (entry.identity.mode === "static" && this.fileDigest(entry.identity.credential.path) !== entry.identity.credential.sha256)
605
+ return `retained credential changed after the cut: ${entry.identity.credential.path}`;
606
+ if (entry.identity.mode === "user" &&
607
+ (this.fileDigest(entry.identity.actorToken.path) !== entry.identity.actorToken.sha256 ||
608
+ this.fileDigest(entry.identity.sentinelCredential.path) !== entry.identity.sentinelCredential.sha256))
609
+ return `retained user identity files changed after the cut for ${entry.name}`;
610
+ if (entry.launch.source.kind === "manifest" && entry.launch.source.runId) {
611
+ const specPath = join(this.workspaceRoot, ".cotal", "run", `${entry.launch.source.runId}.json`);
612
+ if (!entry.launch.source.manifestSha256 || this.fileDigest(specPath) !== entry.launch.source.manifestSha256)
613
+ return `manifest source changed since it became effective: ${specPath}`;
614
+ }
615
+ }
616
+ catch (e) {
617
+ return `retained reference cannot be hashed: ${e.message}`;
618
+ }
619
+ return undefined;
620
+ }
621
+ fileDigest(path) {
622
+ return createHash("sha256").update(readFileSync(path)).digest("hex");
623
+ }
624
+ fileDigestOrEmpty(path) {
625
+ try {
626
+ return this.fileDigest(path);
627
+ }
628
+ catch {
629
+ return "";
630
+ }
631
+ }
632
+ resumeEntry(a) {
633
+ const principal = a.userOwner
634
+ ? parsePrincipalKey(a.id)
635
+ : { owner: DEV_OWNER, actor: a.id };
636
+ if (!principal)
637
+ throw new Error(`managed agent ${a.name} has an invalid principal ${a.id}`);
638
+ const files = agentSecretFilePaths(this.workspaceRoot, a.name);
639
+ const identity = a.userOwner
640
+ ? {
641
+ mode: "user",
642
+ owner: principal.owner,
643
+ actor: principal.actor,
644
+ lifecycleUid: a.lifecycleUid,
645
+ actorToken: { kind: "file", path: files.actorToken, sha256: this.fileDigestOrEmpty(files.actorToken) },
646
+ sentinelCredential: { kind: "file", path: files.sentinelCreds, sha256: this.fileDigestOrEmpty(files.sentinelCreds) },
647
+ health: { kind: "file", path: files.health },
648
+ }
649
+ : this.auth
650
+ ? { mode: "static", id: principal.actor, lifecycleUid: a.lifecycleUid, credential: { kind: "file", path: files.creds, sha256: this.fileDigestOrEmpty(files.creds) } }
651
+ : { mode: "open", id: principal.actor, lifecycleUid: a.lifecycleUid };
652
+ const dependencies = [a.launch.source.configPath];
653
+ if (a.launch.source.kind === "manifest" && a.launch.source.runId)
654
+ dependencies.unshift(join(this.workspaceRoot, ".cotal", "run", `${a.launch.source.runId}.json`));
655
+ return {
656
+ space: this.space,
657
+ name: a.name,
658
+ role: a.role,
659
+ identity,
660
+ launch: {
661
+ connector: a.agent,
662
+ runtime: a.handle.kind,
663
+ cwd: a.launch.cwd,
664
+ source: a.launch.source,
665
+ model: a.launch.model,
666
+ variant: a.launch.variant,
667
+ subscribe: a.launch.subscribe,
668
+ allowSubscribe: a.launch.allowSubscribe,
669
+ allowPublish: a.launch.allowPublish,
670
+ capabilities: a.launch.capabilities,
671
+ transcript: a.launch.transcript,
672
+ shareTools: a.launch.shareTools,
673
+ forkSource: a.launch.forkSource,
674
+ unresolvedLaunchOptionKeys: a.launch.unresolvedLaunchOptionKeys,
675
+ },
676
+ dependencies,
677
+ spawner: a.spawner,
678
+ authorityParent: a.authorityParent,
679
+ startedAt: new Date(a.startedAt).toISOString(),
680
+ };
246
681
  }
247
682
  /** Tear down every managed agent's footprint — the shared teardown for EVERY manager-exit path (#159
248
683
  * B2): graceful {@link stop} AND the fail-closed lease-loss exit ({@link renewLease}). A manager exit is
@@ -262,14 +697,44 @@ export class Manager {
262
697
  this.stopHandle(a, false);
263
698
  }
264
699
  // Deprovision EVERY snapshot entry regardless of whether its stop failed (allSettled + a loud log).
265
- await Promise.allSettled(managed.map((a) => this.deprovision(a).catch((e) => console.error(`deprovision ${a.name} (${a.id}) on shutdown: ${e.message}`))));
700
+ await Promise.allSettled(managed.filter((a) => !a.suppressCleanup).map((a) => this.deprovision(a).catch((e) => console.error(`deprovision ${a.name} (${a.id}) on shutdown: ${e.message}`))));
701
+ }
702
+ async stopRetainedAgentsOnExit() {
703
+ const managed = [...this.agents.values()];
704
+ for (const a of managed)
705
+ a.suppressCleanup = true;
706
+ const failures = [];
707
+ await Promise.all(managed.map(async (a) => {
708
+ try {
709
+ a.handle.stop({ graceful: false });
710
+ }
711
+ catch (e) {
712
+ failures.push(`${a.name}: stop failed: ${e.message}`);
713
+ }
714
+ try {
715
+ await this.awaitHandleExit(a.handle);
716
+ if (this.agents.get(a.name) === a)
717
+ this.agents.delete(a.name);
718
+ }
719
+ catch (e) {
720
+ failures.push(`${a.name}: ${e.message}`);
721
+ }
722
+ }));
723
+ if (failures.length)
724
+ throw new Error(`manager preservation shutdown incomplete: ${failures.join("; ")}`);
266
725
  }
267
726
  async stop() {
268
727
  if (this.leaseTimer)
269
728
  clearInterval(this.leaseTimer);
270
729
  if (this.credRenewTimer)
271
730
  clearInterval(this.credRenewTimer);
272
- await this.teardownManagedAgents(); // reap agents BEFORE releasing the lease/endpoints (#159 B2)
731
+ if (this.maintenanceState === "active" && !this.resumeRequired) {
732
+ await this.teardownManagedAgents(); // normal shutdown stays destructive (#159 B2)
733
+ }
734
+ else {
735
+ // A signal after a partial preservation must never fall back into destructive teardown.
736
+ await this.stopRetainedAgentsOnExit();
737
+ }
273
738
  await this.ep.releaseManagerLease(this.leaseRevision);
274
739
  await this.ep.stop();
275
740
  await this.attach.stop();
@@ -279,9 +744,9 @@ export class Manager {
279
744
  * with the new holder, and exit. We deliberately do NOT re-acquire (a replacement may already be live
280
745
  * while we'd still be serving) and do NOT release the key — it now belongs to that replacement. */
281
746
  async renewLease() {
282
- if (!this.leaseInfo || this.leaseRevision === undefined)
283
- return;
284
747
  try {
748
+ if (!this.leaseInfo || this.leaseRevision === undefined)
749
+ return;
285
750
  this.leaseRevision = await this.ep.renewManagerLease(this.leaseInfo, this.leaseRevision);
286
751
  }
287
752
  catch (e) {
@@ -291,7 +756,10 @@ export class Manager {
291
756
  // Tear down our managed agents' footprints too (#159 B2) — this exit path leaks them otherwise. Do
292
757
  // NOT release the lease key (it may belong to the replacement holder). Best-effort, like ep/attach.
293
758
  try {
294
- await this.teardownManagedAgents();
759
+ if (this.maintenanceState === "active" && !this.resumeRequired)
760
+ await this.teardownManagedAgents();
761
+ else
762
+ await this.stopRetainedAgentsOnExit();
295
763
  }
296
764
  catch { /* best effort */ }
297
765
  try {
@@ -306,6 +774,187 @@ export class Manager {
306
774
  }
307
775
  }
308
776
  async handle(req, tier) {
777
+ if (req.op === "finalizeResume") {
778
+ if (tier !== CONTROL_ADMIN)
779
+ return { ok: false, error: "finalizeResume is admin-only; not allowed on this control subject" };
780
+ let args;
781
+ try {
782
+ args = parseResumeFinalizeArgs(req.args);
783
+ }
784
+ catch (e) {
785
+ return { ok: false, error: e.message };
786
+ }
787
+ if (!this.resumeAttemptId || this.resumeAttemptId !== args.attemptId)
788
+ return { ok: false, error: `manager expects resume attempt ${this.resumeAttemptId ?? "<none>"}, not ${args.attemptId}` };
789
+ if (!this.resumeCommitted || !this.resumeDurableCommitToken)
790
+ return { ok: false, error: `resume attempt ${args.attemptId} has no successful commit to finalize` };
791
+ if (this.resumeDurableCommitToken !== args.durableCommitToken)
792
+ return { ok: false, error: `resume attempt ${args.attemptId} durable commit token does not match` };
793
+ if (this.resumeFinalized)
794
+ return { ok: true, data: { attemptId: args.attemptId, state: "active" } };
795
+ const inventory = this.resumeInventory;
796
+ if (!inventory)
797
+ return { ok: false, error: `resume attempt ${args.attemptId} has no bound inventory` };
798
+ let inactive;
799
+ try {
800
+ inactive = this.resumeLivenessErrors(inventory, this.ep.getRoster());
801
+ }
802
+ catch (e) {
803
+ return { ok: false, error: `resume attempt ${args.attemptId} cannot verify live principals at finalize: ${e.message}` };
804
+ }
805
+ if (inactive.length)
806
+ return { ok: false, error: `resume attempt ${args.attemptId} is not live at finalize: ${inactive.join("; ")}` };
807
+ for (const entry of this.resumeInventory?.agents ?? []) {
808
+ const managed = this.agents.get(entry.name);
809
+ if (managed)
810
+ managed.suppressCleanup = false;
811
+ }
812
+ this.resumeFinalized = true;
813
+ this.resumeRequired = false;
814
+ return { ok: true, data: { attemptId: args.attemptId, state: "active" } };
815
+ }
816
+ if (req.op === "commitResume") {
817
+ if (tier !== CONTROL_ADMIN)
818
+ return { ok: false, error: "commitResume is admin-only; not allowed on this control subject" };
819
+ let attemptId;
820
+ try {
821
+ attemptId = parseResumeCommitArgs(req.args).attemptId;
822
+ }
823
+ catch (e) {
824
+ return { ok: false, error: e.message };
825
+ }
826
+ if (!this.resumeAttemptId || this.resumeAttemptId !== attemptId)
827
+ return { ok: false, error: `manager expects resume attempt ${this.resumeAttemptId ?? "<none>"}, not ${attemptId}` };
828
+ if (this.resumeCommitted)
829
+ return {
830
+ ok: true,
831
+ data: {
832
+ attemptId,
833
+ state: this.resumeFinalized ? "active" : "awaitingFinalize",
834
+ durableCommitToken: this.resumeDurableCommitToken,
835
+ },
836
+ };
837
+ if (this.resumeCommitTask)
838
+ return this.resumeCommitTask;
839
+ const task = this.commitResumeActivation(attemptId);
840
+ this.resumeCommitTask = task;
841
+ try {
842
+ return await task;
843
+ }
844
+ finally {
845
+ if (this.resumeCommitTask === task)
846
+ this.resumeCommitTask = undefined;
847
+ }
848
+ }
849
+ if (req.op === "resumePreserved") {
850
+ if (tier !== CONTROL_ADMIN)
851
+ return { ok: false, error: "resumePreserved is admin-only; not allowed on this control subject" };
852
+ try {
853
+ const args = parseResumeControlArgs(req.args);
854
+ const inventoryDigest = createHash("sha256").update(JSON.stringify(args.inventory)).digest("hex");
855
+ if (!this.resumeAttemptId)
856
+ return { ok: false, error: "resumePreserved requires a manager started with --resume-attempt" };
857
+ if (this.resumeAttemptId !== args.attemptId)
858
+ return { ok: false, error: `manager expects resume attempt ${this.resumeAttemptId}, not ${args.attemptId}` };
859
+ if (this.resumeInventoryDigest && this.resumeInventoryDigest !== inventoryDigest)
860
+ return { ok: false, error: `resume attempt ${args.attemptId} is already bound to a different inventory` };
861
+ if (!this.resumeInventoryDigest) {
862
+ this.resumeInventoryDigest = inventoryDigest;
863
+ this.resumeInventory = args.inventory;
864
+ }
865
+ if (!this.resumeTask && !this.resumeResult) {
866
+ this.resumeTask = this.resumePreserved(args.inventory).then((result) => {
867
+ if (result.ok || this.resumedAgentNames.size > 0)
868
+ this.resumeResult = result;
869
+ return result;
870
+ }).finally(() => {
871
+ this.resumeTask = undefined;
872
+ });
873
+ }
874
+ const result = this.resumeResult ?? await this.resumeTask;
875
+ const data = { attemptId: args.attemptId, state: result.ok ? "awaitingCommit" : "degraded", ...result };
876
+ return result.ok
877
+ ? { ok: true, data }
878
+ : { ok: false, data, error: result.error ?? "retained-agent resume failed" };
879
+ }
880
+ catch (e) {
881
+ return { ok: false, error: e.message };
882
+ }
883
+ }
884
+ if (req.op === "preparePreservation" || req.op === "commitPreservation" || req.op === "abortPreservation") {
885
+ if (tier !== CONTROL_ADMIN)
886
+ return { ok: false, error: `${req.op} is admin-only; not allowed on this control subject` };
887
+ if (this.resumeRequired)
888
+ return { ok: false, error: this.maintenanceError() };
889
+ const attemptId = String(req.args?.attemptId ?? "").trim();
890
+ if (!attemptId)
891
+ return { ok: false, error: `${req.op} requires attemptId` };
892
+ try {
893
+ if (req.op === "abortPreservation") {
894
+ this.abortPreservation(attemptId);
895
+ return { ok: true, data: { attemptId, state: "active" } };
896
+ }
897
+ const result = req.op === "preparePreservation"
898
+ ? await this.preparePreservation(attemptId)
899
+ : await this.commitPreservation(attemptId);
900
+ return result.ok
901
+ ? { ok: true, data: result }
902
+ : {
903
+ ok: false,
904
+ data: result,
905
+ error: `preservation incomplete: ${result.failures.map((f) => `${f.name}: ${f.error}`).join("; ")}`,
906
+ };
907
+ }
908
+ catch (e) {
909
+ return { ok: false, error: e.message };
910
+ }
911
+ }
912
+ const release = this.beginLifecycle();
913
+ if (!release)
914
+ return { ok: false, error: this.maintenanceError() };
915
+ try {
916
+ return await this.handleActive(req, tier);
917
+ }
918
+ finally {
919
+ release();
920
+ }
921
+ }
922
+ async commitResumeActivation(attemptId) {
923
+ if (!this.resumeAwaitingCommit || !this.resumeResult?.ok)
924
+ return { ok: false, error: `resume attempt ${attemptId} has no successful activation to commit` };
925
+ const inventory = this.resumeInventory;
926
+ if (!inventory)
927
+ return { ok: false, error: `resume attempt ${attemptId} has no bound inventory` };
928
+ const authority = await Promise.all(inventory.agents.map(async (entry) => {
929
+ try {
930
+ await this.validateRetainedAuthority(entry);
931
+ return undefined;
932
+ }
933
+ catch (e) {
934
+ return `${entry.name}: ${e.message}`;
935
+ }
936
+ }));
937
+ const drift = authority.filter((error) => error !== undefined);
938
+ if (drift.length)
939
+ return { ok: false, error: `resume attempt ${attemptId} retained authority changed before commit: ${drift.join("; ")}` };
940
+ let inactive;
941
+ try {
942
+ inactive = this.resumeLivenessErrors(inventory, this.ep.getRoster());
943
+ }
944
+ catch (e) {
945
+ return { ok: false, error: `resume attempt ${attemptId} cannot verify live principals: ${e.message}` };
946
+ }
947
+ if (inactive.length)
948
+ return { ok: false, error: `resume attempt ${attemptId} is not live at commit: ${inactive.join("; ")}` };
949
+ this.resumeAwaitingCommit = false;
950
+ this.resumeCommitted = true;
951
+ this.resumeDurableCommitToken ??= randomBytes(32).toString("hex");
952
+ return {
953
+ ok: true,
954
+ data: { attemptId, state: "awaitingFinalize", durableCommitToken: this.resumeDurableCommitToken },
955
+ };
956
+ }
957
+ async handleActive(req, tier) {
309
958
  const args = req.args ?? {};
310
959
  // `req.from.id` is non-forgeable in auth mode: serveControl rejects any request whose payload
311
960
  // `from.id` doesn't match the subject sender (endpoint.ts). In open mode there are no creds, so
@@ -401,6 +1050,55 @@ export class Manager {
401
1050
  managedPrincipal(a) {
402
1051
  return a.userOwner ? a.id : principalKey(DEV_OWNER, a.id).key;
403
1052
  }
1053
+ resumeLivenessErrors(inventory, roster) {
1054
+ const inactive = [];
1055
+ const expectedNames = new Set(inventory.agents.map((entry) => entry.name));
1056
+ for (const name of this.resumedAgentNames)
1057
+ if (!expectedNames.has(name))
1058
+ inactive.push(`${name} is not part of the bound inventory`);
1059
+ for (const entry of inventory.agents) {
1060
+ const managed = this.agents.get(entry.name);
1061
+ if (!managed) {
1062
+ inactive.push(`${entry.name} is no longer managed`);
1063
+ continue;
1064
+ }
1065
+ const expectedId = entry.identity.mode === "user"
1066
+ ? principalKey(entry.identity.owner, entry.identity.actor).key
1067
+ : entry.identity.id;
1068
+ const expectedPrincipal = entry.identity.mode === "user"
1069
+ ? expectedId
1070
+ : principalKey(DEV_OWNER, entry.identity.id).key;
1071
+ if (managed.id !== expectedId || this.managedPrincipal(managed) !== expectedPrincipal) {
1072
+ inactive.push(`${entry.name} no longer holds retained principal ${expectedPrincipal}`);
1073
+ continue;
1074
+ }
1075
+ // The late paths (commit/finalize) must prove the SAME incarnation the incarnation-exact
1076
+ // readiness fence proved (§13.1): a principal-only match lets a wrong/absent-uid presence under
1077
+ // the reused alias satisfy commit/finalize after a readiness timeout, undoing the fence.
1078
+ if (managed.lifecycleUid !== entry.identity.lifecycleUid) {
1079
+ inactive.push(`${entry.name} manager metadata incarnation ${managed.lifecycleUid} drifted from the inventory's ${entry.identity.lifecycleUid}`);
1080
+ continue;
1081
+ }
1082
+ if (managed.handle.name !== entry.name || managed.handle.kind !== entry.launch.runtime) {
1083
+ inactive.push(`${entry.name} is not attached to its exact retained ${entry.launch.runtime} handle`);
1084
+ continue;
1085
+ }
1086
+ try {
1087
+ if (managed.handle.status() !== "running") {
1088
+ inactive.push(`${entry.name} runtime is not running`);
1089
+ continue;
1090
+ }
1091
+ }
1092
+ catch (e) {
1093
+ inactive.push(`${entry.name} runtime status failed: ${e.message}`);
1094
+ continue;
1095
+ }
1096
+ if (!roster.some((presence) => presence.card.id === expectedPrincipal && presence.card.name === entry.name && presence.status !== "offline" &&
1097
+ presence.lifecycleUid === entry.identity.lifecycleUid))
1098
+ inactive.push(`${entry.name} incarnation ${entry.identity.lifecycleUid} (principal ${expectedPrincipal}) is not exactly present`);
1099
+ }
1100
+ return inactive;
1101
+ }
404
1102
  /** Self-despawn (P2b): stop the managed agent whose id == the authenticated caller. The
405
1103
  * no-name self-op can only ever resolve to the caller's OWN managed entry (ids are unique
406
1104
  * per spawn + non-forgeable in auth mode), never a peer — so it's structurally incapable of
@@ -412,7 +1110,7 @@ export class Manager {
412
1110
  return { ok: false, error: `self-stop: caller ${callerId} is not a managed agent` };
413
1111
  const graceful = args.graceful !== false;
414
1112
  this.stopHandle(target, graceful);
415
- this.freeSlot(target, true); // self-despawn is rate-floored (recycle churn)
1113
+ this.trackStoppedHandle(target, true);
416
1114
  return { ok: true, data: { name: target.name, stopped: true, graceful } };
417
1115
  }
418
1116
  // Plane-3 durable join/leave/list ops moved OFF the manager onto the server-side delivery daemon's
@@ -441,6 +1139,49 @@ export class Manager {
441
1139
  console.error(`stop ${a.name} (${a.id}): ${e.message}`);
442
1140
  }
443
1141
  }
1142
+ /** Keep an accepted stop inside the lifecycle drain until the runtime proves the child is gone,
1143
+ * so a maintenance prepare can never fence ahead of a child that is still dying.
1144
+ *
1145
+ * An operator-accepted stop frees its slot at once: `stop` replying ✓ means `ps` no longer lists
1146
+ * the agent. That cannot omit a still-live child from a cut, because runPreparation drains the
1147
+ * lifecycle BEFORE it reads the roster — the exit proof below is what closes the race, not the
1148
+ * slot lingering. A recursive reap (`requireAuthoritativeExit`) instead keeps the slot until the
1149
+ * wait proves exit: nobody asked for those children to be gone, so they stay managed until the
1150
+ * runtime says otherwise, and a runtime that cannot prove exit records an unverified stop. */
1151
+ trackStoppedHandle(a, floor, requireAuthoritativeExit = false) {
1152
+ if (!a.handle.waitForExit) {
1153
+ // Preserve ordinary external-runtime stop behavior, but retain enough evidence for a later
1154
+ // maintenance prepare to fail if that runtime still cannot prove the surface disappeared.
1155
+ this.unverifiedStops.push({
1156
+ name: a.name,
1157
+ id: a.id,
1158
+ handle: a.handle,
1159
+ authoritative: requireAuthoritativeExit,
1160
+ error: requireAuthoritativeExit
1161
+ ? `recursive reap cannot prove exit on runtime "${a.handle.kind}" (AgentHandle.waitForExit is not implemented)`
1162
+ : undefined,
1163
+ });
1164
+ if (requireAuthoritativeExit)
1165
+ return;
1166
+ this.freeSlot(a, floor, true);
1167
+ return;
1168
+ }
1169
+ if (!requireAuthoritativeExit)
1170
+ this.freeSlot(a, floor, true);
1171
+ this.lifecycleInFlight++;
1172
+ void this.awaitHandleExit(a.handle)
1173
+ .then(() => this.freeSlot(a, floor, true)) // no-op once an accepted stop already freed it
1174
+ .catch((e) => {
1175
+ this.unverifiedStops.push({
1176
+ name: a.name,
1177
+ id: a.id,
1178
+ handle: a.handle,
1179
+ authoritative: true,
1180
+ error: `accepted stop could not prove exit: ${e.message}`,
1181
+ });
1182
+ })
1183
+ .finally(() => this.releaseLifecycle());
1184
+ }
444
1185
  /** USER-MODE spawn provisioning (the gate-1 counterpart to the static mint block): resolve the
445
1186
  * OWNER (ctl caller's principal, or the manifest's stamped owner — never a payload field),
446
1187
  * pre-create the principal-keyed durables + ACL row on the ephemeral provisioner, author the
@@ -469,15 +1210,19 @@ export class Manager {
469
1210
  // pass through too (a persona may hold delegable roles) — the ledger's envelope walk still
470
1211
  // attenuates every one of these against the spawner chain.
471
1212
  const scope = (opts.capabilities ?? []).filter((c) => c === "spawn" || c === "admin" || /^role:[A-Za-z0-9_-]+$/.test(c));
472
- const credsDir = join(authDir(this.workspaceRoot), "creds");
473
- const tokenPath = join(credsDir, `${name}.actor-token`);
474
- const sentinelPath = join(credsDir, `${name}.sentinel.creds`);
475
- const healthPath = join(credsDir, `${name}.auth-health.json`);
1213
+ // LOCAL composition, hardcoded: until the manager's entry is store-threaded (the same
1214
+ // later slice as its renewal-owner store, with/after the membership-rw reader migration),
1215
+ // a pure-KMS hosted manager CANNOT read the callout material this grant needs — hosted
1216
+ // user-mode spawn via the manager is UNAVAILABLE, not silently degraded, until then.
1217
+ const secrets = workspaceSecretStore(this.workspaceRoot);
1218
+ const files = agentSecretFilePaths(this.workspaceRoot, name);
1219
+ const { actorToken: tokenPath, sentinelCreds: sentinelPath, health: healthPath } = files;
476
1220
  try {
477
1221
  // The GRANT first — it is the envelope-rule enforcement point (a delegation must sit within
478
1222
  // the spawner's own grant), so a refused delegation exits here having touched nothing beyond
479
1223
  // the ledger: no durables, no broker footprint, nothing for a corrected respawn to race.
480
1224
  const grant = await provider.grantAgent({
1225
+ store: secrets,
481
1226
  dir,
482
1227
  space: this.space,
483
1228
  owner,
@@ -488,17 +1233,23 @@ export class Manager {
488
1233
  role: opts.role,
489
1234
  parent: spawnerPr ? opts.spawner : undefined,
490
1235
  label: opts.label,
1236
+ lifecycleUid: opts.lifecycleUid,
491
1237
  });
492
- // Durables + ACL row, principal-keyed — the same onboarding as static agents minus the mint
493
- // (a user agent's credential is its bearer, minted by the callout per connect).
494
- await this.withProvisioner((prov) => provisionAgentDurables(prov, { owner, actor: name }, {
1238
+ // Durables + ACL row, LIFECYCLE-keyed (SPEC 13.1) — the same onboarding as static agents minus
1239
+ // the mint (a user agent's credential is its bearer, minted by the callout per connect from the
1240
+ // ledger row's recorded lifecycleUid the same value provisioned here).
1241
+ await this.withProvisioner((prov) => provisionAgentDurables(prov, { owner, actor: name, lifecycleUid: opts.lifecycleUid }, {
495
1242
  subscribe: opts.subscribe,
496
1243
  allowSubscribe: opts.allowSubscribe,
497
1244
  role: opts.role,
498
1245
  }));
499
- mkSecretDir(credsDir);
500
- writeSecretFile(tokenPath, grant.actorToken);
501
- writeSecretFile(sentinelPath, grant.sentinelCreds);
1246
+ // The store holds the source of truth; the bearer re-exec (`--token-file`) and the launch's
1247
+ // sentinel handoff read FILES, so materialize both at the canonical paths (under the local
1248
+ // FS composition, a byte-identical rewrite of the keys' own locations).
1249
+ await secrets.put(agentActorTokenKey(name), grant.actorToken);
1250
+ await secrets.put(agentSentinelCredsKey(name), grant.sentinelCreds);
1251
+ await materializeSecretToFile(secrets, agentActorTokenKey(name), tokenPath);
1252
+ await materializeSecretToFile(secrets, agentSentinelCredsKey(name), sentinelPath);
502
1253
  rmSync(healthPath, { force: true }); // a fresh start opens a fresh health window
503
1254
  const bearerCmd = [
504
1255
  // The manager's own invocation prefix (node + loader flags + the cotal entry) — the agent
@@ -525,10 +1276,12 @@ export class Manager {
525
1276
  // may respawn the moment it reads the refusal, and a detached teardown would race (and
526
1277
  // delete) that fresh spawn's just-provisioned durables.
527
1278
  await provider.revokeAgent({ dir, owner, actor: name }).catch(() => { });
1279
+ await secrets.delete(agentActorTokenKey(name)).catch(() => { });
1280
+ await secrets.delete(agentSentinelCredsKey(name)).catch(() => { });
528
1281
  rmSync(tokenPath, { force: true });
529
1282
  rmSync(sentinelPath, { force: true });
530
1283
  rmSync(healthPath, { force: true });
531
- await this.deprovision({ id: principalKey(owner, name).key, name, userOwner: owner }).catch((err) => console.error(`rollback deprovision ${name}: ${err.message}`));
1284
+ await this.deprovision({ id: principalKey(owner, name).key, name, lifecycleUid: opts.lifecycleUid, userOwner: owner }).catch((err) => console.error(`rollback deprovision ${name}: ${err.message}`));
532
1285
  return { error: `agent auth preflight failed for "${name}": ${e.message}` };
533
1286
  }
534
1287
  }
@@ -537,17 +1290,31 @@ export class Manager {
537
1290
  * expires — flooring the RECYCLE, not the call, so both free paths (despawn + exit/reap) are
538
1291
  * covered (P4c). Floor self + own-child despawn and natural exit; NEVER admin despawn (operator
539
1292
  * emergency-kill stays unthrottled) and NEVER the reserved-rollback path (no cold-start paid). */
540
- freeSlot(a, floor) {
1293
+ freeSlot(a, floor, acceptedBeforeFence = false) {
541
1294
  if (this.agents.get(a.name) !== a)
542
1295
  return; // already freed (exit raced despawn, etc.)
543
1296
  this.agents.delete(a.name);
544
1297
  if (floor && Date.now() - a.startedAt < MIN_LIFETIME)
545
1298
  this.cooling.push(a.startedAt + MIN_LIFETIME);
1299
+ // #29 piece 3: on a USER mesh the name is RESERVED PENDING RETIREMENT — despawn started this
1300
+ // lifecycle's FULL teardown (footprint + standing-authority revoke + the auth-side retirement),
1301
+ // and the alias frees only when all of it completes (not the retirement alone). The detached
1302
+ // deprovision below drives it; a failed revoke or an unreachable rail keeps the name held,
1303
+ // re-driven by a retry. Gate on userMode BY
1304
+ // CONSTRUCTION (NEW-1): a static-auth mint has no user-mode lifecycle head to retire, so the
1305
+ // reservation + rail request simply don't apply there (the incidental nkey-parse used to mask
1306
+ // this, but the intent is "user mode only", not "any principal-shaped id").
1307
+ if (this.userMode) {
1308
+ const p = parsePrincipalKey(a.id);
1309
+ if (p)
1310
+ this.retiring.set(a.name, { opId: retireOpId(a.lifecycleUid), lifecycleUid: a.lifecycleUid, owner: p.owner, actor: p.actor, agentId: a.id, userOwner: a.userOwner, startedAt: Date.now() });
1311
+ }
546
1312
  // Auth mode: tear down the departed agent's minted broker footprint + creds file (#159 B2). The
547
1313
  // process is already gone, so this must never block the slot free or throw into the caller — it runs
548
1314
  // detached, and a failure is logged loudly (never swallowed), not retried. The `agents` guard above
549
1315
  // makes this fire exactly once per agent across every free path (despawn / self-stop / reap / exit).
550
- void this.deprovision(a).catch((e) => console.error(`deprovision ${a.name} (${a.id}): ${e.message}`));
1316
+ if (!a.suppressCleanup && (this.maintenanceState === "active" || acceptedBeforeFence))
1317
+ this.trackDeprovision(a);
551
1318
  }
552
1319
  /** Tear down a departed agent's minted footprint (#159 B2, auth mode): its local-principal durables
553
1320
  * (`dm_local-<id>`, `dlv_local-<id>`), its read-ACL row, and its creds file — everything the spawn's
@@ -564,53 +1331,208 @@ export class Manager {
564
1331
  async deprovision(a) {
565
1332
  if (!this.auth)
566
1333
  return; // open mesh mints no creds/durables — nothing to tear down
1334
+ // SINGLE-FLIGHT per (name, lifecycleUid) (INT-2/C): join an in-flight teardown for this exact
1335
+ // lifecycle rather than launching a second concurrent one whose delayed name-keyed revoke could
1336
+ // outlive the hold-clear and delete a successor's row. A fresh trigger after settle re-drives.
1337
+ const key = JSON.stringify([a.name, a.lifecycleUid]); // ASCII-safe, delimiter-collision-free
1338
+ const inflight = this.deprovisioningFlight.get(key);
1339
+ if (inflight)
1340
+ return inflight;
1341
+ const flight = this.driveDeprovision(a).finally(() => {
1342
+ if (this.deprovisioningFlight.get(key) === flight)
1343
+ this.deprovisioningFlight.delete(key);
1344
+ });
1345
+ this.deprovisioningFlight.set(key, flight);
1346
+ return flight;
1347
+ }
1348
+ /** The actual footprint teardown (wrapped by {@link deprovision}'s single-flight). */
1349
+ async driveDeprovision(a) {
1350
+ if (!this.auth)
1351
+ return; // guaranteed by deprovision; re-checked for the deprovisionBroker narrowing
567
1352
  // Drop the local creds file FIRST + unconditionally — it is a usable identity on disk, useless for a
568
1353
  // departed agent, so it must not survive even if the broker teardown below fails or times out. The
569
1354
  // teardown mints its OWN deprovisioner cred (not this file), so removing it early is independent.
570
- rmSync(join(authDir(this.workspaceRoot), "creds", `${a.name}.creds`), { force: true });
1355
+ // Migrated kinds: the store delete is the authoritative removal; the rmSync clears the FS
1356
+ // materialization (a byte-identical no-op under the local composition, real once the manager
1357
+ // is store-threaded onto a non-FS store).
1358
+ const secrets = workspaceSecretStore(this.workspaceRoot);
1359
+ const files = agentSecretFilePaths(this.workspaceRoot, a.name);
1360
+ await secrets.delete(agentCredsKey(a.name));
1361
+ rmSync(files.creds, { force: true });
571
1362
  if (a.userOwner) {
572
1363
  // USER MODE: this teardown IS revocation, not just footprint reduction — the ledger row is
573
1364
  // the agent's standing mint authority, so delete it (next exchange refused, next connect
574
1365
  // denied) and shred the secret/sentinel/health files. A copied actor token dies here; a
575
1366
  // still-LIVE connection ends at its bearer-bound JWT expiry (≤ the agent TTL).
576
- const credsDir = join(authDir(this.workspaceRoot), "creds");
577
- for (const f of [`${a.name}.actor-token`, `${a.name}.sentinel.creds`, `${a.name}.auth-health.json`])
578
- rmSync(join(credsDir, f), { force: true });
1367
+ await secrets.delete(agentActorTokenKey(a.name));
1368
+ await secrets.delete(agentSentinelCredsKey(a.name));
1369
+ for (const f of [files.actorToken, files.sentinelCreds, files.health])
1370
+ rmSync(f, { force: true });
1371
+ // The ledger row IS the agent's STANDING mint authority (a different store from the auth-plane
1372
+ // cred ledger the rail retirement covers): while it lives, a copied actor token can still mint a
1373
+ // fresh connect credential. So a FAILED revoke must NOT be swallowed into a clean terminal (INT-2):
1374
+ // mark the standing authority live on the hold so the retirement can never free the name (a freed
1375
+ // name says "this lifecycle is fully gone" - false while the mint authority stands), and carry a
1376
+ // legible operator copy. A retry re-drives this whole teardown (the same-name-spawn nudge routes
1377
+ // through deprovision, not the rail alone), so the revoke is re-attempted, not stranded.
1378
+ const holdRevoke = this.retiring.get(a.name);
1379
+ if (holdRevoke && holdRevoke.lifecycleUid === a.lifecycleUid)
1380
+ holdRevoke.standingAuthorityLive = true;
579
1381
  try {
580
1382
  await resolveAuthProvider().revokeAgent({
581
1383
  dir: userAuthStateDir(this.workspaceRoot, this.space),
582
1384
  owner: a.userOwner,
583
1385
  actor: a.name,
584
1386
  });
1387
+ const done = this.retiring.get(a.name);
1388
+ if (done && done.lifecycleUid === a.lifecycleUid)
1389
+ done.standingAuthorityLive = false;
585
1390
  }
586
1391
  catch (e) {
1392
+ const h = this.retiring.get(a.name);
1393
+ if (h && h.lifecycleUid === a.lifecycleUid)
1394
+ h.lastError = `the agent's standing mint authority could not be revoked (${e.message}); the name stays held so a copied actor token cannot mint fresh credentials. NEXT: a same-name spawn re-drives the full teardown (including the revoke), or recover the auth state.`;
587
1395
  console.error(`revoke agent grant ${a.name}: ${e.message}`);
588
1396
  }
589
1397
  }
590
- const creds = await mintCreds(this.auth, newIdentity(), "deprovisioner", { deprovisionTarget: a.id });
1398
+ await this.deprovisionBroker(a);
1399
+ // #29 piece 3: after the footprint teardown, ask the AUTH plane to RETIRE the lifecycle over
1400
+ // the auth-admin rail. The rail re-checks the space-manager lease at serve time; the terminal
1401
+ // (or an already-retired answer) clears the name reservation. Failures keep the hold with
1402
+ // their operator copy — legible, retryable, never a silent half-state.
1403
+ await this.requestRetirement(a);
1404
+ }
1405
+ /** Request the auth-side retirement of a departed agent's lifecycle (#29 piece 3): an ephemeral
1406
+ * `retirement-requester` credential (request + reply only), the generic `retireLifecycle` op,
1407
+ * a STABLE opId (derived from the lifecycleUid, so every retry re-drives the SAME operation),
1408
+ * and the four-outcome handling in operator vocabulary. */
1409
+ async requestRetirement(a) {
1410
+ if (!this.userMode)
1411
+ return; // NEW-1: lifecycle retirement is a user-mesh concept; a static mint has no head to retire
1412
+ // SINGLE-FLIGHT per (name, lifecycleUid) (audit #1): the detached deprovision call and every
1413
+ // same-name-spawn nudge for THIS lifecycle share ONE in-flight retirement, so concurrent triggers
1414
+ // never stack independent rail requests that dual-enter runAgentRetirementBarrier. Keyed by
1415
+ // (name, uid), NOT name alone: driveRetirement clears the hold on rail ok but then yields at
1416
+ // `nc.close()` with the flight still stored, so the alias can free and a SUCCESSOR (new uid) spawn.
1417
+ // A name-only key would let that successor's own teardown JOIN the predecessor's still-pending
1418
+ // flight and never send its OWN retirement — leaving the successor's lifecycle unretired. The uid in
1419
+ // the key gives the successor a disjoint flight (mirrors {@link deprovisioningFlight}). A fresh
1420
+ // trigger after settle re-drives (a still-present hold => retirement not yet confirmed).
1421
+ const key = JSON.stringify([a.name, a.lifecycleUid]);
1422
+ const inflight = this.retiringFlight.get(key);
1423
+ if (inflight)
1424
+ return inflight;
1425
+ const flight = this.driveRetirement(a).finally(() => {
1426
+ if (this.retiringFlight.get(key) === flight)
1427
+ this.retiringFlight.delete(key);
1428
+ });
1429
+ this.retiringFlight.set(key, flight);
1430
+ return flight;
1431
+ }
1432
+ /** The rail round-trip for one retirement (wrapped by {@link requestRetirement}'s single-flight). */
1433
+ async driveRetirement(a) {
1434
+ if (!this.auth)
1435
+ return; // guaranteed by requestRetirement; re-checked for the type narrowing below
1436
+ const held = this.retiring.get(a.name);
1437
+ const me = parsePrincipalKey(this.ep.ref().id);
1438
+ const target = parsePrincipalKey(a.id);
1439
+ if (!me || !target) {
1440
+ if (held)
1441
+ held.lastError = "the manager or target principal could not be derived; the retirement was not requested";
1442
+ return;
1443
+ }
1444
+ const uncertain = (why) => `the despawn stopped "${a.name}", but the retirement's completion could NOT be confirmed (${why}). The name stays held - not failed, not done - and a same-name spawn re-drives the same teardown; the auth service also finishes any started retirement on its next boot. NEXT: if the auth rail stays unreachable, recover the stack (\`cotal supervise\`), then re-attempt the same-name spawn.`;
1445
+ try {
1446
+ const creds = await mintCreds(this.auth, newIdentity(), "retirement-requester", { retirementRequester: { owner: me.owner, actor: me.actor } });
1447
+ const nc = await connect({ servers: this.servers ?? DEFAULT_SERVER, authenticator: credsAuthenticator(new TextEncoder().encode(creds)), maxReconnectAttempts: 0 });
1448
+ try {
1449
+ const subject = controlServiceSubject(this.space, CONTROL_AUTH_ADMIN, me.owner, me.actor);
1450
+ const m = await nc.request(subject, JSON.stringify({ op: "retireLifecycle", args: { owner: target.owner, actor: target.actor, lifecycleUid: a.lifecycleUid, opId: retireOpId(a.lifecycleUid) } }), { timeout: 20_000, noMux: true, reply: `${subject}.reply.${randomUUID()}` });
1451
+ const r = m.json();
1452
+ if (r.ok) {
1453
+ // CAS the hold clear (audit #1 ABA): free the alias ONLY if the current hold is still THIS
1454
+ // lifecycle's - a late reply for a retired predecessor must never clear a successor's newer hold.
1455
+ const cur = this.retiring.get(a.name);
1456
+ if (cur && cur.lifecycleUid === a.lifecycleUid) {
1457
+ if (cur.standingAuthorityLive) {
1458
+ // INT-2: the auth-plane lifecycle retired, but the manager-side STANDING mint authority is
1459
+ // not yet revoked (a failed revoke). Freeing the name here would be a false terminal (a
1460
+ // copied token could still mint), so keep the hold with its revoke-failure copy; a retry
1461
+ // re-drives the full teardown (revoke included).
1462
+ console.error(`despawn ${a.name}: the auth-plane lifecycle retired, but the standing mint authority is not yet revoked; the name stays held. ${cur.lastError ?? ""}`);
1463
+ }
1464
+ else {
1465
+ this.retiring.delete(a.name);
1466
+ console.error(`despawn ${a.name}: the agent's retirement completed; the name is free for reuse`);
1467
+ }
1468
+ }
1469
+ else {
1470
+ console.error(`despawn ${a.name}: retirement confirmed for a prior lifecycle of "${a.name}"; the current hold is left intact`);
1471
+ }
1472
+ }
1473
+ else {
1474
+ // The rail's refusal is already the operator copy (lease-loss/stale/foreign-op faces,
1475
+ // full-no-op statements included) - surface it INTACT, never flattened.
1476
+ if (held)
1477
+ held.lastError = r.error ?? "the auth service refused the retirement without a reason";
1478
+ console.error(`despawn ${a.name}: ${r.error ?? "the auth service refused the retirement without a reason"}`);
1479
+ }
1480
+ }
1481
+ finally {
1482
+ await nc.close().catch(() => { });
1483
+ }
1484
+ }
1485
+ catch (e) {
1486
+ const copy = uncertain(e.message);
1487
+ if (held)
1488
+ held.lastError = copy;
1489
+ console.error(`despawn ${a.name}: ${copy}`);
1490
+ }
1491
+ }
1492
+ /** The teardown's ASYNC BROKER PHASE: mint the ephemeral target-pinned deprovisioner cred and
1493
+ * delete the agent's broker footprint (dm_/dlv_ durables + read-ACL row). Split from
1494
+ * {@link deprovision} because it runs LAST in the ordered teardown chain — after the creds/secret
1495
+ * shred and the awaited ledger revoke, which precede it in that same single-flighted chain (the
1496
+ * revoke is awaited and can be deliberately slow, so it is not merely a synchronous prefix). The name
1497
+ * is NOT freed while any teardown phase is still in flight — the hold clears only after the
1498
+ * standing-authority revoke AND the lifecycle retirement both confirm (see {@link driveRetirement}) —
1499
+ * but the deletes here are still lifecycle-uid-pinned so even a replayed/stale teardown can never
1500
+ * reach a same-name successor's footprint (its names embed a different uid). */
1501
+ async deprovisionBroker(a) {
1502
+ // LIFECYCLE-PINNED (SPEC 13.1): both the credential's exact-name grants and the delete names
1503
+ // carry a.lifecycleUid, so a stale/replayed teardown for this retired incarnation is broker-denied
1504
+ // against a same-name successor's footprint (its names embed a different uid).
1505
+ const creds = await mintCreds(this.auth, newIdentity(), "deprovisioner", {
1506
+ deprovisionTarget: { principal: a.id, lifecycleUid: a.lifecycleUid },
1507
+ });
591
1508
  // Bound the detached broker teardown so a wedged broker can't leave the deprovision promise pending
592
1509
  // forever with no log — the timeout rejects into freeSlot's fail-loud `.catch` (paired with the
593
1510
  // helper's own fail-fast connect). The durables/ACL row still fall to space teardown as a backstop.
594
- await withTimeout(deprovisionAgent({ servers: this.servers ?? DEFAULT_SERVER, space: this.space, targetId: a.id, creds }), DEPROVISION_TIMEOUT_MS, `deprovision ${a.name} (${a.id}): broker teardown timed out`);
1511
+ await withTimeout(deprovisionAgent({ servers: this.servers ?? DEFAULT_SERVER, space: this.space, targetId: a.id, lifecycleUid: a.lifecycleUid, creds }), DEPROVISION_TIMEOUT_MS, `deprovision ${a.name} (${a.id}): broker teardown timed out`);
595
1512
  }
596
- /** Reap a parent's children on its exit (P4b): stop + free every agent whose `spawner` is the
597
- * exited agent's id, so orphans don't ratchet the ceiling shut. Recursive a reaped child's
598
- * own children are reaped too. Exit-driven, so each freed slot is rate-floored like a despawn. */
1513
+ /** Reap a parent's children on its exit (P4b). Every descendant remains managed until the runtime's
1514
+ * authoritative wait proves exit; the wait participates in the lifecycle drain, so preservation can
1515
+ * never omit a child that may still be alive. Recursive descendants are scheduled before their parent
1516
+ * slot can disappear. */
599
1517
  reapChildrenOf(parentId) {
600
1518
  for (const child of [...this.agents.values()]) {
601
1519
  if (child.spawner !== parentId)
602
1520
  continue;
1521
+ this.reapChildrenOf(this.managedPrincipal(child));
603
1522
  this.stopHandle(child, false);
604
- this.freeSlot(child, true);
605
- this.reapChildrenOf(child.id);
1523
+ this.trackStoppedHandle(child, true, true);
606
1524
  }
607
1525
  }
608
1526
  /** A managed agent's process exited on its own (crash, /exit, finished). Free its slot
609
1527
  * (rate-floored — exit-driven churn counts) and reap any children it spawned. Idempotent via
610
1528
  * freeSlot's identity guard, so a later graceful-stop SIGKILL firing exit again is a no-op. */
611
1529
  onAgentExit(a) {
1530
+ // Preservation owns the child-stop snapshot. Exit watchers must neither delete that snapshot nor
1531
+ // trigger normal deprovision/reap while the cut is being formed.
1532
+ if (this.maintenanceState !== "active")
1533
+ return;
612
1534
  this.freeSlot(a, true);
613
- this.reapChildrenOf(a.id);
1535
+ this.reapChildrenOf(this.managedPrincipal(a));
614
1536
  }
615
1537
  /** Agent names become `.cotal/agents/<name>.md` paths and mesh identities, so they must be bare
616
1538
  * tokens, never a path — blocks traversal / arbitrary writes from a model-supplied name. */
@@ -623,7 +1545,7 @@ export class Manager {
623
1545
  * in-flight (reserved) slots. Lets a colliding spawn auto-number instead of being rejected, so
624
1546
  * callers never have to invent a unique name. */
625
1547
  uniqueName(base) {
626
- return firstFreeName(base, (n) => this.agents.has(n) || this.reserved.has(n));
1548
+ return firstFreeName(base, (n) => this.agents.has(n) || this.reserved.has(n) || this.retiring.has(n));
627
1549
  }
628
1550
  /** Spawn a teammate by persona ref (`name` loads `.cotal/agents/<name>.md`; the peer presents
629
1551
  * under that file's own `name:`), as if a peer asked via the control plane. Used to pre-spawn the
@@ -835,7 +1757,7 @@ export class Manager {
835
1757
  catch (e) {
836
1758
  return { ok: false, error: e.message };
837
1759
  }
838
- const reply = await this.startAgent(launchAgentToStartOpts(la, configPath, spec.owner), caller);
1760
+ const reply = await this.startAgent(launchAgentToStartOpts(la, configPath, spec.owner, runId), caller);
839
1761
  if (reply.ok)
840
1762
  // `data.name` stays the spawned (numbered) identity — what creds are filed under and the ledger
841
1763
  // keys on; `requested`/`runId`/`hash` give the CLI the manifest name + drift hash for the ledger.
@@ -849,6 +1771,17 @@ export class Manager {
849
1771
  * defaulting to the manager's own id for roster/pre-spawn — recorded for the spawner
850
1772
  * ledger (own-children despawn + reap-on-parent-exit). */
851
1773
  async startAgent(opts, spawner) {
1774
+ const release = this.beginLifecycle();
1775
+ if (!release)
1776
+ return { ok: false, error: this.maintenanceError() };
1777
+ try {
1778
+ return await this.startAgentActive(opts, spawner);
1779
+ }
1780
+ finally {
1781
+ release();
1782
+ }
1783
+ }
1784
+ async startAgentActive(opts, spawner) {
852
1785
  // The spawn argument is a persona REF — a filename in `.cotal/agents` (the unique spawn KEY), or
853
1786
  // a path via `--config`. It is NOT the mesh identity: the identity comes from inside the file
854
1787
  // (`name:`), so a persona can be filed descriptively (review-critic.md) yet present under a
@@ -968,6 +1901,20 @@ export class Manager {
968
1901
  const idErr = this.nameError(identityName);
969
1902
  if (idErr)
970
1903
  return { ok: false, error: opts.resolved ? `launch agent: ${idErr}` : `persona ${configPath}: ${idErr}` };
1904
+ // The alias-reuse gate (#29 piece 3): a name whose previous agent is still retiring REFUSES
1905
+ // legibly (never a silent suffix), and the refusal re-drives the FULL durable teardown so
1906
+ // "retry the spawn" is also the nudge. It routes through `deprovision` (not `requestRetirement`
1907
+ // alone) so a retry re-drives the standing-authority revoke (INT-2) AND the broker cleanup before
1908
+ // any hold-clear (C): the alias must not free while the durable teardown or the revoke is still
1909
+ // outstanding. All the teardown ops are idempotent, and the rail request is single-flighted.
1910
+ const held = this.retiring.get(identityName);
1911
+ if (held !== undefined) {
1912
+ void this.deprovision({ id: held.agentId, name: identityName, lifecycleUid: held.lifecycleUid, userOwner: held.userOwner }).catch(() => { });
1913
+ return {
1914
+ ok: false,
1915
+ error: `the name "${identityName}" is reserved pending retirement: its previous agent's despawn started that lifecycle's teardown (footprint + standing-authority revoke + auth-side retirement), and the name frees only when all of it completes${held.lastError !== undefined ? ` (last attempt: ${held.lastError})` : ""}. NEXT: wait a moment and retry this spawn (retrying re-drives the whole teardown), or pick another name.`,
1916
+ };
1917
+ }
971
1918
  if (variant && !connector.supportsModelVariant)
972
1919
  return { ok: false, error: `${agent} connector does not support model variants (variant)` };
973
1920
  const name = this.uniqueName(identityName);
@@ -1000,6 +1947,10 @@ export class Manager {
1000
1947
  // A stable nkey identity assigned at spawn: the public key is the agent's card.id (threaded via
1001
1948
  // COTAL_ID); the seed is retained to mint matching creds later.
1002
1949
  const identity = newIdentity();
1950
+ // The incarnation's lifecycle UID (SPEC 13.1), minted ONCE per spawn: every lifecycle-keyed
1951
+ // broker resource (dm_/dlv_/chathist_ durables, ACL row, memberships) and the teardown
1952
+ // credential carry it, so a same-name successor's footprint is name-disjoint by construction.
1953
+ const lifecycleUid = mintLifecycleUid();
1003
1954
  // In auth mode, mint the agent's creds from the space signing key and write them where the
1004
1955
  // spawned session reads them (COTAL_CREDS path). Open mesh → no creds. Scope = the resolved
1005
1956
  // subscribe/allowSubscribe (read) + allowPublish (post, default-deny).
@@ -1016,6 +1967,7 @@ export class Manager {
1016
1967
  role,
1017
1968
  capabilities,
1018
1969
  label: ref,
1970
+ lifecycleUid,
1019
1971
  });
1020
1972
  if ("error" in prep) {
1021
1973
  this.reserved.delete(name);
@@ -1023,7 +1975,7 @@ export class Manager {
1023
1975
  }
1024
1976
  userLaunch = prep.launch;
1025
1977
  userOwner = prep.owner;
1026
- provisioned = { id: principalKey(prep.owner, name).key, name, userOwner: prep.owner };
1978
+ provisioned = { id: principalKey(prep.owner, name).key, name, lifecycleUid, userOwner: prep.owner };
1027
1979
  }
1028
1980
  else if (this.auth) {
1029
1981
  // Pre-create the agent's bind-only chat (+ DM + role TASK) durables and mint its scoped creds
@@ -1036,11 +1988,16 @@ export class Manager {
1036
1988
  allowPublish,
1037
1989
  role,
1038
1990
  capabilities,
1991
+ lifecycleUid,
1039
1992
  }));
1040
- credsPath = join(authDir(this.workspaceRoot), "creds", `${name}.creds`);
1041
- mkSecretDir(dirname(credsPath)); // harden the creds dir before the cred lands
1042
- writeSecretFile(credsPath, creds);
1043
- provisioned = { id: identity.id, name }; // footprint now exists — the finally rolls it back if the spawn throws
1993
+ // Store first (the source of truth), then materialize: `buildLaunch` hands the CHILD this
1994
+ // file path, so the cred must exist as a file regardless of the store behind the seam.
1995
+ // LOCAL composition, hardcoded, same posture as provisionUserAgent's grant store above.
1996
+ const secrets = workspaceSecretStore(this.workspaceRoot);
1997
+ credsPath = agentSecretFilePaths(this.workspaceRoot, name).creds;
1998
+ await secrets.put(agentCredsKey(name), creds);
1999
+ await materializeSecretToFile(secrets, agentCredsKey(name), credsPath);
2000
+ provisioned = { id: identity.id, name, lifecycleUid }; // footprint now exists — the finally rolls it back if the spawn throws
1044
2001
  }
1045
2002
  // Personal MCP servers the operator opted to share with manager-spawned agents of this type
1046
2003
  // (cotal config; default none → isolated, the memory-safe default this guards), narrowed by
@@ -1050,6 +2007,11 @@ export class Manager {
1050
2007
  // arbitrary folders/repos. A relative path resolves against the workspace root; omitted → the
1051
2008
  // agent shares the workspace root (the prior, unchanged behavior).
1052
2009
  const cwd = opts.cwd ? resolve(this.workspaceRoot, opts.cwd) : this.workspaceRoot;
2010
+ const configSha256 = this.fileDigest(configPath);
2011
+ const manifestPath = opts.launchRef
2012
+ ? join(this.workspaceRoot, ".cotal", "run", `${opts.launchRef.runId}.json`)
2013
+ : undefined;
2014
+ const manifestSha256 = manifestPath ? this.fileDigest(manifestPath) : undefined;
1053
2015
  const spec = connector.buildLaunch({
1054
2016
  space: this.space,
1055
2017
  name,
@@ -1059,6 +2021,10 @@ export class Manager {
1059
2021
  id: userLaunch ? undefined : identity.id,
1060
2022
  creds: credsPath,
1061
2023
  userAuth: userLaunch,
2024
+ // The incarnation's lifecycle UID: the agent endpoint binds its lifecycle-keyed dm/dlv/
2025
+ // chathist durables by this exact value (its creds pin the same names, so a mismatch fails
2026
+ // at the broker, never silently).
2027
+ lifecycleUid,
1062
2028
  servers: this.servers,
1063
2029
  configPath,
1064
2030
  model,
@@ -1090,11 +2056,41 @@ export class Manager {
1090
2056
  role,
1091
2057
  agent,
1092
2058
  id: userLaunch ? principalKey(userLaunch.owner, name).key : identity.id,
2059
+ lifecycleUid,
1093
2060
  ...(userLaunch ? { userOwner } : { seed: identity.seed }),
1094
2061
  spawner: spawner ?? this.ep.ref().id,
2062
+ authorityParent: userLaunch && spawner && parsePrincipalKey(spawner) ? spawner : undefined,
1095
2063
  startedAt: Date.now(),
1096
2064
  handle,
1097
2065
  control: spec.control,
2066
+ launch: {
2067
+ source: opts.resolved
2068
+ ? {
2069
+ kind: "manifest",
2070
+ runId: opts.launchRef?.runId,
2071
+ requested: opts.launchRef?.requested ?? opts.resolved.name,
2072
+ hash: opts.launchRef?.hash ?? opts.resolved.hash,
2073
+ configPath,
2074
+ configSha256,
2075
+ manifestSha256,
2076
+ }
2077
+ : { kind: "persona", ref, configPath, configSha256 },
2078
+ cwd,
2079
+ model,
2080
+ variant,
2081
+ subscribe,
2082
+ allowSubscribe,
2083
+ allowPublish,
2084
+ capabilities,
2085
+ transcript,
2086
+ shareTools: opts.shareTools,
2087
+ forkSource: opts.resume,
2088
+ // Opaque values may contain secrets. Preserve only their keys and require the referenced
2089
+ // persona/manifest to resolve the values again; imperative overrides have no safe payload.
2090
+ unresolvedLaunchOptionKeys: opts.launchOptions && Object.keys(opts.launchOptions).length
2091
+ ? Object.keys(opts.launchOptions).sort()
2092
+ : undefined,
2093
+ },
1098
2094
  };
1099
2095
  this.agents.set(name, managed);
1100
2096
  // The live slot now owns teardown — freeSlot deprovisions this identity on exit — so the
@@ -1128,10 +2124,402 @@ export class Manager {
1128
2124
  // orphan down (detached, fail-loud) so a failed spawn leaves no creds/durables behind (#159 B).
1129
2125
  if (provisioned) {
1130
2126
  const orphan = provisioned;
1131
- void this.deprovision(orphan).catch((e) => console.error(`deprovision (orphaned spawn) ${orphan.name}: ${e.message}`));
2127
+ this.trackDeprovision(orphan, "(orphaned spawn)");
2128
+ }
2129
+ }
2130
+ }
2131
+ /** Preflight the whole inventory before launching its first process, then adopt each exact retained
2132
+ * principal without provisioning. A later runtime launch failure is reported per-agent, but malformed
2133
+ * or missing inventory material can never produce a partially resumed set. */
2134
+ async resumePreserved(inventory) {
2135
+ const release = this.beginLifecycle(true);
2136
+ if (!release)
2137
+ return { ok: false, agents: [], error: this.maintenanceError() };
2138
+ const batchReservations = [];
2139
+ try {
2140
+ if (inventory.version !== "cotal-manager-resume/v1")
2141
+ return { ok: false, agents: [], error: `unsupported manager resume inventory version ${String(inventory.version)}` };
2142
+ if (inventory.space !== this.space)
2143
+ return { ok: false, agents: [], error: `resume inventory belongs to space "${inventory.space}", not "${this.space}"` };
2144
+ const seen = new Set();
2145
+ const principals = new Set();
2146
+ await this.ep.waitForPresenceSnapshot();
2147
+ const livePrincipals = new Set(this.ep.getRoster()
2148
+ .filter((presence) => presence.status !== "offline")
2149
+ .map((presence) => presence.card.id));
2150
+ if (this.agents.size + this.reserved.size + this.coolingCount() + inventory.agents.length > MAX_AGENTS)
2151
+ return { ok: false, agents: [], error: `resume inventory would exceed manager capacity (${MAX_AGENTS})` };
2152
+ for (const entry of inventory.agents) {
2153
+ if (seen.has(entry.name))
2154
+ return { ok: false, agents: [], error: `resume inventory contains duplicate agent name "${entry.name}"` };
2155
+ seen.add(entry.name);
2156
+ let principal;
2157
+ try {
2158
+ principal = entry.identity.mode === "user"
2159
+ ? principalKey(entry.identity.owner, entry.identity.actor).key
2160
+ : principalKey(DEV_OWNER, entry.identity.id).key;
2161
+ }
2162
+ catch (e) {
2163
+ return { ok: false, agents: [], error: `invalid retained principal for ${entry.name}: ${e.message}` };
2164
+ }
2165
+ if (principals.has(principal))
2166
+ return { ok: false, agents: [], error: `resume inventory contains duplicate principal "${principal}"` };
2167
+ principals.add(principal);
2168
+ if (livePrincipals.has(principal))
2169
+ return { ok: false, agents: [], error: `retained principal "${principal}" is already live and this runtime cannot authoritatively adopt it` };
2170
+ if (this.agents.has(entry.name) || this.reserved.has(entry.name))
2171
+ return { ok: false, agents: [], error: `retained agent "${entry.name}" is already managed or reserved` };
2172
+ }
2173
+ for (const entry of inventory.agents) {
2174
+ this.reserved.add(entry.name);
2175
+ batchReservations.push(entry.name);
2176
+ }
2177
+ const prepared = new Map();
2178
+ const preflight = [];
2179
+ for (const entry of inventory.agents) {
2180
+ const reply = await this.resumePreservedAgent(entry, true, true, prepared);
2181
+ preflight.push({ name: entry.name, reply });
2182
+ }
2183
+ const preflightFailures = preflight.filter(({ reply }) => !reply.ok);
2184
+ if (preflightFailures.length)
2185
+ return {
2186
+ ok: false,
2187
+ agents: preflight,
2188
+ error: `${preflightFailures.length} retained agent${preflightFailures.length === 1 ? "" : "s"} failed preflight`,
2189
+ };
2190
+ const agents = [];
2191
+ for (let i = 0; i < inventory.agents.length; i++) {
2192
+ const entry = inventory.agents[i];
2193
+ const reply = await this.resumePreservedAgent(entry, false, true, prepared);
2194
+ agents.push({ name: entry.name, reply });
2195
+ if (!reply.ok) {
2196
+ for (const skipped of inventory.agents.slice(i + 1))
2197
+ agents.push({ name: skipped.name, reply: { ok: false, error: `not launched because ${entry.name} failed` } });
2198
+ return { ok: false, agents, error: reply.error };
2199
+ }
2200
+ }
2201
+ if (this.resumeAttemptId)
2202
+ this.resumeAwaitingCommit = true;
2203
+ return { ok: true, agents };
2204
+ }
2205
+ finally {
2206
+ for (const name of batchReservations)
2207
+ this.reserved.delete(name);
2208
+ release();
2209
+ }
2210
+ }
2211
+ /** Re-read every retained identity input and its current authority without provisioning. This runs
2212
+ * during whole-inventory preflight, immediately before each individual spawn, and at commit. */
2213
+ async validateRetainedAuthority(entry) {
2214
+ const referenceError = this.inventoryReferenceError(entry);
2215
+ if (referenceError)
2216
+ throw new Error(`retained agent ${entry.name}: ${referenceError}`);
2217
+ if (entry.identity.mode === "open") {
2218
+ if (this.auth || this.userMode)
2219
+ throw new Error(`retained agent ${entry.name} is open-mode but the current manager is authenticated`);
2220
+ return { id: entry.identity.id };
2221
+ }
2222
+ if (entry.identity.mode === "static") {
2223
+ if (!this.auth || this.userMode)
2224
+ throw new Error(`retained agent ${entry.name} is static-auth but the current manager is not`);
2225
+ const expected = resolve(agentSecretFilePaths(this.workspaceRoot, entry.name).creds);
2226
+ if (resolve(entry.identity.credential.path) !== expected)
2227
+ throw new Error(`retained credential reference for ${entry.name} is not the manager-owned path ${expected}`);
2228
+ let credentialText;
2229
+ try {
2230
+ // The lstat guards the FS MATERIALIZATION the child will read at launch; the identity check
2231
+ // runs on the store's value — the source of truth (byte-identical here, the local FS
2232
+ // composition resolves the key to this same path).
2233
+ const st = lstatSync(expected);
2234
+ if (!st.isFile() || st.isSymbolicLink())
2235
+ throw new Error("not a regular non-symlink file");
2236
+ const stored = await workspaceSecretStore(this.workspaceRoot).get(agentCredsKey(entry.name));
2237
+ if (stored === undefined)
2238
+ throw new Error("the credential is not in the secret store");
2239
+ credentialText = stored;
2240
+ const actual = idFromCreds(credentialText);
2241
+ if (actual !== entry.identity.id)
2242
+ throw new Error(`retained credential identity ${actual} does not match inventory principal ${entry.identity.id}`);
1132
2243
  }
2244
+ catch (e) {
2245
+ throw new Error(`retained credential for ${entry.name} is unusable: ${e.message}`);
2246
+ }
2247
+ const accepted = await this.probeStaticCredential(credentialText);
2248
+ if (!accepted.ok)
2249
+ throw new Error(`retained credential for ${entry.name} is not accepted by the current broker (${accepted.reason})`);
2250
+ return { id: entry.identity.id, creds: expected };
2251
+ }
2252
+ if (!this.userMode)
2253
+ throw new Error(`retained agent ${entry.name} is user-auth but the current manager is not`);
2254
+ try {
2255
+ const provider = resolveAuthProvider();
2256
+ // Mirror the static branch's expected-path equality: the store reads below are keyed by
2257
+ // NAME, so a retained record aimed at a foreign path would otherwise pass its digest checks
2258
+ // there while a different secret gets validated here. Canonical paths only.
2259
+ const files = agentSecretFilePaths(this.workspaceRoot, entry.name);
2260
+ if (resolve(entry.identity.actorToken.path) !== resolve(files.actorToken) ||
2261
+ resolve(entry.identity.sentinelCredential.path) !== resolve(files.sentinelCreds))
2262
+ throw new Error(`retained identity references are not the manager-owned paths under ${agentCredsDir(this.workspaceRoot)}`);
2263
+ const secrets = workspaceSecretStore(this.workspaceRoot);
2264
+ const actorToken = await secrets.get(agentActorTokenKey(entry.name));
2265
+ const sentinelCreds = await secrets.get(agentSentinelCredsKey(entry.name));
2266
+ if (actorToken === undefined || sentinelCreds === undefined)
2267
+ throw new Error("the retained actor token / sentinel credential is not in the secret store");
2268
+ const adopted = await provider.validateRetainedAgent({
2269
+ store: secrets,
2270
+ dir: userAuthStateDir(this.workspaceRoot, this.space),
2271
+ space: this.space,
2272
+ owner: entry.identity.owner,
2273
+ actor: entry.identity.actor,
2274
+ actorToken,
2275
+ sentinelCreds,
2276
+ });
2277
+ if (adopted.owner !== entry.identity.owner || adopted.actor !== entry.identity.actor)
2278
+ throw new Error(`auth provider returned a replacement principal; expected ${entry.identity.owner}.${entry.identity.actor}`);
2279
+ // Bind the inventory's uid to the CURRENT authority row BEFORE any spawn: a corrupt or
2280
+ // admin-supplied inventory naming a different incarnation is refused at pre-effect validation,
2281
+ // never left to broker-fail after the child is already running (SPEC §13.1).
2282
+ if (adopted.lifecycleUid !== entry.identity.lifecycleUid)
2283
+ throw new Error(`retained user authority for ${entry.identity.owner}.${entry.identity.actor} is incarnation ${adopted.lifecycleUid}, not the inventory's ${entry.identity.lifecycleUid}; a resume binds the exact recovered uid before any spawn (SPEC 13.1)`);
2284
+ if (!sameStrings(adopted.allowSubscribe, entry.launch.allowSubscribe) ||
2285
+ !sameStrings(adopted.allowPublish, entry.launch.allowPublish) ||
2286
+ !sameStrings(adopted.scope, entry.launch.capabilities) ||
2287
+ adopted.role !== entry.role || adopted.parent !== entry.authorityParent)
2288
+ throw new Error(`retained user authority for ${entry.identity.owner}.${entry.identity.actor} no longer matches the inventory`);
2289
+ return {
2290
+ userAuth: {
2291
+ owner: entry.identity.owner,
2292
+ actor: entry.identity.actor,
2293
+ sentinelCredsPath: entry.identity.sentinelCredential.path,
2294
+ bearerCmd: [
2295
+ process.execPath,
2296
+ ...process.execArgv,
2297
+ process.argv[1],
2298
+ provider.agentBearerCommand,
2299
+ "--dir", userAuthStateDir(this.workspaceRoot, this.space),
2300
+ "--space", this.space,
2301
+ "--owner", entry.identity.owner,
2302
+ "--actor", entry.identity.actor,
2303
+ "--token-file", entry.identity.actorToken.path,
2304
+ "--health-file", entry.identity.health.path,
2305
+ ],
2306
+ },
2307
+ };
2308
+ }
2309
+ catch (e) {
2310
+ throw new Error(`retained user principal ${entry.identity.owner}.${entry.identity.actor} could not be reused: ${e.message}`);
2311
+ }
2312
+ }
2313
+ /** Validate/relaunch one retained inventory entry. Called only through resumePreserved so all
2314
+ * records pass the same preflight before the first child is exposed. */
2315
+ async resumePreservedAgent(entry, preflightOnly = false, batchReserved = false, prepared) {
2316
+ const release = this.beginLifecycle(batchReserved);
2317
+ if (!release)
2318
+ return { ok: false, error: this.maintenanceError() };
2319
+ try {
2320
+ if (entry.space !== this.space)
2321
+ return { ok: false, error: `retained agent ${entry.name} belongs to space "${entry.space}", not "${this.space}"` };
2322
+ if (entry.launch.runtime !== this.runtime.kind)
2323
+ return { ok: false, error: `retained agent ${entry.name} requires runtime "${entry.launch.runtime}", current manager uses "${this.runtime.kind}"` };
2324
+ const nameErr = this.nameError(entry.name);
2325
+ if (nameErr)
2326
+ return { ok: false, error: nameErr };
2327
+ if (this.agents.has(entry.name) || (!batchReserved && this.reserved.has(entry.name)))
2328
+ return { ok: false, error: `retained agent "${entry.name}" is already managed or reserved; same-principal resume never auto-numbers` };
2329
+ if (!batchReserved && this.agents.size + this.reserved.size + this.coolingCount() >= MAX_AGENTS)
2330
+ return { ok: false, error: `at capacity (${MAX_AGENTS} agents incl. in-flight + cooling); same-principal resume refused` };
2331
+ const cached = prepared?.get(entry.name);
2332
+ if (!preflightOnly && cached) {
2333
+ try {
2334
+ // Do not trust the earlier batch preflight across another agent's sequential readiness wait.
2335
+ await this.validateRetainedAuthority(entry);
2336
+ }
2337
+ catch (e) {
2338
+ return { ok: false, error: e.message };
2339
+ }
2340
+ return this.launchPreparedResume(entry, cached, batchReserved);
2341
+ }
2342
+ try {
2343
+ const cwd = lstatSync(entry.launch.cwd);
2344
+ if (!cwd.isDirectory() || cwd.isSymbolicLink())
2345
+ return { ok: false, error: `retained cwd is not a real directory: ${entry.launch.cwd}` };
2346
+ }
2347
+ catch (e) {
2348
+ return { ok: false, error: `retained cwd unavailable: ${entry.launch.cwd} (${e.message})` };
2349
+ }
2350
+ let connector;
2351
+ try {
2352
+ connector = registry.resolve("connector", entry.launch.connector);
2353
+ }
2354
+ catch (e) {
2355
+ return { ok: false, error: e.message };
2356
+ }
2357
+ const missing = (connector.requires ?? []).filter((bin) => !resolveOnPath(bin));
2358
+ if (missing.length)
2359
+ return { ok: false, error: `${connector.name} harness needs ${missing.join(", ")} on PATH - not found` };
2360
+ if (entry.launch.variant && !connector.supportsModelVariant)
2361
+ return { ok: false, error: `${connector.name} connector does not support model variants (variant)` };
2362
+ let launchOptions;
2363
+ if (entry.launch.source.kind === "manifest") {
2364
+ const launchSource = entry.launch.source;
2365
+ if (!launchSource.runId)
2366
+ return { ok: false, error: `retained manifest launch for ${entry.name} has no runId; refusing to guess a .cotal/run source` };
2367
+ let spec;
2368
+ try {
2369
+ const source = launchSpecForRun(this.workspaceRoot, launchSource.runId);
2370
+ if (source.space !== this.space)
2371
+ return { ok: false, error: `retained launch spec space "${source.space}" does not match manager space "${this.space}"` };
2372
+ spec = source.agents.find((a) => a.name === launchSource.requested);
2373
+ }
2374
+ catch (e) {
2375
+ return { ok: false, error: e.message };
2376
+ }
2377
+ if (!spec || spec.hash !== launchSource.hash)
2378
+ return { ok: false, error: `retained manifest agent ${launchSource.requested} is missing or its hash changed; refusing same-principal resume` };
2379
+ launchOptions = spec.launchOptions;
2380
+ }
2381
+ else {
2382
+ try {
2383
+ launchOptions = loadAgentFile(entry.launch.source.configPath).launchOptions;
2384
+ }
2385
+ catch (e) {
2386
+ return { ok: false, error: e.message };
2387
+ }
2388
+ }
2389
+ let authority;
2390
+ try {
2391
+ authority = await this.validateRetainedAuthority(entry);
2392
+ }
2393
+ catch (e) {
2394
+ return { ok: false, error: e.message };
2395
+ }
2396
+ try {
2397
+ const mcpServers = connectorServers(loadCotalConfig(this.workspaceRoot), entry.launch.connector, parseShareSelection(entry.launch.shareTools));
2398
+ const spec = connector.buildLaunch({
2399
+ space: this.space,
2400
+ name: entry.name,
2401
+ role: entry.role,
2402
+ id: authority.id,
2403
+ creds: authority.creds,
2404
+ userAuth: authority.userAuth,
2405
+ // Recover the ORIGINAL incarnation uid (never a fresh mint on resume): the child endpoint
2406
+ // binds its lifecycle-keyed dm/dlv/chathist durables by this exact value, and its creds pin
2407
+ // the same names. Omitting it here (as the pre-fix resume path did) leaves the resumed child
2408
+ // with no COTAL_LIFECYCLE_UID: static/user fail the connector auth gate and open self-mints a
2409
+ // fresh uid that orphans the preserved durables and never matches the readiness fence.
2410
+ lifecycleUid: entry.identity.lifecycleUid,
2411
+ servers: this.servers,
2412
+ configPath: entry.launch.source.configPath,
2413
+ model: entry.launch.model,
2414
+ variant: entry.launch.variant,
2415
+ launchOptions,
2416
+ resume: entry.launch.forkSource,
2417
+ subscribe: entry.launch.subscribe,
2418
+ allowSubscribe: entry.launch.allowSubscribe,
2419
+ allowPublish: entry.launch.allowPublish,
2420
+ capabilities: entry.launch.capabilities,
2421
+ transcript: entry.launch.transcript,
2422
+ mcpServers,
2423
+ workspaceRoot: this.workspaceRoot,
2424
+ });
2425
+ const value = { spec, ...authority };
2426
+ prepared?.set(entry.name, value);
2427
+ if (preflightOnly)
2428
+ return { ok: true, data: { name: entry.name, preflight: true } };
2429
+ return this.launchPreparedResume(entry, value, batchReserved);
2430
+ }
2431
+ catch (e) {
2432
+ return { ok: false, error: e.message };
2433
+ }
2434
+ }
2435
+ finally {
2436
+ release();
1133
2437
  }
1134
2438
  }
2439
+ async launchPreparedResume(entry, prepared, batchReserved) {
2440
+ if (!batchReserved)
2441
+ this.reserved.add(entry.name);
2442
+ try {
2443
+ const handle = this.runtime.spawn(entry.name, prepared.spec, entry.launch.cwd);
2444
+ const managed = {
2445
+ name: entry.name,
2446
+ role: entry.role,
2447
+ agent: entry.launch.connector,
2448
+ id: entry.identity.mode === "user" ? principalKey(entry.identity.owner, entry.identity.actor).key : entry.identity.id,
2449
+ // Recover the ORIGINAL incarnation uid the durables are keyed by (never a fresh mint on resume).
2450
+ lifecycleUid: entry.identity.lifecycleUid,
2451
+ userOwner: entry.identity.mode === "user" ? entry.identity.owner : undefined,
2452
+ spawner: entry.spawner,
2453
+ authorityParent: entry.authorityParent,
2454
+ startedAt: Date.now(),
2455
+ handle,
2456
+ control: prepared.spec.control,
2457
+ launch: {
2458
+ source: entry.launch.source,
2459
+ cwd: entry.launch.cwd,
2460
+ model: entry.launch.model,
2461
+ variant: entry.launch.variant,
2462
+ subscribe: entry.launch.subscribe,
2463
+ allowSubscribe: entry.launch.allowSubscribe,
2464
+ allowPublish: entry.launch.allowPublish,
2465
+ capabilities: entry.launch.capabilities,
2466
+ transcript: entry.launch.transcript,
2467
+ shareTools: entry.launch.shareTools,
2468
+ forkSource: entry.launch.forkSource,
2469
+ },
2470
+ suppressCleanup: true,
2471
+ };
2472
+ this.agents.set(entry.name, managed);
2473
+ if (this.resumeAttemptId)
2474
+ this.resumedAgentNames.add(entry.name);
2475
+ const readiness = await this.awaitReadiness(managed);
2476
+ if (!readiness.ok && !readiness.uncertain)
2477
+ return { ok: false, error: readiness.detail };
2478
+ if (!readiness.ok) {
2479
+ this.watchExit(managed);
2480
+ this.watchResumeAdoption(managed);
2481
+ return { ok: false, error: readiness.detail };
2482
+ }
2483
+ if (!this.resumeAttemptId)
2484
+ managed.suppressCleanup = false;
2485
+ this.watchExit(managed);
2486
+ if (this.agents.get(managed.name) !== managed)
2487
+ return { ok: false, error: `${managed.name} exited immediately after same-principal readiness` };
2488
+ return {
2489
+ ok: true,
2490
+ data: { name: managed.name, role: managed.role, agent: managed.agent, id: managed.id, mode: handle.kind, resumed: true },
2491
+ };
2492
+ }
2493
+ catch (e) {
2494
+ return { ok: false, error: e.message };
2495
+ }
2496
+ finally {
2497
+ if (!batchReserved)
2498
+ this.reserved.delete(entry.name);
2499
+ }
2500
+ }
2501
+ probeStaticCredential(creds) {
2502
+ return probeConnect(this.servers ?? DEFAULT_SERVER, { creds, timeoutMs: 5_000 });
2503
+ }
2504
+ /** An uncertain resume remains non-destructive until exact-principal AND exact-incarnation presence
2505
+ * arrives later. Same predicate as the readiness fence: a principal-only match would let a
2506
+ * wrong/absent-uid presence under the reused alias clear cleanup suppression on another incarnation. */
2507
+ watchResumeAdoption(a) {
2508
+ const wanted = this.managedPrincipal(a);
2509
+ const onPresence = () => {
2510
+ if (this.agents.get(a.name) !== a) {
2511
+ this.ep.off("presence", onPresence);
2512
+ return;
2513
+ }
2514
+ if (!this.ep.getRoster().some((p) => p.card.id === wanted && p.status !== "offline" && p.lifecycleUid === a.lifecycleUid))
2515
+ return;
2516
+ if (!this.resumeRequired)
2517
+ a.suppressCleanup = false;
2518
+ this.ep.off("presence", onPresence);
2519
+ };
2520
+ this.ep.on("presence", onPresence);
2521
+ onPresence();
2522
+ }
1135
2523
  /** #159 B1: wait for a detached launch to reach a REAL outcome before replying — never a liveness-
1136
2524
  * inferring timer. Races three:
1137
2525
  * • the assigned id joins presence (live) → **started** — the honest signal (the manager owns mesh
@@ -1158,7 +2546,16 @@ export class Manager {
1158
2546
  // through managedPrincipal or a static launch can never be seen joining (every static spawn would
1159
2547
  // resolve "uncertain"; caught by the lifecycle e2e).
1160
2548
  const wanted = this.managedPrincipal(a);
1161
- const joined = () => this.ep.getRoster().some((p) => p.card.id === wanted && p.status !== "offline");
2549
+ // READINESS LIFECYCLE FENCE (SPEC 13.1): match the exact principal AND the exact lifecycle uid
2550
+ // the manager minted for THIS spawn (presence carries it, §6/:315). The endpoint's own
2551
+ // register-only broker proof is gated on the CLIENT-authored `card.kind`, which a managed child
2552
+ // holding a valid agent credential could set to "endpoint" to skip - so it is defense-in-depth,
2553
+ // NOT the authority boundary. This equality is: the manager (not the child) owns the expected
2554
+ // uid, so a ghost that advertises a wrong/absent uid never reports STARTED, whatever kind it
2555
+ // claims. The manager threads the uid into EVERY mode's launch (open included), so the child
2556
+ // adopts it over a self-mint and publishes it in presence; the uid is absent only from a peer
2557
+ // the manager never launched (a pure operator/daemon connection that never registers).
2558
+ const joined = () => this.ep.getRoster().some((p) => p.card.id === wanted && p.status !== "offline" && p.lifecycleUid === a.lifecycleUid);
1162
2559
  return await new Promise((resolve) => {
1163
2560
  let done = false;
1164
2561
  let timer;
@@ -1255,7 +2652,7 @@ export class Manager {
1255
2652
  return { ok: false, error: denied };
1256
2653
  const graceful = args.graceful !== false;
1257
2654
  this.stopHandle(a, graceful);
1258
- this.freeSlot(a, !admin); // own-child despawn is rate-floored; admin emergency-kill is not
2655
+ this.trackStoppedHandle(a, !admin);
1259
2656
  return { ok: true, data: { name, stopped: true, graceful } };
1260
2657
  }
1261
2658
  /** Open a short-lived PROVISIONER connection, run the onboarding ops on it, and drain it (closure (ii),
@@ -1392,7 +2789,7 @@ export class Manager {
1392
2789
  // FAIL-CLOSED: a failed record is the failure + repair sentence; a missing/malformed or
1393
2790
  // stale record on a live agent is auth-unknown/auth-stale, NEVER silently healthy.
1394
2791
  const health = a.userOwner
1395
- ? agentAuthState(join(authDir(this.workspaceRoot), "creds", `${a.name}.auth-health.json`))
2792
+ ? agentAuthState(agentSecretFilePaths(this.workspaceRoot, a.name).health)
1396
2793
  : undefined;
1397
2794
  return {
1398
2795
  name: a.name,