@cotal-ai/manager 0.11.6 → 0.12.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/manager.js CHANGED
@@ -1,13 +1,15 @@
1
1
  import { execFile } from "node:child_process";
2
- import { existsSync, rmSync } from "node:fs";
2
+ import { createHash, randomBytes } from "node:crypto";
3
+ import { existsSync, lstatSync, readFileSync, rmSync } from "node:fs";
3
4
  import { join, dirname, resolve } from "node:path";
4
- import { CotalEndpoint, DEFAULT_SERVER, DEV_OWNER, MANAGER_LEASE_TTL_MS, STANDING_RENEWABLE_TTL_SEC, agentFilePath, clearSpaceHistory, connectorServers, deprovisionAgent, firstFreeName, loadAgentFile, loadCotalConfig, mintCreds, mkSecretDir, newIdentity, parsePrincipalKey, parseShareSelection, principalKey, provisionAgent, provisionAgentDurables, registry, resolveAuthProvider, saveAgentFile, writeSecretFile, subjectMatches, CONTROL_PRIVILEGED, CONTROL_SELF_SERVICE, CONTROL_ADMIN, } from "@cotal-ai/core";
5
- import { agentAuthState, authDir, connectorInstallHint, DEFAULT_CONNECTOR, defaultAgentType, findCotalRoot, loadMeshes, loadSpaceAuth, manifestExtensionNames, materializeFromManifest, mergeLaunchOptions, remintDaemonCreds, resolveOnPath, userAuthStateDir, writeRenewalRecord } from "@cotal-ai/workspace";
5
+ import { CotalEndpoint, DEFAULT_SERVER, DEV_OWNER, MANAGER_LEASE_TTL_MS, STANDING_RENEWABLE_TTL_SEC, agentFilePath, clearSpaceHistory, connectorServers, deprovisionAgent, firstFreeName, idFromCreds, loadAgentFile, loadCotalConfig, mintCreds, mkSecretDir, newIdentity, parsePrincipalKey, parseShareSelection, principalKey, probeConnect, provisionAgent, provisionAgentDurables, registry, resolveAuthProvider, saveAgentFile, writeSecretFile, subjectMatches, CONTROL_PRIVILEGED, CONTROL_SELF_SERVICE, CONTROL_ADMIN, } from "@cotal-ai/core";
6
+ import { agentAuthState, authDir, connectorInstallHint, DEFAULT_CONNECTOR, defaultAgentType, findCotalRoot, loadMeshes, loadSpaceAuth, manifestExtensionNames, materializeFromManifest, mergeLaunchOptions, remintDaemonCreds, resolveOnPath, userAuthStateDir, workspaceSecretStore, writeRenewalRecord } from "@cotal-ai/workspace";
6
7
  import { createRuntime, } from "./runtime/index.js";
7
8
  import { AttachEndpoint } from "./attach-endpoint.js";
8
9
  import { launchSpecForRun, materializePersona, launchAgentToStartOpts } from "./launch.js";
9
10
  import { authorizeLaunch, authorizeNamedControl } from "./authorize.js";
10
11
  import { controlShutdown } from "./control-shutdown.js";
12
+ import { parseResumeCommitArgs, parseResumeControlArgs, parseResumeFinalizeArgs } from "./resume.js";
11
13
  /** Concurrency ceiling — the manager refuses to hold more than this many live + in-flight +
12
14
  * cooling slots at once (P4a). Bounds a fork-bomb: spawn is a full agent process per call. */
13
15
  const MAX_AGENTS = 50;
@@ -29,6 +31,9 @@ export const READINESS_TIMEOUT_MS = 30_000;
29
31
  * `.catch`. Generous over the helper's 5s connect timeout to allow the two consumer-deletes + ACL purge
30
32
  * + drain on a healthy-but-slow broker. */
31
33
  const DEPROVISION_TIMEOUT_MS = 15_000;
34
+ /** A hard preservation stop should settle quickly. The manager still waits and reports a partial
35
+ * cut rather than pretending a child is gone. Held in ManagerOptions so fake runtimes can shorten it. */
36
+ const PRESERVE_STOP_TIMEOUT_MS = 10_000;
32
37
  /** Sentinel owner-filter value that matches NO agent's `userOwner` (owner tokens never contain a
33
38
  * dash) — what {@link Manager.psOwnerFilter} returns for an unparseable caller so a malformed
34
39
  * principal fail-closes to an empty `ps` instead of an unbounded one. */
@@ -54,6 +59,9 @@ function withTimeout(p, ms, msg) {
54
59
  });
55
60
  return Promise.race([p.finally(() => clearTimeout(timer)), timeout]);
56
61
  }
62
+ function sameStrings(a, b) {
63
+ return JSON.stringify([...(a ?? [])].sort()) === JSON.stringify([...(b ?? [])].sort());
64
+ }
57
65
  /**
58
66
  * The agent supervisor: a long-lived mesh node that owns agent process lifecycle.
59
67
  * It serves control requests on the "manager" service and spawns/kills agents
@@ -68,6 +76,7 @@ export class Manager {
68
76
  /** See {@link ManagerOptions.installedExtensions}. */
69
77
  installedExtensions;
70
78
  runtime;
79
+ preserveStopTimeoutMs;
71
80
  agents = new Map();
72
81
  /** Names whose spawn is in flight (reserved synchronously before the provision await) — counted
73
82
  * toward the ceiling so two concurrent same-name spawns can't both pass the gate (P4a). */
@@ -92,6 +101,29 @@ export class Manager {
92
101
  leaseTimer;
93
102
  /** The class-2 renewal owner's half-TTL schedule (D5 slice 5); armed only on auth meshes. */
94
103
  credRenewTimer;
104
+ maintenanceState = "active";
105
+ lifecycleInFlight = 0;
106
+ lifecycleDrainWaiters = [];
107
+ preservationTask;
108
+ preparationTask;
109
+ preservationGeneration = 0;
110
+ preservationAttemptId;
111
+ preservationStarted = false;
112
+ preservationFailures = [];
113
+ unverifiedStops = [];
114
+ preservationInventory;
115
+ resumeAttemptId;
116
+ resumeInventoryDigest;
117
+ resumeInventory;
118
+ resumeTask;
119
+ resumeResult;
120
+ resumeRequired = false;
121
+ resumeAwaitingCommit = false;
122
+ resumeCommitted = false;
123
+ resumeCommitTask;
124
+ resumeFinalized = false;
125
+ resumeDurableCommitToken;
126
+ resumedAgentNames = new Set();
95
127
  constructor(opts) {
96
128
  this.space = opts.space;
97
129
  this.servers = opts.servers;
@@ -99,7 +131,17 @@ export class Manager {
99
131
  this.workspaceRoot = opts.workspaceRoot ?? findCotalRoot();
100
132
  this.installedExtensions = opts.installedExtensions ?? false;
101
133
  this.runtime = createRuntime(opts.runtime ?? "auto", `cotal-${this.space}`);
102
- this.attach = new AttachEndpoint((name) => this.agents.get(name)?.handle, () => this.list(),
134
+ this.preserveStopTimeoutMs = opts.preserveStopTimeoutMs ?? PRESERVE_STOP_TIMEOUT_MS;
135
+ if (opts.resumeAttemptId && !/^[A-Za-z0-9_-]{1,128}$/.test(opts.resumeAttemptId))
136
+ throw new Error("resumeAttemptId must be a safe token (letters, digits, _, -; max 128)");
137
+ if (opts.resumeDurableCommitToken && !/^[a-f0-9]{64}$/.test(opts.resumeDurableCommitToken))
138
+ throw new Error("resumeDurableCommitToken must be a lowercase 32-byte token");
139
+ if (opts.resumeDurableCommitToken && !opts.resumeAttemptId)
140
+ throw new Error("resumeDurableCommitToken requires resumeAttemptId");
141
+ this.resumeAttemptId = opts.resumeAttemptId;
142
+ this.resumeRequired = opts.resumeAttemptId !== undefined;
143
+ this.resumeDurableCommitToken = opts.resumeDurableCommitToken;
144
+ this.attach = new AttachEndpoint((name) => this.maintenanceState === "active" && !this.resumeRequired ? this.agents.get(name)?.handle : undefined, () => this.list(),
103
145
  // Initial /feed replay for a connecting console: the current peer roster.
104
146
  () => [{ event: "roster", data: this.ep?.getRoster() ?? [] }], opts.consolePort ?? 0);
105
147
  }
@@ -221,6 +263,9 @@ export class Manager {
221
263
  * (no responder) is recorded honestly: the daemon's 75% source re-read remains the adoption backstop.
222
264
  * Never throws — renewal failure must be LOUD (log + record), not fatal to the supervisor. */
223
265
  async renewDaemonCreds() {
266
+ const release = this.beginLifecycle();
267
+ if (!release)
268
+ return;
224
269
  try {
225
270
  const results = await remintDaemonCreds(this.workspaceRoot);
226
271
  const resigned = results.filter((r) => r.ok);
@@ -243,6 +288,362 @@ export class Manager {
243
288
  catch (e) {
244
289
  console.error(`! credential renewal pass failed: ${e.message}`);
245
290
  }
291
+ finally {
292
+ release();
293
+ }
294
+ }
295
+ /** Admit one lifecycle/control operation while active. The synchronous increment is the fence:
296
+ * preserveState flips state before its first await, so work is either counted or rejected. */
297
+ beginLifecycle(resumeOperation = false) {
298
+ if (this.maintenanceState !== "active" || (this.resumeRequired && !resumeOperation))
299
+ return undefined;
300
+ this.lifecycleInFlight++;
301
+ let released = false;
302
+ return () => {
303
+ if (released)
304
+ return;
305
+ released = true;
306
+ this.releaseLifecycle();
307
+ };
308
+ }
309
+ releaseLifecycle() {
310
+ this.lifecycleInFlight--;
311
+ if (this.lifecycleInFlight !== 0)
312
+ return;
313
+ const waiters = this.lifecycleDrainWaiters;
314
+ this.lifecycleDrainWaiters = [];
315
+ for (const wake of waiters)
316
+ wake();
317
+ }
318
+ /** A cleanup spawned by accepted active-mode work is part of that work for maintenance draining,
319
+ * even where the ordinary control reply remains fire-and-forget. */
320
+ trackDeprovision(a, context = "") {
321
+ this.lifecycleInFlight++;
322
+ void this.deprovision(a)
323
+ .catch((e) => console.error(`deprovision${context ? ` ${context}` : ""} ${a.name} (${a.id}): ${e.message}`))
324
+ .finally(() => this.releaseLifecycle());
325
+ }
326
+ async awaitLifecycleDrain() {
327
+ if (this.lifecycleInFlight === 0)
328
+ return;
329
+ await new Promise((resolve) => this.lifecycleDrainWaiters.push(resolve));
330
+ }
331
+ maintenanceError() {
332
+ if (this.resumeRequired)
333
+ return `manager is waiting for resume attempt ${this.resumeAttemptId}; ordinary lifecycle/control work is fenced`;
334
+ return `manager is in ${this.maintenanceState} mode; new lifecycle/control work is fenced`;
335
+ }
336
+ /** Fence and build the inventory without stopping a child. The coordinator must durably persist
337
+ * this exact plan before calling commitPreservation with the same attempt id. */
338
+ preparePreservation(attemptId) {
339
+ if (!attemptId.trim())
340
+ return Promise.reject(new Error("preservation attemptId is required"));
341
+ if (this.preservationAttemptId && this.preservationAttemptId !== attemptId)
342
+ return Promise.reject(new Error(`manager is fenced for preservation attempt ${this.preservationAttemptId}; refusing different attempt ${attemptId}`));
343
+ if (this.maintenanceState === "preserved" && this.preservationInventory)
344
+ return Promise.resolve({ ok: true, attemptId, state: "preserved", inventory: this.preservationInventory, failures: [] });
345
+ if (this.preparationTask)
346
+ return this.preparationTask;
347
+ if (this.maintenanceState === "active") {
348
+ // The fence lands before any await. Accepted work has already incremented lifecycleInFlight.
349
+ this.maintenanceState = "preserving";
350
+ this.preservationAttemptId = attemptId;
351
+ this.preservationGeneration++;
352
+ if (this.credRenewTimer) {
353
+ clearInterval(this.credRenewTimer);
354
+ this.credRenewTimer = undefined;
355
+ }
356
+ }
357
+ const generation = this.preservationGeneration;
358
+ const task = this.runPreparation(attemptId, generation);
359
+ let wrapped;
360
+ wrapped = task.finally(() => {
361
+ if (this.preservationGeneration === generation && this.preparationTask === wrapped)
362
+ this.preparationTask = undefined;
363
+ });
364
+ this.preparationTask = wrapped;
365
+ return wrapped;
366
+ }
367
+ assertPreservationGeneration(attemptId, generation) {
368
+ if (this.preservationAttemptId !== attemptId || this.preservationGeneration !== generation)
369
+ throw new Error(`preservation attempt ${attemptId} was abandoned before preparation completed`);
370
+ }
371
+ async runPreparation(attemptId, generation) {
372
+ await this.awaitLifecycleDrain();
373
+ this.assertPreservationGeneration(attemptId, generation);
374
+ const inventory = this.preservationInventory ?? {
375
+ version: "cotal-manager-resume/v1",
376
+ space: this.space,
377
+ createdAt: new Date().toISOString(),
378
+ agents: [...this.agents.values()].map((a) => this.resumeEntry(a)),
379
+ };
380
+ const failures = [];
381
+ for (const entry of inventory.agents) {
382
+ const error = this.inventoryReferenceError(entry);
383
+ if (error)
384
+ failures.push({
385
+ name: entry.name,
386
+ id: entry.identity.mode === "user" ? principalKey(entry.identity.owner, entry.identity.actor).key : entry.identity.id,
387
+ error,
388
+ });
389
+ }
390
+ const unverifiedStops = this.unverifiedStops.filter((stopped) => {
391
+ try {
392
+ if (!stopped.authoritative && stopped.handle.status() === "exited")
393
+ return false;
394
+ }
395
+ catch { /* fail closed below */ }
396
+ failures.push({
397
+ name: stopped.name,
398
+ id: stopped.id,
399
+ error: stopped.error ?? `an earlier stop on runtime "${stopped.handle.kind}" cannot prove the child is gone`,
400
+ });
401
+ return true;
402
+ });
403
+ this.assertPreservationGeneration(attemptId, generation);
404
+ // The prepared inventory must round-trip through the EXACT resume control parser (schema and
405
+ // byte cap) NOW, before any child stops: a cut that cannot resume must fail at prepare time,
406
+ // never after listener exposure.
407
+ try {
408
+ parseResumeControlArgs({ attemptId, inventory });
409
+ }
410
+ catch (e) {
411
+ failures.push({ name: "<inventory>", id: attemptId, error: `prepared inventory would be rejected at resume: ${e.message}` });
412
+ }
413
+ this.preservationInventory = inventory;
414
+ this.preservationFailures = failures;
415
+ this.unverifiedStops = unverifiedStops;
416
+ return {
417
+ ok: failures.length === 0,
418
+ attemptId,
419
+ state: "prepared",
420
+ inventory,
421
+ failures: [...failures],
422
+ };
423
+ }
424
+ /** Stop children only after the coordinator has persisted the prepared inventory. Same-attempt
425
+ * retries are idempotent; a different attempt is refused. */
426
+ commitPreservation(attemptId) {
427
+ if (!this.preservationAttemptId || this.preservationAttemptId !== attemptId)
428
+ return Promise.reject(new Error(`preservation attempt ${attemptId} was not prepared by this manager`));
429
+ if (!this.preservationInventory)
430
+ return Promise.reject(new Error(`preservation attempt ${attemptId} has no prepared inventory`));
431
+ if (this.preservationFailures.length)
432
+ return Promise.resolve({
433
+ ok: false,
434
+ attemptId,
435
+ state: "preserving",
436
+ inventory: this.preservationInventory,
437
+ failures: [...this.preservationFailures],
438
+ });
439
+ if (this.maintenanceState === "preserved")
440
+ return Promise.resolve({ ok: true, attemptId, state: "preserved", inventory: this.preservationInventory, failures: [] });
441
+ if (this.preservationTask)
442
+ return this.preservationTask;
443
+ this.preservationStarted = true;
444
+ this.preservationTask = this.runPreservation(attemptId).finally(() => {
445
+ this.preservationTask = undefined;
446
+ });
447
+ return this.preservationTask;
448
+ }
449
+ /** Recover an abandoned prepare before any child stop. Once commit begins, preservation is
450
+ * irreversible and remains fenced until the coordinator records failure/recourse. */
451
+ abortPreservation(attemptId) {
452
+ if (this.preservationAttemptId !== attemptId)
453
+ throw new Error(`preservation attempt ${attemptId} is not the active manager attempt`);
454
+ if (this.preparationTask || this.lifecycleInFlight > 0)
455
+ throw new Error(`preservation attempt ${attemptId} is still preparing or draining accepted lifecycle work and cannot be aborted`);
456
+ if (this.preservationStarted || this.preservationTask || this.maintenanceState === "preserved")
457
+ throw new Error(`preservation attempt ${attemptId} has begun stopping children and cannot return to active mode`);
458
+ this.preservationGeneration++;
459
+ this.maintenanceState = "active";
460
+ this.preservationAttemptId = undefined;
461
+ this.preservationInventory = undefined;
462
+ this.preservationFailures = [];
463
+ if (this.auth && !this.credRenewTimer) {
464
+ this.credRenewTimer = setInterval(() => { void this.renewDaemonCreds(); }, (STANDING_RENEWABLE_TTL_SEC / 2) * 1000);
465
+ this.credRenewTimer.unref?.();
466
+ }
467
+ // Exit watchers were suppressed while the fence stood: reconcile every child that died during
468
+ // preparation now, or its slot/credential footprint would linger unreaped after the abort.
469
+ for (const agent of [...this.agents.values()]) {
470
+ try {
471
+ if (agent.handle.status() === "exited")
472
+ this.onAgentExit(agent);
473
+ }
474
+ catch { /* status unavailable - the exit watcher fires again on real exit */ }
475
+ }
476
+ }
477
+ /** In-process convenience that preserves the crash barrier by awaiting durable persistence between
478
+ * prepare and commit. Wire callers use the explicit two-phase admin operations. */
479
+ async preserveState(opts) {
480
+ const plan = await this.preparePreservation(opts.attemptId);
481
+ if (!plan.ok)
482
+ return { ok: false, attemptId: opts.attemptId, state: "preserving", inventory: plan.inventory, failures: plan.failures };
483
+ await opts.persistInventory(plan.inventory);
484
+ return this.commitPreservation(opts.attemptId);
485
+ }
486
+ async runPreservation(attemptId) {
487
+ const failures = [];
488
+ for (const a of [...this.agents.values()])
489
+ a.suppressCleanup = true;
490
+ await Promise.all([...this.agents.values()].map(async (a) => {
491
+ try {
492
+ // A preservation cut must not run the connector's logical leave/cleanup hooks.
493
+ a.handle.stop({ graceful: false });
494
+ }
495
+ catch (e) {
496
+ failures.push({ name: a.name, id: a.id, error: `stop failed: ${e.message}` });
497
+ return;
498
+ }
499
+ try {
500
+ await this.awaitHandleExit(a.handle);
501
+ if (this.agents.get(a.name) === a)
502
+ this.agents.delete(a.name);
503
+ }
504
+ catch (e) {
505
+ failures.push({ name: a.name, id: a.id, error: e.message });
506
+ }
507
+ }));
508
+ if (failures.length === 0)
509
+ this.maintenanceState = "preserved";
510
+ return {
511
+ ok: failures.length === 0,
512
+ attemptId,
513
+ state: this.maintenanceState === "preserved" ? "preserved" : "preserving",
514
+ inventory: this.preservationInventory,
515
+ failures,
516
+ };
517
+ }
518
+ async awaitHandleExit(handle) {
519
+ if (!handle.waitForExit)
520
+ throw new Error(`runtime "${handle.kind}" cannot prove child exit (AgentHandle.waitForExit is not implemented)`);
521
+ if (handle.status() === "exited")
522
+ return;
523
+ await withTimeout(handle.waitForExit(), this.preserveStopTimeoutMs, `child did not exit within ${this.preserveStopTimeoutMs}ms`);
524
+ if (handle.status() !== "exited")
525
+ throw new Error(`runtime "${handle.kind}" reported exit completion but status is still running`);
526
+ }
527
+ inventoryReferenceError(entry) {
528
+ if (entry.launch.source.kind === "manifest" && !entry.launch.source.runId)
529
+ return "resolved manifest launch has no retained runId";
530
+ if (entry.launch.unresolvedLaunchOptionKeys?.length)
531
+ return `imperative launch options have no non-secret durable source (${entry.launch.unresolvedLaunchOptionKeys.join(", ")})`;
532
+ if (!entry.dependencies.some((path) => resolve(path) === resolve(entry.launch.source.configPath)))
533
+ return `launch config is not declared as a retained dependency: ${entry.launch.source.configPath}`;
534
+ if (entry.launch.source.kind === "manifest" && entry.launch.source.runId) {
535
+ const specPath = join(this.workspaceRoot, ".cotal", "run", `${entry.launch.source.runId}.json`);
536
+ if (!entry.dependencies.some((path) => resolve(path) === resolve(specPath)))
537
+ return `manifest source is not declared as a retained dependency: ${specPath}`;
538
+ }
539
+ const required = [...entry.dependencies];
540
+ if (entry.identity.mode === "static")
541
+ required.push(entry.identity.credential.path);
542
+ if (entry.identity.mode === "user") {
543
+ required.push(entry.identity.actorToken.path, entry.identity.sentinelCredential.path);
544
+ }
545
+ for (const path of required) {
546
+ try {
547
+ const st = lstatSync(path);
548
+ if (!st.isFile() || st.isSymbolicLink())
549
+ return `retained reference is not a regular non-symlink file: ${path}`;
550
+ }
551
+ catch (e) {
552
+ return `retained reference unavailable: ${path} (${e.message})`;
553
+ }
554
+ }
555
+ if (process.platform !== "win32") {
556
+ const secrets = entry.identity.mode === "static"
557
+ ? [entry.identity.credential.path]
558
+ : entry.identity.mode === "user"
559
+ ? [entry.identity.actorToken.path, entry.identity.sentinelCredential.path]
560
+ : [];
561
+ for (const path of secrets)
562
+ if ((lstatSync(path).mode & 0o077) !== 0)
563
+ return `retained identity file is not private (expected 0600): ${path}`;
564
+ }
565
+ try {
566
+ if (this.fileDigest(entry.launch.source.configPath) !== entry.launch.source.configSha256)
567
+ return `launch config changed since it became effective: ${entry.launch.source.configPath}`;
568
+ if (entry.identity.mode === "static" && this.fileDigest(entry.identity.credential.path) !== entry.identity.credential.sha256)
569
+ return `retained credential changed after the cut: ${entry.identity.credential.path}`;
570
+ if (entry.identity.mode === "user" &&
571
+ (this.fileDigest(entry.identity.actorToken.path) !== entry.identity.actorToken.sha256 ||
572
+ this.fileDigest(entry.identity.sentinelCredential.path) !== entry.identity.sentinelCredential.sha256))
573
+ return `retained user identity files changed after the cut for ${entry.name}`;
574
+ if (entry.launch.source.kind === "manifest" && entry.launch.source.runId) {
575
+ const specPath = join(this.workspaceRoot, ".cotal", "run", `${entry.launch.source.runId}.json`);
576
+ if (!entry.launch.source.manifestSha256 || this.fileDigest(specPath) !== entry.launch.source.manifestSha256)
577
+ return `manifest source changed since it became effective: ${specPath}`;
578
+ }
579
+ }
580
+ catch (e) {
581
+ return `retained reference cannot be hashed: ${e.message}`;
582
+ }
583
+ return undefined;
584
+ }
585
+ fileDigest(path) {
586
+ return createHash("sha256").update(readFileSync(path)).digest("hex");
587
+ }
588
+ fileDigestOrEmpty(path) {
589
+ try {
590
+ return this.fileDigest(path);
591
+ }
592
+ catch {
593
+ return "";
594
+ }
595
+ }
596
+ resumeEntry(a) {
597
+ const principal = a.userOwner
598
+ ? parsePrincipalKey(a.id)
599
+ : { owner: DEV_OWNER, actor: a.id };
600
+ if (!principal)
601
+ throw new Error(`managed agent ${a.name} has an invalid principal ${a.id}`);
602
+ const credsDir = join(authDir(this.workspaceRoot), "creds");
603
+ const staticCredsPath = join(credsDir, `${a.name}.creds`);
604
+ const actorTokenPath = join(credsDir, `${a.name}.actor-token`);
605
+ const sentinelPath = join(credsDir, `${a.name}.sentinel.creds`);
606
+ const identity = a.userOwner
607
+ ? {
608
+ mode: "user",
609
+ owner: principal.owner,
610
+ actor: principal.actor,
611
+ actorToken: { kind: "file", path: actorTokenPath, sha256: this.fileDigestOrEmpty(actorTokenPath) },
612
+ sentinelCredential: { kind: "file", path: sentinelPath, sha256: this.fileDigestOrEmpty(sentinelPath) },
613
+ health: { kind: "file", path: join(credsDir, `${a.name}.auth-health.json`) },
614
+ }
615
+ : this.auth
616
+ ? { mode: "static", id: principal.actor, credential: { kind: "file", path: staticCredsPath, sha256: this.fileDigestOrEmpty(staticCredsPath) } }
617
+ : { mode: "open", id: principal.actor };
618
+ const dependencies = [a.launch.source.configPath];
619
+ if (a.launch.source.kind === "manifest" && a.launch.source.runId)
620
+ dependencies.unshift(join(this.workspaceRoot, ".cotal", "run", `${a.launch.source.runId}.json`));
621
+ return {
622
+ space: this.space,
623
+ name: a.name,
624
+ role: a.role,
625
+ identity,
626
+ launch: {
627
+ connector: a.agent,
628
+ runtime: a.handle.kind,
629
+ cwd: a.launch.cwd,
630
+ source: a.launch.source,
631
+ model: a.launch.model,
632
+ variant: a.launch.variant,
633
+ subscribe: a.launch.subscribe,
634
+ allowSubscribe: a.launch.allowSubscribe,
635
+ allowPublish: a.launch.allowPublish,
636
+ capabilities: a.launch.capabilities,
637
+ transcript: a.launch.transcript,
638
+ shareTools: a.launch.shareTools,
639
+ forkSource: a.launch.forkSource,
640
+ unresolvedLaunchOptionKeys: a.launch.unresolvedLaunchOptionKeys,
641
+ },
642
+ dependencies,
643
+ spawner: a.spawner,
644
+ authorityParent: a.authorityParent,
645
+ startedAt: new Date(a.startedAt).toISOString(),
646
+ };
246
647
  }
247
648
  /** Tear down every managed agent's footprint — the shared teardown for EVERY manager-exit path (#159
248
649
  * B2): graceful {@link stop} AND the fail-closed lease-loss exit ({@link renewLease}). A manager exit is
@@ -262,14 +663,44 @@ export class Manager {
262
663
  this.stopHandle(a, false);
263
664
  }
264
665
  // Deprovision EVERY snapshot entry regardless of whether its stop failed (allSettled + a loud log).
265
- await Promise.allSettled(managed.map((a) => this.deprovision(a).catch((e) => console.error(`deprovision ${a.name} (${a.id}) on shutdown: ${e.message}`))));
666
+ await Promise.allSettled(managed.filter((a) => !a.suppressCleanup).map((a) => this.deprovision(a).catch((e) => console.error(`deprovision ${a.name} (${a.id}) on shutdown: ${e.message}`))));
667
+ }
668
+ async stopRetainedAgentsOnExit() {
669
+ const managed = [...this.agents.values()];
670
+ for (const a of managed)
671
+ a.suppressCleanup = true;
672
+ const failures = [];
673
+ await Promise.all(managed.map(async (a) => {
674
+ try {
675
+ a.handle.stop({ graceful: false });
676
+ }
677
+ catch (e) {
678
+ failures.push(`${a.name}: stop failed: ${e.message}`);
679
+ }
680
+ try {
681
+ await this.awaitHandleExit(a.handle);
682
+ if (this.agents.get(a.name) === a)
683
+ this.agents.delete(a.name);
684
+ }
685
+ catch (e) {
686
+ failures.push(`${a.name}: ${e.message}`);
687
+ }
688
+ }));
689
+ if (failures.length)
690
+ throw new Error(`manager preservation shutdown incomplete: ${failures.join("; ")}`);
266
691
  }
267
692
  async stop() {
268
693
  if (this.leaseTimer)
269
694
  clearInterval(this.leaseTimer);
270
695
  if (this.credRenewTimer)
271
696
  clearInterval(this.credRenewTimer);
272
- await this.teardownManagedAgents(); // reap agents BEFORE releasing the lease/endpoints (#159 B2)
697
+ if (this.maintenanceState === "active" && !this.resumeRequired) {
698
+ await this.teardownManagedAgents(); // normal shutdown stays destructive (#159 B2)
699
+ }
700
+ else {
701
+ // A signal after a partial preservation must never fall back into destructive teardown.
702
+ await this.stopRetainedAgentsOnExit();
703
+ }
273
704
  await this.ep.releaseManagerLease(this.leaseRevision);
274
705
  await this.ep.stop();
275
706
  await this.attach.stop();
@@ -279,9 +710,9 @@ export class Manager {
279
710
  * with the new holder, and exit. We deliberately do NOT re-acquire (a replacement may already be live
280
711
  * while we'd still be serving) and do NOT release the key — it now belongs to that replacement. */
281
712
  async renewLease() {
282
- if (!this.leaseInfo || this.leaseRevision === undefined)
283
- return;
284
713
  try {
714
+ if (!this.leaseInfo || this.leaseRevision === undefined)
715
+ return;
285
716
  this.leaseRevision = await this.ep.renewManagerLease(this.leaseInfo, this.leaseRevision);
286
717
  }
287
718
  catch (e) {
@@ -291,7 +722,10 @@ export class Manager {
291
722
  // Tear down our managed agents' footprints too (#159 B2) — this exit path leaks them otherwise. Do
292
723
  // NOT release the lease key (it may belong to the replacement holder). Best-effort, like ep/attach.
293
724
  try {
294
- await this.teardownManagedAgents();
725
+ if (this.maintenanceState === "active" && !this.resumeRequired)
726
+ await this.teardownManagedAgents();
727
+ else
728
+ await this.stopRetainedAgentsOnExit();
295
729
  }
296
730
  catch { /* best effort */ }
297
731
  try {
@@ -306,6 +740,187 @@ export class Manager {
306
740
  }
307
741
  }
308
742
  async handle(req, tier) {
743
+ if (req.op === "finalizeResume") {
744
+ if (tier !== CONTROL_ADMIN)
745
+ return { ok: false, error: "finalizeResume is admin-only; not allowed on this control subject" };
746
+ let args;
747
+ try {
748
+ args = parseResumeFinalizeArgs(req.args);
749
+ }
750
+ catch (e) {
751
+ return { ok: false, error: e.message };
752
+ }
753
+ if (!this.resumeAttemptId || this.resumeAttemptId !== args.attemptId)
754
+ return { ok: false, error: `manager expects resume attempt ${this.resumeAttemptId ?? "<none>"}, not ${args.attemptId}` };
755
+ if (!this.resumeCommitted || !this.resumeDurableCommitToken)
756
+ return { ok: false, error: `resume attempt ${args.attemptId} has no successful commit to finalize` };
757
+ if (this.resumeDurableCommitToken !== args.durableCommitToken)
758
+ return { ok: false, error: `resume attempt ${args.attemptId} durable commit token does not match` };
759
+ if (this.resumeFinalized)
760
+ return { ok: true, data: { attemptId: args.attemptId, state: "active" } };
761
+ const inventory = this.resumeInventory;
762
+ if (!inventory)
763
+ return { ok: false, error: `resume attempt ${args.attemptId} has no bound inventory` };
764
+ let inactive;
765
+ try {
766
+ inactive = this.resumeLivenessErrors(inventory, this.ep.getRoster());
767
+ }
768
+ catch (e) {
769
+ return { ok: false, error: `resume attempt ${args.attemptId} cannot verify live principals at finalize: ${e.message}` };
770
+ }
771
+ if (inactive.length)
772
+ return { ok: false, error: `resume attempt ${args.attemptId} is not live at finalize: ${inactive.join("; ")}` };
773
+ for (const entry of this.resumeInventory?.agents ?? []) {
774
+ const managed = this.agents.get(entry.name);
775
+ if (managed)
776
+ managed.suppressCleanup = false;
777
+ }
778
+ this.resumeFinalized = true;
779
+ this.resumeRequired = false;
780
+ return { ok: true, data: { attemptId: args.attemptId, state: "active" } };
781
+ }
782
+ if (req.op === "commitResume") {
783
+ if (tier !== CONTROL_ADMIN)
784
+ return { ok: false, error: "commitResume is admin-only; not allowed on this control subject" };
785
+ let attemptId;
786
+ try {
787
+ attemptId = parseResumeCommitArgs(req.args).attemptId;
788
+ }
789
+ catch (e) {
790
+ return { ok: false, error: e.message };
791
+ }
792
+ if (!this.resumeAttemptId || this.resumeAttemptId !== attemptId)
793
+ return { ok: false, error: `manager expects resume attempt ${this.resumeAttemptId ?? "<none>"}, not ${attemptId}` };
794
+ if (this.resumeCommitted)
795
+ return {
796
+ ok: true,
797
+ data: {
798
+ attemptId,
799
+ state: this.resumeFinalized ? "active" : "awaitingFinalize",
800
+ durableCommitToken: this.resumeDurableCommitToken,
801
+ },
802
+ };
803
+ if (this.resumeCommitTask)
804
+ return this.resumeCommitTask;
805
+ const task = this.commitResumeActivation(attemptId);
806
+ this.resumeCommitTask = task;
807
+ try {
808
+ return await task;
809
+ }
810
+ finally {
811
+ if (this.resumeCommitTask === task)
812
+ this.resumeCommitTask = undefined;
813
+ }
814
+ }
815
+ if (req.op === "resumePreserved") {
816
+ if (tier !== CONTROL_ADMIN)
817
+ return { ok: false, error: "resumePreserved is admin-only; not allowed on this control subject" };
818
+ try {
819
+ const args = parseResumeControlArgs(req.args);
820
+ const inventoryDigest = createHash("sha256").update(JSON.stringify(args.inventory)).digest("hex");
821
+ if (!this.resumeAttemptId)
822
+ return { ok: false, error: "resumePreserved requires a manager started with --resume-attempt" };
823
+ if (this.resumeAttemptId !== args.attemptId)
824
+ return { ok: false, error: `manager expects resume attempt ${this.resumeAttemptId}, not ${args.attemptId}` };
825
+ if (this.resumeInventoryDigest && this.resumeInventoryDigest !== inventoryDigest)
826
+ return { ok: false, error: `resume attempt ${args.attemptId} is already bound to a different inventory` };
827
+ if (!this.resumeInventoryDigest) {
828
+ this.resumeInventoryDigest = inventoryDigest;
829
+ this.resumeInventory = args.inventory;
830
+ }
831
+ if (!this.resumeTask && !this.resumeResult) {
832
+ this.resumeTask = this.resumePreserved(args.inventory).then((result) => {
833
+ if (result.ok || this.resumedAgentNames.size > 0)
834
+ this.resumeResult = result;
835
+ return result;
836
+ }).finally(() => {
837
+ this.resumeTask = undefined;
838
+ });
839
+ }
840
+ const result = this.resumeResult ?? await this.resumeTask;
841
+ const data = { attemptId: args.attemptId, state: result.ok ? "awaitingCommit" : "degraded", ...result };
842
+ return result.ok
843
+ ? { ok: true, data }
844
+ : { ok: false, data, error: result.error ?? "retained-agent resume failed" };
845
+ }
846
+ catch (e) {
847
+ return { ok: false, error: e.message };
848
+ }
849
+ }
850
+ if (req.op === "preparePreservation" || req.op === "commitPreservation" || req.op === "abortPreservation") {
851
+ if (tier !== CONTROL_ADMIN)
852
+ return { ok: false, error: `${req.op} is admin-only; not allowed on this control subject` };
853
+ if (this.resumeRequired)
854
+ return { ok: false, error: this.maintenanceError() };
855
+ const attemptId = String(req.args?.attemptId ?? "").trim();
856
+ if (!attemptId)
857
+ return { ok: false, error: `${req.op} requires attemptId` };
858
+ try {
859
+ if (req.op === "abortPreservation") {
860
+ this.abortPreservation(attemptId);
861
+ return { ok: true, data: { attemptId, state: "active" } };
862
+ }
863
+ const result = req.op === "preparePreservation"
864
+ ? await this.preparePreservation(attemptId)
865
+ : await this.commitPreservation(attemptId);
866
+ return result.ok
867
+ ? { ok: true, data: result }
868
+ : {
869
+ ok: false,
870
+ data: result,
871
+ error: `preservation incomplete: ${result.failures.map((f) => `${f.name}: ${f.error}`).join("; ")}`,
872
+ };
873
+ }
874
+ catch (e) {
875
+ return { ok: false, error: e.message };
876
+ }
877
+ }
878
+ const release = this.beginLifecycle();
879
+ if (!release)
880
+ return { ok: false, error: this.maintenanceError() };
881
+ try {
882
+ return await this.handleActive(req, tier);
883
+ }
884
+ finally {
885
+ release();
886
+ }
887
+ }
888
+ async commitResumeActivation(attemptId) {
889
+ if (!this.resumeAwaitingCommit || !this.resumeResult?.ok)
890
+ return { ok: false, error: `resume attempt ${attemptId} has no successful activation to commit` };
891
+ const inventory = this.resumeInventory;
892
+ if (!inventory)
893
+ return { ok: false, error: `resume attempt ${attemptId} has no bound inventory` };
894
+ const authority = await Promise.all(inventory.agents.map(async (entry) => {
895
+ try {
896
+ await this.validateRetainedAuthority(entry);
897
+ return undefined;
898
+ }
899
+ catch (e) {
900
+ return `${entry.name}: ${e.message}`;
901
+ }
902
+ }));
903
+ const drift = authority.filter((error) => error !== undefined);
904
+ if (drift.length)
905
+ return { ok: false, error: `resume attempt ${attemptId} retained authority changed before commit: ${drift.join("; ")}` };
906
+ let inactive;
907
+ try {
908
+ inactive = this.resumeLivenessErrors(inventory, this.ep.getRoster());
909
+ }
910
+ catch (e) {
911
+ return { ok: false, error: `resume attempt ${attemptId} cannot verify live principals: ${e.message}` };
912
+ }
913
+ if (inactive.length)
914
+ return { ok: false, error: `resume attempt ${attemptId} is not live at commit: ${inactive.join("; ")}` };
915
+ this.resumeAwaitingCommit = false;
916
+ this.resumeCommitted = true;
917
+ this.resumeDurableCommitToken ??= randomBytes(32).toString("hex");
918
+ return {
919
+ ok: true,
920
+ data: { attemptId, state: "awaitingFinalize", durableCommitToken: this.resumeDurableCommitToken },
921
+ };
922
+ }
923
+ async handleActive(req, tier) {
309
924
  const args = req.args ?? {};
310
925
  // `req.from.id` is non-forgeable in auth mode: serveControl rejects any request whose payload
311
926
  // `from.id` doesn't match the subject sender (endpoint.ts). In open mode there are no creds, so
@@ -401,6 +1016,47 @@ export class Manager {
401
1016
  managedPrincipal(a) {
402
1017
  return a.userOwner ? a.id : principalKey(DEV_OWNER, a.id).key;
403
1018
  }
1019
+ resumeLivenessErrors(inventory, roster) {
1020
+ const inactive = [];
1021
+ const expectedNames = new Set(inventory.agents.map((entry) => entry.name));
1022
+ for (const name of this.resumedAgentNames)
1023
+ if (!expectedNames.has(name))
1024
+ inactive.push(`${name} is not part of the bound inventory`);
1025
+ for (const entry of inventory.agents) {
1026
+ const managed = this.agents.get(entry.name);
1027
+ if (!managed) {
1028
+ inactive.push(`${entry.name} is no longer managed`);
1029
+ continue;
1030
+ }
1031
+ const expectedId = entry.identity.mode === "user"
1032
+ ? principalKey(entry.identity.owner, entry.identity.actor).key
1033
+ : entry.identity.id;
1034
+ const expectedPrincipal = entry.identity.mode === "user"
1035
+ ? expectedId
1036
+ : principalKey(DEV_OWNER, entry.identity.id).key;
1037
+ if (managed.id !== expectedId || this.managedPrincipal(managed) !== expectedPrincipal) {
1038
+ inactive.push(`${entry.name} no longer holds retained principal ${expectedPrincipal}`);
1039
+ continue;
1040
+ }
1041
+ if (managed.handle.name !== entry.name || managed.handle.kind !== entry.launch.runtime) {
1042
+ inactive.push(`${entry.name} is not attached to its exact retained ${entry.launch.runtime} handle`);
1043
+ continue;
1044
+ }
1045
+ try {
1046
+ if (managed.handle.status() !== "running") {
1047
+ inactive.push(`${entry.name} runtime is not running`);
1048
+ continue;
1049
+ }
1050
+ }
1051
+ catch (e) {
1052
+ inactive.push(`${entry.name} runtime status failed: ${e.message}`);
1053
+ continue;
1054
+ }
1055
+ if (!roster.some((presence) => presence.card.id === expectedPrincipal && presence.card.name === entry.name && presence.status !== "offline"))
1056
+ inactive.push(`${entry.name} principal ${expectedPrincipal} is not exactly present`);
1057
+ }
1058
+ return inactive;
1059
+ }
404
1060
  /** Self-despawn (P2b): stop the managed agent whose id == the authenticated caller. The
405
1061
  * no-name self-op can only ever resolve to the caller's OWN managed entry (ids are unique
406
1062
  * per spawn + non-forgeable in auth mode), never a peer — so it's structurally incapable of
@@ -412,7 +1068,7 @@ export class Manager {
412
1068
  return { ok: false, error: `self-stop: caller ${callerId} is not a managed agent` };
413
1069
  const graceful = args.graceful !== false;
414
1070
  this.stopHandle(target, graceful);
415
- this.freeSlot(target, true); // self-despawn is rate-floored (recycle churn)
1071
+ this.trackStoppedHandle(target, true);
416
1072
  return { ok: true, data: { name: target.name, stopped: true, graceful } };
417
1073
  }
418
1074
  // Plane-3 durable join/leave/list ops moved OFF the manager onto the server-side delivery daemon's
@@ -441,6 +1097,49 @@ export class Manager {
441
1097
  console.error(`stop ${a.name} (${a.id}): ${e.message}`);
442
1098
  }
443
1099
  }
1100
+ /** Keep an accepted stop inside the lifecycle drain until the runtime proves the child is gone,
1101
+ * so a maintenance prepare can never fence ahead of a child that is still dying.
1102
+ *
1103
+ * An operator-accepted stop frees its slot at once: `stop` replying ✓ means `ps` no longer lists
1104
+ * the agent. That cannot omit a still-live child from a cut, because runPreparation drains the
1105
+ * lifecycle BEFORE it reads the roster — the exit proof below is what closes the race, not the
1106
+ * slot lingering. A recursive reap (`requireAuthoritativeExit`) instead keeps the slot until the
1107
+ * wait proves exit: nobody asked for those children to be gone, so they stay managed until the
1108
+ * runtime says otherwise, and a runtime that cannot prove exit records an unverified stop. */
1109
+ trackStoppedHandle(a, floor, requireAuthoritativeExit = false) {
1110
+ if (!a.handle.waitForExit) {
1111
+ // Preserve ordinary external-runtime stop behavior, but retain enough evidence for a later
1112
+ // maintenance prepare to fail if that runtime still cannot prove the surface disappeared.
1113
+ this.unverifiedStops.push({
1114
+ name: a.name,
1115
+ id: a.id,
1116
+ handle: a.handle,
1117
+ authoritative: requireAuthoritativeExit,
1118
+ error: requireAuthoritativeExit
1119
+ ? `recursive reap cannot prove exit on runtime "${a.handle.kind}" (AgentHandle.waitForExit is not implemented)`
1120
+ : undefined,
1121
+ });
1122
+ if (requireAuthoritativeExit)
1123
+ return;
1124
+ this.freeSlot(a, floor, true);
1125
+ return;
1126
+ }
1127
+ if (!requireAuthoritativeExit)
1128
+ this.freeSlot(a, floor, true);
1129
+ this.lifecycleInFlight++;
1130
+ void this.awaitHandleExit(a.handle)
1131
+ .then(() => this.freeSlot(a, floor, true)) // no-op once an accepted stop already freed it
1132
+ .catch((e) => {
1133
+ this.unverifiedStops.push({
1134
+ name: a.name,
1135
+ id: a.id,
1136
+ handle: a.handle,
1137
+ authoritative: true,
1138
+ error: `accepted stop could not prove exit: ${e.message}`,
1139
+ });
1140
+ })
1141
+ .finally(() => this.releaseLifecycle());
1142
+ }
444
1143
  /** USER-MODE spawn provisioning (the gate-1 counterpart to the static mint block): resolve the
445
1144
  * OWNER (ctl caller's principal, or the manifest's stamped owner — never a payload field),
446
1145
  * pre-create the principal-keyed durables + ACL row on the ephemeral provisioner, author the
@@ -478,6 +1177,11 @@ export class Manager {
478
1177
  // the spawner's own grant), so a refused delegation exits here having touched nothing beyond
479
1178
  // the ledger: no durables, no broker footprint, nothing for a corrected respawn to race.
480
1179
  const grant = await provider.grantAgent({
1180
+ // LOCAL composition, hardcoded: until the manager's entry is store-threaded (the same
1181
+ // later slice as its renewal-owner store, with/after the membership-rw reader migration),
1182
+ // a pure-KMS hosted manager CANNOT read the callout material this grant needs — hosted
1183
+ // user-mode spawn via the manager is UNAVAILABLE, not silently degraded, until then.
1184
+ store: workspaceSecretStore(this.workspaceRoot),
481
1185
  dir,
482
1186
  space: this.space,
483
1187
  owner,
@@ -537,7 +1241,7 @@ export class Manager {
537
1241
  * expires — flooring the RECYCLE, not the call, so both free paths (despawn + exit/reap) are
538
1242
  * covered (P4c). Floor self + own-child despawn and natural exit; NEVER admin despawn (operator
539
1243
  * emergency-kill stays unthrottled) and NEVER the reserved-rollback path (no cold-start paid). */
540
- freeSlot(a, floor) {
1244
+ freeSlot(a, floor, acceptedBeforeFence = false) {
541
1245
  if (this.agents.get(a.name) !== a)
542
1246
  return; // already freed (exit raced despawn, etc.)
543
1247
  this.agents.delete(a.name);
@@ -547,7 +1251,8 @@ export class Manager {
547
1251
  // process is already gone, so this must never block the slot free or throw into the caller — it runs
548
1252
  // detached, and a failure is logged loudly (never swallowed), not retried. The `agents` guard above
549
1253
  // makes this fire exactly once per agent across every free path (despawn / self-stop / reap / exit).
550
- void this.deprovision(a).catch((e) => console.error(`deprovision ${a.name} (${a.id}): ${e.message}`));
1254
+ if (!a.suppressCleanup && (this.maintenanceState === "active" || acceptedBeforeFence))
1255
+ this.trackDeprovision(a);
551
1256
  }
552
1257
  /** Tear down a departed agent's minted footprint (#159 B2, auth mode): its local-principal durables
553
1258
  * (`dm_local-<id>`, `dlv_local-<id>`), its read-ACL row, and its creds file — everything the spawn's
@@ -593,24 +1298,29 @@ export class Manager {
593
1298
  // helper's own fail-fast connect). The durables/ACL row still fall to space teardown as a backstop.
594
1299
  await withTimeout(deprovisionAgent({ servers: this.servers ?? DEFAULT_SERVER, space: this.space, targetId: a.id, creds }), DEPROVISION_TIMEOUT_MS, `deprovision ${a.name} (${a.id}): broker teardown timed out`);
595
1300
  }
596
- /** Reap a parent's children on its exit (P4b): stop + free every agent whose `spawner` is the
597
- * exited agent's id, so orphans don't ratchet the ceiling shut. Recursive a reaped child's
598
- * own children are reaped too. Exit-driven, so each freed slot is rate-floored like a despawn. */
1301
+ /** Reap a parent's children on its exit (P4b). Every descendant remains managed until the runtime's
1302
+ * authoritative wait proves exit; the wait participates in the lifecycle drain, so preservation can
1303
+ * never omit a child that may still be alive. Recursive descendants are scheduled before their parent
1304
+ * slot can disappear. */
599
1305
  reapChildrenOf(parentId) {
600
1306
  for (const child of [...this.agents.values()]) {
601
1307
  if (child.spawner !== parentId)
602
1308
  continue;
1309
+ this.reapChildrenOf(this.managedPrincipal(child));
603
1310
  this.stopHandle(child, false);
604
- this.freeSlot(child, true);
605
- this.reapChildrenOf(child.id);
1311
+ this.trackStoppedHandle(child, true, true);
606
1312
  }
607
1313
  }
608
1314
  /** A managed agent's process exited on its own (crash, /exit, finished). Free its slot
609
1315
  * (rate-floored — exit-driven churn counts) and reap any children it spawned. Idempotent via
610
1316
  * freeSlot's identity guard, so a later graceful-stop SIGKILL firing exit again is a no-op. */
611
1317
  onAgentExit(a) {
1318
+ // Preservation owns the child-stop snapshot. Exit watchers must neither delete that snapshot nor
1319
+ // trigger normal deprovision/reap while the cut is being formed.
1320
+ if (this.maintenanceState !== "active")
1321
+ return;
612
1322
  this.freeSlot(a, true);
613
- this.reapChildrenOf(a.id);
1323
+ this.reapChildrenOf(this.managedPrincipal(a));
614
1324
  }
615
1325
  /** Agent names become `.cotal/agents/<name>.md` paths and mesh identities, so they must be bare
616
1326
  * tokens, never a path — blocks traversal / arbitrary writes from a model-supplied name. */
@@ -835,7 +1545,7 @@ export class Manager {
835
1545
  catch (e) {
836
1546
  return { ok: false, error: e.message };
837
1547
  }
838
- const reply = await this.startAgent(launchAgentToStartOpts(la, configPath, spec.owner), caller);
1548
+ const reply = await this.startAgent(launchAgentToStartOpts(la, configPath, spec.owner, runId), caller);
839
1549
  if (reply.ok)
840
1550
  // `data.name` stays the spawned (numbered) identity — what creds are filed under and the ledger
841
1551
  // keys on; `requested`/`runId`/`hash` give the CLI the manifest name + drift hash for the ledger.
@@ -849,6 +1559,17 @@ export class Manager {
849
1559
  * defaulting to the manager's own id for roster/pre-spawn — recorded for the spawner
850
1560
  * ledger (own-children despawn + reap-on-parent-exit). */
851
1561
  async startAgent(opts, spawner) {
1562
+ const release = this.beginLifecycle();
1563
+ if (!release)
1564
+ return { ok: false, error: this.maintenanceError() };
1565
+ try {
1566
+ return await this.startAgentActive(opts, spawner);
1567
+ }
1568
+ finally {
1569
+ release();
1570
+ }
1571
+ }
1572
+ async startAgentActive(opts, spawner) {
852
1573
  // The spawn argument is a persona REF — a filename in `.cotal/agents` (the unique spawn KEY), or
853
1574
  // a path via `--config`. It is NOT the mesh identity: the identity comes from inside the file
854
1575
  // (`name:`), so a persona can be filed descriptively (review-critic.md) yet present under a
@@ -1050,6 +1771,11 @@ export class Manager {
1050
1771
  // arbitrary folders/repos. A relative path resolves against the workspace root; omitted → the
1051
1772
  // agent shares the workspace root (the prior, unchanged behavior).
1052
1773
  const cwd = opts.cwd ? resolve(this.workspaceRoot, opts.cwd) : this.workspaceRoot;
1774
+ const configSha256 = this.fileDigest(configPath);
1775
+ const manifestPath = opts.launchRef
1776
+ ? join(this.workspaceRoot, ".cotal", "run", `${opts.launchRef.runId}.json`)
1777
+ : undefined;
1778
+ const manifestSha256 = manifestPath ? this.fileDigest(manifestPath) : undefined;
1053
1779
  const spec = connector.buildLaunch({
1054
1780
  space: this.space,
1055
1781
  name,
@@ -1092,9 +1818,38 @@ export class Manager {
1092
1818
  id: userLaunch ? principalKey(userLaunch.owner, name).key : identity.id,
1093
1819
  ...(userLaunch ? { userOwner } : { seed: identity.seed }),
1094
1820
  spawner: spawner ?? this.ep.ref().id,
1821
+ authorityParent: userLaunch && spawner && parsePrincipalKey(spawner) ? spawner : undefined,
1095
1822
  startedAt: Date.now(),
1096
1823
  handle,
1097
1824
  control: spec.control,
1825
+ launch: {
1826
+ source: opts.resolved
1827
+ ? {
1828
+ kind: "manifest",
1829
+ runId: opts.launchRef?.runId,
1830
+ requested: opts.launchRef?.requested ?? opts.resolved.name,
1831
+ hash: opts.launchRef?.hash ?? opts.resolved.hash,
1832
+ configPath,
1833
+ configSha256,
1834
+ manifestSha256,
1835
+ }
1836
+ : { kind: "persona", ref, configPath, configSha256 },
1837
+ cwd,
1838
+ model,
1839
+ variant,
1840
+ subscribe,
1841
+ allowSubscribe,
1842
+ allowPublish,
1843
+ capabilities,
1844
+ transcript,
1845
+ shareTools: opts.shareTools,
1846
+ forkSource: opts.resume,
1847
+ // Opaque values may contain secrets. Preserve only their keys and require the referenced
1848
+ // persona/manifest to resolve the values again; imperative overrides have no safe payload.
1849
+ unresolvedLaunchOptionKeys: opts.launchOptions && Object.keys(opts.launchOptions).length
1850
+ ? Object.keys(opts.launchOptions).sort()
1851
+ : undefined,
1852
+ },
1098
1853
  };
1099
1854
  this.agents.set(name, managed);
1100
1855
  // The live slot now owns teardown — freeSlot deprovisions this identity on exit — so the
@@ -1128,9 +1883,370 @@ export class Manager {
1128
1883
  // orphan down (detached, fail-loud) so a failed spawn leaves no creds/durables behind (#159 B).
1129
1884
  if (provisioned) {
1130
1885
  const orphan = provisioned;
1131
- void this.deprovision(orphan).catch((e) => console.error(`deprovision (orphaned spawn) ${orphan.name}: ${e.message}`));
1886
+ this.trackDeprovision(orphan, "(orphaned spawn)");
1887
+ }
1888
+ }
1889
+ }
1890
+ /** Preflight the whole inventory before launching its first process, then adopt each exact retained
1891
+ * principal without provisioning. A later runtime launch failure is reported per-agent, but malformed
1892
+ * or missing inventory material can never produce a partially resumed set. */
1893
+ async resumePreserved(inventory) {
1894
+ const release = this.beginLifecycle(true);
1895
+ if (!release)
1896
+ return { ok: false, agents: [], error: this.maintenanceError() };
1897
+ const batchReservations = [];
1898
+ try {
1899
+ if (inventory.version !== "cotal-manager-resume/v1")
1900
+ return { ok: false, agents: [], error: `unsupported manager resume inventory version ${String(inventory.version)}` };
1901
+ if (inventory.space !== this.space)
1902
+ return { ok: false, agents: [], error: `resume inventory belongs to space "${inventory.space}", not "${this.space}"` };
1903
+ const seen = new Set();
1904
+ const principals = new Set();
1905
+ await this.ep.waitForPresenceSnapshot();
1906
+ const livePrincipals = new Set(this.ep.getRoster()
1907
+ .filter((presence) => presence.status !== "offline")
1908
+ .map((presence) => presence.card.id));
1909
+ if (this.agents.size + this.reserved.size + this.coolingCount() + inventory.agents.length > MAX_AGENTS)
1910
+ return { ok: false, agents: [], error: `resume inventory would exceed manager capacity (${MAX_AGENTS})` };
1911
+ for (const entry of inventory.agents) {
1912
+ if (seen.has(entry.name))
1913
+ return { ok: false, agents: [], error: `resume inventory contains duplicate agent name "${entry.name}"` };
1914
+ seen.add(entry.name);
1915
+ let principal;
1916
+ try {
1917
+ principal = entry.identity.mode === "user"
1918
+ ? principalKey(entry.identity.owner, entry.identity.actor).key
1919
+ : principalKey(DEV_OWNER, entry.identity.id).key;
1920
+ }
1921
+ catch (e) {
1922
+ return { ok: false, agents: [], error: `invalid retained principal for ${entry.name}: ${e.message}` };
1923
+ }
1924
+ if (principals.has(principal))
1925
+ return { ok: false, agents: [], error: `resume inventory contains duplicate principal "${principal}"` };
1926
+ principals.add(principal);
1927
+ if (livePrincipals.has(principal))
1928
+ return { ok: false, agents: [], error: `retained principal "${principal}" is already live and this runtime cannot authoritatively adopt it` };
1929
+ if (this.agents.has(entry.name) || this.reserved.has(entry.name))
1930
+ return { ok: false, agents: [], error: `retained agent "${entry.name}" is already managed or reserved` };
1931
+ }
1932
+ for (const entry of inventory.agents) {
1933
+ this.reserved.add(entry.name);
1934
+ batchReservations.push(entry.name);
1935
+ }
1936
+ const prepared = new Map();
1937
+ const preflight = [];
1938
+ for (const entry of inventory.agents) {
1939
+ const reply = await this.resumePreservedAgent(entry, true, true, prepared);
1940
+ preflight.push({ name: entry.name, reply });
1941
+ }
1942
+ const preflightFailures = preflight.filter(({ reply }) => !reply.ok);
1943
+ if (preflightFailures.length)
1944
+ return {
1945
+ ok: false,
1946
+ agents: preflight,
1947
+ error: `${preflightFailures.length} retained agent${preflightFailures.length === 1 ? "" : "s"} failed preflight`,
1948
+ };
1949
+ const agents = [];
1950
+ for (let i = 0; i < inventory.agents.length; i++) {
1951
+ const entry = inventory.agents[i];
1952
+ const reply = await this.resumePreservedAgent(entry, false, true, prepared);
1953
+ agents.push({ name: entry.name, reply });
1954
+ if (!reply.ok) {
1955
+ for (const skipped of inventory.agents.slice(i + 1))
1956
+ agents.push({ name: skipped.name, reply: { ok: false, error: `not launched because ${entry.name} failed` } });
1957
+ return { ok: false, agents, error: reply.error };
1958
+ }
1959
+ }
1960
+ if (this.resumeAttemptId)
1961
+ this.resumeAwaitingCommit = true;
1962
+ return { ok: true, agents };
1963
+ }
1964
+ finally {
1965
+ for (const name of batchReservations)
1966
+ this.reserved.delete(name);
1967
+ release();
1968
+ }
1969
+ }
1970
+ /** Re-read every retained identity input and its current authority without provisioning. This runs
1971
+ * during whole-inventory preflight, immediately before each individual spawn, and at commit. */
1972
+ async validateRetainedAuthority(entry) {
1973
+ const referenceError = this.inventoryReferenceError(entry);
1974
+ if (referenceError)
1975
+ throw new Error(`retained agent ${entry.name}: ${referenceError}`);
1976
+ if (entry.identity.mode === "open") {
1977
+ if (this.auth || this.userMode)
1978
+ throw new Error(`retained agent ${entry.name} is open-mode but the current manager is authenticated`);
1979
+ return { id: entry.identity.id };
1980
+ }
1981
+ if (entry.identity.mode === "static") {
1982
+ if (!this.auth || this.userMode)
1983
+ throw new Error(`retained agent ${entry.name} is static-auth but the current manager is not`);
1984
+ const expected = resolve(authDir(this.workspaceRoot), "creds", `${entry.name}.creds`);
1985
+ if (resolve(entry.identity.credential.path) !== expected)
1986
+ throw new Error(`retained credential reference for ${entry.name} is not the manager-owned path ${expected}`);
1987
+ let credentialText;
1988
+ try {
1989
+ const st = lstatSync(expected);
1990
+ if (!st.isFile() || st.isSymbolicLink())
1991
+ throw new Error("not a regular non-symlink file");
1992
+ credentialText = readFileSync(expected, "utf8");
1993
+ const actual = idFromCreds(credentialText);
1994
+ if (actual !== entry.identity.id)
1995
+ throw new Error(`retained credential identity ${actual} does not match inventory principal ${entry.identity.id}`);
1996
+ }
1997
+ catch (e) {
1998
+ throw new Error(`retained credential for ${entry.name} is unusable: ${e.message}`);
1999
+ }
2000
+ const accepted = await this.probeStaticCredential(credentialText);
2001
+ if (!accepted.ok)
2002
+ throw new Error(`retained credential for ${entry.name} is not accepted by the current broker (${accepted.reason})`);
2003
+ return { id: entry.identity.id, creds: expected };
2004
+ }
2005
+ if (!this.userMode)
2006
+ throw new Error(`retained agent ${entry.name} is user-auth but the current manager is not`);
2007
+ try {
2008
+ const provider = resolveAuthProvider();
2009
+ const actorToken = readFileSync(entry.identity.actorToken.path, "utf8");
2010
+ const sentinelCreds = readFileSync(entry.identity.sentinelCredential.path, "utf8");
2011
+ const adopted = await provider.validateRetainedAgent({
2012
+ store: workspaceSecretStore(this.workspaceRoot),
2013
+ dir: userAuthStateDir(this.workspaceRoot, this.space),
2014
+ space: this.space,
2015
+ owner: entry.identity.owner,
2016
+ actor: entry.identity.actor,
2017
+ actorToken,
2018
+ sentinelCreds,
2019
+ });
2020
+ if (adopted.owner !== entry.identity.owner || adopted.actor !== entry.identity.actor)
2021
+ throw new Error(`auth provider returned a replacement principal; expected ${entry.identity.owner}.${entry.identity.actor}`);
2022
+ if (!sameStrings(adopted.allowSubscribe, entry.launch.allowSubscribe) ||
2023
+ !sameStrings(adopted.allowPublish, entry.launch.allowPublish) ||
2024
+ !sameStrings(adopted.scope, entry.launch.capabilities) ||
2025
+ adopted.role !== entry.role || adopted.parent !== entry.authorityParent)
2026
+ throw new Error(`retained user authority for ${entry.identity.owner}.${entry.identity.actor} no longer matches the inventory`);
2027
+ return {
2028
+ userAuth: {
2029
+ owner: entry.identity.owner,
2030
+ actor: entry.identity.actor,
2031
+ sentinelCredsPath: entry.identity.sentinelCredential.path,
2032
+ bearerCmd: [
2033
+ process.execPath,
2034
+ ...process.execArgv,
2035
+ process.argv[1],
2036
+ provider.agentBearerCommand,
2037
+ "--dir", userAuthStateDir(this.workspaceRoot, this.space),
2038
+ "--space", this.space,
2039
+ "--owner", entry.identity.owner,
2040
+ "--actor", entry.identity.actor,
2041
+ "--token-file", entry.identity.actorToken.path,
2042
+ "--health-file", entry.identity.health.path,
2043
+ ],
2044
+ },
2045
+ };
2046
+ }
2047
+ catch (e) {
2048
+ throw new Error(`retained user principal ${entry.identity.owner}.${entry.identity.actor} could not be reused: ${e.message}`);
2049
+ }
2050
+ }
2051
+ /** Validate/relaunch one retained inventory entry. Called only through resumePreserved so all
2052
+ * records pass the same preflight before the first child is exposed. */
2053
+ async resumePreservedAgent(entry, preflightOnly = false, batchReserved = false, prepared) {
2054
+ const release = this.beginLifecycle(batchReserved);
2055
+ if (!release)
2056
+ return { ok: false, error: this.maintenanceError() };
2057
+ try {
2058
+ if (entry.space !== this.space)
2059
+ return { ok: false, error: `retained agent ${entry.name} belongs to space "${entry.space}", not "${this.space}"` };
2060
+ if (entry.launch.runtime !== this.runtime.kind)
2061
+ return { ok: false, error: `retained agent ${entry.name} requires runtime "${entry.launch.runtime}", current manager uses "${this.runtime.kind}"` };
2062
+ const nameErr = this.nameError(entry.name);
2063
+ if (nameErr)
2064
+ return { ok: false, error: nameErr };
2065
+ if (this.agents.has(entry.name) || (!batchReserved && this.reserved.has(entry.name)))
2066
+ return { ok: false, error: `retained agent "${entry.name}" is already managed or reserved; same-principal resume never auto-numbers` };
2067
+ if (!batchReserved && this.agents.size + this.reserved.size + this.coolingCount() >= MAX_AGENTS)
2068
+ return { ok: false, error: `at capacity (${MAX_AGENTS} agents incl. in-flight + cooling); same-principal resume refused` };
2069
+ const cached = prepared?.get(entry.name);
2070
+ if (!preflightOnly && cached) {
2071
+ try {
2072
+ // Do not trust the earlier batch preflight across another agent's sequential readiness wait.
2073
+ await this.validateRetainedAuthority(entry);
2074
+ }
2075
+ catch (e) {
2076
+ return { ok: false, error: e.message };
2077
+ }
2078
+ return this.launchPreparedResume(entry, cached, batchReserved);
2079
+ }
2080
+ try {
2081
+ const cwd = lstatSync(entry.launch.cwd);
2082
+ if (!cwd.isDirectory() || cwd.isSymbolicLink())
2083
+ return { ok: false, error: `retained cwd is not a real directory: ${entry.launch.cwd}` };
2084
+ }
2085
+ catch (e) {
2086
+ return { ok: false, error: `retained cwd unavailable: ${entry.launch.cwd} (${e.message})` };
2087
+ }
2088
+ let connector;
2089
+ try {
2090
+ connector = registry.resolve("connector", entry.launch.connector);
2091
+ }
2092
+ catch (e) {
2093
+ return { ok: false, error: e.message };
2094
+ }
2095
+ const missing = (connector.requires ?? []).filter((bin) => !resolveOnPath(bin));
2096
+ if (missing.length)
2097
+ return { ok: false, error: `${connector.name} harness needs ${missing.join(", ")} on PATH - not found` };
2098
+ if (entry.launch.variant && !connector.supportsModelVariant)
2099
+ return { ok: false, error: `${connector.name} connector does not support model variants (variant)` };
2100
+ let launchOptions;
2101
+ if (entry.launch.source.kind === "manifest") {
2102
+ const launchSource = entry.launch.source;
2103
+ if (!launchSource.runId)
2104
+ return { ok: false, error: `retained manifest launch for ${entry.name} has no runId; refusing to guess a .cotal/run source` };
2105
+ let spec;
2106
+ try {
2107
+ const source = launchSpecForRun(this.workspaceRoot, launchSource.runId);
2108
+ if (source.space !== this.space)
2109
+ return { ok: false, error: `retained launch spec space "${source.space}" does not match manager space "${this.space}"` };
2110
+ spec = source.agents.find((a) => a.name === launchSource.requested);
2111
+ }
2112
+ catch (e) {
2113
+ return { ok: false, error: e.message };
2114
+ }
2115
+ if (!spec || spec.hash !== launchSource.hash)
2116
+ return { ok: false, error: `retained manifest agent ${launchSource.requested} is missing or its hash changed; refusing same-principal resume` };
2117
+ launchOptions = spec.launchOptions;
2118
+ }
2119
+ else {
2120
+ try {
2121
+ launchOptions = loadAgentFile(entry.launch.source.configPath).launchOptions;
2122
+ }
2123
+ catch (e) {
2124
+ return { ok: false, error: e.message };
2125
+ }
2126
+ }
2127
+ let authority;
2128
+ try {
2129
+ authority = await this.validateRetainedAuthority(entry);
2130
+ }
2131
+ catch (e) {
2132
+ return { ok: false, error: e.message };
2133
+ }
2134
+ try {
2135
+ const mcpServers = connectorServers(loadCotalConfig(this.workspaceRoot), entry.launch.connector, parseShareSelection(entry.launch.shareTools));
2136
+ const spec = connector.buildLaunch({
2137
+ space: this.space,
2138
+ name: entry.name,
2139
+ role: entry.role,
2140
+ id: authority.id,
2141
+ creds: authority.creds,
2142
+ userAuth: authority.userAuth,
2143
+ servers: this.servers,
2144
+ configPath: entry.launch.source.configPath,
2145
+ model: entry.launch.model,
2146
+ variant: entry.launch.variant,
2147
+ launchOptions,
2148
+ resume: entry.launch.forkSource,
2149
+ subscribe: entry.launch.subscribe,
2150
+ allowSubscribe: entry.launch.allowSubscribe,
2151
+ allowPublish: entry.launch.allowPublish,
2152
+ capabilities: entry.launch.capabilities,
2153
+ transcript: entry.launch.transcript,
2154
+ mcpServers,
2155
+ workspaceRoot: this.workspaceRoot,
2156
+ });
2157
+ const value = { spec, ...authority };
2158
+ prepared?.set(entry.name, value);
2159
+ if (preflightOnly)
2160
+ return { ok: true, data: { name: entry.name, preflight: true } };
2161
+ return this.launchPreparedResume(entry, value, batchReserved);
2162
+ }
2163
+ catch (e) {
2164
+ return { ok: false, error: e.message };
1132
2165
  }
1133
2166
  }
2167
+ finally {
2168
+ release();
2169
+ }
2170
+ }
2171
+ async launchPreparedResume(entry, prepared, batchReserved) {
2172
+ if (!batchReserved)
2173
+ this.reserved.add(entry.name);
2174
+ try {
2175
+ const handle = this.runtime.spawn(entry.name, prepared.spec, entry.launch.cwd);
2176
+ const managed = {
2177
+ name: entry.name,
2178
+ role: entry.role,
2179
+ agent: entry.launch.connector,
2180
+ id: entry.identity.mode === "user" ? principalKey(entry.identity.owner, entry.identity.actor).key : entry.identity.id,
2181
+ userOwner: entry.identity.mode === "user" ? entry.identity.owner : undefined,
2182
+ spawner: entry.spawner,
2183
+ authorityParent: entry.authorityParent,
2184
+ startedAt: Date.now(),
2185
+ handle,
2186
+ control: prepared.spec.control,
2187
+ launch: {
2188
+ source: entry.launch.source,
2189
+ cwd: entry.launch.cwd,
2190
+ model: entry.launch.model,
2191
+ variant: entry.launch.variant,
2192
+ subscribe: entry.launch.subscribe,
2193
+ allowSubscribe: entry.launch.allowSubscribe,
2194
+ allowPublish: entry.launch.allowPublish,
2195
+ capabilities: entry.launch.capabilities,
2196
+ transcript: entry.launch.transcript,
2197
+ shareTools: entry.launch.shareTools,
2198
+ forkSource: entry.launch.forkSource,
2199
+ },
2200
+ suppressCleanup: true,
2201
+ };
2202
+ this.agents.set(entry.name, managed);
2203
+ if (this.resumeAttemptId)
2204
+ this.resumedAgentNames.add(entry.name);
2205
+ const readiness = await this.awaitReadiness(managed);
2206
+ if (!readiness.ok && !readiness.uncertain)
2207
+ return { ok: false, error: readiness.detail };
2208
+ if (!readiness.ok) {
2209
+ this.watchExit(managed);
2210
+ this.watchResumeAdoption(managed);
2211
+ return { ok: false, error: readiness.detail };
2212
+ }
2213
+ if (!this.resumeAttemptId)
2214
+ managed.suppressCleanup = false;
2215
+ this.watchExit(managed);
2216
+ if (this.agents.get(managed.name) !== managed)
2217
+ return { ok: false, error: `${managed.name} exited immediately after same-principal readiness` };
2218
+ return {
2219
+ ok: true,
2220
+ data: { name: managed.name, role: managed.role, agent: managed.agent, id: managed.id, mode: handle.kind, resumed: true },
2221
+ };
2222
+ }
2223
+ catch (e) {
2224
+ return { ok: false, error: e.message };
2225
+ }
2226
+ finally {
2227
+ if (!batchReserved)
2228
+ this.reserved.delete(entry.name);
2229
+ }
2230
+ }
2231
+ probeStaticCredential(creds) {
2232
+ return probeConnect(this.servers ?? DEFAULT_SERVER, { creds, timeoutMs: 5_000 });
2233
+ }
2234
+ /** An uncertain resume remains non-destructive until exact-principal presence arrives later. */
2235
+ watchResumeAdoption(a) {
2236
+ const wanted = this.managedPrincipal(a);
2237
+ const onPresence = () => {
2238
+ if (this.agents.get(a.name) !== a) {
2239
+ this.ep.off("presence", onPresence);
2240
+ return;
2241
+ }
2242
+ if (!this.ep.getRoster().some((p) => p.card.id === wanted && p.status !== "offline"))
2243
+ return;
2244
+ if (!this.resumeRequired)
2245
+ a.suppressCleanup = false;
2246
+ this.ep.off("presence", onPresence);
2247
+ };
2248
+ this.ep.on("presence", onPresence);
2249
+ onPresence();
1134
2250
  }
1135
2251
  /** #159 B1: wait for a detached launch to reach a REAL outcome before replying — never a liveness-
1136
2252
  * inferring timer. Races three:
@@ -1255,7 +2371,7 @@ export class Manager {
1255
2371
  return { ok: false, error: denied };
1256
2372
  const graceful = args.graceful !== false;
1257
2373
  this.stopHandle(a, graceful);
1258
- this.freeSlot(a, !admin); // own-child despawn is rate-floored; admin emergency-kill is not
2374
+ this.trackStoppedHandle(a, !admin);
1259
2375
  return { ok: true, data: { name, stopped: true, graceful } };
1260
2376
  }
1261
2377
  /** Open a short-lived PROVISIONER connection, run the onboarding ops on it, and drain it (closure (ii),