@cotal-ai/manager 0.57.0 → 0.59.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. package/README.md +13 -0
  2. package/dist/commands.d.ts.map +1 -1
  3. package/dist/commands.js +98 -16
  4. package/dist/commands.js.map +1 -1
  5. package/dist/console/session-bundle.js +1 -1
  6. package/dist/endpoint-evict.d.ts +9 -4
  7. package/dist/endpoint-evict.d.ts.map +1 -1
  8. package/dist/endpoint-evict.js +82 -26
  9. package/dist/endpoint-evict.js.map +1 -1
  10. package/dist/holder-liveness.d.ts +4 -0
  11. package/dist/holder-liveness.d.ts.map +1 -1
  12. package/dist/holder-liveness.js +95 -3
  13. package/dist/holder-liveness.js.map +1 -1
  14. package/dist/index.d.ts +5 -1
  15. package/dist/index.d.ts.map +1 -1
  16. package/dist/index.js +3 -0
  17. package/dist/index.js.map +1 -1
  18. package/dist/manager-service-contract.d.ts +15 -6
  19. package/dist/manager-service-contract.d.ts.map +1 -1
  20. package/dist/manager-service-contract.js +84 -13
  21. package/dist/manager-service-contract.js.map +1 -1
  22. package/dist/manager.d.ts +165 -67
  23. package/dist/manager.d.ts.map +1 -1
  24. package/dist/manager.js +706 -233
  25. package/dist/manager.js.map +1 -1
  26. package/dist/remote-authority.d.ts +54 -3
  27. package/dist/remote-authority.d.ts.map +1 -1
  28. package/dist/remote-authority.js +168 -2
  29. package/dist/remote-authority.js.map +1 -1
  30. package/dist/resume.d.ts.map +1 -1
  31. package/dist/resume.js +3 -0
  32. package/dist/resume.js.map +1 -1
  33. package/dist/run-hosting.d.ts +66 -1
  34. package/dist/run-hosting.d.ts.map +1 -1
  35. package/dist/run-hosting.js +126 -14
  36. package/dist/run-hosting.js.map +1 -1
  37. package/dist/runtime/custodial-pty.d.ts +12 -9
  38. package/dist/runtime/custodial-pty.d.ts.map +1 -1
  39. package/dist/runtime/custodial-pty.js +27 -16
  40. package/dist/runtime/custodial-pty.js.map +1 -1
  41. package/dist/runtime/index.d.ts +2 -0
  42. package/dist/runtime/index.d.ts.map +1 -1
  43. package/dist/runtime/index.js +148 -5
  44. package/dist/runtime/index.js.map +1 -1
  45. package/dist/runtime/pty.d.ts +3 -4
  46. package/dist/runtime/pty.d.ts.map +1 -1
  47. package/dist/runtime/pty.js +13 -6
  48. package/dist/runtime/pty.js.map +1 -1
  49. package/package.json +15 -13
  50. package/dist/ui.d.ts +0 -10
  51. package/dist/ui.d.ts.map +0 -1
  52. package/dist/ui.js +0 -10
  53. package/dist/ui.js.map +0 -1
package/dist/manager.js CHANGED
@@ -4,11 +4,11 @@ import { hostname } from "node:os";
4
4
  import { credsAuthenticator } from "@nats-io/transport-node";
5
5
  import { existsSync, lstatSync, readFileSync, realpathSync, rmSync, statSync } from "node:fs";
6
6
  import { isAbsolute, join, dirname, resolve } from "node:path";
7
- import { CotalEndpoint, DEFAULT_SERVER, DEV_OWNER, MANAGER_LEASE_RENEW_MS, STANDING_RENEWABLE_TTL_SEC, assertLifecycleToken, divergentSecretStoreNotice, parseDaemonStoreAnswer, parseSecretStoreIdentity, sameSecretStoreIdentity, agentFilePath, clearSpaceHistory, dialerFor, connectorServers, spawnEnvAllow, deprovisionAgent, firstFreeName, spawnNameError, idFromCreds, inspectCredHealth, credsRenewalDelayMs, composeWirePersona, loadAgentFile, listPersonaCatalog, personaCatalogDescription, personaCatalogReadable, loadCotalConfig, mintCreds, mintGeneration, mintAcceptedToken, openIssuedStore, acceptedBucket, issuedBucket, mintLifecycleUid, managedRetirementOpId, mkSecretDir, newIdentity, actionContext, parsePrincipalKey, parseShareSelection, principalKey, probeConnect, provisionAgent, provisionAgentDurables, registry, resolveAuthProvider, saveAgentFile, subjectMatches, AUTH_ENDPOINT, EP_CMD_RETIRE_LIFECYCLE, epRequestSubject, epCallerReplyFilter, parseEpSubject, controlServiceSubject, eventChannelPrincipal, } from "@cotal-ai/core";
7
+ import { CotalEndpoint, accountFromCreds, credsClaims, DEFAULT_SERVER, DEV_OWNER, MANAGER_LEASE_RENEW_MS, STANDING_RENEWABLE_TTL_SEC, assertLifecycleToken, discardLaunchArtifacts, divergentSecretStoreNotice, parseDaemonStoreAnswer, parseSecretStoreIdentity, sameSecretStoreIdentity, agentFilePath, clearSpaceHistory, dialerFor, connectorServers, spawnEnvAllow, deprovisionAgent, DeprovisionError, isConcreteChannel, deprovisionTargetPrincipal, firstFreeName, spawnNameError, idFromCreds, inspectCredHealth, credsRenewalDelayMs, composeWirePersona, loadAgentFile, listPersonaCatalog, personaCatalogDescription, personaCatalogReadable, loadCotalConfig, mintCreds, mintGeneration, mintAcceptedToken, openIssuedStore, acceptedBucket, issuedBucket, mintLifecycleUid, managedRetirementOpId, mkSecretDir, newIdentity, actionContext, parsePrincipalKey, parseShareSelection, principalKey, probeConnect, provisionAgent, provisionAgentDurables, registry, resolveAuthProvider, saveAgentFile, subjectMatches, AUTH_ENDPOINT, EP_CMD_RETIRE_LIFECYCLE, controlServiceSubject, eventChannelPrincipal, resolveService, invokeCommand, } from "@cotal-ai/core";
8
8
  import { agentAuthState, agentCredsDir, agentLifecycleSecretFilePaths, agentSecretFilePaths, agentSecretKeyForFile, authDir, connectorInstallHint, DEFAULT_CONNECTOR, defaultAgentType, DELIVERY_CREDS_KIND, extensionConnectors, findCotalRoot, getSpaceAuth, hasUserAuthState, loadExtensionsManifest, loadManagerInstanceIdentity, loadMeshes, localTrustOfSpace, manifestExtensionNames, materializeFromManifest, materializeSecretToFile, MEMBERSHIP_RW_CREDS_KIND, mergeLaunchOptions, remintDaemonCreds, resolveOnPath, createManagerInstanceIdentity, spaceMaterialKey, SYSTEM_CREDS_FILES, userAuthStateDir, workspaceSecretStore, writeRenewalRecord } from "@cotal-ai/workspace";
9
9
  import { createRuntime, isCustodialRuntime, requireRuntimeAdopt, requireRuntimeReap, RuntimeReapUnproven, } from "./runtime/index.js";
10
10
  import { AttachEndpoint } from "./attach-endpoint.js";
11
- import { makeManagerEndpointEvictionEvidence, makeManagerEndpointEvictor } from "./endpoint-evict.js";
11
+ import { makeManagerEndpointEvictionEvidence, makeManagerEndpointEvictor, makeManagerEndpointHolderEvictor } from "./endpoint-evict.js";
12
12
  import { makeManagerHolderLivenessProbe } from "./holder-liveness.js";
13
13
  import { GateReconcileRefused, reconcileEndpointGate } from "./reconcile-gate.js";
14
14
  import { launchSpecForRun, materializePersona, launchAgentToStartOpts, parseLaunchSpec, persistLaunchSpec } from "./launch.js";
@@ -197,61 +197,6 @@ function parseTurnNote(raw) {
197
197
  function turnHoldToken(goalId) {
198
198
  return createHash("sha256").update(`${goalId}:turn-deadline`, "utf8").digest("base64url").slice(0, 43);
199
199
  }
200
- /** One ep request/reply round-trip on the caller's OWN reply-plane filter (§13.2). The responder
201
- * derives the reply subject from the authenticated request, so there is no caller-selected reply
202
- * target to honour; the caller binds the answer off the reply SUBJECT — endpoint and nonce, both
203
- * broker-pinned by the responder's serve publish grant. A reply on another nonce belongs to a
204
- * different request (the rail is shared) and is ignored rather than allowed to fail this one. */
205
- export async function epAwaitReply(nc, space, caller, nonce, requestId, requestSubject, body, timeoutMs) {
206
- let sub;
207
- let timer;
208
- try {
209
- const got = new Promise((resolve, reject) => {
210
- sub = nc.subscribe(epCallerReplyFilter(space, caller), {
211
- callback: (err, msg) => {
212
- if (err) {
213
- reject(new Error(`the retirement reply subscription failed: ${err.message}`));
214
- return;
215
- }
216
- const parsed = parseEpSubject(msg.subject);
217
- if (!parsed || parsed.plane !== "reply" || parsed.endpoint !== AUTH_ENDPOINT || parsed.nonce !== nonce)
218
- return;
219
- let body;
220
- try {
221
- body = JSON.parse(new TextDecoder().decode(msg.data));
222
- }
223
- catch {
224
- return;
225
- } // a malformed body on our nonce is not an answer; keep waiting
226
- // REQUIRE THE ID ECHO. Binding on (endpoint, nonce) alone would accept a malformed or
227
- // WRONG-ID `{ok:true}` and clear a retirement hold on it. Ignore rather than fail: the
228
- // rail is shared across concurrent requests and a responder may publish at any nonce, so
229
- // an unmatched reply is someone else's answer, never grounds to fail the honest one.
230
- if (body.id !== requestId)
231
- return;
232
- resolve(body);
233
- },
234
- });
235
- timer = setTimeout(() => reject(new Error("timeout")), timeoutMs);
236
- nc.publish(requestSubject, new TextEncoder().encode(body));
237
- });
238
- return await got;
239
- }
240
- finally {
241
- if (timer)
242
- clearTimeout(timer);
243
- try {
244
- sub?.unsubscribe();
245
- }
246
- catch { /* connection already down */ }
247
- }
248
- }
249
- /**
250
- * The agent supervisor: a long-lived mesh node that owns agent process lifecycle.
251
- * It serves control requests on the "manager" service and spawns/kills agents
252
- * through a pluggable {@link Runtime} (pty by default). It does NOT proxy agent
253
- * mesh traffic — terminal I/O streams over its own attach endpoint instead.
254
- */
255
200
  /**
256
201
  * Event channels in `channels` that do NOT belong to `{owner, actor}`.
257
202
  *
@@ -292,6 +237,12 @@ function isAbsentDeliveryAdmin(msg) {
292
237
  return false;
293
238
  return /no responders|\b503\b/i.test(msg);
294
239
  }
240
+ /**
241
+ * The agent supervisor: a long-lived mesh node that owns agent process lifecycle.
242
+ * It serves control requests on the "manager" service and spawns/kills agents
243
+ * through a pluggable {@link Runtime} (pty by default). It does NOT proxy agent
244
+ * mesh traffic — terminal I/O streams over its own attach endpoint instead.
245
+ */
295
246
  export class Manager {
296
247
  space;
297
248
  servers;
@@ -317,8 +268,13 @@ export class Manager {
317
268
  endpointServeExecutorExpiresInSeconds;
318
269
  agents = new Map();
319
270
  /** Names whose spawn is in flight (reserved synchronously before the provision await) — counted
320
- * toward the ceiling so two concurrent same-name spawns can't both pass the gate (P4a). */
271
+ * toward the ceiling (once per name, see {@link Manager.occupancy}) so two concurrent same-name
272
+ * spawns can't both pass the gate (P4a). */
321
273
  reserved = new Set();
274
+ /** Reserved names whose launch already put its seat in `agents`. The name stays reserved until
275
+ * the launch settles, so no other spawn can take it, but the ceiling counts that seat through
276
+ * `agents` and, if it frees young, its cooling stamp, never through the reservation (#906). */
277
+ reservedLive = new Set();
322
278
  /** Expiry stamps (`startedAt + MIN_LIFETIME`) for slots that freed while still young — a
323
279
  * count-only, lazily-pruned recycle floor (P4c). Pruned + summed into the ceiling gate. */
324
280
  cooling = [];
@@ -370,6 +326,11 @@ export class Manager {
370
326
  * every (re)connect, so a renewal is adopted without re-registration). Absent on open meshes,
371
327
  * in user mode (the named 1a follow-up), and before registration completes. */
372
328
  serviceServe;
329
+ /** #1577: serve connections whose transport is down while the client reconnects them. Such a
330
+ * connection is open (`isClosed()` is false under unbounded reconnects) yet the broker holds
331
+ * none of its rails, so the liveness responder must not grade it `bound`. Keyed by the
332
+ * connection, so a re-dial that swaps in a fresh one starts from its own state. */
333
+ serveTransportDown = new WeakSet();
373
334
  /** P2 item 2 (spawn-as-action): the SELF-MEDIATED goal-writer connection + ActionContext — a
374
335
  * standing connection DISJOINT from the serve credential (Q2), scoped to exactly this endpoint's
375
336
  * goal bind/terminal facts + goal-record writes ({@link goalWriterGrants}). Auth mode mints the
@@ -424,10 +385,11 @@ export class Manager {
424
385
  /** P2 item 2 (M4): the live spawn goal ref for each managed agent name, so a despawn MID-GOAL
425
386
  * drives the cancel path (transition -> cancel terminal). Cleared when the goal terminalizes. */
426
387
  agentGoals = new Map();
427
- /** The turn relay's same-incarnation idempotency map ({@link goalAcceptances}'s twin): a
428
- * same-goalId retry serves the identical acceptance; cross-incarnation retries rebuild from
429
- * the goal-index entry (its acceptance floor + note). */
430
388
  /**
389
+ * The turn relay's same-incarnation idempotency map ({@link goalAcceptances}'s twin): a
390
+ * same-goalId retry serves the identical acceptance; cross-incarnation retries rebuild from
391
+ * the goal-index entry (its acceptance floor + note).
392
+ *
431
393
  * One entry per turn this incarnation accepted: the acceptance a duplicate submission is served
432
394
  * from, and — once the turn settles — the answer a RETRIED yield is served from.
433
395
  *
@@ -467,6 +429,8 @@ export class Manager {
467
429
  * lifecycle authority: every re-drive re-reads broker KV and re-runs planStaticSlotResume. */
468
430
  staticReconcileItems = new Map();
469
431
  staticReconcileLastSweep;
432
+ staticReconcileLastSweepResources;
433
+ lastDeprovisionResources;
470
434
  staticReconcileSweepsInFlight = 0;
471
435
  staticReconcileDrainWaiters = [];
472
436
  staticReconcileStopping = false;
@@ -549,6 +513,11 @@ export class Manager {
549
513
  resumedAgentNames = new Set();
550
514
  remoteAuthority;
551
515
  remoteExecutorCreds;
516
+ remoteSupervisorCreds;
517
+ remoteBundleRenewal;
518
+ /** A refused or failed standing candidate is cleanup debt, never a silent loss: the last-good
519
+ * family stays live and this records what the host may have signed but the manager never adopted. */
520
+ remoteRenewalDebt;
552
521
  /** Every broker dial this manager makes, through ONE transport decision.
553
522
  *
554
523
  * `this.servers` is whatever the mesh record held (`cotal supervise` reads it from there), so it
@@ -561,6 +530,10 @@ export class Manager {
561
530
  return dialerFor(servers)({ ...opts, servers });
562
531
  }
563
532
  constructor(opts) {
533
+ if (opts.pooled && (!opts.remoteAuthority || opts.runtime === undefined || opts.runtime === "auto" || opts.runtime === "pty"))
534
+ throw new Error("pooled control requires signerless remote authority and an explicit non-PTY runtime; local custodial execution is forbidden");
535
+ if (opts.pooled && !opts.remoteAuthority?.renewStandingBundle)
536
+ throw new Error("pooled control requires a closed host-issued all-duty renewal callback before construction");
564
537
  this.space = opts.space;
565
538
  this.servers = opts.servers;
566
539
  this.name = opts.name ?? "manager";
@@ -569,6 +542,10 @@ export class Manager {
569
542
  this.maxSessions = opts.maxSessions;
570
543
  this.remoteAuthority = opts.remoteAuthority;
571
544
  this.remoteExecutorCreds = opts.remoteAuthority?.executorCreds;
545
+ this.remoteSupervisorCreds = opts.remoteAuthority?.supervisorCreds;
546
+ if (opts.pooled && (!opts.remoteAuthority?.accountPublicKey ||
547
+ accountFromCreds(opts.remoteAuthority.supervisorCreds) !== opts.remoteAuthority.accountPublicKey))
548
+ throw new Error("pooled control requires a proved assigned account for its initial supervisor credential");
572
549
  if (opts.remoteAuthority)
573
550
  this.managerLifecycleUid = opts.remoteAuthority.lifecycleUid;
574
551
  this.secrets = opts.secretStore ?? workspaceSecretStore(this.workspaceRoot);
@@ -577,6 +554,8 @@ export class Manager {
577
554
  : { kind: "fs", root: resolve(this.workspaceRoot) };
578
555
  this.installedExtensions = opts.installedExtensions ?? false;
579
556
  this.runtime = createRuntime(opts.runtime ?? "auto", `cotal-${this.space}`);
557
+ if (opts.pooled && isCustodialRuntime(this.runtime))
558
+ throw new Error(`pooled control refuses custodial runtime "${this.runtime.kind}" before starting agents`);
580
559
  this.preserveStopTimeoutMs = opts.preserveStopTimeoutMs ?? PRESERVE_STOP_TIMEOUT_MS;
581
560
  this.endpointServeExecutorExpiresInSeconds = opts.endpointServeExecutorExpiresInSeconds;
582
561
  if (opts.resumeAttemptId && !/^[A-Za-z0-9_-]{1,128}$/.test(opts.resumeAttemptId))
@@ -688,8 +667,8 @@ export class Manager {
688
667
  case "user":
689
668
  return (`\`cotal actor grant <reader> --owner ${owner} --scope '' --allow-subscribe '${channel}' ` +
690
669
  `--allow-publish ''\`, with every field spelled out: \`actor grant\` is an upsert of the ` +
691
- `WHOLE row and an omitted flag means the WIDE default (\`>\` read, \`>\` post, \`spawn,role:default\` ` +
692
- `scope), not "leave it alone".`);
670
+ `WHOLE row, and an omitted flag is refused, or with \`--full\` is the WIDE default (\`>\` read, \`>\` post, \`spawn,role:default\` ` +
671
+ `scope).`);
693
672
  case "static":
694
673
  return (`\`cotal mint <reader> --profile agent --allow-subscribe ${channel} --provision\`, where there ` +
695
674
  `is no actor ledger for \`actor grant\` to write to. The AGENT profile, not the observer ` +
@@ -792,7 +771,7 @@ export class Manager {
792
771
  if (this.remoteAuthority) {
793
772
  const remote = this.remoteAuthority;
794
773
  id = remote.identities.supervisor.id;
795
- creds = async () => remote.supervisorCreds;
774
+ creds = async () => this.remoteSupervisorCreds;
796
775
  }
797
776
  else if (this.auth) {
798
777
  const identity = newIdentity();
@@ -836,6 +815,10 @@ export class Manager {
836
815
  const reportEndpoint = (e) => console.error(`! manager endpoint: ${e.message}`);
837
816
  this.ep.on("error", reportEndpoint);
838
817
  this.ep.on("warning", reportEndpoint);
818
+ // Every roster change and every view transition, so a seat that is offline in a fresh snapshot
819
+ // (recorded with no presence event) is logged the moment the view can support the verdict.
820
+ this.ep.on("roster", () => this.logSeatMeshTransitions());
821
+ this.ep.on("presence-view", () => this.logSeatMeshTransitions());
839
822
  await this.ep.start();
840
823
  await this.ep.setActivity(`supervisor (${this.runtime.kind})`);
841
824
  // Per-instance liveness lease (P2 item 3 — the old per-space singleton is DEMOTED per D9). Acquire
@@ -861,6 +844,31 @@ export class Manager {
861
844
  }
862
845
  this.leaseTimer = setInterval(() => { void this.renewLease(); }, MANAGER_LEASE_RENEW_MS);
863
846
  this.leaseTimer.unref?.();
847
+ // THE PEER-READABLE LIVENESS RESPONDER for the manager plane (#1577). Bound HERE — after the
848
+ // lease, before registration — precisely so it is up during the window in which the manager is
849
+ // NOT yet serving. A responder that only appeared once everything worked could never report the
850
+ // broken state it exists to report, which is the reported defect in a new place.
851
+ //
852
+ // WHAT IT GRADES is `serviceServe`, the v0.4 service endpoint's serve handle, which is assigned
853
+ // only after `serveEndpoint` has bound this instance's rails, AND that handle's connection. A
854
+ // closed serve connection holds no subscriptions, while `onServeConnectionClosed` keeps the
855
+ // handle through its re-dial waits, so the handle alone would answer `bound` with every rail
856
+ // gone. The re-dial swaps in the fresh connection and handle together, so `bound` returns only
857
+ // once the rails are served again. A DISCONNECTED serve connection is the same fault without the
858
+ // close: it reconnects without limit, so it stays open while the broker holds none of its rails,
859
+ // and this probe arrives on the separate supervisor connection that is still up. So the grade
860
+ // also requires the serve transport to be connected. That is a RESPONDER fact. It is
861
+ // deliberately NOT "this process is alive": this process is trivially alive whenever it answers
862
+ // at all, so grading on that would make the surface a `pgrep` that also lies in both directions
863
+ // — the exact instrument the reporter's evidence table indicts. Between lease acquisition and a
864
+ // successful registration the honest answer is `unbound`, and that is what a peer is told.
865
+ //
866
+ // The lease row is never consulted and never sent: presence is the whole ask, and the row
867
+ // carries the operator's workspace root and pid.
868
+ this.ep.serveLiveness("manager", () => {
869
+ const s = this.serviceServe;
870
+ return s?.handle && !s.nc.isClosed() && !this.serveTransportDown.has(s.nc) ? "bound" : "unbound";
871
+ });
864
872
  // Unit B (static §13.1): after this instance holds its lease, collect the durable static rows
865
873
  // now, but do not let their exact-op terminals make the whole space unreachable. The service
866
874
  // comes up below, then the sweep overlaps the remaining registration work. Two-window
@@ -895,8 +903,38 @@ export class Manager {
895
903
  this.credRenewTimer.unref?.();
896
904
  }
897
905
  else if (this.remoteAuthority) {
898
- await this.renewRemoteExecutor();
899
- this.credRenewTimer = setInterval(() => { void this.renewRemoteExecutor(); }, credRenewIntervalMs(5 * 60));
906
+ if (this.remoteAuthority.renewStandingBundle)
907
+ await this.renewRemoteStandingBundle();
908
+ else
909
+ await this.renewRemoteExecutor();
910
+ let standingTtl = STANDING_RENEWABLE_TTL_SEC;
911
+ if (this.remoteSupervisorCreds) {
912
+ try {
913
+ const claims = credsClaims(this.remoteSupervisorCreds);
914
+ if (typeof claims.exp === "number" && typeof claims.iat === "number" && claims.exp > claims.iat) {
915
+ standingTtl = claims.exp - claims.iat;
916
+ }
917
+ }
918
+ catch { }
919
+ }
920
+ const intervalMs = Math.min(credRenewIntervalMs(5 * 60), credRenewIntervalMs(standingTtl));
921
+ this.credRenewTimer = setInterval(async () => {
922
+ try {
923
+ if (this.remoteAuthority?.renewStandingBundle)
924
+ await this.renewRemoteStandingBundle();
925
+ else
926
+ await this.renewRemoteExecutor();
927
+ }
928
+ catch (e) {
929
+ console.error(`! remote manager renewal pass failed: ${e.message}`);
930
+ }
931
+ try {
932
+ await this.runHosting?.renew();
933
+ }
934
+ catch (e) {
935
+ console.error(`! remote run-hosting renewal pass failed: ${e.message}`);
936
+ }
937
+ }, intervalMs);
900
938
  this.credRenewTimer.unref?.();
901
939
  }
902
940
  // stop() fences before it waits for an accepted startup reconciliation terminal. Once that
@@ -929,9 +967,26 @@ export class Manager {
929
967
  // incarnation takes back every run a dead predecessor was driving before it accepts new ones.
930
968
  // The serve surface is already live by here, so the family itself holds the gate: `runHost()`
931
969
  // refuses `run-start`/`run-resume` as `unavailable` until the host exists and `RunHosting`
932
- // refuses them until its reconcile has returned. A user-auth mesh stands no host up at all
933
- // (`runHost()` names why); a remote-authority manager holds no signer to mint with.
934
- if (!this.remoteAuthority && !this.userMode) {
970
+ // refuses them until its reconcile has returned. A remote user-auth manager uses its four
971
+ // registered issuer callbacks instead of holding a signer; a local user mesh stands none up.
972
+ const remoteRuns = this.remoteAuthority?.runHosting;
973
+ if (this.remoteAuthority && remoteRuns) {
974
+ this.runHosting = new RunHosting({
975
+ space: this.space,
976
+ servers: this.servers,
977
+ endpoint: MANAGER_ENDPOINT,
978
+ instanceId: this.managerInstanceId,
979
+ holder: { id: this.remoteAuthority.identities.supervisor.id, lifecycleUid: this.managerLifecycleUid },
980
+ auth: undefined,
981
+ log: (line) => console.error(line),
982
+ admitRun: remoteRuns.admitRun,
983
+ issueAttempt: remoteRuns.issueAttempt,
984
+ issueOperator: remoteRuns.issueOperator,
985
+ renewRun: remoteRuns.renewRun,
986
+ });
987
+ await this.runHosting.reconcile();
988
+ }
989
+ else if (!this.remoteAuthority && !this.userMode) {
935
990
  this.runHosting = new RunHosting({
936
991
  space: this.space,
937
992
  servers: this.servers,
@@ -1339,6 +1394,94 @@ export class Manager {
1339
1394
  throw e;
1340
1395
  }
1341
1396
  }
1397
+ /** An active signerless manager renews its standing duties together. The issuer stages the closed
1398
+ * credential family first; broker preflight proves every JWT and existing caller-held nkey
1399
+ * before any mutable holder changes. A failed candidate leaves all current creds intact. */
1400
+ async renewRemoteStandingBundle(force = false) {
1401
+ if (this.remoteBundleRenewal)
1402
+ return this.remoteBundleRenewal;
1403
+ const renewal = this.renewRemoteStandingBundleOnce(force);
1404
+ this.remoteBundleRenewal = renewal;
1405
+ try {
1406
+ await renewal;
1407
+ }
1408
+ finally {
1409
+ if (this.remoteBundleRenewal === renewal)
1410
+ this.remoteBundleRenewal = undefined;
1411
+ }
1412
+ }
1413
+ /** A signerless manager's closed standing connection re-dials through the all-duty family only.
1414
+ * A healthy holder re-dials as is. An unhealthy one waits for the closed renewal and adoption,
1415
+ * and a refusal fails this attempt with last-good and debt kept. Nothing is minted locally. */
1416
+ async renewRemoteBeforeRedial(label, read) {
1417
+ if (inspectCredHealth(read() ?? "").state === "healthy")
1418
+ return;
1419
+ await this.renewRemoteStandingBundle(true);
1420
+ if (inspectCredHealth(read() ?? "").state !== "healthy")
1421
+ throw new Error(`the ${label} credential is not healthy after all-duty renewal; last-good is kept and nothing is minted locally`);
1422
+ }
1423
+ async renewRemoteStandingBundleOnce(force) {
1424
+ const remote = this.remoteAuthority;
1425
+ if (!remote?.renewStandingBundle || !this.serviceServe)
1426
+ return;
1427
+ const current = [this.remoteSupervisorCreds, this.remoteExecutorCreds, this.serviceServe.creds,
1428
+ this.goalWriterCreds, this.sessionLedgerCreds];
1429
+ if (!force && current.every((cred) => cred && inspectCredHealth(cred).state === "healthy"))
1430
+ return;
1431
+ const processEpoch = this.serviceServe.grant.epoch;
1432
+ let candidate;
1433
+ try {
1434
+ candidate = await remote.renewStandingBundle(processEpoch);
1435
+ const names = ["supervisor", "executor", "serve", "goalWriter", "sessionLedger"];
1436
+ if (Object.keys(candidate).sort().join(",") !== [...names].sort().join(","))
1437
+ throw new Error("host returned an incomplete or widened standing credential family");
1438
+ for (const name of names) {
1439
+ if (typeof candidate[name] !== "string" || inspectCredHealth(candidate[name]).state === "unreadable")
1440
+ throw new Error(`host returned an unreadable ${name} credential`);
1441
+ if (!remote.accountPublicKey || accountFromCreds(candidate[name]) !== remote.accountPublicKey)
1442
+ throw new Error(`host returned a ${name} credential for another account`);
1443
+ if (credsClaims(candidate[name]).sub !== remote.identities[name].id)
1444
+ throw new Error(`host returned a ${name} credential for a different nkey than the held identity`);
1445
+ const staged = candidate[name];
1446
+ const nc = await this.dial({ authenticator: (nonce) => credsAuthenticator(new TextEncoder().encode(staged))(nonce), maxReconnectAttempts: 0 });
1447
+ await nc.close();
1448
+ }
1449
+ if (this.leaseStopping || !this.serviceServe || this.serviceServe.grant.epoch !== processEpoch)
1450
+ throw new Error("manager changed its active serve epoch while standing renewal was in flight");
1451
+ this.remoteSupervisorCreds = candidate.supervisor;
1452
+ this.remoteExecutorCreds = candidate.executor;
1453
+ this.serviceServe.creds = candidate.serve;
1454
+ this.goalWriterCreds = candidate.goalWriter;
1455
+ this.sessionLedgerCreds = candidate.sessionLedger;
1456
+ if (this.goalWriter)
1457
+ this.goalWriter.creds = candidate.goalWriter;
1458
+ if (this.sessionLedgerConn)
1459
+ this.sessionLedgerConn.creds = candidate.sessionLedger;
1460
+ // Adoption is committed above; a connection that cannot come back yet is a push failure the
1461
+ // endpoint's own reconnect loop retries with the adopted credential, not unadopted debt.
1462
+ this.remoteRenewalDebt = undefined;
1463
+ await Promise.all([
1464
+ this.ep.reconnect().catch((e) => console.error(`! remote manager supervisor reconnect after renewal: ${e.message}`)),
1465
+ this.pushRenewedCreds(this.serviceServe.nc),
1466
+ ...(this.goalWriter ? [this.pushRenewedCreds(this.goalWriter.nc)] : []),
1467
+ ...(this.sessionLedgerConn ? [this.pushRenewedCreds(this.sessionLedgerConn.nc)] : []),
1468
+ ]);
1469
+ }
1470
+ catch (e) {
1471
+ const unadoptedSubjects = candidate && typeof candidate === "object"
1472
+ ? Object.values(candidate).flatMap((cred) => { try {
1473
+ return typeof cred === "string" && credsClaims(cred).sub ? [credsClaims(cred).sub] : [];
1474
+ }
1475
+ catch {
1476
+ return [];
1477
+ } })
1478
+ : [];
1479
+ this.remoteRenewalDebt = { at: Date.now(), processEpoch, reason: e.message, unadoptedSubjects };
1480
+ console.error(`! remote manager all-duty renewal: ${e.message} - no unsigned or local replacement is available`);
1481
+ if (force)
1482
+ throw e;
1483
+ }
1484
+ }
1342
1485
  /** Warn, on every renewal pass, when a $SYS credential is at or past its renewal point.
1343
1486
  *
1344
1487
  * The manager is the renewal owner for every credential it CAN re-sign, and these two are the ones
@@ -1736,6 +1879,7 @@ export class Manager {
1736
1879
  events: a.launch.events,
1737
1880
  shareTools: a.launch.shareTools,
1738
1881
  forkSource: a.launch.forkSource,
1882
+ ...(this.forkProvenance(a) ? { resumed: this.forkProvenance(a) } : {}),
1739
1883
  sessionId: a.restart?.armed ? this.readManagedSession(a) : a.launch.sessionId,
1740
1884
  // Additive and only when the connector supplied one. A seat with no pointer records no
1741
1885
  // field, which is what keeps an older inventory and a fresh one the same document shape.
@@ -1861,38 +2005,64 @@ export class Manager {
1861
2005
  // revision it has since moved is a CAS the broker correctly refuses, silently, forever.
1862
2006
  await this.leaseRenewTask?.catch(() => { });
1863
2007
  let releaseFailures = [];
1864
- if (this.maintenanceState === "active" && !this.resumeRequired) {
1865
- if (options.withAgents)
1866
- await this.teardownManagedAgents();
1867
- else
1868
- releaseFailures = this.releaseManagedAgents();
2008
+ let seatFailure;
2009
+ try {
2010
+ if (this.maintenanceState === "active" && !this.resumeRequired) {
2011
+ if (options.withAgents)
2012
+ await this.teardownManagedAgents();
2013
+ else
2014
+ releaseFailures = this.releaseManagedAgents();
2015
+ }
2016
+ else {
2017
+ // A signal after a partial preservation must never fall back into destructive teardown.
2018
+ await this.stopRetainedAgentsOnExit();
2019
+ }
1869
2020
  }
1870
- else {
1871
- // A signal after a partial preservation must never fall back into destructive teardown.
1872
- await this.stopRetainedAgentsOnExit();
2021
+ catch (e) {
2022
+ seatFailure = e;
2023
+ }
2024
+ // Every step below runs even when an earlier one fails, so a failed stop still closes this
2025
+ // process's connections and listener (#2290). Failures are collected and thrown at the end.
2026
+ const failures = [];
2027
+ const step = async (name, run) => {
2028
+ try {
2029
+ await run();
2030
+ }
2031
+ catch (e) {
2032
+ failures.push(`${name}: ${e.message}`);
2033
+ }
2034
+ };
2035
+ // A seat that may still be alive keeps this instance's leases and registration (see
2036
+ // teardownManagedAgents): they lapse on their TTL instead of being handed to a successor.
2037
+ if (!seatFailure) {
2038
+ await step("release manager lease", () => this.releaseOwnManagerLease());
2039
+ // #1634: hand the renewal lease back on a clean stop so a sibling picks it up on its next pass
2040
+ // rather than waiting out the bucket TTL. A non-owner holds no revision and this is a no-op.
2041
+ await step("release renewal lease", () => this.ep.releaseDaemonRenewalLease());
1873
2042
  }
1874
- await this.releaseOwnManagerLease();
1875
- // #1634: hand the renewal lease back on a clean stop so a sibling picks it up on its next pass
1876
- // rather than waiting out the bucket TTL. A non-owner holds no revision and this is a no-op.
1877
- await this.ep.releaseDaemonRenewalLease();
1878
2043
  // Capture BEFORE the serve loop is torn down: `stopServiceServe` clears the state, and what is
1879
2044
  // being asked here is "did this process register a service instance", which only the state
1880
2045
  // before teardown can answer. Deregistration runs AFTER the serve loop has drained, so no
1881
2046
  // in-flight command can write a status back onto the record it just removed.
1882
2047
  const registered = this.serviceServe !== undefined;
1883
- await this.stopServiceServe();
2048
+ await step("stop service serve", () => this.stopServiceServe());
1884
2049
  // The drives AFTER the serve loop (no new `run-start` can land) and BEFORE deregistration: a
1885
2050
  // released run's status write is the last thing this incarnation says about it.
1886
- await this.runHosting?.stop();
2051
+ await step("stop run hosting", () => this.runHosting?.stop());
1887
2052
  this.runHosting = undefined;
1888
- if (registered)
1889
- await this.deregisterServiceOnStop();
1890
- await this.stopGoalWriter();
1891
- await this.stopSessionPlane();
1892
- await this.ep.stop();
1893
- await this.attach.stop();
2053
+ if (registered && !seatFailure)
2054
+ await step("deregister service", () => this.deregisterServiceOnStop());
2055
+ await step("stop goal writer", () => this.stopGoalWriter());
2056
+ await step("stop session plane", () => this.stopSessionPlane());
2057
+ await step("stop endpoint", () => this.ep.stop());
2058
+ await step("stop attach endpoint", () => this.attach.stop());
2059
+ const also = failures.length ? `; teardown also failed: ${failures.join("; ")}` : "";
2060
+ if (seatFailure)
2061
+ throw new Error(`${seatFailure.message}${also}`);
1894
2062
  if (releaseFailures.length)
1895
- throw new Error(`manager shutdown could not release every detachable seat: ${releaseFailures.join("; ")}`);
2063
+ throw new Error(`manager shutdown could not release every detachable seat: ${releaseFailures.join("; ")}${also}`);
2064
+ if (failures.length)
2065
+ throw new Error(`manager shutdown incomplete: ${failures.join("; ")}`);
1896
2066
  }
1897
2067
  /** Release this instance's liveness lease key on a clean stop, at the broker's own revision
1898
2068
  * rather than a cached one that a renew still in flight moments ago may have already moved past.
@@ -2329,37 +2499,9 @@ export class Manager {
2329
2499
  throw new EpEnvelopeError("permission-denied", `an any-mode ${ctx.subject.command} is operator reach; the caller's current ledger grant does not carry "admin" (SPEC 13.2)`);
2330
2500
  return true;
2331
2501
  }
2332
- /** The v0.4 typed command table (P2 item 1, slice 1b): every ordinary handler runs the SHARED
2333
- * admission chokepoint ({@link serveGated}) and then delegates to the SAME op core the ctl
2334
- * door dispatches (checklist 8: one core, two thin doors). The resume/preservation family
2335
- * deliberately BYPASSES serveGated — exactly as it sits before {@link admitControl} on the ctl
2336
- * door (those ops must run while `resumeRequired` fences ordinary work) — riding its own state
2337
- * fences; its ep gate is the admin-grade `manager.admin` capability grant (the 1b rule: static
2338
- * admin-class commands are capability-gated + untargeted, never a fabricated ledger mode).
2339
- *
2340
- * TIER SEMANTICS on the ep door (the 1c grant-migration table): the tier lives in the CALLER'S
2341
- * GRANT, refined per-op exactly as the ctl doors refine their subject tier. Owner-mode
2342
- * `despawn`/`attach` keep the privileged semantics (`admin=false`, own-domain via
2343
- * {@link authorizeNamed}) — every spawn-capable agent holds those rows. ANY-mode requests are
2344
- * the operator instrument's cross-agent reach (rev 3): the any-mode subject row is mintable
2345
- * only under operator policy (§13.2), so on a static mesh holding it IS the admin tier, and in
2346
- * user mode the caller's CURRENT ledger scope must still carry `admin`
2347
- * ({@link epAdminReach}, the same fresh-read authority `psOwnerFilter` consults). The
2348
- * `manager.admin` family (purge + the resume/preservation ops) is capability-gated at mint AND
2349
- * re-checked at serve time via {@link epAdminReach} (the `adminGated` wrapper) so a user's
2350
- * revoked scope demotes the next call. `launch` is OWNER-EQUALITY on this door for everyone
2351
- * (freelance HIGH #2): the deploy path is its only consumer and stamps the caller's own owner,
2352
- * so cross-owner launch was a ctl-tier incidental never exercised, and keying it on the actor's
2353
- * ledger scope broke the deployer-view attenuation - uniform owner-equality is the safe tier.
2354
- * TWO DELIBERATE NARROWINGS vs the ctl doors (NOT bit-exact parity, panel-accepted): (1)
2355
- * `define-persona` is `admin=false` for everyone (own-persona discipline; no ep consumer needs
2356
- * cross-owner persona writes - an operator redefines via config, not the wire), where the ctl
2357
- * admin tier allowed operator cross-owner redefine; (2) launch is owner-equality-only, above.
2358
- * Both are least-privilege reductions, never widenings. */
2359
2502
  /** The run host, or one of three refusals. A remote-authority manager holds no space signer, so
2360
- * it cannot mint the per-run driver credential SPEC 14.6 requires, and hosting on any other
2361
- * identity would be the fallback this tree does not take: `unimplemented`, for good. A
2362
- * user-auth mesh is `unimplemented` too, for a different reason it names: a hosted run's seats
2503
+ * it cannot mint the per-run driver credential SPEC 14.6 requires without all four closed
2504
+ * host callbacks. A local user-auth mesh is `unimplemented`: a hosted run's seats
2363
2505
  * are spawned, turned and despawned by a caller derived from the run id under the static
2364
2506
  * owner, which the user-mode spawn door refuses (no `u_` owner), so a program would fail at
2365
2507
  * its first seat; no path to `--local` is offered there since a user bearer holds no run rows
@@ -2369,9 +2511,9 @@ export class Manager {
2369
2511
  runHost() {
2370
2512
  if (this.runHosting)
2371
2513
  return this.runHosting;
2372
- if (this.remoteAuthority)
2373
- throw new EpEnvelopeError("unimplemented", "this manager does not host workflow runs: a remote-authority manager mints no run-driver credentials (SPEC 14.6); drive the run from a terminal with `cotal run start --local --file <program>`");
2374
- if (this.userMode)
2514
+ if (this.remoteAuthority && !this.remoteAuthority.runHosting)
2515
+ throw new EpEnvelopeError("unimplemented", "this manager does not host workflow runs: its issuing host supplied no closed run callbacks, and a remote-authority manager mints no run-driver credentials (SPEC 14.6); drive the run from a terminal with `cotal run start --local --file <program>`");
2516
+ if (this.userMode && !this.remoteAuthority?.runHosting)
2375
2517
  throw new EpEnvelopeError("unimplemented", `user-auth space "${this.space}" hosts no workflow runs yet: a hosted run's seats would be spawned under the static owner, which a user mesh refuses; run programs on a static-auth mesh`);
2376
2518
  throw new EpEnvelopeError("unavailable", "the manager is still booting its workflow-run host; retry shortly (SPEC 14.3)");
2377
2519
  }
@@ -2413,6 +2555,44 @@ export class Manager {
2413
2555
  if (p.run !== args.runId || p.step !== args.stepKey || addressedToken !== open.token)
2414
2556
  throw new EpEnvelopeError("permission-denied", `run-answer is allowed to ${agent.name} only for the open pause named by its pending ask or escalation relay; ${args.runId} ${args.stepKey} is not that pause`);
2415
2557
  }
2558
+ /** A baseline seat may amend only an answer recorded under its own name: it changes its own
2559
+ * position, never another answerer's. Explicit `run` holders and unmanaged operator instruments
2560
+ * keep the reach they have to answer, as {@link authorizeRunAnswer} does. */
2561
+ authorizeRunAmend(ctx, accepted) {
2562
+ const caller = principalKey(ctx.subject.caller.owner, ctx.subject.caller.actor).key;
2563
+ const agent = [...this.agents.values()].find((a) => this.managedPrincipal(a) === caller && a.lifecycleUid === ctx.subject.caller.uid);
2564
+ if (agent === undefined || agent.launch.capabilities?.includes("run"))
2565
+ return;
2566
+ if (accepted.by !== agent.name)
2567
+ throw new EpEnvelopeError("permission-denied", `run-answer amend is allowed to ${agent.name} only for an answer recorded under its own name; this step's accepted answer is ${JSON.stringify(accepted.by)}'s`);
2568
+ }
2569
+ /** The v0.4 typed command table (P2 item 1, slice 1b): every ordinary handler runs the SHARED
2570
+ * admission chokepoint ({@link serveGated}) and then delegates to the SAME op core the ctl
2571
+ * door dispatches (checklist 8: one core, two thin doors). The resume/preservation family
2572
+ * deliberately BYPASSES serveGated — exactly as it sits before {@link admitControl} on the ctl
2573
+ * door (those ops must run while `resumeRequired` fences ordinary work) — riding its own state
2574
+ * fences; its ep gate is the admin-grade `manager.admin` capability grant (the 1b rule: static
2575
+ * admin-class commands are capability-gated + untargeted, never a fabricated ledger mode).
2576
+ *
2577
+ * TIER SEMANTICS on the ep door (the 1c grant-migration table): the tier lives in the CALLER'S
2578
+ * GRANT, refined per-op exactly as the ctl doors refine their subject tier. Owner-mode
2579
+ * `despawn`/`attach` keep the privileged semantics (`admin=false`, own-domain via
2580
+ * {@link authorizeNamed}) — every spawn-capable agent holds those rows. ANY-mode requests are
2581
+ * the operator instrument's cross-agent reach (rev 3): the any-mode subject row is mintable
2582
+ * only under operator policy (§13.2), so on a static mesh holding it IS the admin tier, and in
2583
+ * user mode the caller's CURRENT ledger scope must still carry `admin`
2584
+ * ({@link epAdminReach}, the same fresh-read authority `psOwnerFilter` consults). The
2585
+ * `manager.admin` family (purge + the resume/preservation ops) is capability-gated at mint AND
2586
+ * re-checked at serve time via {@link epAdminReach} (the `adminGated` wrapper) so a user's
2587
+ * revoked scope demotes the next call. `launch` is OWNER-EQUALITY on this door for everyone
2588
+ * (freelance HIGH #2): the deploy path is its only consumer and stamps the caller's own owner,
2589
+ * so cross-owner launch was a ctl-tier incidental never exercised, and keying it on the actor's
2590
+ * ledger scope broke the deployer-view attenuation - uniform owner-equality is the safe tier.
2591
+ * TWO DELIBERATE NARROWINGS vs the ctl doors (NOT bit-exact parity, panel-accepted): (1)
2592
+ * `define-persona` is `admin=false` for everyone (own-persona discipline; no ep consumer needs
2593
+ * cross-owner persona writes - an operator redefines via config, not the wire), where the ctl
2594
+ * admin tier allowed operator cross-owner redefine; (2) launch is owner-equality-only, above.
2595
+ * Both are least-privilege reductions, never widenings. */
2416
2596
  managerServiceDefs() {
2417
2597
  const args = (ctx) => (ctx.request.args ?? {});
2418
2598
  const callerOf = (ctx) => principalKey(ctx.subject.caller.owner, ctx.subject.caller.actor).key;
@@ -2601,6 +2781,8 @@ export class Manager {
2601
2781
  runResume: (ctx) => this.serveGated(ctx, () => this.runHost().resume(args(ctx))),
2602
2782
  runAnswer: (ctx) => this.serveGated(ctx, () => {
2603
2783
  const input = args(ctx);
2784
+ if (input.amend === true)
2785
+ return this.runHost().amend(input, this.runAnswerer(ctx), (accepted) => this.authorizeRunAmend(ctx, accepted));
2604
2786
  return this.runHost().answer(input, this.runAnswerer(ctx), (open) => this.authorizeRunAnswer(ctx, input, open));
2605
2787
  }),
2606
2788
  runStatus: (ctx) => this.serveGated(ctx, () => this.runHost().status(args(ctx))),
@@ -3035,11 +3217,6 @@ export class Manager {
3035
3217
  return { error: `agent auth preflight failed for "${name}": ${e.message}` };
3036
3218
  }
3037
3219
  }
3038
- /** Drop a live agent's slot. When `floor` is set and the agent died young (lived less than
3039
- * MIN_LIFETIME), push a cooling stamp so the freed slot still counts toward the ceiling until it
3040
- * expires — flooring the RECYCLE, not the call, so both free paths (despawn + exit/reap) are
3041
- * covered (P4c). Floor self + own-child despawn and natural exit; NEVER admin despawn (operator
3042
- * emergency-kill stays unthrottled) and NEVER the reserved-rollback path (no cold-start paid). */
3043
3220
  /**
3044
3221
  * WHY A SEAT LEFT — one line, at the one place every free path passes through.
3045
3222
  *
@@ -3062,7 +3239,7 @@ export class Manager {
3062
3239
  const info = a.handle.exitInfo?.();
3063
3240
  detail = info === undefined
3064
3241
  ? `exit detail unavailable from runtime "${a.handle.kind}"`
3065
- : `exit code ${info.code ?? "unknown"}${info.signal === undefined ? "" : `, signal ${info.signal}`}`;
3242
+ : `exit code ${info.code ?? "unknown"}${info.signal === undefined ? "" : `, signal ${info.signal}`}${info.diagnostic ? `; last connector diagnostic: ${info.diagnostic}` : ""}`;
3066
3243
  }
3067
3244
  catch (e) {
3068
3245
  // A runtime that throws while being asked has told us something real; it must not take the
@@ -3071,6 +3248,49 @@ export class Manager {
3071
3248
  }
3072
3249
  console.error(`seat reaped: ${a.name} (${a.id}, uid ${a.lifecycleUid}${a.handle.pid !== undefined ? `, pid ${a.handle.pid}` : ""}) - ${causeText}; ${detail}`);
3073
3250
  }
3251
+ /**
3252
+ * A SEAT THAT LEAVES THE MESH WHILE ITS SLOT IS HELD GETS A LINE, AND SO DOES ITS RETURN.
3253
+ *
3254
+ * Seats once read `running · mesh offline` for many hours with no manager log line, so a watchdog
3255
+ * keyed on process liveness saw nothing wrong and nobody could say when each seat dropped (#1208).
3256
+ * The line follows the same state `ps` reads, checked on every roster change and view transition,
3257
+ * because not every offline record arrives as a presence event: after a reconnect the fresh snapshot
3258
+ * records an already-offline seat quietly. Only a current view is a verdict, so a stale or
3259
+ * unpopulated one logs nothing. The match is {@link isOwnPresence}. A seat being stopped is left to
3260
+ * the reap line. The heartbeat is the seat's own record and nothing validated it, so an unreadable
3261
+ * one prints as unknown instead of throwing inside the watch.
3262
+ */
3263
+ logSeatMeshTransitions() {
3264
+ if (typeof this.ep.presenceView === "function" && this.ep.presenceView().state !== "current")
3265
+ return;
3266
+ const roster = new Map(this.ep.getRoster().map((p) => [p.card.id, p]));
3267
+ for (const a of this.agents.values()) {
3268
+ const seen = roster.get(this.managedPrincipal(a));
3269
+ if (a.terminalizing || !this.isOwnPresence(a, seen))
3270
+ continue;
3271
+ const who = `${a.name} (${a.id}, uid ${a.lifecycleUid}${a.handle.pid !== undefined ? `, pid ${a.handle.pid}` : ""})`;
3272
+ if (seen.status === "offline" && !a.offMesh) {
3273
+ a.offMesh = true;
3274
+ const ts = seen.ts;
3275
+ const last = typeof ts === "number" && !Number.isNaN(new Date(ts).getTime()) ? new Date(ts).toISOString() : "unknown";
3276
+ console.error(`seat offline on the mesh: ${who} - last heartbeat ${last}; process ${a.handle.status()}`);
3277
+ }
3278
+ else if (seen.status !== "offline" && a.offMesh) {
3279
+ a.offMesh = false;
3280
+ console.error(`seat back on the mesh: ${who}`);
3281
+ }
3282
+ }
3283
+ }
3284
+ /** Whether a roster record is this seat's own: the readiness fence's exact principal and
3285
+ * incarnation, so a same-named foreign peer or an older incarnation never speaks for it (#1208). */
3286
+ isOwnPresence(a, p) {
3287
+ return p !== undefined && p.card.id === this.managedPrincipal(a) && p.lifecycleUid === a.lifecycleUid;
3288
+ }
3289
+ /** Drop a live agent's slot. When `floor` is set and the agent died young (lived less than
3290
+ * MIN_LIFETIME), push a cooling stamp so the freed slot still counts toward the ceiling until it
3291
+ * expires — flooring the RECYCLE, not the call, so both free paths (despawn + exit/reap) are
3292
+ * covered (P4c). Floor self + own-child despawn and natural exit; NEVER admin despawn (operator
3293
+ * emergency-kill stays unthrottled) and NEVER the reserved-rollback path (no cold-start paid). */
3074
3294
  freeSlot(a, floor, cause, acceptedBeforeFence = false) {
3075
3295
  if (this.agents.get(a.name) !== a)
3076
3296
  return; // already freed (exit raced despawn, etc.)
@@ -3101,14 +3321,14 @@ export class Manager {
3101
3321
  if (this.userMode) {
3102
3322
  const p = parsePrincipalKey(a.id);
3103
3323
  if (p)
3104
- this.retiring.set(a.name, { opId: retireOpId(a.lifecycleUid), lifecycleUid: a.lifecycleUid, owner: p.owner, actor: p.actor, agentId: a.id, userOwner: a.userOwner, secretPaths: a.secretPaths, startedAt: Date.now() });
3324
+ this.retiring.set(a.name, { opId: retireOpId(a.lifecycleUid), lifecycleUid: a.lifecycleUid, owner: p.owner, actor: p.actor, agentId: a.id, userOwner: a.userOwner, secretPaths: a.secretPaths, launch: { allowSubscribe: a.launch.allowSubscribe }, startedAt: Date.now() });
3105
3325
  }
3106
3326
  else if (this.auth) {
3107
3327
  // Unit B: a STATIC lifecycle now also holds its name pending its own terminal (the F1
3108
3328
  // static retirement the detached deprovision below drives) — the alias frees only when the
3109
3329
  // gate+head terminal completes, exactly the user-mode discipline. The wire principal is the
3110
3330
  // incarnation-unique nkey (F5-bind); owner is the dev owner.
3111
- this.retiring.set(a.name, { opId: retireOpId(a.lifecycleUid), lifecycleUid: a.lifecycleUid, owner: DEV_OWNER, actor: a.id, agentId: a.id, secretPaths: a.secretPaths, runtime: a.handle?.reference, startedAt: Date.now() });
3331
+ this.retiring.set(a.name, { opId: retireOpId(a.lifecycleUid), lifecycleUid: a.lifecycleUid, owner: DEV_OWNER, actor: a.id, agentId: a.id, secretPaths: a.secretPaths, launch: { allowSubscribe: a.launch.allowSubscribe }, runtime: a.handle?.reference, startedAt: Date.now() });
3112
3332
  }
3113
3333
  // Auth mode: tear down the departed agent's minted broker footprint + creds file (#159 B2). The
3114
3334
  // process is already gone, so this must never block the slot free or throw into the caller — it runs
@@ -3232,7 +3452,21 @@ export class Manager {
3232
3452
  console.error(`revoke agent grant ${a.name}: ${e.message}`);
3233
3453
  }
3234
3454
  }
3235
- await this.deprovisionBroker(a);
3455
+ try {
3456
+ const res = await this.deprovisionBroker(a);
3457
+ const h = this.retiring.get(a.name);
3458
+ if (h && h.lifecycleUid === a.lifecycleUid)
3459
+ h.lastResources = res;
3460
+ }
3461
+ catch (e) {
3462
+ const h = this.retiring.get(a.name);
3463
+ if (h && h.lifecycleUid === a.lifecycleUid) {
3464
+ h.lastError = e.message;
3465
+ if (e instanceof DeprovisionError)
3466
+ h.lastResources = e.accounting;
3467
+ }
3468
+ throw e;
3469
+ }
3236
3470
  // #29 piece 3: after the footprint teardown, ask the AUTH plane to RETIRE the lifecycle over
3237
3471
  // the auth endpoint rail. The rail re-checks the SERVE-ISSUANCE GATE at serve time (not the
3238
3472
  // space-manager lease - that check was replaced in 02794b2f) and refuses unless the registration
@@ -3324,46 +3558,44 @@ export class Manager {
3324
3558
  : await mintCreds(this.auth, requestIdentity, "retirement-requester", {
3325
3559
  retirementRequester: { ...caller, target: retirementTarget },
3326
3560
  });
3327
- const nc = await this.dial({ authenticator: credsAuthenticator(new TextEncoder().encode(creds)), maxReconnectAttempts: 0 });
3561
+ // The requester grant subscribes `_INBOX_<id>.>` only (provision.ts): the endpoint path's
3562
+ // JetStream API requests (the contract-store read inside resolveService) must reply under it, not under the client default `_INBOX.`.
3563
+ const nc = await this.dial({ authenticator: credsAuthenticator(new TextEncoder().encode(creds)), inboxPrefix: `_INBOX_${requestIdentity.id}`, maxReconnectAttempts: 0 });
3328
3564
  try {
3329
- // §13.2 nonce: >=128 bits of CSPRNG entropy, base64url (the `endpoint-invoke` idiom).
3330
- const nonce = randomBytes(24).toString("base64url");
3331
- // The caller-chosen request id the reply MUST echo (the minimal correctness guard from the
3332
- // not-yet-migrated endpoint envelope - see the residual in the auth listener).
3333
- const requestId = randomBytes(16).toString("base64url");
3334
- const subject = epRequestSubject(this.space, {
3335
- route: { mode: "one" },
3336
- endpoint: AUTH_ENDPOINT,
3337
- command: EP_CMD_RETIRE_LIFECYCLE,
3338
- // The TARGET rides the subject (authz mode `handle`, arity 3) — broker-enforced, and
3339
- // pinned by the grant minted above, so it is not a body claim the caller can vary.
3340
- target: { mode: "handle", tOwner: target.owner, tActor: target.actor, tUid: a.lifecycleUid },
3341
- caller,
3342
- nonce,
3565
+ // The service resolves FRESH on every attempt (never cached on `this`): a plane restart
3566
+ // mid-despawn then lands on the new epoch through an ordinary resolve, rather than a
3567
+ // caller pinned to a now-superseded incarnation.
3568
+ const service = await resolveService(nc, this.space, AUTH_ENDPOINT, caller, { deadlineMs: 10_000 });
3569
+ // The TARGET rides the invoke's own target (authz mode `exact`, arity 3) — broker-enforced,
3570
+ // and pinned by the grant minted above, so it is not a body claim the caller can vary.
3571
+ const attributed = await invokeCommand(nc, this.space, service, EP_CMD_RETIRE_LIFECYCLE, {
3572
+ opId,
3573
+ serveEndpoint: MANAGER_ENDPOINT, serveInstanceId: this.managerInstanceId, serveEpoch,
3574
+ }, {
3575
+ target: { mode: "exact", owner: target.owner, actor: target.actor, lifecycleUid: a.lifecycleUid },
3576
+ deadlineMs: 20_000,
3343
3577
  });
3344
- // The responder derives its OWN reply subject from this request (caller triple + nonce,
3345
- // prefixed with the RESPONDER's instance identity), so a caller-supplied `reply` header
3346
- // would be ignored — that is the confused-deputy boundary being structural. The caller
3347
- // therefore reads its own reply-plane filter and binds the answer off the reply SUBJECT.
3348
- const m = await epAwaitReply(nc, this.space, caller, nonce, requestId, subject,
3349
- // Declare THIS manager instance's serve identity. Since #350 these SELECT the gate row;
3350
- // they no longer authorize — the rail refuses unless the row they name belongs to this
3351
- // caller's own subject-derived principal. A superseded predecessor (same instanceId, OLD
3352
- // epoch after a restart) is still refused by the epoch comparison.
3353
- JSON.stringify({ id: requestId, op: "retireLifecycle", args: {
3354
- opId,
3355
- serveEndpoint: MANAGER_ENDPOINT, serveInstanceId: this.managerInstanceId, serveEpoch,
3356
- } }), 20_000);
3357
- const r = m;
3578
+ const r = attributed.reply;
3358
3579
  if (r.ok) {
3359
3580
  this.confirmRetirement(a);
3360
3581
  }
3582
+ else if (r.error?.code === "expired") {
3583
+ // A stale-epoch refusal: the responder answered, but at another epoch than this resolve
3584
+ // bound (a plane restart mid-despawn landed on the new epoch) - the retry copy, never a
3585
+ // hard despawn failure line, since a same-name spawn (or the auth service's own boot
3586
+ // resume) re-drives the same teardown.
3587
+ const copy = uncertain(r.error.message);
3588
+ if (held)
3589
+ held.lastError = copy;
3590
+ console.error(`despawn ${a.name}: ${copy}`);
3591
+ }
3361
3592
  else {
3362
3593
  // The rail's refusal is already the operator copy (lease-loss/stale/foreign-op faces,
3363
3594
  // full-no-op statements included) - surface it INTACT, never flattened.
3595
+ const message = r.error?.message ?? "the auth service refused the retirement without a reason";
3364
3596
  if (held)
3365
- held.lastError = r.error ?? "the auth service refused the retirement without a reason";
3366
- console.error(`despawn ${a.name}: ${r.error ?? "the auth service refused the retirement without a reason"}`);
3597
+ held.lastError = message;
3598
+ console.error(`despawn ${a.name}: ${message}`);
3367
3599
  }
3368
3600
  }
3369
3601
  finally {
@@ -3414,13 +3646,59 @@ export class Manager {
3414
3646
  // LIFECYCLE-PINNED (SPEC 13.1): both the credential's exact-name grants and the delete names
3415
3647
  // carry a.lifecycleUid, so a stale/replayed teardown for this retired incarnation is broker-denied
3416
3648
  // against a same-name successor's footprint (its names embed a different uid).
3649
+ // Durable membership rows are lifecycle-keyed per concrete channel. Start with the launch's
3650
+ // concrete read channels, then include wildcard-covered and unnamed channels from the inventory.
3651
+ const memberChannels = new Set((a.launch?.allowSubscribe ?? []).filter(isConcreteChannel));
3652
+ // Complete the set from the delivery daemon's lifecycle-exact inventory, which covers rows on
3653
+ // wildcard-granted channels and teardowns that carry no launch spec. An inventory that cannot be
3654
+ // read leaves those rows RETAINED and says so; it never widens the delete beyond exact keys.
3655
+ const inventory = await this.lifecycleMembershipInventory(a);
3656
+ for (const ch of inventory.channels)
3657
+ memberChannels.add(ch);
3658
+ if (!inventory.complete)
3659
+ console.error(`deprovision ${a.name} (${a.id}): membership inventory incomplete (${inventory.reason}); rows on channels outside its launch policy stay RETAINED for uid ${a.lifecycleUid}`);
3417
3660
  const creds = await mintCreds(this.auth, newIdentity(), "deprovisioner", {
3418
- deprovisionTarget: { principal: a.id, lifecycleUid: a.lifecycleUid },
3661
+ deprovisionTarget: { principal: a.id, lifecycleUid: a.lifecycleUid, memberChannels: [...memberChannels] },
3419
3662
  });
3420
3663
  // Bound the detached broker teardown so a wedged broker can't leave the deprovision promise pending
3421
3664
  // forever with no log — the timeout rejects into freeSlot's fail-loud `.catch` (paired with the
3422
3665
  // helper's own fail-fast connect). The durables/ACL row still fall to space teardown as a backstop.
3423
- await withTimeout(deprovisionAgent({ servers: this.servers ?? DEFAULT_SERVER, space: this.space, targetId: a.id, lifecycleUid: a.lifecycleUid, creds }), DEPROVISION_TIMEOUT_MS, `deprovision ${a.name} (${a.id}): broker teardown timed out`);
3666
+ let accounting;
3667
+ try {
3668
+ accounting = await withTimeout(deprovisionAgent({
3669
+ servers: this.servers ?? DEFAULT_SERVER,
3670
+ space: this.space,
3671
+ targetId: a.id,
3672
+ lifecycleUid: a.lifecycleUid,
3673
+ memberChannels: [...memberChannels],
3674
+ creds,
3675
+ }), DEPROVISION_TIMEOUT_MS, `deprovision ${a.name} (${a.id}): broker teardown timed out`);
3676
+ this.lastDeprovisionResources = accounting;
3677
+ }
3678
+ catch (e) {
3679
+ if (e instanceof DeprovisionError) {
3680
+ this.lastDeprovisionResources = e.accounting;
3681
+ }
3682
+ throw e;
3683
+ }
3684
+ if (!inventory.complete)
3685
+ throw new Error(`membership inventory incomplete (${inventory.reason}); rows on channels outside its launch policy stay RETAINED for uid ${a.lifecycleUid}`);
3686
+ return accounting;
3687
+ }
3688
+ /** The retiring lifecycle's durable membership channels, read from the delivery daemon's
3689
+ * `lifecycleMemberships` admin verb (read-only, lifecycle-exact). Incomplete on any refusal. */
3690
+ async lifecycleMembershipInventory(a) {
3691
+ const t = deprovisionTargetPrincipal({ principal: a.id, lifecycleUid: a.lifecycleUid });
3692
+ try {
3693
+ const r = await this.ep.requestDeliveryAdmin("lifecycleMemberships", { principal: principalKey(t.owner, t.actor).key, lifecycleUid: t.lifecycleUid }, 5_000);
3694
+ const data = r.data;
3695
+ if (!r.ok || data?.complete !== true || !Array.isArray(data.channels) || !data.channels.every((c) => typeof c === "string" && isConcreteChannel(c)))
3696
+ return { complete: false, channels: [], reason: r.ok ? "malformed inventory reply" : (r.error ?? "refused") };
3697
+ return { complete: true, channels: data.channels };
3698
+ }
3699
+ catch (e) {
3700
+ return { complete: false, channels: [], reason: e.message };
3701
+ }
3424
3702
  }
3425
3703
  /** Reap a parent's children on its exit (P4b). Every descendant remains managed until the runtime's
3426
3704
  * authoritative wait proves exit; the wait participates in the lifecycle drain, so preservation can
@@ -3471,6 +3749,28 @@ export class Manager {
3471
3749
  `(${error.message}). Reload the live Pi seat before the preservation cut; refusing to resume fresh and lose its context.`);
3472
3750
  }
3473
3751
  }
3752
+ /** A resumed seat's fork provenance (#1500). The seat forks after it launches, so this reads the
3753
+ * record its connector named once the seat has written it, and keeps it on the launch so the
3754
+ * resume document carries it across a manager restart. Until the seat has written it, or when the
3755
+ * record cannot be read or does not name this seat's source, it is absent: `ps` then shows the
3756
+ * source id alone rather than failing the listing for every other seat. */
3757
+ forkProvenance(a) {
3758
+ if (a.launch.resumed || !a.launch.resumeRecordPath || !a.launch.forkSource)
3759
+ return a.launch.resumed;
3760
+ let record;
3761
+ try {
3762
+ record = JSON.parse(readFileSync(a.launch.resumeRecordPath, "utf8"));
3763
+ }
3764
+ catch {
3765
+ return undefined;
3766
+ }
3767
+ const r = (typeof record === "object" && record !== null ? record : {});
3768
+ if (r.source !== a.launch.forkSource || typeof r.transcriptSha256 !== "string" || !/^[a-f0-9]{64}$/.test(r.transcriptSha256) ||
3769
+ !(r.title === undefined || r.title === null || (typeof r.title === "string" && r.title.length <= 1024)))
3770
+ return undefined;
3771
+ a.launch.resumed = { source: r.source, ...(typeof r.title === "string" && r.title ? { title: r.title } : {}), transcriptSha256: r.transcriptSha256 };
3772
+ return a.launch.resumed;
3773
+ }
3474
3774
  readManagedSession(a) {
3475
3775
  return this.readManagedSessionState(a).sessionId;
3476
3776
  }
@@ -4181,9 +4481,9 @@ export class Manager {
4181
4481
  // SYNCHRONOUS (existsSync / registry / accessSync / readFileSync — no await), so the gate stays
4182
4482
  // atomic: the capacity snapshot and the reserve land in one tick (P4a/P4c), and two concurrent
4183
4483
  // spawns can't overshoot the ceiling or pick the same name.
4184
- const cooling = this.coolingCount(); // prune expired stamps, then count live cooling slots
4185
- if (this.agents.size + this.reserved.size + cooling >= MAX_AGENTS)
4186
- return { ok: false, error: `at capacity (${MAX_AGENTS} agents incl. in-flight + cooling); despawn one or wait` };
4484
+ const occupancy = this.occupancy(); // prunes expired cooling stamps; counts each name once
4485
+ if (occupancy.used >= MAX_AGENTS)
4486
+ return { ok: false, error: this.capacityRefusal(occupancy) };
4187
4487
  // Harness preflight before reserving a slot or minting — a missing `claude`/`opencode` binary
4188
4488
  // fails here with a clear name, not obscurely at process spawn. No fallback. All synchronous, so
4189
4489
  // the reserve gate stays atomic. (The connector itself was resolved up top, before the capacity gate.)
@@ -4328,7 +4628,7 @@ export class Manager {
4328
4628
  return { ok: false, error: `the name "${identityName}" is still reconciling at manager startup; its prior lifecycle terminal owns this alias until it completes. Retry shortly.` };
4329
4629
  const held = this.retiring.get(identityName);
4330
4630
  if (held !== undefined) {
4331
- void this.deprovision({ id: held.agentId, name: identityName, lifecycleUid: held.lifecycleUid, userOwner: held.userOwner, secretPaths: held.secretPaths }).catch(() => { });
4631
+ void this.deprovision({ id: held.agentId, name: identityName, lifecycleUid: held.lifecycleUid, userOwner: held.userOwner, secretPaths: held.secretPaths, launch: held.launch }).catch(() => { });
4332
4632
  const err = lifecycleBlocked("failed-precondition", `the name "${identityName}" is reserved pending retirement: its previous agent's despawn started that lifecycle's teardown (footprint + standing-authority revoke + auth-side retirement), and the name frees only when all of it completes${held.lastError !== undefined ? ` (last attempt: ${held.lastError})` : ""}. NEXT: wait a moment and retry this spawn (retrying re-drives the whole teardown), or pick another name.`, { blockedOp: "retirement", headState: "retiring", opId: held.opId, remedy: "retry" });
4333
4633
  return { ok: false, error: renderLifecycleBlocked(err.message, err), details: err.details };
4334
4634
  }
@@ -4673,6 +4973,7 @@ export class Manager {
4673
4973
  events,
4674
4974
  shareTools: opts.shareTools,
4675
4975
  forkSource: opts.resume,
4976
+ ...(opts.resume !== undefined && spec.resumeRecordPath ? { resumeRecordPath: spec.resumeRecordPath } : {}),
4676
4977
  // Opaque values may contain secrets. Preserve only their keys and require the referenced
4677
4978
  // persona/manifest to resolve the values again; imperative overrides have no safe payload.
4678
4979
  unresolvedLaunchOptionKeys: opts.launchOptions && Object.keys(opts.launchOptions).length
@@ -4707,6 +5008,7 @@ export class Manager {
4707
5008
  });
4708
5009
  }
4709
5010
  this.agents.set(name, managed);
5011
+ this.reservedLive.add(name);
4710
5012
  // The live slot now owns teardown — freeSlot deprovisions this identity on exit — so the
4711
5013
  // orphan-rollback in `finally` no longer applies to it.
4712
5014
  provisioned = undefined;
@@ -4776,6 +5078,7 @@ export class Manager {
4776
5078
  }
4777
5079
  finally {
4778
5080
  this.reserved.delete(name);
5081
+ this.reservedLive.delete(name);
4779
5082
  // Minted but never handed to a live slot (buildLaunch / runtime.spawn threw after mint) → tear the
4780
5083
  // orphan down (detached, fail-loud) so a failed spawn leaves no creds/durables behind (#159 B).
4781
5084
  if (provisioned) {
@@ -4792,6 +5095,7 @@ export class Manager {
4792
5095
  if (!release)
4793
5096
  return { ok: false, agents: [], error: this.maintenanceError() };
4794
5097
  const batchReservations = [];
5098
+ const prepared = new Map();
4795
5099
  try {
4796
5100
  if (inventory.version !== "cotal-manager-resume/v1")
4797
5101
  return { ok: false, agents: [], error: `unsupported manager resume inventory version ${String(inventory.version)}` };
@@ -4801,8 +5105,9 @@ export class Manager {
4801
5105
  const principals = new Set();
4802
5106
  await this.ep.waitForPresenceSnapshot();
4803
5107
  const liveRoster = this.ep.getRoster().filter((presence) => presence.status !== "offline");
4804
- if (this.agents.size + this.reserved.size + this.coolingCount() + inventory.agents.length > MAX_AGENTS)
4805
- return { ok: false, agents: [], error: `resume inventory would exceed manager capacity (${MAX_AGENTS})` };
5108
+ const occupancy = this.occupancy();
5109
+ if (occupancy.used + inventory.agents.length > MAX_AGENTS)
5110
+ return { ok: false, agents: [], error: `resume inventory of ${inventory.agents.length} would exceed manager capacity (${this.occupancyText(occupancy)})` };
4806
5111
  for (const entry of inventory.agents) {
4807
5112
  if (seen.has(entry.name))
4808
5113
  return { ok: false, agents: [], error: `resume inventory contains duplicate agent name "${entry.name}"` };
@@ -4858,7 +5163,6 @@ export class Manager {
4858
5163
  this.reserved.add(entry.name);
4859
5164
  batchReservations.push(entry.name);
4860
5165
  }
4861
- const prepared = new Map();
4862
5166
  const preflight = [];
4863
5167
  for (const entry of inventory.agents) {
4864
5168
  const reply = await this.resumePreservedAgent(entry, true, true, prepared);
@@ -4887,8 +5191,22 @@ export class Manager {
4887
5191
  return { ok: true, agents };
4888
5192
  }
4889
5193
  finally {
4890
- for (const name of batchReservations)
5194
+ for (const name of batchReservations) {
4891
5195
  this.reserved.delete(name);
5196
+ this.reservedLive.delete(name);
5197
+ }
5198
+ // Preflight builds every launch before any child starts, so a refused batch, a failed launch,
5199
+ // or the agents skipped after it leave built specs no runtime ever received.
5200
+ for (const { spec, spawned } of prepared.values()) {
5201
+ if (spawned)
5202
+ continue;
5203
+ try {
5204
+ discardLaunchArtifacts(spec.artifacts);
5205
+ }
5206
+ catch (e) {
5207
+ console.error(`! resume: ${e.message}`);
5208
+ }
5209
+ }
4892
5210
  release();
4893
5211
  }
4894
5212
  }
@@ -5072,8 +5390,9 @@ export class Manager {
5072
5390
  return { ok: false, error: nameErr };
5073
5391
  if (this.agents.has(entry.name) || (!batchReserved && this.reserved.has(entry.name)))
5074
5392
  return { ok: false, error: `retained agent "${entry.name}" is already managed or reserved; same-principal resume never auto-numbers` };
5075
- if (!batchReserved && this.agents.size + this.reserved.size + this.coolingCount() >= MAX_AGENTS)
5076
- return { ok: false, error: `at capacity (${MAX_AGENTS} agents incl. in-flight + cooling); same-principal resume refused` };
5393
+ const occupancy = batchReserved ? undefined : this.occupancy();
5394
+ if (occupancy && occupancy.used >= MAX_AGENTS)
5395
+ return { ok: false, error: `${this.capacityRefusal(occupancy)}; same-principal resume refused` };
5077
5396
  const cached = prepared?.get(entry.name);
5078
5397
  if (!preflightOnly && cached) {
5079
5398
  try {
@@ -5232,6 +5551,7 @@ export class Manager {
5232
5551
  const custody = this.reserveCustody();
5233
5552
  if (custody)
5234
5553
  await this.recordSlotCustody({ name: entry.name, id: resumeId, lifecycleUid: entry.identity.lifecycleUid }, custody);
5554
+ prepared.spawned = true;
5235
5555
  const handle = await this.spawnCustodied(entry.name, prepared.spec, entry.launch.cwd, custody);
5236
5556
  const managed = {
5237
5557
  name: entry.name,
@@ -5268,6 +5588,8 @@ export class Manager {
5268
5588
  events: entry.launch.events,
5269
5589
  shareTools: entry.launch.shareTools,
5270
5590
  forkSource: entry.launch.forkSource,
5591
+ ...(entry.launch.resumed ? { resumed: entry.launch.resumed } : {}),
5592
+ ...(prepared.launchOpts.resume !== undefined && prepared.spec.resumeRecordPath ? { resumeRecordPath: prepared.spec.resumeRecordPath } : {}),
5271
5593
  sessionId: entry.launch.sessionId,
5272
5594
  },
5273
5595
  ...(prepared.spec.sessionStatePath
@@ -5276,6 +5598,7 @@ export class Manager {
5276
5598
  suppressCleanup: true,
5277
5599
  };
5278
5600
  this.agents.set(entry.name, managed);
5601
+ this.reservedLive.add(entry.name);
5279
5602
  if (this.resumeAttemptId)
5280
5603
  this.resumedAgentNames.add(entry.name);
5281
5604
  await this.recordSlotRuntime(managed);
@@ -5312,8 +5635,10 @@ export class Manager {
5312
5635
  return { ok: false, error: e.message };
5313
5636
  }
5314
5637
  finally {
5315
- if (!batchReserved)
5638
+ if (!batchReserved) {
5316
5639
  this.reserved.delete(entry.name);
5640
+ this.reservedLive.delete(entry.name);
5641
+ }
5317
5642
  }
5318
5643
  }
5319
5644
  probeStaticCredential(creds) {
@@ -5517,6 +5842,48 @@ export class Manager {
5517
5842
  this.cooling = this.cooling.filter((stamp) => stamp > now);
5518
5843
  return this.cooling.length;
5519
5844
  }
5845
+ /** What holds the ceiling's slots, one count per seat (#906). A launching seat sits in `reserved`
5846
+ * from accept until its launch settles, in `agents` once its process exists, and in `cooling`
5847
+ * if it then frees young, so a reservation stops counting once it reaches `agents`. `managed` is
5848
+ * what this manager's `ps` lists, `reserved` is the accepted launches that have no process yet,
5849
+ * and `launching` is every unsettled launch that still holds a slot through `reserved` or
5850
+ * `agents`. A launch whose seat already freed holds no slot but its cooling stamp, so its
5851
+ * failure can free nothing. */
5852
+ occupancy() {
5853
+ const cooling = this.coolingCount();
5854
+ let reserved = 0;
5855
+ let launching = 0;
5856
+ for (const name of this.reserved) {
5857
+ const placed = this.reservedLive.has(name);
5858
+ if (!placed)
5859
+ reserved++;
5860
+ if (!placed || this.agents.has(name))
5861
+ launching++;
5862
+ }
5863
+ const managed = this.agents.size;
5864
+ return {
5865
+ managed,
5866
+ reserved,
5867
+ cooling,
5868
+ launching,
5869
+ used: managed + reserved + cooling,
5870
+ nextCoolingMs: cooling ? Math.min(...this.cooling) - Date.now() : undefined,
5871
+ };
5872
+ }
5873
+ occupancyText(o) {
5874
+ return `${o.used} of ${MAX_AGENTS} slots: ${o.managed} managed, ${o.reserved} reserved, ${o.cooling} cooling`;
5875
+ }
5876
+ /** The capacity refusal states the split it enforces and whether waiting can free a slot: a
5877
+ * cooling slot frees at a known time, an unsettled launch frees one only if it fails, and a
5878
+ * managed seat frees one only when it is stopped. */
5879
+ capacityRefusal(o) {
5880
+ const remedy = o.nextCoolingMs !== undefined
5881
+ ? `waiting frees a cooling slot in ${Math.ceil(o.nextCoolingMs / 1000)}s, or despawn one`
5882
+ : o.launching
5883
+ ? `waiting frees a slot only if an unsettled launch fails (${o.launching} pending); otherwise despawn one`
5884
+ : "waiting frees no slot; despawn one";
5885
+ return `at capacity (${this.occupancyText(o)}); ${remedy}`;
5886
+ }
5520
5887
  async opStop(args, caller, admin) {
5521
5888
  const name = String(args.name ?? "").trim();
5522
5889
  const a = this.agents.get(name);
@@ -5623,8 +5990,14 @@ export class Manager {
5623
5990
  * This is NEVER the manager's standing seed/supervisor connection. */
5624
5991
  async withEndpointServeExecutor(fn) {
5625
5992
  const identity = this.remoteAuthority?.identities.executor ?? newIdentity();
5626
- if (this.remoteAuthority && (!this.remoteExecutorCreds || inspectCredHealth(this.remoteExecutorCreds).state !== "healthy"))
5627
- await this.renewRemoteExecutor(true);
5993
+ if (this.remoteAuthority && (!this.remoteExecutorCreds || inspectCredHealth(this.remoteExecutorCreds).state !== "healthy")) {
5994
+ // Shutdown has drained and discarded the standing serve context. Renew only its scoped
5995
+ // maintenance executor through the existing host operation; never resurrect standing duties.
5996
+ if (this.leaseStopping)
5997
+ await this.renewRemoteExecutor(true);
5998
+ else
5999
+ await (this.remoteAuthority.renewStandingBundle ? this.renewRemoteStandingBundle(true) : this.renewRemoteExecutor(true));
6000
+ }
5628
6001
  const creds = this.remoteExecutorCreds ?? (this.auth
5629
6002
  ? await mintCreds(this.auth, identity, "endpoint-serve-executor", {
5630
6003
  endpointServeExecutor: { endpoint: MANAGER_ENDPOINT, instanceId: this.managerInstanceId },
@@ -5670,7 +6043,7 @@ export class Manager {
5670
6043
  return {
5671
6044
  instanceId: this.managerInstanceId,
5672
6045
  runtime: this.runtime.kind,
5673
- custody: process.platform === "linux" && this.runtime.kind === "pty" ? "custodied" : "legacy",
6046
+ custody: this.runtime.kind === "pty" && this.runtime.supportsRelease === true ? "custodied" : "legacy",
5674
6047
  agentCount: this.agents.size,
5675
6048
  uptimeMs: Date.now() - this.startedAtMs,
5676
6049
  connectors: this.connectorStatuses.map((row) => ({ ...row, binaries: { ...row.binaries } })),
@@ -5721,7 +6094,7 @@ export class Manager {
5721
6094
  report = await reconcileEndpointGate({
5722
6095
  kv: authKv, space: this.space, endpoint: MANAGER_ENDPOINT, instanceId,
5723
6096
  probeHolder: makeManagerHolderLivenessProbe({ space: this.space, servers, auth, log }),
5724
- evict: makeManagerEndpointEvictor({ space: this.space, servers, auth, log }),
6097
+ evictHolders: makeManagerEndpointHolderEvictor({ space: this.space, servers, auth, log }),
5725
6098
  log,
5726
6099
  recordsKv,
5727
6100
  });
@@ -5755,7 +6128,17 @@ export class Manager {
5755
6128
  inboxPrefix: `_INBOX_${state.identity.id}`,
5756
6129
  maxReconnectAttempts: -1,
5757
6130
  });
5758
- nc.closed().then((err) => { void this.onServeConnectionClosed(err ?? undefined); });
6131
+ // Track the transport from the first moment: nats.js reports `disconnect` when the socket
6132
+ // drops and `reconnect` only after it has re-sent this connection's subscriptions. If the
6133
+ // status stream itself fails, the transport can no longer be observed, so it counts as down.
6134
+ void (async () => {
6135
+ for await (const st of nc.status()) {
6136
+ if (st.type === "disconnect")
6137
+ this.serveTransportDown.add(nc);
6138
+ else if (st.type === "reconnect")
6139
+ this.serveTransportDown.delete(nc);
6140
+ }
6141
+ })().catch(() => this.serveTransportDown.add(nc));
5759
6142
  let handle;
5760
6143
  try {
5761
6144
  // The 1b typed surface + the derived `describe`. The descriptor stays PUBLIC in static
@@ -5770,6 +6153,13 @@ export class Manager {
5770
6153
  // durable slot rows mirror it). Static mode carries no mapping-revision dimension, so
5771
6154
  // the revision is the constant 0 — a caller that pins a revision pins 0.
5772
6155
  resolveTarget: (t) => {
6156
+ if (this.remoteAuthority) {
6157
+ const key = principalKey(t.owner, t.actor).key;
6158
+ for (const a of this.agents.values())
6159
+ if (a.id === key)
6160
+ return { lifecycleUid: a.lifecycleUid, mappingRevision: 0 };
6161
+ return undefined;
6162
+ }
5773
6163
  if (t.owner === DEV_OWNER) {
5774
6164
  for (const a of this.agents.values())
5775
6165
  if (!a.userOwner && a.id === t.actor)
@@ -5789,6 +6179,9 @@ export class Manager {
5789
6179
  await nc.drain().catch(() => nc.close());
5790
6180
  throw e;
5791
6181
  }
6182
+ // Wired only once the connection serves: a connection this method drains itself after a
6183
+ // refused serve is not a fault to recover, and the caller already sees the refusal.
6184
+ nc.closed().then((err) => { void this.onServeConnectionClosed(err ?? undefined); });
5792
6185
  return { nc, handle };
5793
6186
  }
5794
6187
  /** #2073: re-mint the SAME `endpoint-serve` identity through the SAME scoped one-shot executor and
@@ -5823,7 +6216,9 @@ export class Manager {
5823
6216
  const recovered = await this.boundedReconnect("service endpoint", async () => {
5824
6217
  if (!s)
5825
6218
  return false;
5826
- if (inspectCredHealth(s.creds ?? "").state !== "healthy" && s.creds !== undefined) {
6219
+ if (this.remoteAuthority)
6220
+ await this.renewRemoteBeforeRedial("serve", () => s.creds);
6221
+ else if (inspectCredHealth(s.creds ?? "").state !== "healthy" && s.creds !== undefined) {
5827
6222
  s.creds = await this.remintServeCreds(s);
5828
6223
  this.scheduleServeRenewal(s.creds);
5829
6224
  }
@@ -5867,28 +6262,9 @@ export class Manager {
5867
6262
  grant: remote.serveGrant,
5868
6263
  creds: remote.serveCreds,
5869
6264
  };
5870
- const enc = new TextEncoder();
5871
- const nc = await this.dial({
5872
- authenticator: (nonce) => credsAuthenticator(enc.encode(state.creds))(nonce),
5873
- inboxPrefix: `_INBOX_${state.identity.id}`,
5874
- maxReconnectAttempts: -1,
5875
- });
5876
- try {
5877
- state.handle = serveEndpoint(nc, this.space, state.grant, this.managerServiceDefs(), { public: true }, {
5878
- resolveTarget: (t) => {
5879
- const key = principalKey(t.owner, t.actor).key;
5880
- for (const a of this.agents.values())
5881
- if (a.id === key)
5882
- return { lifecycleUid: a.lifecycleUid, mappingRevision: 0 };
5883
- return undefined;
5884
- },
5885
- ...(this.omitClassCommands() ? { omitClassCommands: this.omitClassCommands() } : {}),
5886
- });
5887
- }
5888
- catch (e) {
5889
- await nc.drain().catch(() => nc.close());
5890
- throw e;
5891
- }
6265
+ // The shared dialer wires the closed-connection handler, which re-dials through the all-duty family.
6266
+ const { nc, handle } = await this.dialServeConnection(state);
6267
+ state.handle = handle;
5892
6268
  state.nc = nc;
5893
6269
  this.serviceServe = state;
5894
6270
  console.error(`remote manager service endpoint activated: ${MANAGER_ENDPOINT}/${this.managerInstanceId} (epoch ${state.grant.epoch})`);
@@ -6136,15 +6512,6 @@ export class Manager {
6136
6512
  });
6137
6513
  return creds;
6138
6514
  }
6139
- /** P2 item 2 (spawn-as-action): stand up the standing self-mediated goal-writer connection +
6140
- * ActionContext. Mode-dual, mirroring {@link registerManagerService}: an AUTH mesh uses the
6141
- * scoped `goal-writer` credential already minted + family-STAGED inside registration's run block
6142
- * ({@link mintAndStageGoalWriter} — DISJOINT grant from the serve cred, SHARED §13.1 revocation
6143
- * family); an OPEN mesh uses a bare connection (no credential system to mint from - the broker
6144
- * enforces nothing). The connection presents the CURRENT credential on every (re)connect (a
6145
- * renewal is adopted without reconnecting the whole endpoint); the ActionContext bonds its
6146
- * KV + JS + JSM to this one connection and space (SPEC 13.4), so a composition mixup cannot splice
6147
- * goal state across brokers. */
6148
6515
  /** #2073: dial the goal-writer's ONE standing connection and rebuild everything derived from it
6149
6516
  * (the fence-resolving action context, the issuance-gate reader) — shared by the initial start
6150
6517
  * and the closed-connection recovery ({@link onGoalWriterConnectionClosed}), so a redial never
@@ -6198,7 +6565,9 @@ export class Manager {
6198
6565
  const recovered = await this.boundedReconnect("goal-writer", async () => {
6199
6566
  if (!gw)
6200
6567
  return false;
6201
- if (gw.creds !== undefined && inspectCredHealth(gw.creds).state !== "healthy") {
6568
+ if (this.remoteAuthority)
6569
+ await this.renewRemoteBeforeRedial("goal-writer", () => gw.creds);
6570
+ else if (gw.creds !== undefined && inspectCredHealth(gw.creds).state !== "healthy") {
6202
6571
  gw.creds = await this.withEndpointServeExecutor(({ authKv }) => this.mintAndStageGoalWriter(authKv));
6203
6572
  this.goalWriterCreds = gw.creds;
6204
6573
  }
@@ -6210,6 +6579,15 @@ export class Manager {
6210
6579
  return;
6211
6580
  console.error("! manager goal-writer could not be re-dialed; leaving the credential to the next renewal pass");
6212
6581
  }
6582
+ /** P2 item 2 (spawn-as-action): stand up the standing self-mediated goal-writer connection +
6583
+ * ActionContext. Mode-dual, mirroring {@link registerManagerService}: an AUTH mesh uses the
6584
+ * scoped `goal-writer` credential already minted + family-STAGED inside registration's run block
6585
+ * ({@link mintAndStageGoalWriter} — DISJOINT grant from the serve cred, SHARED §13.1 revocation
6586
+ * family); an OPEN mesh uses a bare connection (no credential system to mint from - the broker
6587
+ * enforces nothing). The connection presents the CURRENT credential on every (re)connect (a
6588
+ * renewal is adopted without reconnecting the whole endpoint); the ActionContext bonds its
6589
+ * KV + JS + JSM to this one connection and space (SPEC 13.4), so a composition mixup cannot splice
6590
+ * goal state across brokers. */
6213
6591
  async startGoalWriter() {
6214
6592
  const identity = (this.auth || this.remoteAuthority) ? this.goalWriterIdentity : newIdentity();
6215
6593
  // The mutable holder captured by the authenticator (mirrors the serve connection): a half-TTL
@@ -6271,18 +6649,6 @@ export class Manager {
6271
6649
  });
6272
6650
  return creds;
6273
6651
  }
6274
- /** P2 item 6: stand up the ONE §13.6 session plane on its own standing connection. Mode-dual,
6275
- * mirroring {@link startGoalWriter}: an AUTH mesh presents the scoped `session-ledger` cred
6276
- * already minted + family-staged inside registration's run block ({@link mintAndStageSessionLedger});
6277
- * an OPEN mesh uses a bare connection (no credential system to mint from). The connection presents
6278
- * the CURRENT credential on every (re)connect, so a half-TTL renewal is adopted without reconnecting.
6279
- *
6280
- * The offer SIGNER is a per-incarnation in-memory keypair: the static collapsed path mints AND
6281
- * redeems the offer in one call ({@link ManagerSessionPlane.establishAttach}), so the manager
6282
- * self-signs and self-verifies its own §13.6 grants and the keypair never leaves the process — a
6283
- * holder never verifies the signature (it presents the grant back over the rail; the broker's
6284
- * per-session caller cred is the holder's real fence). The plane's ledger lives in the DEDICATED
6285
- * sessions bucket (createEndpointStreams provisioned it at registration). */
6286
6652
  /** #2073: dial the session-ledger's ONE standing connection — shared by the initial start and the
6287
6653
  * closed-connection recovery ({@link onSessionLedgerConnectionClosed}), the same shape {@link
6288
6654
  * dialGoalWriter} uses for its rail. The KV/JetStream clients the session plane opened from the
@@ -6313,7 +6679,9 @@ export class Manager {
6313
6679
  const recovered = await this.boundedReconnect("session-ledger", async () => {
6314
6680
  if (!sw || !identity)
6315
6681
  return false;
6316
- if (sw.creds !== undefined && inspectCredHealth(sw.creds).state !== "healthy") {
6682
+ if (this.remoteAuthority)
6683
+ await this.renewRemoteBeforeRedial("session-ledger", () => sw.creds);
6684
+ else if (sw.creds !== undefined && inspectCredHealth(sw.creds).state !== "healthy") {
6317
6685
  sw.creds = await this.withEndpointServeExecutor(({ authKv }) => this.mintAndStageSessionLedger(authKv));
6318
6686
  this.sessionLedgerCreds = sw.creds;
6319
6687
  }
@@ -6325,6 +6693,18 @@ export class Manager {
6325
6693
  return;
6326
6694
  console.error("! manager session-ledger could not be re-dialed; leaving the credential to the next renewal pass");
6327
6695
  }
6696
+ /** P2 item 6: stand up the ONE §13.6 session plane on its own standing connection. Mode-dual,
6697
+ * mirroring {@link startGoalWriter}: an AUTH mesh presents the scoped `session-ledger` cred
6698
+ * already minted + family-staged inside registration's run block ({@link mintAndStageSessionLedger});
6699
+ * an OPEN mesh uses a bare connection (no credential system to mint from). The connection presents
6700
+ * the CURRENT credential on every (re)connect, so a half-TTL renewal is adopted without reconnecting.
6701
+ *
6702
+ * The offer SIGNER is a per-incarnation in-memory keypair: the static collapsed path mints AND
6703
+ * redeems the offer in one call ({@link ManagerSessionPlane.establishAttach}), so the manager
6704
+ * self-signs and self-verifies its own §13.6 grants and the keypair never leaves the process — a
6705
+ * holder never verifies the signature (it presents the grant back over the rail; the broker's
6706
+ * per-session caller cred is the holder's real fence). The plane's ledger lives in the DEDICATED
6707
+ * sessions bucket (createEndpointStreams provisioned it at registration). */
6328
6708
  async startSessionPlane() {
6329
6709
  const identity = (this.auth || this.remoteAuthority) ? this.sessionLedgerIdentity : newIdentity();
6330
6710
  // The mutable holder captured by the authenticator (mirrors the goal-writer): a half-TTL renewal
@@ -7921,6 +8301,10 @@ export class Manager {
7921
8301
  finally {
7922
8302
  await nc.drain().catch(() => nc.close());
7923
8303
  }
8304
+ let retiredSlots = 0;
8305
+ let preservedForeign = 0;
8306
+ let preservedAdopted = 0;
8307
+ let deferredSlots = 0;
7924
8308
  const terminalRows = [];
7925
8309
  for (const row of slotRows) {
7926
8310
  if (row.phase === "retired") {
@@ -7928,6 +8312,7 @@ export class Manager {
7928
8312
  // regardless of which instance owned it, so a sibling-retired incarnation's copied credential
7929
8313
  // is refused at this control surface too. Ownership gates only the DESTRUCTIVE sweep below.
7930
8314
  this.retiredPrincipals.add(principalKey(row.owner, row.actor).key);
8315
+ retiredSlots++;
7931
8316
  continue;
7932
8317
  }
7933
8318
  // 3b-2 RECONCILE OWNERSHIP (multi-manager-per-space): a manager adjudicates ONLY the non-retired
@@ -7936,19 +8321,26 @@ export class Manager {
7936
8321
  // hazard, now cross-instance). A legacy row (pre-3b-2, no owner recorded) predates multi-manager,
7937
8322
  // so this manager is its legitimate single-manager-past successor and reconciles it. An orphaned
7938
8323
  // sibling row is reclaimed only by an explicit operator CAS takeover (ruling 1), never here.
7939
- if (row.ownerInstanceId !== undefined && row.ownerInstanceId !== this.managerInstanceId)
8324
+ if (row.ownerInstanceId !== undefined && row.ownerInstanceId !== this.managerInstanceId) {
8325
+ preservedForeign++;
7940
8326
  continue;
8327
+ }
7941
8328
  // ADOPTION is genuine membership: a slot backed by a live managed agent THIS process owns
7942
8329
  // at the SAME uid is never an orphan (empty at boot; exactly the adopted set at the
7943
8330
  // post-adoption sweep — the fix for the F3 resume hole).
7944
8331
  const live = this.agents.get(row.alias);
7945
8332
  const adopted = live !== undefined && live.lifecycleUid === row.lifecycleUid;
8333
+ if (adopted) {
8334
+ preservedAdopted++;
8335
+ }
7946
8336
  // Boot sweep with a resume PENDING: an active slot may yet be adopted (the resume path runs
7947
8337
  // AFTER this boot sweep), so DEFER it — the post-adoption sweep terminalizes any the resume
7948
8338
  // did not claim. provisioning/terminalizing NEVER defer (they are crashed operations, never
7949
8339
  // an agent to adopt). At `postAdoption` (or a non-resume boot) nothing defers.
7950
- if (!postAdoption && row.phase === "active" && !adopted && this.resumeRequired)
8340
+ if (!postAdoption && row.phase === "active" && !adopted && this.resumeRequired) {
8341
+ deferredSlots++;
7951
8342
  continue;
8343
+ }
7952
8344
  if (planStaticSlotResume(row, adopted) !== "none")
7953
8345
  terminalRows.push(row);
7954
8346
  }
@@ -7957,6 +8349,17 @@ export class Manager {
7957
8349
  // alias, but the initial serial sweep must close every discovery-to-await gap up front.
7958
8350
  for (const row of terminalRows)
7959
8351
  this.reconcilingAliases.add(row.alias);
8352
+ const sweepDeprovision = {
8353
+ examined: 0,
8354
+ deleted: 0,
8355
+ absent: 0,
8356
+ refused: 0,
8357
+ acknowledged: 0,
8358
+ disappeared: 0,
8359
+ consumers: { examined: 0, deleted: 0, absent: 0, refused: 0, acknowledged: 0, disappeared: 0 },
8360
+ acls: { examined: 0, deleted: 0, absent: 0, refused: 0, disappeared: 0 },
8361
+ members: { examined: 0, deleted: 0, absent: 0, refused: 0, disappeared: 0 },
8362
+ };
7960
8363
  for (const row of terminalRows) {
7961
8364
  if (this.staticReconcileStopping)
7962
8365
  break;
@@ -7976,7 +8379,33 @@ export class Manager {
7976
8379
  };
7977
8380
  this.staticReconcileItems.set(key, item);
7978
8381
  }
8382
+ this.lastDeprovisionResources = undefined;
7979
8383
  const succeeded = await this.attemptStaticReconcile(key, item, row);
8384
+ const acc = this.lastDeprovisionResources;
8385
+ if (acc) {
8386
+ sweepDeprovision.examined += acc.examined;
8387
+ sweepDeprovision.deleted = sweepDeprovision.deleted === null || acc.deleted === null ? null : sweepDeprovision.deleted + acc.deleted;
8388
+ sweepDeprovision.absent += acc.absent;
8389
+ sweepDeprovision.refused += acc.refused;
8390
+ sweepDeprovision.acknowledged += acc.acknowledged;
8391
+ sweepDeprovision.disappeared += acc.disappeared;
8392
+ sweepDeprovision.consumers.examined += acc.consumers.examined;
8393
+ sweepDeprovision.consumers.deleted = sweepDeprovision.consumers.deleted === null || acc.consumers.deleted === null ? null : sweepDeprovision.consumers.deleted + acc.consumers.deleted;
8394
+ sweepDeprovision.consumers.absent += acc.consumers.absent;
8395
+ sweepDeprovision.consumers.refused += acc.consumers.refused;
8396
+ sweepDeprovision.consumers.acknowledged += acc.consumers.acknowledged;
8397
+ sweepDeprovision.consumers.disappeared += acc.consumers.disappeared;
8398
+ sweepDeprovision.acls.examined += acc.acls.examined;
8399
+ sweepDeprovision.acls.deleted += acc.acls.deleted;
8400
+ sweepDeprovision.acls.absent += acc.acls.absent;
8401
+ sweepDeprovision.acls.refused += acc.acls.refused;
8402
+ sweepDeprovision.acls.disappeared += acc.acls.disappeared;
8403
+ sweepDeprovision.members.examined += acc.members.examined;
8404
+ sweepDeprovision.members.deleted += acc.members.deleted;
8405
+ sweepDeprovision.members.absent += acc.members.absent;
8406
+ sweepDeprovision.members.refused += acc.members.refused;
8407
+ sweepDeprovision.members.disappeared += acc.members.disappeared;
8408
+ }
7980
8409
  if (succeeded)
7981
8410
  sweep.succeeded++;
7982
8411
  else
@@ -7984,6 +8413,18 @@ export class Manager {
7984
8413
  this.reconcilingAliases.delete(row.alias);
7985
8414
  }
7986
8415
  sweep.completedAt = new Date().toISOString();
8416
+ this.staticReconcileLastSweepResources = {
8417
+ slots: {
8418
+ examined: slotRows.length,
8419
+ retired: retiredSlots,
8420
+ preservedForeign,
8421
+ preservedAdopted,
8422
+ deferred: deferredSlots,
8423
+ terminalized: sweep.succeeded,
8424
+ failed: sweep.failed,
8425
+ },
8426
+ deprovision: sweepDeprovision,
8427
+ };
7987
8428
  for (const row of terminalRows) {
7988
8429
  const key = this.staticReconcileKey(row);
7989
8430
  const item = this.staticReconcileItems.get(key);
@@ -8404,8 +8845,9 @@ export class Manager {
8404
8845
  throw e;
8405
8846
  }
8406
8847
  }
8407
- /** Managed agents cross-referenced with live presence (the manager sees the roster). */
8408
- /** `ownerFilter`: restrict to agents whose spawn-time stored `userOwner` equals it (the ps/status
8848
+ /** Managed agents cross-referenced with live presence (the manager sees the roster).
8849
+ *
8850
+ * `ownerFilter`: restrict to agents whose spawn-time stored `userOwner` equals it (the ps/status
8409
8851
  * owner-domain bound); undefined = unbounded. {@link NO_OWNER_MATCHES} matches nothing. */
8410
8852
  list(ownerFilter) {
8411
8853
  const roster = new Map(this.ep.getRoster().map((p) => [p.card.name, p]));
@@ -8427,8 +8869,33 @@ export class Manager {
8427
8869
  : undefined;
8428
8870
  // The connector-reported provider from presence (#785): the live roster card's `meta.provider`,
8429
8871
  // when it is a non-empty string. Absent when the connector reported none — never fabricated.
8430
- const providerMeta = roster.get(a.name)?.card.meta?.provider;
8872
+ const seen = roster.get(a.name);
8873
+ const providerMeta = seen?.card.meta?.provider;
8431
8874
  const provider = typeof providerMeta === "string" && providerMeta ? providerMeta : undefined;
8875
+ // The harness-reported condition beside the mesh status (#618): a turn that died upstream reads
8876
+ // `waiting` with `rate_limit` here, not a bare `waiting`. Absent when the connector reported none.
8877
+ // Projected field by field like `provider`: a malformed peer row must not fail this reply's schema.
8878
+ const reported = roster.get(a.name)?.condition;
8879
+ const condition = reported && typeof reported.code === "string" && reported.code
8880
+ ? {
8881
+ code: reported.code,
8882
+ ...(typeof reported.source === "string" ? { source: reported.source } : {}),
8883
+ ...(typeof reported.message === "string" ? { message: reported.message } : {}),
8884
+ ...(typeof reported.since === "number" && Number.isFinite(reported.since) ? { since: reported.since } : {}),
8885
+ }
8886
+ : undefined;
8887
+ // The seat's last harness-reported work progress (#618). `ts` is only its heartbeat, so a turn
8888
+ // that stopped advancing reads as an old `activeAt` beside a live process. Absent when none was reported.
8889
+ const activeAtReported = roster.get(a.name)?.activeAt;
8890
+ const activeAt = typeof activeAtReported === "number" && Number.isFinite(activeAtReported) ? activeAtReported : undefined;
8891
+ // How long an offline verdict has stood (#1208): the seat's last heartbeat before it went
8892
+ // offline. Only beside a verdict, since a stale or unpopulated view dates nothing, only from the
8893
+ // seat's own record, since `seen` is matched by name and a same-named foreign peer or an older
8894
+ // incarnation must not date this seat, and only when the seat's unvalidated `ts` fits the row
8895
+ // schema, so one odd record cannot fail the reply.
8896
+ const offlineSince = view.state === "current" && this.isOwnPresence(a, seen) && seen.status === "offline" && Number.isSafeInteger(seen.ts) && seen.ts >= 0
8897
+ ? seen.ts
8898
+ : undefined;
8432
8899
  return {
8433
8900
  name: a.name,
8434
8901
  // The spawned agent's id (nkey, or the user-mode principal) — lets an operator tool (e.g.
@@ -8441,10 +8908,13 @@ export class Manager {
8441
8908
  mode: a.handle.kind,
8442
8909
  status: a.handle.status(),
8443
8910
  uptimeMs: Date.now() - a.startedAt,
8444
- mesh: roster.get(a.name)?.status ?? "absent",
8911
+ mesh: seen?.status ?? "absent",
8912
+ ...(condition ? { condition } : {}),
8913
+ ...(activeAt !== undefined ? { activeAt } : {}),
8445
8914
  // `current` is the only state in which `mesh` is a verdict; the other two are the
8446
8915
  // observer's own condition and travel on the row (older CLIs ignore the field).
8447
8916
  meshView: view.state,
8917
+ ...(offlineSince !== undefined ? { offlineSince } : {}),
8448
8918
  // The incarnation coordinate (SPEC 13.1) — with `id`, exactly what a v0.4 caller needs to
8449
8919
  // build a targeted (`despawn`/`attach`) request against THIS incarnation.
8450
8920
  lifecycleUid: a.lifecycleUid,
@@ -8460,6 +8930,9 @@ export class Manager {
8460
8930
  spawner: a.spawner,
8461
8931
  instanceId: this.managerInstanceId,
8462
8932
  host: hostname(),
8933
+ // #1500: the session a `--resume` seat forked, with its title and transcript hash once the
8934
+ // seat has recorded them.
8935
+ ...(a.launch.forkSource ? { resume: this.forkProvenance(a) ?? { source: a.launch.forkSource } } : {}),
8463
8936
  ...(health && health.state !== "ok" ? { authHealth: health.state, authReason: health.reason } : {}),
8464
8937
  };
8465
8938
  });