@cotal-ai/manager 0.44.0 → 0.46.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/manager.js CHANGED
@@ -6,7 +6,7 @@ import { existsSync, lstatSync, readFileSync, rmSync } from "node:fs";
6
6
  import { join, dirname, resolve } from "node:path";
7
7
  import { CotalEndpoint, DEFAULT_SERVER, DEV_OWNER, MANAGER_LEASE_RENEW_MS, STANDING_RENEWABLE_TTL_SEC, agentFilePath, clearSpaceHistory, connectorServers, spawnEnvAllow, deprovisionAgent, firstFreeName, spawnNameError, idFromCreds, inspectCredHealth, loadAgentFile, listPersonaCatalog, personaCatalogDescription, personaCatalogReadable, loadCotalConfig, mintCreds, mintLifecycleUid, mkSecretDir, newIdentity, actionContext, parsePrincipalKey, parseShareSelection, principalKey, probeConnect, provisionAgent, provisionAgentDurables, registry, resolveAuthProvider, saveAgentFile, subjectMatches, AUTH_ENDPOINT, EP_CMD_RETIRE_LIFECYCLE, epRequestSubject, epCallerReplyFilter, parseEpSubject, controlServiceSubject, eventChannelPrincipal, } from "@cotal-ai/core";
8
8
  import { agentAuthState, agentCredsDir, agentLifecycleSecretFilePaths, agentSecretFilePaths, agentSecretKeyForFile, authDir, connectorInstallHint, DEFAULT_CONNECTOR, defaultAgentType, DELIVERY_CREDS_KIND, extensionConnectors, findCotalRoot, getSpaceAuth, hasUserAuthState, loadExtensionsManifest, loadManagerInstanceIdentity, loadMeshes, manifestExtensionNames, materializeFromManifest, materializeSecretToFile, MEMBERSHIP_RW_CREDS_KIND, mergeLaunchOptions, remintDaemonCreds, resolveOnPath, saveManagerInstanceIdentity, spaceMaterialKey, SYSTEM_CREDS_FILES, userAuthStateDir, workspaceSecretStore, writeRenewalRecord } from "@cotal-ai/workspace";
9
- import { createRuntime, } from "./runtime/index.js";
9
+ import { createRuntime, requireRuntimeAdopt, } from "./runtime/index.js";
10
10
  import { AttachEndpoint } from "./attach-endpoint.js";
11
11
  import { makeManagerEndpointEvictionEvidence, makeManagerEndpointEvictor } from "./endpoint-evict.js";
12
12
  import { makeManagerHolderLivenessProbe } from "./holder-liveness.js";
@@ -14,6 +14,7 @@ import { GateReconcileRefused, reconcileEndpointGate } from "./reconcile-gate.js
14
14
  import { launchSpecForRun, materializePersona, launchAgentToStartOpts, parseLaunchSpec, persistLaunchSpec } from "./launch.js";
15
15
  import { authorizeLaunch, authorizeNamedControl } from "./authorize.js";
16
16
  import { controlShutdown } from "./control-shutdown.js";
17
+ import { RunHosting } from "./run-hosting.js";
17
18
  import { controlSession } from "./control-session.js";
18
19
  import { parseResumeCommitArgs, parseResumeControlArgs, parseResumeFinalizeArgs } from "./resume.js";
19
20
  // Unit B (the static §13.1 lifecycle executor): the shared grammar/stores from core plus the
@@ -30,7 +31,7 @@ import { ManagerSessionPlane, openSessionLedgerKv } from "./session/index.js";
30
31
  // the register/authorize/serve seams, all driven over a scoped one-shot executor connection.
31
32
  import { provisionEndpointGateOpen, endpointRegistrationBarrier, serveIssuanceGateKv, commitSiblingIssuance, markLedgerRowRevoked, epcredRowKey, epgateKey, parseEndpointGate, registerServiceInstance, deregisterServiceInstance, authorizeServeGrant, writeServiceStatus, SERVICE_READY, serveEndpoint, bindGoal, createGoal, transitionGoal, commitGoalResult, settleGoalUncertain, readGoalResult, readGoalStatus, readGoalSpec, recordGoalIndex, readGoalIndex, clearGoalIndex, listGoalIndex, mintCheckpoint, resumeCheckpoint, readCheckpointSettle, readCheckpointSpec, expireCheckpoint, GOAL_TERMINAL_STATES, goalRefOf, goalProgressTopic, epeSubject, submissionFingerprint, EpEnvelopeError, lifecycleBlocked, renderLifecycleBlocked, } from "@cotal-ai/core";
32
33
  import { MANAGER_ENDPOINT, managerClusterArtifacts, managerCommandDefs, managerContractArtifactValues } from "./manager-service-contract.js";
33
- import { staticLifecycleTransport, activateStaticLifecycle, runStaticTerminal, readStaticSlot, casStaticSlot, recordSlotCredential, appendStaticCredentialRow, planStaticSlotResume, } from "./static-lifecycle.js";
34
+ import { staticLifecycleTransport, activateStaticLifecycle, runStaticTerminal, readStaticSlot, casStaticSlot, recordSlotCredential, appendStaticCredentialRow, planStaticSlotResume, observeStaticSlot, renderStaticSlotObservation, StaticSlotReadError, STATIC_SLOT_READ_FAILED_DETAIL, } from "./static-lifecycle.js";
34
35
  /** Concurrency ceiling — the manager refuses to hold more than this many live + in-flight +
35
36
  * cooling slots at once (P4a). Bounds a fork-bomb: spawn is a full agent process per call. */
36
37
  const MAX_AGENTS = 50;
@@ -136,6 +137,8 @@ const FREE_SLOT_CAUSE_TEXT = {
136
137
  "process-exit": "its own process exited and this manager did not stop it",
137
138
  "pi-crash-loop": "this manager retired it after a Pi crash loop",
138
139
  "pi-recovery-failed": "this manager retired it after Pi session recovery failed",
140
+ "supervise-crash-loop": "this manager retired it after its supervise restart budget was spent",
141
+ "supervise-recovery-failed": "this manager retired it after a supervised restart failed",
139
142
  "session-bind-failed": "this manager stopped it: its host session could not be bound at launch",
140
143
  "resume-session-rebind-failed": "this manager stopped it: its host session could not be rebound on resume",
141
144
  };
@@ -319,6 +322,10 @@ export class Manager {
319
322
  * manager reads its OWN `epgate.<e>.<iid>` epoch over this connection before a terminal commit
320
323
  * and skips a superseded commit (the fast-fail belt paired with the (b) barrier-revoke fence). */
321
324
  goalWriter;
325
+ /** The workflow-run host (SPEC 14.3): the drives this incarnation holds, each on its own
326
+ * per-run credential and connection. Absent under a remote authority, which mints no driver
327
+ * credentials, so the `run-*` family refuses there rather than connecting on a weaker identity. */
328
+ runHosting;
322
329
  /** P2 item 2 must-5 (b): the STABLE goal-writer identity (auth mode) — minted once at
323
330
  * registration alongside the serve identity; a renewal re-mints the SAME nkey with a fresh
324
331
  * bounded exp and re-stages its distinct credId into the §13.1 revocation family. The current
@@ -495,6 +502,10 @@ export class Manager {
495
502
  get runtimeKind() {
496
503
  return this.runtime.kind;
497
504
  }
505
+ /** Reattach this manager's runtime to a durable handle. Refuses by name when adopt is absent. */
506
+ adoptRuntimeHandle(reference) {
507
+ return requireRuntimeAdopt(this.runtime, reference);
508
+ }
498
509
  /** The console page URL (manager-hosted, loopback). */
499
510
  get consoleUrl() {
500
511
  return this.attach.consoleUrl();
@@ -699,6 +710,25 @@ export class Manager {
699
710
  // BEFORE spawn-as-action begins accepting (the goalReconcileDone gate) — a fresh incarnation
700
711
  // never drops a goal a dead predecessor accepted. Never fatal; the gate opens either way.
701
712
  await this.reconcileGoalIndex();
713
+ // SPEC 14.3: the manager hosts workflow runs. Stood up AFTER registration (it names this
714
+ // instance's coordinates) and reconciled AFTER the goal index, for the same reason: a fresh
715
+ // incarnation takes back every run a dead predecessor was driving before it accepts new ones.
716
+ // The serve surface is already live by here, so the family itself holds the gate: `runHost()`
717
+ // refuses `run-start`/`run-resume` as `unavailable` until the host exists and `RunHosting`
718
+ // refuses them until its reconcile has returned. A user-auth mesh stands no host up at all
719
+ // (`runHost()` names why); a remote-authority manager holds no signer to mint with.
720
+ if (!this.remoteAuthority && !this.userMode) {
721
+ this.runHosting = new RunHosting({
722
+ space: this.space,
723
+ servers: this.servers,
724
+ endpoint: MANAGER_ENDPOINT,
725
+ instanceId: this.managerInstanceId,
726
+ holder: { id: this.ep.ref().id, lifecycleUid: this.managerLifecycleUid },
727
+ auth: this.auth,
728
+ log: (line) => console.error(line),
729
+ });
730
+ await this.runHosting.reconcile();
731
+ }
702
732
  // Plane-3 (durable backstop) is NOT the manager's job — the manager only manages agent lifecycle.
703
733
  // The server-side delivery daemon hosts the fan-out writer + trusted reader, owns the durable
704
734
  // membership registry, and serves the runtime durable join/leave/list ops (on `ctl.delivery`). The
@@ -835,6 +865,9 @@ export class Manager {
835
865
  // scoped executor. Without this the standing session-ledger connection dies at its TTL and
836
866
  // `attach` stops establishing sessions until a restart. The connection's authenticator presents
837
867
  // the refreshed credential on its next (re)connect.
868
+ // SPEC 14.6: every hosted drive's per-run `run-driver` credential is the manager's to renew
869
+ // for the same nkey; a run parked in a pause for days must not die at the credential's TTL.
870
+ await this.runHosting?.renew();
838
871
  if (this.sessionLedgerConn && this.sessionLedgerCreds && this.auth) {
839
872
  const sw = this.sessionLedgerConn;
840
873
  try {
@@ -1336,6 +1369,10 @@ export class Manager {
1336
1369
  // in-flight command can write a status back onto the record it just removed.
1337
1370
  const registered = this.serviceServe !== undefined;
1338
1371
  await this.stopServiceServe();
1372
+ // The drives AFTER the serve loop (no new `run-start` can land) and BEFORE deregistration: a
1373
+ // released run's status write is the last thing this incarnation says about it.
1374
+ await this.runHosting?.stop();
1375
+ this.runHosting = undefined;
1339
1376
  if (registered)
1340
1377
  await this.deregisterServiceOnStop();
1341
1378
  await this.stopGoalWriter();
@@ -1759,6 +1796,35 @@ export class Manager {
1759
1796
  * cross-owner persona writes - an operator redefines via config, not the wire), where the ctl
1760
1797
  * admin tier allowed operator cross-owner redefine; (2) launch is owner-equality-only, above.
1761
1798
  * Both are least-privilege reductions, never widenings. */
1799
+ /** The run host, or one of three refusals. A remote-authority manager holds no space signer, so
1800
+ * it cannot mint the per-run driver credential SPEC 14.6 requires, and hosting on any other
1801
+ * identity would be the fallback this tree does not take: `unimplemented`, for good. A
1802
+ * user-auth mesh is `unimplemented` too, for a different reason it names: a hosted run's seats
1803
+ * are spawned, turned and despawned by a caller derived from the run id under the static
1804
+ * owner, which the user-mode spawn door refuses (no `u_` owner), so a program would fail at
1805
+ * its first seat; no path to `--local` is offered there since a user bearer holds no run rows
1806
+ * either. An ordinary manager whose host is not standing yet is still booting (the serve
1807
+ * surface comes up before the host): `unavailable`, retry. The three are told apart, since
1808
+ * the first sentence steers a caller to `--local` and the others must not. */
1809
+ runHost() {
1810
+ if (this.runHosting)
1811
+ return this.runHosting;
1812
+ if (this.remoteAuthority)
1813
+ throw new EpEnvelopeError("unimplemented", "this manager does not host workflow runs: a remote-authority manager mints no run-driver credentials (SPEC 14.6); drive the run from a terminal with `cotal run start --local --file <program>`");
1814
+ if (this.userMode)
1815
+ throw new EpEnvelopeError("unimplemented", `user-auth space "${this.space}" hosts no workflow runs yet: a hosted run's seats would be spawned under the static owner, which a user mesh refuses; run programs on a static-auth mesh`);
1816
+ throw new EpEnvelopeError("unavailable", "the manager is still booting its workflow-run host; retry shortly (SPEC 14.3)");
1817
+ }
1818
+ /** The answerer a `run-answer` records (SPEC 14.5): the caller as this manager knows them. A
1819
+ * managed seat is named by its persona name; any other authenticated caller (an operator
1820
+ * instrument, a logged-in user) by its principal. Never the request body's word. */
1821
+ runAnswerer(ctx) {
1822
+ const caller = principalKey(ctx.subject.caller.owner, ctx.subject.caller.actor).key;
1823
+ for (const a of this.agents.values())
1824
+ if (this.managedPrincipal(a) === caller)
1825
+ return a.name;
1826
+ return caller;
1827
+ }
1762
1828
  managerServiceDefs() {
1763
1829
  const args = (ctx) => (ctx.request.args ?? {});
1764
1830
  const callerOf = (ctx) => principalKey(ctx.subject.caller.owner, ctx.subject.caller.actor).key;
@@ -1794,8 +1860,34 @@ export class Manager {
1794
1860
  inspect: (ctx) => this.serveGated(ctx, async () => {
1795
1861
  const name = String(args(ctx).name ?? "").trim();
1796
1862
  const row = this.list(await this.psOwnerFilter(callerOf(ctx), false)).find((x) => x.name === name);
1797
- if (!row)
1863
+ if (!row) {
1864
+ // The live hit above stays entirely local. Only a miss widens into durable static state,
1865
+ // where `not-found` is honest only after the slot read itself succeeds and returns absent.
1866
+ // User-mode lifecycles have no manager-local mgrslot row; their per-name projection remains
1867
+ // the live map until the auth service exposes an equivalent mediated reader.
1868
+ if (this.auth && !this.userMode) {
1869
+ const recordsKv = this.goalWriter?.ctx.kv;
1870
+ if (!recordsKv)
1871
+ throw new EpEnvelopeError("unavailable", `the durable static slot store is not ready while inspecting "${name}"`, [
1872
+ { kind: STATIC_SLOT_READ_FAILED_DETAIL, name, record: "slot", operation: "read" },
1873
+ ]);
1874
+ let observed;
1875
+ try {
1876
+ observed = await observeStaticSlot(recordsKv, DEV_OWNER, name, this.managerInstanceId);
1877
+ }
1878
+ catch (error) {
1879
+ throw new EpEnvelopeError("unavailable", `the durable static slot state for "${name}" could not be read: ${error.message}`, [
1880
+ { kind: STATIC_SLOT_READ_FAILED_DETAIL, name, record: error instanceof StaticSlotReadError ? error.record : "slot-or-head", operation: "read" },
1881
+ ]);
1882
+ }
1883
+ // A retired row affirms that no agent exists now, so it keeps the old `not-found`
1884
+ // result. The changed refusal is reserved for durable NONTERMINAL state that contradicts
1885
+ // the live-map miss and needs operator attention.
1886
+ if (observed !== undefined && observed.slotPhase !== "retired")
1887
+ throw new EpEnvelopeError("failed-precondition", `no live agent "${name}"; durable state contradicts absence ${renderStaticSlotObservation(observed)}`, [observed]);
1888
+ }
1798
1889
  throw new EpEnvelopeError("not-found", `no agent "${name}"`);
1890
+ }
1799
1891
  return row;
1800
1892
  }),
1801
1893
  models: (ctx) => this.serveGated(ctx, async () => {
@@ -1873,6 +1965,13 @@ export class Manager {
1873
1965
  resumePreserved: (ctx) => adminGated(ctx, async () => unwrap(await this.opResumePreserved(args(ctx)))),
1874
1966
  commitResume: (ctx) => adminGated(ctx, async () => unwrap(await this.opCommitResume(args(ctx)))),
1875
1967
  finalizeResume: (ctx) => adminGated(ctx, async () => unwrap(await this.opFinalizeResume(args(ctx)))),
1968
+ // The workflow-run family (SPEC 14.3): the manager hosts the driver. Reach is the broker's
1969
+ // (`run` capability / privileged instrument rows); the serve gate is the maintenance fence.
1970
+ runStart: (ctx) => this.serveGated(ctx, () => this.runHost().start(args(ctx))),
1971
+ runResume: (ctx) => this.serveGated(ctx, () => this.runHost().resume(args(ctx))),
1972
+ runAnswer: (ctx) => this.serveGated(ctx, () => this.runHost().answer(args(ctx), this.runAnswerer(ctx))),
1973
+ runStatus: (ctx) => this.serveGated(ctx, () => this.runHost().status(args(ctx))),
1974
+ runPs: (ctx) => this.serveGated(ctx, () => this.runHost().list(args(ctx))),
1876
1975
  preparePreservation: (ctx) => adminGated(ctx, async () => unwrap(await this.opPreservationCtl("preparePreservation", args(ctx)))),
1877
1976
  commitPreservation: (ctx) => adminGated(ctx, async () => unwrap(await this.opPreservationCtl("commitPreservation", args(ctx)))),
1878
1977
  abortPreservation: (ctx) => adminGated(ctx, async () => unwrap(await this.opPreservationCtl("abortPreservation", args(ctx)))),
@@ -2101,7 +2200,7 @@ export class Manager {
2101
2200
  // vocabulary as static capabilities; the broker maps them to the ctl tiers. `role:<r>` tokens
2102
2201
  // pass through too (a persona may hold delegable roles) — the ledger's envelope walk still
2103
2202
  // attenuates every one of these against the spawner chain.
2104
- const scope = (opts.capabilities ?? []).filter((c) => c === "spawn" || c === "admin" || /^role:[A-Za-z0-9_-]+$/.test(c));
2203
+ const scope = (opts.capabilities ?? []).filter((c) => c === "spawn" || c === "run" || c === "admin" || /^role:[A-Za-z0-9_-]+$/.test(c));
2105
2204
  // The manager's ONE store (injected for a hosted composition, workstation FS locally). A hosted
2106
2205
  // user-mode spawn reads the callout material from it — the same store the auth-store kinds
2107
2206
  // (callout/issuer/…) were migrated onto — so this is no longer a local-only path.
@@ -2635,9 +2734,10 @@ export class Manager {
2635
2734
  }
2636
2735
  throw new Error(`replacement did not prove session ${expected} (${last})`);
2637
2736
  }
2638
- /** Restart one continuation-capable managed process in place. Identity, lifecycle, credentials,
2639
- * durables, children, and the manager row remain owned; only the process handle/control endpoint
2640
- * change. A fourth crash inside two minutes is a loop and falls through to normal retirement. */
2737
+ /** Restart one managed process in place. Identity, lifecycle, credentials, durables, children,
2738
+ * and the manager row remain owned; only the process handle/control endpoint change. Budget
2739
+ * comes from `restart.policy` when spawn carried `supervise`; otherwise the Pi session-recovery
2740
+ * constants. Spending the budget falls through to normal retirement. */
2641
2741
  recoverManagedSession(a) {
2642
2742
  const restart = a.restart;
2643
2743
  if (!restart || !restart.armed || restart.recovering || a.terminalizing)
@@ -2646,12 +2746,15 @@ export class Manager {
2646
2746
  if (!release)
2647
2747
  return; // preservation owns the cut once the lifecycle fence closes
2648
2748
  const now = Date.now();
2649
- restart.crashes = restart.crashes.filter((at) => now - at < SESSION_RESTART_WINDOW_MS);
2749
+ const limit = restart.policy?.restarts ?? SESSION_RESTART_LIMIT;
2750
+ const windowMs = restart.policy?.windowMs ?? SESSION_RESTART_WINDOW_MS;
2751
+ const supervised = restart.policy !== undefined;
2752
+ restart.crashes = restart.crashes.filter((at) => now - at < windowMs);
2650
2753
  restart.crashes.push(now);
2651
- if (restart.crashes.length > SESSION_RESTART_LIMIT) {
2652
- console.error(`! ${a.name}: Pi crash loop (${restart.crashes.length} crashes in ${SESSION_RESTART_WINDOW_MS / 1000}s) - retiring the managed seat`);
2754
+ if (restart.crashes.length > limit) {
2755
+ console.error(`! ${a.name}: ${supervised ? "supervised" : "Pi"} crash loop (${restart.crashes.length} crashes in ${windowMs / 1000}s) - retiring the managed seat`);
2653
2756
  restart.armed = false;
2654
- this.freeSlot(a, true, "pi-crash-loop");
2757
+ this.freeSlot(a, true, supervised ? "supervise-crash-loop" : "pi-crash-loop");
2655
2758
  this.reapChildrenOf(this.managedPrincipal(a));
2656
2759
  release();
2657
2760
  return;
@@ -2660,21 +2763,40 @@ export class Manager {
2660
2763
  void (async () => {
2661
2764
  let replacement;
2662
2765
  try {
2663
- const sessionId = this.readManagedSession(a);
2664
2766
  const connector = await this.resolveConnector(a.agent);
2665
- if (!connector.supportsSessionContinuation)
2666
- throw new Error(`connector ${connector.name} no longer declares same-session continuation`);
2667
- const opts = {
2668
- ...restart.opts,
2669
- resume: undefined,
2670
- prompt: undefined,
2671
- continueSession: sessionId,
2672
- };
2767
+ const continueSession = connector.supportsSessionContinuation ? this.readManagedSession(a) : undefined;
2768
+ const opts = continueSession !== undefined
2769
+ ? { ...restart.opts, resume: undefined, prompt: undefined, continueSession }
2770
+ : { ...restart.opts, resume: undefined, prompt: undefined };
2673
2771
  const spec = connector.buildLaunch(opts);
2772
+ const wanted = this.managedPrincipal(a);
2773
+ const joinedAfter = this.ep.getRoster()
2774
+ .filter((p) => p.card.id === wanted && p.lifecycleUid === a.lifecycleUid)
2775
+ .reduce((max, p) => Math.max(max, p.ts), 0) + 1;
2674
2776
  const handle = this.runtime.spawn(a.name, spec, a.launch.cwd);
2675
2777
  replacement = handle;
2676
2778
  restart.sessionStatePath = spec.sessionStatePath ?? restart.sessionStatePath;
2677
- await this.awaitRecoveredSession(a, sessionId, handle, spec.control);
2779
+ if (continueSession !== undefined)
2780
+ await this.awaitRecoveredSession(a, continueSession, handle, spec.control);
2781
+ else {
2782
+ const previousHandle = a.handle;
2783
+ const previousControl = a.control;
2784
+ a.handle = handle;
2785
+ a.control = spec.control;
2786
+ try {
2787
+ const readiness = await this.awaitReadiness(a, connector.readinessTimeoutMs ?? this.readinessTimeoutMs, {
2788
+ reapOnExit: false,
2789
+ joinedAfter,
2790
+ });
2791
+ if (!readiness.ok)
2792
+ throw new Error(readiness.detail);
2793
+ }
2794
+ catch (error) {
2795
+ a.handle = previousHandle;
2796
+ a.control = previousControl;
2797
+ throw error;
2798
+ }
2799
+ }
2678
2800
  if (this.agents.get(a.name) !== a || a.terminalizing) {
2679
2801
  try {
2680
2802
  handle.stop({ graceful: false });
@@ -2687,7 +2809,10 @@ export class Manager {
2687
2809
  replacement = undefined;
2688
2810
  restart.opts = opts;
2689
2811
  restart.recovering = false;
2690
- console.error(`! ${a.name}: recovered Pi session ${sessionId} after crash (${restart.crashes.length}/${SESSION_RESTART_LIMIT})`);
2812
+ if (continueSession !== undefined)
2813
+ console.error(`! ${a.name}: recovered Pi session ${continueSession} after crash (${restart.crashes.length}/${limit})`);
2814
+ else
2815
+ console.error(`! ${a.name}: restarted under the same lifecycle after crash (${restart.crashes.length}/${limit})`);
2691
2816
  this.watchExit(a);
2692
2817
  }
2693
2818
  catch (error) {
@@ -2698,14 +2823,14 @@ export class Manager {
2698
2823
  tail = this.tail(await (replacement ?? a.handle).attach().backlog());
2699
2824
  }
2700
2825
  catch { /* runtime has no readable tail */ }
2701
- console.error(`! ${a.name}: Pi session recovery failed: ${error.message}${tail ? ` - last output: ${tail}` : ""} - retiring the managed seat`);
2702
- // The replacement may be alive but unable to prove the expected session. Stop it BEFORE
2826
+ console.error(`! ${a.name}: ${supervised ? "supervised restart" : "Pi session recovery"} failed: ${error.message}${tail ? ` - last output: ${tail}` : ""} - retiring the managed seat`);
2827
+ // The replacement may be alive but unable to prove readiness. Stop it BEFORE
2703
2828
  // retiring credentials/durables; otherwise an untracked process survives under torn auth.
2704
2829
  try {
2705
2830
  replacement?.stop({ graceful: false });
2706
2831
  }
2707
2832
  catch { /* terminal cleanup continues */ }
2708
- this.freeSlot(a, true, "pi-recovery-failed");
2833
+ this.freeSlot(a, true, supervised ? "supervise-recovery-failed" : "pi-recovery-failed");
2709
2834
  this.reapChildrenOf(this.managedPrincipal(a));
2710
2835
  }
2711
2836
  finally {
@@ -2714,13 +2839,23 @@ export class Manager {
2714
2839
  })();
2715
2840
  }
2716
2841
  /** A managed agent's process exited on its own (crash, /exit, finished). Continuation-capable Pi
2717
- * seats restart in place after readiness; every other exit follows the existing terminal path. */
2842
+ * seats restart in place after readiness; a spawn carrying `supervise` restarts any connector
2843
+ * the same way, without classifying a session-state file. Every other exit follows the existing
2844
+ * terminal path. */
2718
2845
  onAgentExit(a) {
2719
2846
  // Preservation owns the child-stop snapshot. Exit watchers must neither delete that snapshot nor
2720
2847
  // trigger normal deprovision/reap while the cut is being formed.
2721
2848
  if (this.maintenanceState !== "active")
2722
2849
  return;
2850
+ // A replacement is proving readiness under this row. Its own wait owns a failed
2851
+ // relaunch; treating that exit as a seat death would free the slot mid-recovery.
2852
+ if (a.restart?.recovering && !a.terminalizing)
2853
+ return;
2723
2854
  if (a.restart?.armed && !a.terminalizing) {
2855
+ if (a.restart.policy !== undefined) {
2856
+ this.recoverManagedSession(a);
2857
+ return;
2858
+ }
2724
2859
  try {
2725
2860
  if (this.readManagedSessionState(a).status === "running") {
2726
2861
  this.recoverManagedSession(a);
@@ -2808,6 +2943,21 @@ export class Manager {
2808
2943
  // scalar/array (the CLI never does). Core doesn't interpret the keys; the connector validates them.
2809
2944
  if (args.launchOptions !== undefined && (typeof args.launchOptions !== "object" || args.launchOptions === null || Array.isArray(args.launchOptions)))
2810
2945
  return Promise.resolve({ ok: false, error: "launchOptions: expected a key:value mapping" });
2946
+ let supervise;
2947
+ if (args.supervise !== undefined) {
2948
+ const raw = args.supervise;
2949
+ if (typeof raw !== "object" || raw === null || Array.isArray(raw))
2950
+ return Promise.resolve({ ok: false, error: "supervise: expected { restarts, windowMs }" });
2951
+ const rec = raw;
2952
+ const extra = Object.keys(rec).filter((k) => k !== "restarts" && k !== "windowMs");
2953
+ if (extra.length > 0)
2954
+ return Promise.resolve({ ok: false, error: `supervise: unknown key ${extra[0]}; it takes restarts and windowMs` });
2955
+ if (typeof rec.restarts !== "number" || !Number.isInteger(rec.restarts) || rec.restarts < 1)
2956
+ return Promise.resolve({ ok: false, error: "supervise.restarts: expected a positive integer" });
2957
+ if (typeof rec.windowMs !== "number" || !Number.isInteger(rec.windowMs) || rec.windowMs < 1)
2958
+ return Promise.resolve({ ok: false, error: "supervise.windowMs: expected a positive integer" });
2959
+ supervise = { restarts: rec.restarts, windowMs: rec.windowMs };
2960
+ }
2811
2961
  // ACL overrides arrive as string arrays or not at all — a malformed value is a bad request,
2812
2962
  // not something to coerce (no fallbacks).
2813
2963
  const strList = (v, flag) => {
@@ -2844,6 +2994,7 @@ export class Manager {
2844
2994
  allowSubscribe,
2845
2995
  allowPublish,
2846
2996
  shareTools: args.shareTools !== undefined ? String(args.shareTools) : undefined,
2997
+ ...(supervise !== undefined ? { supervise } : {}),
2847
2998
  }, caller, hooks);
2848
2999
  }
2849
3000
  /** Resolve a connector by agent type. Library composition (installedExtensions off) → a registry
@@ -3165,6 +3316,16 @@ export class Manager {
3165
3316
  // reject-before-side-effects window as the harness preflight above; buildLaunch stays the backstop.
3166
3317
  if (opts.resume && !connector.supportsResume)
3167
3318
  return { ok: false, error: `${agent} connector does not support resuming an existing session (resume)` };
3319
+ // A restart policy this host cannot honour is refused at accept, never accepted and ignored.
3320
+ // External runtimes (tmux/cmux/orca/herdr) attach to a process they do not own and stream no
3321
+ // exit, so a name cannot be respawned in place. User-mode seats have no static slot that
3322
+ // keeps the incarnation owned across a process death, so the same refusal applies there.
3323
+ if (opts.supervise !== undefined) {
3324
+ if (this.runtime.kind !== "pty")
3325
+ return { ok: false, error: `supervise is a restart policy this host cannot enforce: runtime "${this.runtime.kind}" cannot respawn a name in place` };
3326
+ if (this.userMode)
3327
+ return { ok: false, error: "supervise is a restart policy this host cannot enforce: a user-mode seat has no static slot to keep the incarnation owned across a process death" };
3328
+ }
3168
3329
  // Resolve the launch profile: IDENTITY (free-form `name:`) + role + read/post ACL + capabilities
3169
3330
  // + model/variant. Either from a fully-resolved manifest launch object (`opts.resolved`, whose `config`
3170
3331
  // is a materialized transient persona — the file is NOT the access authority), or from the
@@ -3577,8 +3738,17 @@ export class Manager {
3577
3738
  ? Object.keys(opts.launchOptions).sort()
3578
3739
  : undefined,
3579
3740
  },
3580
- ...(connector.supportsSessionContinuation
3581
- ? { restart: { opts: launchOpts, sessionStatePath: spec.sessionStatePath, crashes: [], recovering: false, armed: false } }
3741
+ ...(connector.supportsSessionContinuation || opts.supervise !== undefined
3742
+ ? {
3743
+ restart: {
3744
+ opts: launchOpts,
3745
+ sessionStatePath: spec.sessionStatePath,
3746
+ crashes: [],
3747
+ recovering: false,
3748
+ armed: false,
3749
+ ...(opts.supervise !== undefined ? { policy: opts.supervise } : {}),
3750
+ },
3751
+ }
3582
3752
  : {}),
3583
3753
  };
3584
3754
  // Unit B: the DURABLE slot takes the `active` phase before the in-memory row takes the
@@ -3621,16 +3791,21 @@ export class Manager {
3621
3791
  return { ok: false, error: readiness.detail };
3622
3792
  }
3623
3793
  if (managed.restart) {
3624
- try {
3625
- await this.armSessionRecovery(managed);
3626
- managed.launch.sessionId = this.readManagedSession(managed);
3794
+ if (connector.supportsSessionContinuation) {
3795
+ try {
3796
+ await this.armSessionRecovery(managed);
3797
+ managed.launch.sessionId = this.readManagedSession(managed);
3798
+ }
3799
+ catch (error) {
3800
+ const detail = `${managed.name} joined, but its exact host session could not be bound for supervised recovery: ${error.message}`;
3801
+ this.stopHandle(managed, false);
3802
+ this.freeSlot(managed, true, "session-bind-failed");
3803
+ await hooks?.onOutcome?.({ kind: "failed", data: { error: detail } });
3804
+ return { ok: false, error: detail };
3805
+ }
3627
3806
  }
3628
- catch (error) {
3629
- const detail = `${managed.name} joined, but its exact host session could not be bound for supervised recovery: ${error.message}`;
3630
- this.stopHandle(managed, false);
3631
- this.freeSlot(managed, true, "session-bind-failed");
3632
- await hooks?.onOutcome?.({ kind: "failed", data: { error: detail } });
3633
- return { ok: false, error: detail };
3807
+ else {
3808
+ managed.restart.armed = true;
3634
3809
  }
3635
3810
  }
3636
3811
  this.watchExit(managed);
@@ -4184,7 +4359,7 @@ export class Manager {
4184
4359
  * `"presence"` event is only a wake; the roster is re-read as the source of truth (subscribe-then-check
4185
4360
  * catches a join/exit that landed before we subscribed). Runtimes that stream no exit signal (external surfaces,
4186
4361
  * whose `attach()` throws) race presence-vs-backstop only — better than the old "assume up". */
4187
- async awaitReadiness(a, readinessTimeoutMs) {
4362
+ async awaitReadiness(a, readinessTimeoutMs, opts = {}) {
4188
4363
  let session;
4189
4364
  try {
4190
4365
  session = a.handle.attach();
@@ -4206,16 +4381,25 @@ export class Manager {
4206
4381
  // claims. The manager threads the uid into EVERY mode's launch (open included), so the child
4207
4382
  // adopts it over a self-mint and publishes it in presence; the uid is absent only from a peer
4208
4383
  // the manager never launched (a pure operator/daemon connection that never registers).
4209
- const joined = () => this.ep.getRoster().some((p) => p.card.id === wanted && p.status !== "offline" && p.lifecycleUid === a.lifecycleUid);
4384
+ // A supervised restart keeps the same principal+uid, so a SIGKILL'd child's still-live
4385
+ // presence row would otherwise satisfy this equality. `joinedAfter` is one millisecond
4386
+ // past that row's last heartbeat: only a later heartbeat counts as THIS replacement joining.
4387
+ const joined = () => this.ep.getRoster().some((p) => p.card.id === wanted
4388
+ && p.status !== "offline"
4389
+ && p.lifecycleUid === a.lifecycleUid
4390
+ && (opts.joinedAfter === undefined || p.ts >= opts.joinedAfter));
4210
4391
  return await new Promise((resolve) => {
4211
4392
  let done = false;
4212
4393
  let timer;
4213
4394
  let unsubExit = () => { };
4395
+ let joinedPoll;
4214
4396
  const finish = (r) => {
4215
4397
  if (done)
4216
4398
  return;
4217
4399
  done = true;
4218
4400
  clearTimeout(timer);
4401
+ if (joinedPoll !== undefined)
4402
+ clearInterval(joinedPoll);
4219
4403
  this.ep.off("presence", onPresence);
4220
4404
  unsubExit();
4221
4405
  resolve(r);
@@ -4224,6 +4408,11 @@ export class Manager {
4224
4408
  if (joined())
4225
4409
  finish({ ok: true });
4226
4410
  };
4411
+ if (opts.joinedAfter !== undefined) {
4412
+ // Heartbeats bump roster.ts without emitting "presence" (same status/uid/activity).
4413
+ // A supervised restart needs that bump, so poll joined() rather than waiting on the event.
4414
+ joinedPoll = setInterval(onPresence, 50);
4415
+ }
4227
4416
  // Process exit → failed. Clear the backstop FIRST (synchronously) so it can't resolve UNCERTAIN while
4228
4417
  // the backlog reads async — the process is known dead, that's a failure, not an unknown. Reap through
4229
4418
  // onAgentExit so a child the launcher spawned in the window is reaped too.
@@ -4256,7 +4445,8 @@ export class Manager {
4256
4445
  const deliberate = a.terminalizing === true;
4257
4446
  void (async () => {
4258
4447
  const tail = this.tail(await s.backlog());
4259
- this.onAgentExit(a);
4448
+ if (opts.reapOnExit !== false)
4449
+ this.onAgentExit(a);
4260
4450
  // A DELIBERATE STOP IS NOT A LAUNCH FAILURE. The despawn path owns this goal's terminal
4261
4451
  // and commits `cancel`; reporting `failed` here races it and, when it wins, tells the
4262
4452
  // caller the agent died on launch when in fact an operator cancelled it. The process
@@ -4439,6 +4629,7 @@ export class Manager {
4439
4629
  return {
4440
4630
  instanceId: this.managerInstanceId,
4441
4631
  runtime: this.runtime.kind,
4632
+ custody: "legacy",
4442
4633
  agentCount: this.agents.size,
4443
4634
  uptimeMs: Date.now() - this.startedAtMs,
4444
4635
  connectors: this.connectorStatuses.map((row) => ({ ...row, binaries: { ...row.binaries } })),