@cotal-ai/manager 0.44.0 → 0.46.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +2 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -1
- package/dist/index.js.map +1 -1
- package/dist/launch.js +2 -2
- package/dist/launch.js.map +1 -1
- package/dist/manager-service-contract.d.ts +17 -1
- package/dist/manager-service-contract.d.ts.map +1 -1
- package/dist/manager-service-contract.js +122 -3
- package/dist/manager-service-contract.js.map +1 -1
- package/dist/manager.d.ts +39 -7
- package/dist/manager.d.ts.map +1 -1
- package/dist/manager.js +231 -40
- package/dist/manager.js.map +1 -1
- package/dist/run-hosting.d.ts +114 -0
- package/dist/run-hosting.d.ts.map +1 -0
- package/dist/run-hosting.js +462 -0
- package/dist/run-hosting.js.map +1 -0
- package/dist/runtime/index.d.ts +3 -1
- package/dist/runtime/index.d.ts.map +1 -1
- package/dist/runtime/index.js +6 -0
- package/dist/runtime/index.js.map +1 -1
- package/dist/static-lifecycle.d.ts +35 -1
- package/dist/static-lifecycle.d.ts.map +1 -1
- package/dist/static-lifecycle.js +105 -7
- package/dist/static-lifecycle.js.map +1 -1
- package/package.json +9 -9
package/dist/manager.js
CHANGED
|
@@ -6,7 +6,7 @@ import { existsSync, lstatSync, readFileSync, rmSync } from "node:fs";
|
|
|
6
6
|
import { join, dirname, resolve } from "node:path";
|
|
7
7
|
import { CotalEndpoint, DEFAULT_SERVER, DEV_OWNER, MANAGER_LEASE_RENEW_MS, STANDING_RENEWABLE_TTL_SEC, agentFilePath, clearSpaceHistory, connectorServers, spawnEnvAllow, deprovisionAgent, firstFreeName, spawnNameError, idFromCreds, inspectCredHealth, loadAgentFile, listPersonaCatalog, personaCatalogDescription, personaCatalogReadable, loadCotalConfig, mintCreds, mintLifecycleUid, mkSecretDir, newIdentity, actionContext, parsePrincipalKey, parseShareSelection, principalKey, probeConnect, provisionAgent, provisionAgentDurables, registry, resolveAuthProvider, saveAgentFile, subjectMatches, AUTH_ENDPOINT, EP_CMD_RETIRE_LIFECYCLE, epRequestSubject, epCallerReplyFilter, parseEpSubject, controlServiceSubject, eventChannelPrincipal, } from "@cotal-ai/core";
|
|
8
8
|
import { agentAuthState, agentCredsDir, agentLifecycleSecretFilePaths, agentSecretFilePaths, agentSecretKeyForFile, authDir, connectorInstallHint, DEFAULT_CONNECTOR, defaultAgentType, DELIVERY_CREDS_KIND, extensionConnectors, findCotalRoot, getSpaceAuth, hasUserAuthState, loadExtensionsManifest, loadManagerInstanceIdentity, loadMeshes, manifestExtensionNames, materializeFromManifest, materializeSecretToFile, MEMBERSHIP_RW_CREDS_KIND, mergeLaunchOptions, remintDaemonCreds, resolveOnPath, saveManagerInstanceIdentity, spaceMaterialKey, SYSTEM_CREDS_FILES, userAuthStateDir, workspaceSecretStore, writeRenewalRecord } from "@cotal-ai/workspace";
|
|
9
|
-
import { createRuntime, } from "./runtime/index.js";
|
|
9
|
+
import { createRuntime, requireRuntimeAdopt, } from "./runtime/index.js";
|
|
10
10
|
import { AttachEndpoint } from "./attach-endpoint.js";
|
|
11
11
|
import { makeManagerEndpointEvictionEvidence, makeManagerEndpointEvictor } from "./endpoint-evict.js";
|
|
12
12
|
import { makeManagerHolderLivenessProbe } from "./holder-liveness.js";
|
|
@@ -14,6 +14,7 @@ import { GateReconcileRefused, reconcileEndpointGate } from "./reconcile-gate.js
|
|
|
14
14
|
import { launchSpecForRun, materializePersona, launchAgentToStartOpts, parseLaunchSpec, persistLaunchSpec } from "./launch.js";
|
|
15
15
|
import { authorizeLaunch, authorizeNamedControl } from "./authorize.js";
|
|
16
16
|
import { controlShutdown } from "./control-shutdown.js";
|
|
17
|
+
import { RunHosting } from "./run-hosting.js";
|
|
17
18
|
import { controlSession } from "./control-session.js";
|
|
18
19
|
import { parseResumeCommitArgs, parseResumeControlArgs, parseResumeFinalizeArgs } from "./resume.js";
|
|
19
20
|
// Unit B (the static §13.1 lifecycle executor): the shared grammar/stores from core plus the
|
|
@@ -30,7 +31,7 @@ import { ManagerSessionPlane, openSessionLedgerKv } from "./session/index.js";
|
|
|
30
31
|
// the register/authorize/serve seams, all driven over a scoped one-shot executor connection.
|
|
31
32
|
import { provisionEndpointGateOpen, endpointRegistrationBarrier, serveIssuanceGateKv, commitSiblingIssuance, markLedgerRowRevoked, epcredRowKey, epgateKey, parseEndpointGate, registerServiceInstance, deregisterServiceInstance, authorizeServeGrant, writeServiceStatus, SERVICE_READY, serveEndpoint, bindGoal, createGoal, transitionGoal, commitGoalResult, settleGoalUncertain, readGoalResult, readGoalStatus, readGoalSpec, recordGoalIndex, readGoalIndex, clearGoalIndex, listGoalIndex, mintCheckpoint, resumeCheckpoint, readCheckpointSettle, readCheckpointSpec, expireCheckpoint, GOAL_TERMINAL_STATES, goalRefOf, goalProgressTopic, epeSubject, submissionFingerprint, EpEnvelopeError, lifecycleBlocked, renderLifecycleBlocked, } from "@cotal-ai/core";
|
|
32
33
|
import { MANAGER_ENDPOINT, managerClusterArtifacts, managerCommandDefs, managerContractArtifactValues } from "./manager-service-contract.js";
|
|
33
|
-
import { staticLifecycleTransport, activateStaticLifecycle, runStaticTerminal, readStaticSlot, casStaticSlot, recordSlotCredential, appendStaticCredentialRow, planStaticSlotResume, } from "./static-lifecycle.js";
|
|
34
|
+
import { staticLifecycleTransport, activateStaticLifecycle, runStaticTerminal, readStaticSlot, casStaticSlot, recordSlotCredential, appendStaticCredentialRow, planStaticSlotResume, observeStaticSlot, renderStaticSlotObservation, StaticSlotReadError, STATIC_SLOT_READ_FAILED_DETAIL, } from "./static-lifecycle.js";
|
|
34
35
|
/** Concurrency ceiling — the manager refuses to hold more than this many live + in-flight +
|
|
35
36
|
* cooling slots at once (P4a). Bounds a fork-bomb: spawn is a full agent process per call. */
|
|
36
37
|
const MAX_AGENTS = 50;
|
|
@@ -136,6 +137,8 @@ const FREE_SLOT_CAUSE_TEXT = {
|
|
|
136
137
|
"process-exit": "its own process exited and this manager did not stop it",
|
|
137
138
|
"pi-crash-loop": "this manager retired it after a Pi crash loop",
|
|
138
139
|
"pi-recovery-failed": "this manager retired it after Pi session recovery failed",
|
|
140
|
+
"supervise-crash-loop": "this manager retired it after its supervise restart budget was spent",
|
|
141
|
+
"supervise-recovery-failed": "this manager retired it after a supervised restart failed",
|
|
139
142
|
"session-bind-failed": "this manager stopped it: its host session could not be bound at launch",
|
|
140
143
|
"resume-session-rebind-failed": "this manager stopped it: its host session could not be rebound on resume",
|
|
141
144
|
};
|
|
@@ -319,6 +322,10 @@ export class Manager {
|
|
|
319
322
|
* manager reads its OWN `epgate.<e>.<iid>` epoch over this connection before a terminal commit
|
|
320
323
|
* and skips a superseded commit (the fast-fail belt paired with the (b) barrier-revoke fence). */
|
|
321
324
|
goalWriter;
|
|
325
|
+
/** The workflow-run host (SPEC 14.3): the drives this incarnation holds, each on its own
|
|
326
|
+
* per-run credential and connection. Absent under a remote authority, which mints no driver
|
|
327
|
+
* credentials, so the `run-*` family refuses there rather than connecting on a weaker identity. */
|
|
328
|
+
runHosting;
|
|
322
329
|
/** P2 item 2 must-5 (b): the STABLE goal-writer identity (auth mode) — minted once at
|
|
323
330
|
* registration alongside the serve identity; a renewal re-mints the SAME nkey with a fresh
|
|
324
331
|
* bounded exp and re-stages its distinct credId into the §13.1 revocation family. The current
|
|
@@ -495,6 +502,10 @@ export class Manager {
|
|
|
495
502
|
get runtimeKind() {
|
|
496
503
|
return this.runtime.kind;
|
|
497
504
|
}
|
|
505
|
+
/** Reattach this manager's runtime to a durable handle. Refuses by name when adopt is absent. */
|
|
506
|
+
adoptRuntimeHandle(reference) {
|
|
507
|
+
return requireRuntimeAdopt(this.runtime, reference);
|
|
508
|
+
}
|
|
498
509
|
/** The console page URL (manager-hosted, loopback). */
|
|
499
510
|
get consoleUrl() {
|
|
500
511
|
return this.attach.consoleUrl();
|
|
@@ -699,6 +710,25 @@ export class Manager {
|
|
|
699
710
|
// BEFORE spawn-as-action begins accepting (the goalReconcileDone gate) — a fresh incarnation
|
|
700
711
|
// never drops a goal a dead predecessor accepted. Never fatal; the gate opens either way.
|
|
701
712
|
await this.reconcileGoalIndex();
|
|
713
|
+
// SPEC 14.3: the manager hosts workflow runs. Stood up AFTER registration (it names this
|
|
714
|
+
// instance's coordinates) and reconciled AFTER the goal index, for the same reason: a fresh
|
|
715
|
+
// incarnation takes back every run a dead predecessor was driving before it accepts new ones.
|
|
716
|
+
// The serve surface is already live by here, so the family itself holds the gate: `runHost()`
|
|
717
|
+
// refuses `run-start`/`run-resume` as `unavailable` until the host exists and `RunHosting`
|
|
718
|
+
// refuses them until its reconcile has returned. A user-auth mesh stands no host up at all
|
|
719
|
+
// (`runHost()` names why); a remote-authority manager holds no signer to mint with.
|
|
720
|
+
if (!this.remoteAuthority && !this.userMode) {
|
|
721
|
+
this.runHosting = new RunHosting({
|
|
722
|
+
space: this.space,
|
|
723
|
+
servers: this.servers,
|
|
724
|
+
endpoint: MANAGER_ENDPOINT,
|
|
725
|
+
instanceId: this.managerInstanceId,
|
|
726
|
+
holder: { id: this.ep.ref().id, lifecycleUid: this.managerLifecycleUid },
|
|
727
|
+
auth: this.auth,
|
|
728
|
+
log: (line) => console.error(line),
|
|
729
|
+
});
|
|
730
|
+
await this.runHosting.reconcile();
|
|
731
|
+
}
|
|
702
732
|
// Plane-3 (durable backstop) is NOT the manager's job — the manager only manages agent lifecycle.
|
|
703
733
|
// The server-side delivery daemon hosts the fan-out writer + trusted reader, owns the durable
|
|
704
734
|
// membership registry, and serves the runtime durable join/leave/list ops (on `ctl.delivery`). The
|
|
@@ -835,6 +865,9 @@ export class Manager {
|
|
|
835
865
|
// scoped executor. Without this the standing session-ledger connection dies at its TTL and
|
|
836
866
|
// `attach` stops establishing sessions until a restart. The connection's authenticator presents
|
|
837
867
|
// the refreshed credential on its next (re)connect.
|
|
868
|
+
// SPEC 14.6: every hosted drive's per-run `run-driver` credential is the manager's to renew
|
|
869
|
+
// for the same nkey; a run parked in a pause for days must not die at the credential's TTL.
|
|
870
|
+
await this.runHosting?.renew();
|
|
838
871
|
if (this.sessionLedgerConn && this.sessionLedgerCreds && this.auth) {
|
|
839
872
|
const sw = this.sessionLedgerConn;
|
|
840
873
|
try {
|
|
@@ -1336,6 +1369,10 @@ export class Manager {
|
|
|
1336
1369
|
// in-flight command can write a status back onto the record it just removed.
|
|
1337
1370
|
const registered = this.serviceServe !== undefined;
|
|
1338
1371
|
await this.stopServiceServe();
|
|
1372
|
+
// The drives AFTER the serve loop (no new `run-start` can land) and BEFORE deregistration: a
|
|
1373
|
+
// released run's status write is the last thing this incarnation says about it.
|
|
1374
|
+
await this.runHosting?.stop();
|
|
1375
|
+
this.runHosting = undefined;
|
|
1339
1376
|
if (registered)
|
|
1340
1377
|
await this.deregisterServiceOnStop();
|
|
1341
1378
|
await this.stopGoalWriter();
|
|
@@ -1759,6 +1796,35 @@ export class Manager {
|
|
|
1759
1796
|
* cross-owner persona writes - an operator redefines via config, not the wire), where the ctl
|
|
1760
1797
|
* admin tier allowed operator cross-owner redefine; (2) launch is owner-equality-only, above.
|
|
1761
1798
|
* Both are least-privilege reductions, never widenings. */
|
|
1799
|
+
/** The run host, or one of three refusals. A remote-authority manager holds no space signer, so
|
|
1800
|
+
* it cannot mint the per-run driver credential SPEC 14.6 requires, and hosting on any other
|
|
1801
|
+
* identity would be the fallback this tree does not take: `unimplemented`, for good. A
|
|
1802
|
+
* user-auth mesh is `unimplemented` too, for a different reason it names: a hosted run's seats
|
|
1803
|
+
* are spawned, turned and despawned by a caller derived from the run id under the static
|
|
1804
|
+
* owner, which the user-mode spawn door refuses (no `u_` owner), so a program would fail at
|
|
1805
|
+
* its first seat; no path to `--local` is offered there since a user bearer holds no run rows
|
|
1806
|
+
* either. An ordinary manager whose host is not standing yet is still booting (the serve
|
|
1807
|
+
* surface comes up before the host): `unavailable`, retry. The three are told apart, since
|
|
1808
|
+
* the first sentence steers a caller to `--local` and the others must not. */
|
|
1809
|
+
runHost() {
|
|
1810
|
+
if (this.runHosting)
|
|
1811
|
+
return this.runHosting;
|
|
1812
|
+
if (this.remoteAuthority)
|
|
1813
|
+
throw new EpEnvelopeError("unimplemented", "this manager does not host workflow runs: a remote-authority manager mints no run-driver credentials (SPEC 14.6); drive the run from a terminal with `cotal run start --local --file <program>`");
|
|
1814
|
+
if (this.userMode)
|
|
1815
|
+
throw new EpEnvelopeError("unimplemented", `user-auth space "${this.space}" hosts no workflow runs yet: a hosted run's seats would be spawned under the static owner, which a user mesh refuses; run programs on a static-auth mesh`);
|
|
1816
|
+
throw new EpEnvelopeError("unavailable", "the manager is still booting its workflow-run host; retry shortly (SPEC 14.3)");
|
|
1817
|
+
}
|
|
1818
|
+
/** The answerer a `run-answer` records (SPEC 14.5): the caller as this manager knows them. A
|
|
1819
|
+
* managed seat is named by its persona name; any other authenticated caller (an operator
|
|
1820
|
+
* instrument, a logged-in user) by its principal. Never the request body's word. */
|
|
1821
|
+
runAnswerer(ctx) {
|
|
1822
|
+
const caller = principalKey(ctx.subject.caller.owner, ctx.subject.caller.actor).key;
|
|
1823
|
+
for (const a of this.agents.values())
|
|
1824
|
+
if (this.managedPrincipal(a) === caller)
|
|
1825
|
+
return a.name;
|
|
1826
|
+
return caller;
|
|
1827
|
+
}
|
|
1762
1828
|
managerServiceDefs() {
|
|
1763
1829
|
const args = (ctx) => (ctx.request.args ?? {});
|
|
1764
1830
|
const callerOf = (ctx) => principalKey(ctx.subject.caller.owner, ctx.subject.caller.actor).key;
|
|
@@ -1794,8 +1860,34 @@ export class Manager {
|
|
|
1794
1860
|
inspect: (ctx) => this.serveGated(ctx, async () => {
|
|
1795
1861
|
const name = String(args(ctx).name ?? "").trim();
|
|
1796
1862
|
const row = this.list(await this.psOwnerFilter(callerOf(ctx), false)).find((x) => x.name === name);
|
|
1797
|
-
if (!row)
|
|
1863
|
+
if (!row) {
|
|
1864
|
+
// The live hit above stays entirely local. Only a miss widens into durable static state,
|
|
1865
|
+
// where `not-found` is honest only after the slot read itself succeeds and returns absent.
|
|
1866
|
+
// User-mode lifecycles have no manager-local mgrslot row; their per-name projection remains
|
|
1867
|
+
// the live map until the auth service exposes an equivalent mediated reader.
|
|
1868
|
+
if (this.auth && !this.userMode) {
|
|
1869
|
+
const recordsKv = this.goalWriter?.ctx.kv;
|
|
1870
|
+
if (!recordsKv)
|
|
1871
|
+
throw new EpEnvelopeError("unavailable", `the durable static slot store is not ready while inspecting "${name}"`, [
|
|
1872
|
+
{ kind: STATIC_SLOT_READ_FAILED_DETAIL, name, record: "slot", operation: "read" },
|
|
1873
|
+
]);
|
|
1874
|
+
let observed;
|
|
1875
|
+
try {
|
|
1876
|
+
observed = await observeStaticSlot(recordsKv, DEV_OWNER, name, this.managerInstanceId);
|
|
1877
|
+
}
|
|
1878
|
+
catch (error) {
|
|
1879
|
+
throw new EpEnvelopeError("unavailable", `the durable static slot state for "${name}" could not be read: ${error.message}`, [
|
|
1880
|
+
{ kind: STATIC_SLOT_READ_FAILED_DETAIL, name, record: error instanceof StaticSlotReadError ? error.record : "slot-or-head", operation: "read" },
|
|
1881
|
+
]);
|
|
1882
|
+
}
|
|
1883
|
+
// A retired row affirms that no agent exists now, so it keeps the old `not-found`
|
|
1884
|
+
// result. The changed refusal is reserved for durable NONTERMINAL state that contradicts
|
|
1885
|
+
// the live-map miss and needs operator attention.
|
|
1886
|
+
if (observed !== undefined && observed.slotPhase !== "retired")
|
|
1887
|
+
throw new EpEnvelopeError("failed-precondition", `no live agent "${name}"; durable state contradicts absence ${renderStaticSlotObservation(observed)}`, [observed]);
|
|
1888
|
+
}
|
|
1798
1889
|
throw new EpEnvelopeError("not-found", `no agent "${name}"`);
|
|
1890
|
+
}
|
|
1799
1891
|
return row;
|
|
1800
1892
|
}),
|
|
1801
1893
|
models: (ctx) => this.serveGated(ctx, async () => {
|
|
@@ -1873,6 +1965,13 @@ export class Manager {
|
|
|
1873
1965
|
resumePreserved: (ctx) => adminGated(ctx, async () => unwrap(await this.opResumePreserved(args(ctx)))),
|
|
1874
1966
|
commitResume: (ctx) => adminGated(ctx, async () => unwrap(await this.opCommitResume(args(ctx)))),
|
|
1875
1967
|
finalizeResume: (ctx) => adminGated(ctx, async () => unwrap(await this.opFinalizeResume(args(ctx)))),
|
|
1968
|
+
// The workflow-run family (SPEC 14.3): the manager hosts the driver. Reach is the broker's
|
|
1969
|
+
// (`run` capability / privileged instrument rows); the serve gate is the maintenance fence.
|
|
1970
|
+
runStart: (ctx) => this.serveGated(ctx, () => this.runHost().start(args(ctx))),
|
|
1971
|
+
runResume: (ctx) => this.serveGated(ctx, () => this.runHost().resume(args(ctx))),
|
|
1972
|
+
runAnswer: (ctx) => this.serveGated(ctx, () => this.runHost().answer(args(ctx), this.runAnswerer(ctx))),
|
|
1973
|
+
runStatus: (ctx) => this.serveGated(ctx, () => this.runHost().status(args(ctx))),
|
|
1974
|
+
runPs: (ctx) => this.serveGated(ctx, () => this.runHost().list(args(ctx))),
|
|
1876
1975
|
preparePreservation: (ctx) => adminGated(ctx, async () => unwrap(await this.opPreservationCtl("preparePreservation", args(ctx)))),
|
|
1877
1976
|
commitPreservation: (ctx) => adminGated(ctx, async () => unwrap(await this.opPreservationCtl("commitPreservation", args(ctx)))),
|
|
1878
1977
|
abortPreservation: (ctx) => adminGated(ctx, async () => unwrap(await this.opPreservationCtl("abortPreservation", args(ctx)))),
|
|
@@ -2101,7 +2200,7 @@ export class Manager {
|
|
|
2101
2200
|
// vocabulary as static capabilities; the broker maps them to the ctl tiers. `role:<r>` tokens
|
|
2102
2201
|
// pass through too (a persona may hold delegable roles) — the ledger's envelope walk still
|
|
2103
2202
|
// attenuates every one of these against the spawner chain.
|
|
2104
|
-
const scope = (opts.capabilities ?? []).filter((c) => c === "spawn" || c === "admin" || /^role:[A-Za-z0-9_-]+$/.test(c));
|
|
2203
|
+
const scope = (opts.capabilities ?? []).filter((c) => c === "spawn" || c === "run" || c === "admin" || /^role:[A-Za-z0-9_-]+$/.test(c));
|
|
2105
2204
|
// The manager's ONE store (injected for a hosted composition, workstation FS locally). A hosted
|
|
2106
2205
|
// user-mode spawn reads the callout material from it — the same store the auth-store kinds
|
|
2107
2206
|
// (callout/issuer/…) were migrated onto — so this is no longer a local-only path.
|
|
@@ -2635,9 +2734,10 @@ export class Manager {
|
|
|
2635
2734
|
}
|
|
2636
2735
|
throw new Error(`replacement did not prove session ${expected} (${last})`);
|
|
2637
2736
|
}
|
|
2638
|
-
/** Restart one
|
|
2639
|
-
*
|
|
2640
|
-
*
|
|
2737
|
+
/** Restart one managed process in place. Identity, lifecycle, credentials, durables, children,
|
|
2738
|
+
* and the manager row remain owned; only the process handle/control endpoint change. Budget
|
|
2739
|
+
* comes from `restart.policy` when spawn carried `supervise`; otherwise the Pi session-recovery
|
|
2740
|
+
* constants. Spending the budget falls through to normal retirement. */
|
|
2641
2741
|
recoverManagedSession(a) {
|
|
2642
2742
|
const restart = a.restart;
|
|
2643
2743
|
if (!restart || !restart.armed || restart.recovering || a.terminalizing)
|
|
@@ -2646,12 +2746,15 @@ export class Manager {
|
|
|
2646
2746
|
if (!release)
|
|
2647
2747
|
return; // preservation owns the cut once the lifecycle fence closes
|
|
2648
2748
|
const now = Date.now();
|
|
2649
|
-
|
|
2749
|
+
const limit = restart.policy?.restarts ?? SESSION_RESTART_LIMIT;
|
|
2750
|
+
const windowMs = restart.policy?.windowMs ?? SESSION_RESTART_WINDOW_MS;
|
|
2751
|
+
const supervised = restart.policy !== undefined;
|
|
2752
|
+
restart.crashes = restart.crashes.filter((at) => now - at < windowMs);
|
|
2650
2753
|
restart.crashes.push(now);
|
|
2651
|
-
if (restart.crashes.length >
|
|
2652
|
-
console.error(`! ${a.name}: Pi crash loop (${restart.crashes.length} crashes in ${
|
|
2754
|
+
if (restart.crashes.length > limit) {
|
|
2755
|
+
console.error(`! ${a.name}: ${supervised ? "supervised" : "Pi"} crash loop (${restart.crashes.length} crashes in ${windowMs / 1000}s) - retiring the managed seat`);
|
|
2653
2756
|
restart.armed = false;
|
|
2654
|
-
this.freeSlot(a, true, "pi-crash-loop");
|
|
2757
|
+
this.freeSlot(a, true, supervised ? "supervise-crash-loop" : "pi-crash-loop");
|
|
2655
2758
|
this.reapChildrenOf(this.managedPrincipal(a));
|
|
2656
2759
|
release();
|
|
2657
2760
|
return;
|
|
@@ -2660,21 +2763,40 @@ export class Manager {
|
|
|
2660
2763
|
void (async () => {
|
|
2661
2764
|
let replacement;
|
|
2662
2765
|
try {
|
|
2663
|
-
const sessionId = this.readManagedSession(a);
|
|
2664
2766
|
const connector = await this.resolveConnector(a.agent);
|
|
2665
|
-
|
|
2666
|
-
|
|
2667
|
-
|
|
2668
|
-
...restart.opts,
|
|
2669
|
-
resume: undefined,
|
|
2670
|
-
prompt: undefined,
|
|
2671
|
-
continueSession: sessionId,
|
|
2672
|
-
};
|
|
2767
|
+
const continueSession = connector.supportsSessionContinuation ? this.readManagedSession(a) : undefined;
|
|
2768
|
+
const opts = continueSession !== undefined
|
|
2769
|
+
? { ...restart.opts, resume: undefined, prompt: undefined, continueSession }
|
|
2770
|
+
: { ...restart.opts, resume: undefined, prompt: undefined };
|
|
2673
2771
|
const spec = connector.buildLaunch(opts);
|
|
2772
|
+
const wanted = this.managedPrincipal(a);
|
|
2773
|
+
const joinedAfter = this.ep.getRoster()
|
|
2774
|
+
.filter((p) => p.card.id === wanted && p.lifecycleUid === a.lifecycleUid)
|
|
2775
|
+
.reduce((max, p) => Math.max(max, p.ts), 0) + 1;
|
|
2674
2776
|
const handle = this.runtime.spawn(a.name, spec, a.launch.cwd);
|
|
2675
2777
|
replacement = handle;
|
|
2676
2778
|
restart.sessionStatePath = spec.sessionStatePath ?? restart.sessionStatePath;
|
|
2677
|
-
|
|
2779
|
+
if (continueSession !== undefined)
|
|
2780
|
+
await this.awaitRecoveredSession(a, continueSession, handle, spec.control);
|
|
2781
|
+
else {
|
|
2782
|
+
const previousHandle = a.handle;
|
|
2783
|
+
const previousControl = a.control;
|
|
2784
|
+
a.handle = handle;
|
|
2785
|
+
a.control = spec.control;
|
|
2786
|
+
try {
|
|
2787
|
+
const readiness = await this.awaitReadiness(a, connector.readinessTimeoutMs ?? this.readinessTimeoutMs, {
|
|
2788
|
+
reapOnExit: false,
|
|
2789
|
+
joinedAfter,
|
|
2790
|
+
});
|
|
2791
|
+
if (!readiness.ok)
|
|
2792
|
+
throw new Error(readiness.detail);
|
|
2793
|
+
}
|
|
2794
|
+
catch (error) {
|
|
2795
|
+
a.handle = previousHandle;
|
|
2796
|
+
a.control = previousControl;
|
|
2797
|
+
throw error;
|
|
2798
|
+
}
|
|
2799
|
+
}
|
|
2678
2800
|
if (this.agents.get(a.name) !== a || a.terminalizing) {
|
|
2679
2801
|
try {
|
|
2680
2802
|
handle.stop({ graceful: false });
|
|
@@ -2687,7 +2809,10 @@ export class Manager {
|
|
|
2687
2809
|
replacement = undefined;
|
|
2688
2810
|
restart.opts = opts;
|
|
2689
2811
|
restart.recovering = false;
|
|
2690
|
-
|
|
2812
|
+
if (continueSession !== undefined)
|
|
2813
|
+
console.error(`! ${a.name}: recovered Pi session ${continueSession} after crash (${restart.crashes.length}/${limit})`);
|
|
2814
|
+
else
|
|
2815
|
+
console.error(`! ${a.name}: restarted under the same lifecycle after crash (${restart.crashes.length}/${limit})`);
|
|
2691
2816
|
this.watchExit(a);
|
|
2692
2817
|
}
|
|
2693
2818
|
catch (error) {
|
|
@@ -2698,14 +2823,14 @@ export class Manager {
|
|
|
2698
2823
|
tail = this.tail(await (replacement ?? a.handle).attach().backlog());
|
|
2699
2824
|
}
|
|
2700
2825
|
catch { /* runtime has no readable tail */ }
|
|
2701
|
-
console.error(`! ${a.name}: Pi session recovery failed: ${error.message}${tail ? ` - last output: ${tail}` : ""} - retiring the managed seat`);
|
|
2702
|
-
// The replacement may be alive but unable to prove
|
|
2826
|
+
console.error(`! ${a.name}: ${supervised ? "supervised restart" : "Pi session recovery"} failed: ${error.message}${tail ? ` - last output: ${tail}` : ""} - retiring the managed seat`);
|
|
2827
|
+
// The replacement may be alive but unable to prove readiness. Stop it BEFORE
|
|
2703
2828
|
// retiring credentials/durables; otherwise an untracked process survives under torn auth.
|
|
2704
2829
|
try {
|
|
2705
2830
|
replacement?.stop({ graceful: false });
|
|
2706
2831
|
}
|
|
2707
2832
|
catch { /* terminal cleanup continues */ }
|
|
2708
|
-
this.freeSlot(a, true, "pi-recovery-failed");
|
|
2833
|
+
this.freeSlot(a, true, supervised ? "supervise-recovery-failed" : "pi-recovery-failed");
|
|
2709
2834
|
this.reapChildrenOf(this.managedPrincipal(a));
|
|
2710
2835
|
}
|
|
2711
2836
|
finally {
|
|
@@ -2714,13 +2839,23 @@ export class Manager {
|
|
|
2714
2839
|
})();
|
|
2715
2840
|
}
|
|
2716
2841
|
/** A managed agent's process exited on its own (crash, /exit, finished). Continuation-capable Pi
|
|
2717
|
-
* seats restart in place after readiness;
|
|
2842
|
+
* seats restart in place after readiness; a spawn carrying `supervise` restarts any connector
|
|
2843
|
+
* the same way, without classifying a session-state file. Every other exit follows the existing
|
|
2844
|
+
* terminal path. */
|
|
2718
2845
|
onAgentExit(a) {
|
|
2719
2846
|
// Preservation owns the child-stop snapshot. Exit watchers must neither delete that snapshot nor
|
|
2720
2847
|
// trigger normal deprovision/reap while the cut is being formed.
|
|
2721
2848
|
if (this.maintenanceState !== "active")
|
|
2722
2849
|
return;
|
|
2850
|
+
// A replacement is proving readiness under this row. Its own wait owns a failed
|
|
2851
|
+
// relaunch; treating that exit as a seat death would free the slot mid-recovery.
|
|
2852
|
+
if (a.restart?.recovering && !a.terminalizing)
|
|
2853
|
+
return;
|
|
2723
2854
|
if (a.restart?.armed && !a.terminalizing) {
|
|
2855
|
+
if (a.restart.policy !== undefined) {
|
|
2856
|
+
this.recoverManagedSession(a);
|
|
2857
|
+
return;
|
|
2858
|
+
}
|
|
2724
2859
|
try {
|
|
2725
2860
|
if (this.readManagedSessionState(a).status === "running") {
|
|
2726
2861
|
this.recoverManagedSession(a);
|
|
@@ -2808,6 +2943,21 @@ export class Manager {
|
|
|
2808
2943
|
// scalar/array (the CLI never does). Core doesn't interpret the keys; the connector validates them.
|
|
2809
2944
|
if (args.launchOptions !== undefined && (typeof args.launchOptions !== "object" || args.launchOptions === null || Array.isArray(args.launchOptions)))
|
|
2810
2945
|
return Promise.resolve({ ok: false, error: "launchOptions: expected a key:value mapping" });
|
|
2946
|
+
let supervise;
|
|
2947
|
+
if (args.supervise !== undefined) {
|
|
2948
|
+
const raw = args.supervise;
|
|
2949
|
+
if (typeof raw !== "object" || raw === null || Array.isArray(raw))
|
|
2950
|
+
return Promise.resolve({ ok: false, error: "supervise: expected { restarts, windowMs }" });
|
|
2951
|
+
const rec = raw;
|
|
2952
|
+
const extra = Object.keys(rec).filter((k) => k !== "restarts" && k !== "windowMs");
|
|
2953
|
+
if (extra.length > 0)
|
|
2954
|
+
return Promise.resolve({ ok: false, error: `supervise: unknown key ${extra[0]}; it takes restarts and windowMs` });
|
|
2955
|
+
if (typeof rec.restarts !== "number" || !Number.isInteger(rec.restarts) || rec.restarts < 1)
|
|
2956
|
+
return Promise.resolve({ ok: false, error: "supervise.restarts: expected a positive integer" });
|
|
2957
|
+
if (typeof rec.windowMs !== "number" || !Number.isInteger(rec.windowMs) || rec.windowMs < 1)
|
|
2958
|
+
return Promise.resolve({ ok: false, error: "supervise.windowMs: expected a positive integer" });
|
|
2959
|
+
supervise = { restarts: rec.restarts, windowMs: rec.windowMs };
|
|
2960
|
+
}
|
|
2811
2961
|
// ACL overrides arrive as string arrays or not at all — a malformed value is a bad request,
|
|
2812
2962
|
// not something to coerce (no fallbacks).
|
|
2813
2963
|
const strList = (v, flag) => {
|
|
@@ -2844,6 +2994,7 @@ export class Manager {
|
|
|
2844
2994
|
allowSubscribe,
|
|
2845
2995
|
allowPublish,
|
|
2846
2996
|
shareTools: args.shareTools !== undefined ? String(args.shareTools) : undefined,
|
|
2997
|
+
...(supervise !== undefined ? { supervise } : {}),
|
|
2847
2998
|
}, caller, hooks);
|
|
2848
2999
|
}
|
|
2849
3000
|
/** Resolve a connector by agent type. Library composition (installedExtensions off) → a registry
|
|
@@ -3165,6 +3316,16 @@ export class Manager {
|
|
|
3165
3316
|
// reject-before-side-effects window as the harness preflight above; buildLaunch stays the backstop.
|
|
3166
3317
|
if (opts.resume && !connector.supportsResume)
|
|
3167
3318
|
return { ok: false, error: `${agent} connector does not support resuming an existing session (resume)` };
|
|
3319
|
+
// A restart policy this host cannot honour is refused at accept, never accepted and ignored.
|
|
3320
|
+
// External runtimes (tmux/cmux/orca/herdr) attach to a process they do not own and stream no
|
|
3321
|
+
// exit, so a name cannot be respawned in place. User-mode seats have no static slot that
|
|
3322
|
+
// keeps the incarnation owned across a process death, so the same refusal applies there.
|
|
3323
|
+
if (opts.supervise !== undefined) {
|
|
3324
|
+
if (this.runtime.kind !== "pty")
|
|
3325
|
+
return { ok: false, error: `supervise is a restart policy this host cannot enforce: runtime "${this.runtime.kind}" cannot respawn a name in place` };
|
|
3326
|
+
if (this.userMode)
|
|
3327
|
+
return { ok: false, error: "supervise is a restart policy this host cannot enforce: a user-mode seat has no static slot to keep the incarnation owned across a process death" };
|
|
3328
|
+
}
|
|
3168
3329
|
// Resolve the launch profile: IDENTITY (free-form `name:`) + role + read/post ACL + capabilities
|
|
3169
3330
|
// + model/variant. Either from a fully-resolved manifest launch object (`opts.resolved`, whose `config`
|
|
3170
3331
|
// is a materialized transient persona — the file is NOT the access authority), or from the
|
|
@@ -3577,8 +3738,17 @@ export class Manager {
|
|
|
3577
3738
|
? Object.keys(opts.launchOptions).sort()
|
|
3578
3739
|
: undefined,
|
|
3579
3740
|
},
|
|
3580
|
-
...(connector.supportsSessionContinuation
|
|
3581
|
-
? {
|
|
3741
|
+
...(connector.supportsSessionContinuation || opts.supervise !== undefined
|
|
3742
|
+
? {
|
|
3743
|
+
restart: {
|
|
3744
|
+
opts: launchOpts,
|
|
3745
|
+
sessionStatePath: spec.sessionStatePath,
|
|
3746
|
+
crashes: [],
|
|
3747
|
+
recovering: false,
|
|
3748
|
+
armed: false,
|
|
3749
|
+
...(opts.supervise !== undefined ? { policy: opts.supervise } : {}),
|
|
3750
|
+
},
|
|
3751
|
+
}
|
|
3582
3752
|
: {}),
|
|
3583
3753
|
};
|
|
3584
3754
|
// Unit B: the DURABLE slot takes the `active` phase before the in-memory row takes the
|
|
@@ -3621,16 +3791,21 @@ export class Manager {
|
|
|
3621
3791
|
return { ok: false, error: readiness.detail };
|
|
3622
3792
|
}
|
|
3623
3793
|
if (managed.restart) {
|
|
3624
|
-
|
|
3625
|
-
|
|
3626
|
-
|
|
3794
|
+
if (connector.supportsSessionContinuation) {
|
|
3795
|
+
try {
|
|
3796
|
+
await this.armSessionRecovery(managed);
|
|
3797
|
+
managed.launch.sessionId = this.readManagedSession(managed);
|
|
3798
|
+
}
|
|
3799
|
+
catch (error) {
|
|
3800
|
+
const detail = `${managed.name} joined, but its exact host session could not be bound for supervised recovery: ${error.message}`;
|
|
3801
|
+
this.stopHandle(managed, false);
|
|
3802
|
+
this.freeSlot(managed, true, "session-bind-failed");
|
|
3803
|
+
await hooks?.onOutcome?.({ kind: "failed", data: { error: detail } });
|
|
3804
|
+
return { ok: false, error: detail };
|
|
3805
|
+
}
|
|
3627
3806
|
}
|
|
3628
|
-
|
|
3629
|
-
|
|
3630
|
-
this.stopHandle(managed, false);
|
|
3631
|
-
this.freeSlot(managed, true, "session-bind-failed");
|
|
3632
|
-
await hooks?.onOutcome?.({ kind: "failed", data: { error: detail } });
|
|
3633
|
-
return { ok: false, error: detail };
|
|
3807
|
+
else {
|
|
3808
|
+
managed.restart.armed = true;
|
|
3634
3809
|
}
|
|
3635
3810
|
}
|
|
3636
3811
|
this.watchExit(managed);
|
|
@@ -4184,7 +4359,7 @@ export class Manager {
|
|
|
4184
4359
|
* `"presence"` event is only a wake; the roster is re-read as the source of truth (subscribe-then-check
|
|
4185
4360
|
* catches a join/exit that landed before we subscribed). Runtimes that stream no exit signal (external surfaces,
|
|
4186
4361
|
* whose `attach()` throws) race presence-vs-backstop only — better than the old "assume up". */
|
|
4187
|
-
async awaitReadiness(a, readinessTimeoutMs) {
|
|
4362
|
+
async awaitReadiness(a, readinessTimeoutMs, opts = {}) {
|
|
4188
4363
|
let session;
|
|
4189
4364
|
try {
|
|
4190
4365
|
session = a.handle.attach();
|
|
@@ -4206,16 +4381,25 @@ export class Manager {
|
|
|
4206
4381
|
// claims. The manager threads the uid into EVERY mode's launch (open included), so the child
|
|
4207
4382
|
// adopts it over a self-mint and publishes it in presence; the uid is absent only from a peer
|
|
4208
4383
|
// the manager never launched (a pure operator/daemon connection that never registers).
|
|
4209
|
-
|
|
4384
|
+
// A supervised restart keeps the same principal+uid, so a SIGKILL'd child's still-live
|
|
4385
|
+
// presence row would otherwise satisfy this equality. `joinedAfter` is one millisecond
|
|
4386
|
+
// past that row's last heartbeat: only a later heartbeat counts as THIS replacement joining.
|
|
4387
|
+
const joined = () => this.ep.getRoster().some((p) => p.card.id === wanted
|
|
4388
|
+
&& p.status !== "offline"
|
|
4389
|
+
&& p.lifecycleUid === a.lifecycleUid
|
|
4390
|
+
&& (opts.joinedAfter === undefined || p.ts >= opts.joinedAfter));
|
|
4210
4391
|
return await new Promise((resolve) => {
|
|
4211
4392
|
let done = false;
|
|
4212
4393
|
let timer;
|
|
4213
4394
|
let unsubExit = () => { };
|
|
4395
|
+
let joinedPoll;
|
|
4214
4396
|
const finish = (r) => {
|
|
4215
4397
|
if (done)
|
|
4216
4398
|
return;
|
|
4217
4399
|
done = true;
|
|
4218
4400
|
clearTimeout(timer);
|
|
4401
|
+
if (joinedPoll !== undefined)
|
|
4402
|
+
clearInterval(joinedPoll);
|
|
4219
4403
|
this.ep.off("presence", onPresence);
|
|
4220
4404
|
unsubExit();
|
|
4221
4405
|
resolve(r);
|
|
@@ -4224,6 +4408,11 @@ export class Manager {
|
|
|
4224
4408
|
if (joined())
|
|
4225
4409
|
finish({ ok: true });
|
|
4226
4410
|
};
|
|
4411
|
+
if (opts.joinedAfter !== undefined) {
|
|
4412
|
+
// Heartbeats bump roster.ts without emitting "presence" (same status/uid/activity).
|
|
4413
|
+
// A supervised restart needs that bump, so poll joined() rather than waiting on the event.
|
|
4414
|
+
joinedPoll = setInterval(onPresence, 50);
|
|
4415
|
+
}
|
|
4227
4416
|
// Process exit → failed. Clear the backstop FIRST (synchronously) so it can't resolve UNCERTAIN while
|
|
4228
4417
|
// the backlog reads async — the process is known dead, that's a failure, not an unknown. Reap through
|
|
4229
4418
|
// onAgentExit so a child the launcher spawned in the window is reaped too.
|
|
@@ -4256,7 +4445,8 @@ export class Manager {
|
|
|
4256
4445
|
const deliberate = a.terminalizing === true;
|
|
4257
4446
|
void (async () => {
|
|
4258
4447
|
const tail = this.tail(await s.backlog());
|
|
4259
|
-
|
|
4448
|
+
if (opts.reapOnExit !== false)
|
|
4449
|
+
this.onAgentExit(a);
|
|
4260
4450
|
// A DELIBERATE STOP IS NOT A LAUNCH FAILURE. The despawn path owns this goal's terminal
|
|
4261
4451
|
// and commits `cancel`; reporting `failed` here races it and, when it wins, tells the
|
|
4262
4452
|
// caller the agent died on launch when in fact an operator cancelled it. The process
|
|
@@ -4439,6 +4629,7 @@ export class Manager {
|
|
|
4439
4629
|
return {
|
|
4440
4630
|
instanceId: this.managerInstanceId,
|
|
4441
4631
|
runtime: this.runtime.kind,
|
|
4632
|
+
custody: "legacy",
|
|
4442
4633
|
agentCount: this.agents.size,
|
|
4443
4634
|
uptimeMs: Date.now() - this.startedAtMs,
|
|
4444
4635
|
connectors: this.connectorStatuses.map((row) => ({ ...row, binaries: { ...row.binaries } })),
|