@cotal-ai/manager 0.16.0 → 0.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -1
- package/dist/attach-endpoint.d.ts +53 -59
- package/dist/attach-endpoint.d.ts.map +1 -1
- package/dist/attach-endpoint.js +120 -244
- package/dist/attach-endpoint.js.map +1 -1
- package/dist/commands.js +88 -8
- package/dist/commands.js.map +1 -1
- package/dist/console/app.js +68 -30
- package/dist/console/index.html +3 -2
- package/dist/console/session-bundle.js +9736 -0
- package/dist/console-crypto-shim.d.ts +16 -0
- package/dist/console-crypto-shim.d.ts.map +1 -0
- package/dist/console-crypto-shim.js +20 -0
- package/dist/console-crypto-shim.js.map +1 -0
- package/dist/console-session-entry.d.ts +2 -0
- package/dist/console-session-entry.d.ts.map +1 -0
- package/dist/console-session-entry.js +26 -0
- package/dist/console-session-entry.js.map +1 -0
- package/dist/endpoint-evict.d.ts +29 -0
- package/dist/endpoint-evict.d.ts.map +1 -0
- package/dist/endpoint-evict.js +80 -0
- package/dist/endpoint-evict.js.map +1 -0
- package/dist/holder-liveness.d.ts +40 -0
- package/dist/holder-liveness.d.ts.map +1 -0
- package/dist/holder-liveness.js +98 -0
- package/dist/holder-liveness.js.map +1 -0
- package/dist/manager-service-contract.d.ts +132 -0
- package/dist/manager-service-contract.d.ts.map +1 -0
- package/dist/manager-service-contract.js +332 -0
- package/dist/manager-service-contract.js.map +1 -0
- package/dist/manager.d.ts +519 -22
- package/dist/manager.d.ts.map +1 -1
- package/dist/manager.js +2273 -262
- package/dist/manager.js.map +1 -1
- package/dist/reconcile-gate.d.ts +82 -0
- package/dist/reconcile-gate.d.ts.map +1 -0
- package/dist/reconcile-gate.js +118 -0
- package/dist/reconcile-gate.js.map +1 -0
- package/dist/runtime/pty.d.ts.map +1 -1
- package/dist/runtime/pty.js +15 -2
- package/dist/runtime/pty.js.map +1 -1
- package/dist/session/bridge.d.ts +63 -0
- package/dist/session/bridge.d.ts.map +1 -0
- package/dist/session/bridge.js +157 -0
- package/dist/session/bridge.js.map +1 -0
- package/dist/session/establish.d.ts +212 -0
- package/dist/session/establish.d.ts.map +1 -0
- package/dist/session/establish.js +174 -0
- package/dist/session/establish.js.map +1 -0
- package/dist/session/index.d.ts +12 -0
- package/dist/session/index.d.ts.map +1 -0
- package/dist/session/index.js +15 -0
- package/dist/session/index.js.map +1 -0
- package/dist/session/plane.d.ts +120 -0
- package/dist/session/plane.d.ts.map +1 -0
- package/dist/session/plane.js +327 -0
- package/dist/session/plane.js.map +1 -0
- package/dist/static-lifecycle.d.ts +97 -0
- package/dist/static-lifecycle.d.ts.map +1 -0
- package/dist/static-lifecycle.js +262 -0
- package/dist/static-lifecycle.js.map +1 -0
- package/package.json +10 -6
package/dist/manager.js
CHANGED
|
@@ -3,14 +3,30 @@ import { createHash, randomUUID, randomBytes } from "node:crypto";
|
|
|
3
3
|
import { connect, credsAuthenticator } from "@nats-io/transport-node";
|
|
4
4
|
import { existsSync, lstatSync, readFileSync, rmSync } from "node:fs";
|
|
5
5
|
import { join, dirname, resolve } from "node:path";
|
|
6
|
-
import { CotalEndpoint, DEFAULT_SERVER, DEV_OWNER, MANAGER_LEASE_TTL_MS, STANDING_RENEWABLE_TTL_SEC, agentFilePath, clearSpaceHistory, connectorServers, deprovisionAgent, firstFreeName, idFromCreds, loadAgentFile, loadCotalConfig, mintCreds, mintLifecycleUid, mkSecretDir, newIdentity, parsePrincipalKey, parseShareSelection, principalKey, probeConnect, provisionAgent, provisionAgentDurables, registry, resolveAuthProvider, saveAgentFile, subjectMatches,
|
|
7
|
-
import {
|
|
6
|
+
import { CotalEndpoint, DEFAULT_SERVER, DEV_OWNER, MANAGER_LEASE_TTL_MS, STANDING_RENEWABLE_TTL_SEC, agentFilePath, clearSpaceHistory, connectorServers, deprovisionAgent, firstFreeName, idFromCreds, inspectCredHealth, loadAgentFile, loadCotalConfig, mintCreds, mintLifecycleUid, mkSecretDir, newIdentity, actionContext, parsePrincipalKey, parseShareSelection, principalKey, probeConnect, provisionAgent, provisionAgentDurables, registry, resolveAuthProvider, saveAgentFile, subjectMatches, AUTH_ENDPOINT, EP_CMD_RETIRE_LIFECYCLE, epRequestSubject, epCallerReplyFilter, parseEpSubject, controlServiceSubject, } from "@cotal-ai/core";
|
|
7
|
+
import { agentAuthState, agentCredsDir, agentLifecycleSecretFilePaths, agentSecretFilePaths, agentSecretKeyForFile, authDir, connectorInstallHint, DEFAULT_CONNECTOR, defaultAgentType, DELIVERY_CREDS_KEY, findCotalRoot, getSpaceAuth, hasUserAuthState, loadManagerInstanceIdentity, loadMeshes, manifestExtensionNames, materializeFromManifest, materializeSecretToFile, MEMBERSHIP_RW_CREDS_KEY, mergeLaunchOptions, remintDaemonCreds, resolveOnPath, saveManagerInstanceIdentity, SYSTEM_CREDS_FILES, userAuthStateDir, workspaceSecretStore, writeRenewalRecord } from "@cotal-ai/workspace";
|
|
8
8
|
import { createRuntime, } from "./runtime/index.js";
|
|
9
|
-
import { AttachEndpoint
|
|
9
|
+
import { AttachEndpoint } from "./attach-endpoint.js";
|
|
10
|
+
import { makeManagerEndpointEvictor } from "./endpoint-evict.js";
|
|
10
11
|
import { launchSpecForRun, materializePersona, launchAgentToStartOpts, parseLaunchSpec, persistLaunchSpec } from "./launch.js";
|
|
11
12
|
import { authorizeLaunch, authorizeNamedControl } from "./authorize.js";
|
|
12
13
|
import { controlShutdown } from "./control-shutdown.js";
|
|
13
14
|
import { parseResumeCommitArgs, parseResumeControlArgs, parseResumeFinalizeArgs } from "./resume.js";
|
|
15
|
+
// Unit B (the static §13.1 lifecycle executor): the shared grammar/stores from core plus the
|
|
16
|
+
// manager-side adapter (transport + slot orchestration + the F1 terminal) — see static-lifecycle.ts.
|
|
17
|
+
import { jetstreamManager } from "@nats-io/jetstream";
|
|
18
|
+
import { Kvm } from "@nats-io/kv";
|
|
19
|
+
import { recordsBucket, epAuthBucket, ensureAuthorityStores, ensureContractStore, createEndpointStreams, contractStoreContext, publishContractArtifact, contractArtifactCanonicalBytes, standaloneConnectOpts, STATIC_SLOT_PREFIX, rawDigest, STANDING_RENEWABLE_TTL_SEC as MANAGED_STATIC_TTL_SEC, newArtifactSigner, sessionsBucket, SESSION_GRANT_MAX_TTL_MS, } from "@cotal-ai/core";
|
|
20
|
+
// P2 item 6: the manager's ONE §13.6 session plane — offer mint + one-use redeem + PTY-bridge
|
|
21
|
+
// standup for `attach`, over a dedicated standing session-LEDGER connection (the byte rails ride
|
|
22
|
+
// per-session credentials on their own short-lived connections).
|
|
23
|
+
import { ManagerSessionPlane, openSessionLedgerKv } from "./session/index.js";
|
|
24
|
+
// P2 item 1 (1a-serve): the manager as an ordinary v0.4 `service` endpoint — the §13.1
|
|
25
|
+
// endpoint-serve credential subsystem (gate provisioning, registration barrier, mint fence) plus
|
|
26
|
+
// the register/authorize/serve seams, all driven over a scoped one-shot executor connection.
|
|
27
|
+
import { provisionEndpointGateOpen, endpointRegistrationBarrier, serveIssuanceGateKv, commitSiblingIssuance, markLedgerRowRevoked, epcredRowKey, epgateKey, registerServiceInstance, authorizeServeGrant, writeServiceStatus, SERVICE_READY, serveEndpoint, bindGoal, createGoal, transitionGoal, commitGoalResult, settleGoalUncertain, readGoalResult, readGoalStatus, readGoalSpec, recordGoalIndex, readGoalIndex, clearGoalIndex, listGoalIndex, GOAL_TERMINAL_STATES, goalRefOf, goalProgressTopic, epeSubject, submissionFingerprint, EpEnvelopeError, } from "@cotal-ai/core";
|
|
28
|
+
import { MANAGER_ENDPOINT, managerClusterArtifacts, managerCommandDefs, managerContractArtifactValues } from "./manager-service-contract.js";
|
|
29
|
+
import { staticLifecycleTransport, activateStaticLifecycle, runStaticTerminal, readStaticSlot, casStaticSlot, recordSlotCredential, appendStaticCredentialRow, planStaticSlotResume, } from "./static-lifecycle.js";
|
|
14
30
|
/** Concurrency ceiling — the manager refuses to hold more than this many live + in-flight +
|
|
15
31
|
* cooling slots at once (P4a). Bounds a fork-bomb: spawn is a full agent process per call. */
|
|
16
32
|
const MAX_AGENTS = 50;
|
|
@@ -75,17 +91,65 @@ function withTimeout(p, ms, msg) {
|
|
|
75
91
|
function sameStrings(a, b) {
|
|
76
92
|
return JSON.stringify([...(a ?? [])].sort()) === JSON.stringify([...(b ?? [])].sort());
|
|
77
93
|
}
|
|
78
|
-
/**
|
|
79
|
-
*
|
|
80
|
-
*
|
|
81
|
-
*
|
|
82
|
-
*
|
|
83
|
-
|
|
94
|
+
/** One ep request/reply round-trip on the caller's OWN reply-plane filter (§13.2). The responder
|
|
95
|
+
* derives the reply subject from the authenticated request, so there is no caller-selected reply
|
|
96
|
+
* target to honour; the caller binds the answer off the reply SUBJECT — endpoint and nonce, both
|
|
97
|
+
* broker-pinned by the responder's serve publish grant. A reply on another nonce belongs to a
|
|
98
|
+
* different request (the rail is shared) and is ignored rather than allowed to fail this one. */
|
|
99
|
+
export async function epAwaitReply(nc, space, caller, nonce, requestId, requestSubject, body, timeoutMs) {
|
|
100
|
+
let sub;
|
|
101
|
+
let timer;
|
|
102
|
+
try {
|
|
103
|
+
const got = new Promise((resolve, reject) => {
|
|
104
|
+
sub = nc.subscribe(epCallerReplyFilter(space, caller), {
|
|
105
|
+
callback: (err, msg) => {
|
|
106
|
+
if (err) {
|
|
107
|
+
reject(new Error(`the retirement reply subscription failed: ${err.message}`));
|
|
108
|
+
return;
|
|
109
|
+
}
|
|
110
|
+
const parsed = parseEpSubject(msg.subject);
|
|
111
|
+
if (!parsed || parsed.plane !== "reply" || parsed.endpoint !== AUTH_ENDPOINT || parsed.nonce !== nonce)
|
|
112
|
+
return;
|
|
113
|
+
let body;
|
|
114
|
+
try {
|
|
115
|
+
body = JSON.parse(new TextDecoder().decode(msg.data));
|
|
116
|
+
}
|
|
117
|
+
catch {
|
|
118
|
+
return;
|
|
119
|
+
} // a malformed body on our nonce is not an answer; keep waiting
|
|
120
|
+
// REQUIRE THE ID ECHO. Binding on (endpoint, nonce) alone would accept a malformed or
|
|
121
|
+
// WRONG-ID `{ok:true}` and clear a retirement hold on it. Ignore rather than fail: the
|
|
122
|
+
// rail is shared across concurrent requests and a responder may publish at any nonce, so
|
|
123
|
+
// an unmatched reply is someone else's answer, never grounds to fail the honest one.
|
|
124
|
+
if (body.id !== requestId)
|
|
125
|
+
return;
|
|
126
|
+
resolve(body);
|
|
127
|
+
},
|
|
128
|
+
});
|
|
129
|
+
timer = setTimeout(() => reject(new Error("timeout")), timeoutMs);
|
|
130
|
+
nc.publish(requestSubject, new TextEncoder().encode(body));
|
|
131
|
+
});
|
|
132
|
+
return await got;
|
|
133
|
+
}
|
|
134
|
+
finally {
|
|
135
|
+
if (timer)
|
|
136
|
+
clearTimeout(timer);
|
|
137
|
+
try {
|
|
138
|
+
sub?.unsubscribe();
|
|
139
|
+
}
|
|
140
|
+
catch { /* connection already down */ }
|
|
141
|
+
}
|
|
142
|
+
}
|
|
84
143
|
export class Manager {
|
|
85
144
|
space;
|
|
86
145
|
servers;
|
|
146
|
+
/** P2 item 6: the broker ws listener port (loopback) `cotal up` allocated, for the console session
|
|
147
|
+
* client's wsUrl. Undefined ⇒ no console session client (POST /session 503s). */
|
|
148
|
+
wsPort;
|
|
87
149
|
name;
|
|
88
150
|
workspaceRoot;
|
|
151
|
+
/** P2 item 6: the operator-set global live-session ceiling (see {@link ManagerOptions.maxSessions}). */
|
|
152
|
+
maxSessions;
|
|
89
153
|
/** The ONE secret store for every kind this manager touches (daemon-cred remint + agent kinds).
|
|
90
154
|
* See {@link ManagerOptions.secretStore}. */
|
|
91
155
|
secrets;
|
|
@@ -115,6 +179,82 @@ export class Manager {
|
|
|
115
179
|
* same-name SUCCESSOR (which can spawn after the hold clears but before this flight's `nc.close`
|
|
116
180
|
* yield settles) never joins the predecessor's rail request and skips its own retirement. */
|
|
117
181
|
retiringFlight = new Map();
|
|
182
|
+
/** Wire principals of RETIRED static incarnations (Unit B, F5(a)): populated at every completed
|
|
183
|
+
* static terminal and from the boot sweep's retired slot rows, so a copied credential of a
|
|
184
|
+
* retired incarnation is refused at the control surface even across a manager restart. The
|
|
185
|
+
* durable truth is the slot row + principal-keyed head; this set is the in-memory index of it
|
|
186
|
+
* (one string per retired incarnation — bounded by lifecycle count, never pruned in-process). */
|
|
187
|
+
retiredPrincipals = new Set();
|
|
188
|
+
/** This manager process's own incarnation uid (SPEC 13.1; minted once per supervisor process,
|
|
189
|
+
* never reused across restarts) — the endpoint's presence key AND the `managerInstance` audit
|
|
190
|
+
* coordinate every static activation records. */
|
|
191
|
+
managerLifecycleUid = mintLifecycleUid();
|
|
192
|
+
/** The persisted LOGICAL instance id (SPEC 13.6 item 7, P2 item 3): STABLE across restart, so a
|
|
193
|
+
* restart re-registers the SAME id with an ADVANCED epoch through the §13.1 gate (the successor
|
|
194
|
+
* fences the predecessor's epoch — the (i) fence bites on a real restart). It is the registration
|
|
195
|
+
* instanceId, the served-status id, the goal spec's executor.instanceId, the epe route, and the
|
|
196
|
+
* resolveExecutorEpoch key — DISTINCT from {@link managerLifecycleUid} (per-process: the presence
|
|
197
|
+
* node uid + the managerInstance audit coordinate every static activation records). Set in start(). */
|
|
198
|
+
managerInstanceId;
|
|
199
|
+
/** The persisted serve nkey identity: reusing the SAME principal across restart keeps
|
|
200
|
+
* {@link provisionEndpointGateOpen} idempotent (no core barrier change) and gives verified
|
|
201
|
+
* eviction a stable target (the predecessor's connections under this principal). Set in start(). */
|
|
202
|
+
managerServeIdentity;
|
|
203
|
+
/** P2 item 1 (1a-serve): the manager's v0.4 service-endpoint serve state — the serve handle +
|
|
204
|
+
* its dedicated connection, the STABLE serve identity (renewals re-mint the same nkey), the
|
|
205
|
+
* branded serve grant, and the CURRENT credential (the connection's authenticator reads it on
|
|
206
|
+
* every (re)connect, so a renewal is adopted without re-registration). Absent on open meshes,
|
|
207
|
+
* in user mode (the named 1a follow-up), and before registration completes. */
|
|
208
|
+
serviceServe;
|
|
209
|
+
/** P2 item 2 (spawn-as-action): the SELF-MEDIATED goal-writer connection + ActionContext — a
|
|
210
|
+
* standing connection DISJOINT from the serve credential (Q2), scoped to exactly this endpoint's
|
|
211
|
+
* goal bind/terminal facts + goal-record writes ({@link goalWriterGrants}). Auth mode mints the
|
|
212
|
+
* `goal-writer` cred; an open mesh uses a bare connection (no credential system to mint from).
|
|
213
|
+
* `gate` (auth mode) is the own-issuance-gate READER for the must-5 (a) currency belt — the
|
|
214
|
+
* manager reads its OWN `epgate.<e>.<iid>` epoch over this connection before a terminal commit
|
|
215
|
+
* and skips a superseded commit (the fast-fail belt paired with the (b) barrier-revoke fence). */
|
|
216
|
+
goalWriter;
|
|
217
|
+
/** P2 item 2 must-5 (b): the STABLE goal-writer identity (auth mode) — minted once at
|
|
218
|
+
* registration alongside the serve identity; a renewal re-mints the SAME nkey with a fresh
|
|
219
|
+
* bounded exp and re-stages its distinct credId into the §13.1 revocation family. The current
|
|
220
|
+
* goal-writer credential is minted INSIDE {@link registerManagerService}'s run block (fence
|
|
221
|
+
* live) and stashed here for {@link startGoalWriter} to build the standing connection from. */
|
|
222
|
+
goalWriterIdentity;
|
|
223
|
+
goalWriterCreds;
|
|
224
|
+
/** P2 item 6: the manager's ONE §13.6 session plane — offer mint + one-use redeem + PTY-bridge
|
|
225
|
+
* standup for `attach`. The face's establisher and the CLI attach handler both call THIS one
|
|
226
|
+
* plane; the manager never constructs a second. Undefined until {@link startSessionPlane}. */
|
|
227
|
+
sessionPlane;
|
|
228
|
+
/** P2 item 6: the standing session-LEDGER connection + its mutable creds holder (the authenticator
|
|
229
|
+
* presents the refreshed cred on the next reconnect after a half-TTL renewal — the goal-writer
|
|
230
|
+
* precedent). Auth mode only; an open mesh runs the plane over a bare connection. */
|
|
231
|
+
sessionLedgerConn;
|
|
232
|
+
/** P2 item 6: credentialId → the nkey that credential was minted for, for the live per-session
|
|
233
|
+
* SERVING credentials. The §13.1 ledger row records the holder principal, and the row is written
|
|
234
|
+
* at stage time (after the mint), so the two steps need this one hop. Entries are dropped at
|
|
235
|
+
* revoke; a session that never staged drops its entry when the manager exits. */
|
|
236
|
+
sessionServingKeys = new Map();
|
|
237
|
+
/** P2 item 6: the STABLE session-LEDGER identity (auth mode) — minted once at registration
|
|
238
|
+
* alongside the serve + goal-writer identities; a renewal re-mints the SAME nkey with a fresh
|
|
239
|
+
* bounded exp and re-stages its distinct credId into the §13.1 revocation family. The current
|
|
240
|
+
* credential is minted INSIDE {@link registerManagerService}'s run block and stashed here. */
|
|
241
|
+
sessionLedgerIdentity;
|
|
242
|
+
sessionLedgerCreds;
|
|
243
|
+
/** P2 item 2: the acceptance replied for each in-flight goalId this incarnation accepted, so an
|
|
244
|
+
* idempotent same-goalId retry serves the IDENTICAL acceptance (same allocated name/triple) without
|
|
245
|
+
* a second spawn. Durable cross-incarnation reconstruction rides the must-5 goal-index; here the
|
|
246
|
+
* live map covers same-incarnation retries, with the committed result fact as the fallback. */
|
|
247
|
+
goalAcceptances = new Map();
|
|
248
|
+
/** P2 item 2 must-5 Q-B: the boot reconcile of the durable goal index runs ONCE at start (a
|
|
249
|
+
* fresh incarnation inherits the endpoint's accepted-but-unterminal goals from any predecessor).
|
|
250
|
+
* Spawn-as-action REFUSES to accept until it completes, so the sweep never races a live goal's
|
|
251
|
+
* acceptance (settling one mid-flight would steal its real terminal). */
|
|
252
|
+
goalReconcileDone = false;
|
|
253
|
+
/** P2 item 2 (M4): the live spawn goal ref for each managed agent name, so a despawn MID-GOAL
|
|
254
|
+
* drives the cancel path (transition -> cancel terminal). Cleared when the goal terminalizes. */
|
|
255
|
+
agentGoals = new Map();
|
|
256
|
+
/** Process start, for the served `status` uptime. */
|
|
257
|
+
startedAtMs = Date.now();
|
|
118
258
|
/** SINGLE-FLIGHT guard for {@link deprovision} (INT-2/C): one in-flight teardown per
|
|
119
259
|
* (name, lifecycleUid). The detached freeSlot teardown and every same-name-spawn nudge that
|
|
120
260
|
* re-drives it JOIN one promise instead of launching a SECOND, concurrent teardown. Without it,
|
|
@@ -168,6 +308,7 @@ export class Manager {
|
|
|
168
308
|
this.servers = opts.servers;
|
|
169
309
|
this.name = opts.name ?? "manager";
|
|
170
310
|
this.workspaceRoot = opts.workspaceRoot ?? findCotalRoot();
|
|
311
|
+
this.maxSessions = opts.maxSessions;
|
|
171
312
|
this.secrets = opts.secretStore ?? workspaceSecretStore(this.workspaceRoot);
|
|
172
313
|
this.installedExtensions = opts.installedExtensions ?? false;
|
|
173
314
|
this.runtime = createRuntime(opts.runtime ?? "auto", `cotal-${this.space}`);
|
|
@@ -181,16 +322,19 @@ export class Manager {
|
|
|
181
322
|
this.resumeAttemptId = opts.resumeAttemptId;
|
|
182
323
|
this.resumeRequired = opts.resumeAttemptId !== undefined;
|
|
183
324
|
this.resumeDurableCommitToken = opts.resumeDurableCommitToken;
|
|
184
|
-
this.
|
|
325
|
+
this.wsPort = opts.wsPort;
|
|
326
|
+
this.attach = new AttachEndpoint(() => this.list(),
|
|
185
327
|
// Initial /feed replay for a connecting console: the current peer roster.
|
|
186
328
|
() => [{ event: "roster", data: this.ep?.getRoster() ?? [] }], opts.consolePort ?? 0,
|
|
329
|
+
// P2 item 6: the console's mesh §13.6 session establisher — injected ONLY when a broker ws
|
|
330
|
+
// listener exists (cotal up allocated a wsPort). Never a second plane; it drives THE plane.
|
|
331
|
+
opts.wsPort !== undefined ? (name) => this.establishConsoleSession(name) : undefined,
|
|
187
332
|
// Loopback unless the OPERATOR said otherwise. A broker *dial* address is not a manager *bind*
|
|
188
333
|
// address: deriving one from the other breaks every topology where they differ (a manager
|
|
189
334
|
// supervising a broker on another host cannot bind that host's address at all, and a failover
|
|
190
|
-
// list's first entry need not be the server actually selected)
|
|
191
|
-
//
|
|
192
|
-
//
|
|
193
|
-
// embedded Manager, a bare `cotal supervise` — keeps the loopback-only endpoint it always had.
|
|
335
|
+
// list's first entry need not be the server actually selected). Exposure is therefore an
|
|
336
|
+
// explicit decision; every other caller — an embedded Manager, a bare `cotal supervise` —
|
|
337
|
+
// keeps the loopback-only console it always had.
|
|
194
338
|
opts.attachHost ?? "127.0.0.1");
|
|
195
339
|
}
|
|
196
340
|
get runtimeKind() {
|
|
@@ -220,6 +364,24 @@ export class Manager {
|
|
|
220
364
|
throw new Error(`space "${this.space}" has user-auth state on disk but no mesh registry entry - a user-mode manager needs the authoritative record (\`cotal up\` writes it before the control plane); \`cotal up --user-auth\` this space, or remove the stale ${userAuthStateDir(this.workspaceRoot, this.space)}`);
|
|
221
365
|
if (this.userMode && !this.auth)
|
|
222
366
|
throw new Error(`space "${this.space}" has user-auth state but no auth.json under ${authDir(this.workspaceRoot)} - the pre-flip manager still needs the space trust bundle; re-run \`cotal up --user-auth\` here`);
|
|
367
|
+
// P2 item 3 (SPEC 13.6 item 7): the LOGICAL instance id + serve identity PERSIST across restart
|
|
368
|
+
// (a space-scoped manager identity file under .cotal). A restart re-registers the SAME id with an
|
|
369
|
+
// ADVANCED epoch (the successor fences the predecessor); a fresh mint over a malformed file is
|
|
370
|
+
// refused loud (no-fallbacks - a restart never silently becomes a fresh instance). A second
|
|
371
|
+
// manager in a DIFFERENT workspace root is a DIFFERENT logical id by construction (its own state
|
|
372
|
+
// dir) - two managers in ONE space are two workspace roots.
|
|
373
|
+
{
|
|
374
|
+
const persisted = loadManagerInstanceIdentity(this.workspaceRoot, this.space);
|
|
375
|
+
if (persisted !== undefined) {
|
|
376
|
+
this.managerInstanceId = persisted.instanceId;
|
|
377
|
+
this.managerServeIdentity = persisted.serveIdentity;
|
|
378
|
+
}
|
|
379
|
+
else {
|
|
380
|
+
this.managerInstanceId = mintLifecycleUid();
|
|
381
|
+
this.managerServeIdentity = newIdentity();
|
|
382
|
+
saveManagerInstanceIdentity(this.workspaceRoot, this.space, { instanceId: this.managerInstanceId, serveIdentity: this.managerServeIdentity });
|
|
383
|
+
}
|
|
384
|
+
}
|
|
223
385
|
let creds;
|
|
224
386
|
let id;
|
|
225
387
|
if (this.auth) {
|
|
@@ -244,9 +406,10 @@ export class Manager {
|
|
|
244
406
|
creds,
|
|
245
407
|
// The supervisor registers on the roster, and an authed presence-registering endpoint is
|
|
246
408
|
// lifecycle-keyed (SPEC 13.1, fail-before-presence). The manager process is the top of its
|
|
247
|
-
// own launch chain (the operator command IS its launcher)
|
|
248
|
-
//
|
|
249
|
-
|
|
409
|
+
// own launch chain (the operator command IS its launcher): its incarnation uid is the
|
|
410
|
+
// per-process `managerLifecycleUid` field (also the `managerInstance` audit coordinate on
|
|
411
|
+
// every static activation, Unit B).
|
|
412
|
+
lifecycleUid: this.managerLifecycleUid,
|
|
250
413
|
// The supervisor serves control + watches presence; it never consumes chat/dm/task
|
|
251
414
|
// (no message handler). consume:false avoids binding consumers it doesn't use — and
|
|
252
415
|
// under auth avoids trying to bind its own DM/task durables that nothing pre-created.
|
|
@@ -263,38 +426,39 @@ export class Manager {
|
|
|
263
426
|
this.ep.on("error", (e) => console.error(`! manager endpoint: ${e.message}`));
|
|
264
427
|
await this.ep.start();
|
|
265
428
|
await this.ep.setActivity(`supervisor (${this.runtime.kind})`);
|
|
266
|
-
//
|
|
267
|
-
//
|
|
268
|
-
//
|
|
269
|
-
|
|
429
|
+
// Per-instance liveness lease (P2 item 3 — the old per-space singleton is DEMOTED per D9). Acquire
|
|
430
|
+
// THIS logical instance's own key (atomic CAS create). A DIFFERENT instance (a second manager in a
|
|
431
|
+
// second workspace root) has a distinct id ⇒ a distinct key ⇒ it coexists; the create THROWS only
|
|
432
|
+
// when the SAME instance id is already live (a same-root double-start, or a restart racing the
|
|
433
|
+
// crashed predecessor's not-yet-expired key), and we REFUSE loud. A crashed holder's key auto-expires
|
|
434
|
+
// (bucket TTL). Losing this key later stops THIS instance only, never the space (security pin 6).
|
|
435
|
+
this.leaseInfo = { holder: this.ep.ref().id, instanceId: this.managerInstanceId, runtime: this.runtime.kind, root: resolve(this.workspaceRoot), pid: process.pid };
|
|
270
436
|
try {
|
|
271
437
|
this.leaseRevision = await this.ep.acquireManagerLease(this.leaseInfo);
|
|
272
438
|
}
|
|
273
439
|
catch (e) {
|
|
274
|
-
//
|
|
275
|
-
// failure to surface, not a silent "held" — keep the cause so it isn't misread as a conflict.
|
|
440
|
+
// Our OWN instance id already holds a live key ⇒ refuse. Anything else (e.g. a KV/JS error) is a
|
|
441
|
+
// real failure to surface, not a silent "held" — keep the cause so it isn't misread as a conflict.
|
|
276
442
|
const held = await this.ep.readManagerLease().catch(() => undefined);
|
|
277
443
|
await this.ep.stop();
|
|
278
444
|
await this.attach.stop();
|
|
279
445
|
throw new Error(held
|
|
280
|
-
? `
|
|
446
|
+
? `manager instance ${this.managerInstanceId} already serves space "${this.space}" from this workspace root (${held.runtime}, pid ${held.pid}, root ${held.root}) - stop it first before restarting the same instance`
|
|
281
447
|
: `could not acquire the manager lease for space "${this.space}": ${e.message}`);
|
|
282
448
|
}
|
|
283
449
|
this.leaseTimer = setInterval(() => { void this.renewLease(); }, MANAGER_LEASE_TTL_MS / 2);
|
|
284
450
|
this.leaseTimer.unref?.();
|
|
285
|
-
//
|
|
286
|
-
//
|
|
287
|
-
//
|
|
288
|
-
//
|
|
289
|
-
//
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
// (
|
|
293
|
-
//
|
|
294
|
-
//
|
|
295
|
-
|
|
296
|
-
this.ep.serveControl(CONTROL_SELF_SERVICE, (req) => this.handle(req, CONTROL_SELF_SERVICE), { boundReply: true });
|
|
297
|
-
this.ep.serveControl(CONTROL_ADMIN, (req) => this.handle(req, CONTROL_ADMIN), { boundReply: true });
|
|
451
|
+
// Unit B (static §13.1): ensure the two authority stores exist with their normative shape,
|
|
452
|
+
// then sweep the durable slot rows and reconcile — re-drive any crashed activation/terminal
|
|
453
|
+
// (exact-op) and terminalize dead-but-active slots (F3 "no active orphan"). Runs ONLY under
|
|
454
|
+
// the just-acquired lease (a refused second manager must never sweep-terminal live slots) and
|
|
455
|
+
// BEFORE control serving, so no spawn races the reconciliation.
|
|
456
|
+
if (this.auth && !this.userMode)
|
|
457
|
+
await this.reconcileStaticLifecycles();
|
|
458
|
+
// P2 item 1 (1d): the manager serves NO ctl tiers - its whole control surface is the v0.4
|
|
459
|
+
// service endpoint registered below. The old three-tier rail (self/manager/admin) is deleted;
|
|
460
|
+
// `ctl.delivery`/`ctl.delivery-admin` (the delivery daemon) and `ctl.auth-admin` (the auth
|
|
461
|
+
// plane) are separate services and keep their rails.
|
|
298
462
|
// D5 slice 5 class 2: the manager is the CLASS-2 RENEWAL OWNER — the one control-plane process
|
|
299
463
|
// that is resident in EVERY mesh mode (foreground `up`, `up --detach`, same-root refresh) and
|
|
300
464
|
// holds the signer. Ordered initial pass NOW (ensureControlPlane starts delivery BEFORE the
|
|
@@ -306,6 +470,24 @@ export class Manager {
|
|
|
306
470
|
this.credRenewTimer = setInterval(() => { void this.renewDaemonCreds(); }, (STANDING_RENEWABLE_TTL_SEC / 2) * 1000);
|
|
307
471
|
this.credRenewTimer.unref?.();
|
|
308
472
|
}
|
|
473
|
+
// P2 item 1: register the manager as an ordinary v0.4 `service` endpoint (SPEC §13.7/§13.9)
|
|
474
|
+
// and serve its typed command surface on the ep rails - since 1d the ONLY control door, in
|
|
475
|
+
// EVERY mesh mode. Static + user meshes mint the scoped executor + endpoint-serve credential;
|
|
476
|
+
// an open mesh runs the same gate/registration ceremony over bare connections and never mints
|
|
477
|
+
// (there is no credential system - the broker enforces nothing, matching the old open-mesh ctl
|
|
478
|
+
// trust). Fail-loud: a manager that cannot register does not start half-registered.
|
|
479
|
+
await this.registerManagerService();
|
|
480
|
+
// P2 item 2: stand up the standing goal-writer connection for spawn-as-action — AFTER
|
|
481
|
+
// registration (it writes this endpoint's goal facts/records), disjoint from the serve cred.
|
|
482
|
+
await this.startGoalWriter();
|
|
483
|
+
// P2 item 6: stand up the ONE §13.6 session plane for `attach` — AFTER registration too (it
|
|
484
|
+
// rides the serve grant's epoch + the family-staged session-ledger cred), on its own standing
|
|
485
|
+
// connection disjoint from both the serve and goal-writer creds.
|
|
486
|
+
await this.startSessionPlane();
|
|
487
|
+
// P2 item 2 must-5 Q-B: reconcile any accepted-but-unterminal goals inherited from a predecessor
|
|
488
|
+
// BEFORE spawn-as-action begins accepting (the goalReconcileDone gate) — a fresh incarnation
|
|
489
|
+
// never drops a goal a dead predecessor accepted. Never fatal; the gate opens either way.
|
|
490
|
+
await this.reconcileGoalIndex();
|
|
309
491
|
// Plane-3 (durable backstop) is NOT the manager's job — the manager only manages agent lifecycle.
|
|
310
492
|
// The server-side delivery daemon hosts the fan-out writer + trusted reader, owns the durable
|
|
311
493
|
// membership registry, and serves the runtime durable join/leave/list ops (on `ctl.delivery`). The
|
|
@@ -364,6 +546,97 @@ export class Manager {
|
|
|
364
546
|
// `writeRenewalRecord` redacts the ephemeral fingerprint at the persistence boundary (covering
|
|
365
547
|
// the `doctor auth --fix` writer too), so the results pass straight through.
|
|
366
548
|
writeRenewalRecord(this.workspaceRoot, { ts: new Date().toISOString(), owner: "manager", results, adoption });
|
|
549
|
+
this.warnOnSystemCredExpiry();
|
|
550
|
+
// F5(b) (Unit B): the MANAGER is the renewal owner for its managed-static agent creds —
|
|
551
|
+
// supervisor-side PUSH remint for recorded LIVE slots (the child JWT is never proof of
|
|
552
|
+
// incarnation; a copied credential cannot drive this and is stranded at its own row's TTL).
|
|
553
|
+
// Same class-2 mechanics as the daemon creds: re-sign the file for the SAME nkey; the
|
|
554
|
+
// agent endpoint's 75% source re-read adopts it.
|
|
555
|
+
if (!this.userMode) {
|
|
556
|
+
for (const a of [...this.agents.values()]) {
|
|
557
|
+
// The `terminalizing` test here is an OPTIMISATION, NOT THE GUARD. There are awaits below
|
|
558
|
+
// it, so an agent can latch mid-iteration and this filter will have already let it
|
|
559
|
+
// through — what actually refuses is the same test at the top of
|
|
560
|
+
// {@link renewManagedStaticCred}, which every renewal on this path goes through.
|
|
561
|
+
// COUPLING: deleting or weakening that check silently promotes this line from an
|
|
562
|
+
// optimisation into the whole guard, and nothing fails at the moment of the change.
|
|
563
|
+
if (a.userOwner || a.terminalizing || !a.seed || !a.secretPaths?.creds)
|
|
564
|
+
continue;
|
|
565
|
+
try {
|
|
566
|
+
const stored = await this.secrets.get(agentSecretKeyForFile(a.secretPaths.creds));
|
|
567
|
+
if (stored === undefined)
|
|
568
|
+
continue; // no materialized cred (never minted here) - nothing to renew
|
|
569
|
+
const health = inspectCredHealth(stored);
|
|
570
|
+
if (health.state === "healthy")
|
|
571
|
+
continue;
|
|
572
|
+
if (health.state === "unbounded" || health.state === "unreadable") {
|
|
573
|
+
console.error(`! managed cred renewal ${a.name}: credential is ${health.state}${health.error ? ` (${health.error})` : ""} - not renewed (a pre-TTL credential stays as minted until respawn)`);
|
|
574
|
+
continue;
|
|
575
|
+
}
|
|
576
|
+
await this.renewManagedStaticCred(a);
|
|
577
|
+
}
|
|
578
|
+
catch (e) {
|
|
579
|
+
console.error(`! managed cred renewal ${a.name}: ${e.message} - the agent dies loud at this cred's expiry unless it is reminted`);
|
|
580
|
+
}
|
|
581
|
+
}
|
|
582
|
+
}
|
|
583
|
+
// P2 item 1 (checklist 7): the manager is the `endpoint-serve` renewal owner for its OWN
|
|
584
|
+
// service credential — re-mint the SAME serve identity with a fresh bounded exp THROUGH the
|
|
585
|
+
// §13.1 mint fence over a scoped one-shot executor (every renewal stages a distinct ledger
|
|
586
|
+
// row and wins the gate CAS; never the standing connection). The serve connection's
|
|
587
|
+
// authenticator presents the refreshed credential on its next (re)connect.
|
|
588
|
+
if (this.serviceServe?.creds && this.auth) {
|
|
589
|
+
const s = this.serviceServe;
|
|
590
|
+
const authRef = this.auth;
|
|
591
|
+
try {
|
|
592
|
+
const health = inspectCredHealth(this.serviceServe.creds);
|
|
593
|
+
if (health.state !== "healthy") {
|
|
594
|
+
s.creds = await this.withEndpointServeExecutor(({ authKv }) => mintCreds(authRef, s.identity, "endpoint-serve", {
|
|
595
|
+
serveIssuance: serveIssuanceGateKv(authKv, this.space, { endpoint: MANAGER_ENDPOINT, instanceId: this.managerInstanceId }),
|
|
596
|
+
endpointServe: s.grant,
|
|
597
|
+
}));
|
|
598
|
+
}
|
|
599
|
+
}
|
|
600
|
+
catch (e) {
|
|
601
|
+
console.error(`! endpoint-serve renewal: ${e.message} - the manager's service endpoint dies loud at this cred's expiry unless it is re-registered`);
|
|
602
|
+
}
|
|
603
|
+
}
|
|
604
|
+
// P2 item 2 must-5 (b): the manager is also the goal-writer's renewal owner — re-mint the SAME
|
|
605
|
+
// goal-writer nkey with a fresh bounded exp AND re-stage its new credId into the §13.1 family,
|
|
606
|
+
// through the scoped executor (never the standing seed). Without this the standing goal-writer
|
|
607
|
+
// connection dies at its TTL and spawn-as-action stops accepting until a restart. The
|
|
608
|
+
// connection's authenticator presents the refreshed credential on its next (re)connect.
|
|
609
|
+
if (this.goalWriter && this.goalWriterCreds && this.auth) {
|
|
610
|
+
const gw = this.goalWriter;
|
|
611
|
+
try {
|
|
612
|
+
if (inspectCredHealth(this.goalWriterCreds).state !== "healthy") {
|
|
613
|
+
const fresh = await this.withEndpointServeExecutor(({ authKv }) => this.mintAndStageGoalWriter(authKv));
|
|
614
|
+
this.goalWriterCreds = fresh;
|
|
615
|
+
gw.creds = fresh;
|
|
616
|
+
}
|
|
617
|
+
}
|
|
618
|
+
catch (e) {
|
|
619
|
+
console.error(`! goal-writer renewal: ${e.message} - spawn-as-action stops accepting at this cred's expiry unless the manager restarts`);
|
|
620
|
+
}
|
|
621
|
+
}
|
|
622
|
+
// P2 item 6: the manager is also the session-ledger's renewal owner — re-mint the SAME nkey
|
|
623
|
+
// with a fresh bounded exp AND re-stage its new credId into the §13.1 family, through the
|
|
624
|
+
// scoped executor. Without this the standing session-ledger connection dies at its TTL and
|
|
625
|
+
// `attach` stops establishing sessions until a restart. The connection's authenticator presents
|
|
626
|
+
// the refreshed credential on its next (re)connect.
|
|
627
|
+
if (this.sessionLedgerConn && this.sessionLedgerCreds && this.auth) {
|
|
628
|
+
const sw = this.sessionLedgerConn;
|
|
629
|
+
try {
|
|
630
|
+
if (inspectCredHealth(this.sessionLedgerCreds).state !== "healthy") {
|
|
631
|
+
const fresh = await this.withEndpointServeExecutor(({ authKv }) => this.mintAndStageSessionLedger(authKv));
|
|
632
|
+
this.sessionLedgerCreds = fresh;
|
|
633
|
+
sw.creds = fresh;
|
|
634
|
+
}
|
|
635
|
+
}
|
|
636
|
+
catch (e) {
|
|
637
|
+
console.error(`! session-ledger renewal: ${e.message} - attach stops establishing sessions at this cred's expiry unless the manager restarts`);
|
|
638
|
+
}
|
|
639
|
+
}
|
|
367
640
|
}
|
|
368
641
|
catch (e) {
|
|
369
642
|
console.error(`! credential renewal pass failed: ${e.message}`);
|
|
@@ -372,6 +645,38 @@ export class Manager {
|
|
|
372
645
|
release();
|
|
373
646
|
}
|
|
374
647
|
}
|
|
648
|
+
/** Warn, on every renewal pass, when a $SYS credential is at or past its renewal point.
|
|
649
|
+
*
|
|
650
|
+
* The manager is the renewal owner for every credential it CAN re-sign, and these two are the ones
|
|
651
|
+
* it cannot: they are `rotation-renewed`, so no resident process re-mints them and they simply die
|
|
652
|
+
* on their 30-day horizon. Before this, a mesh that never ran `doctor auth` got no signal at all,
|
|
653
|
+
* it discovered the expiry as an "Authorization Violation" in the delivery log and a refused
|
|
654
|
+
* membership adoption, weeks after the warning would have been actionable (#338). The pass runs
|
|
655
|
+
* every half-TTL of the 24h class, so this repeats about twice a day for the ~7 days between the
|
|
656
|
+
* renewal point and expiry: loud enough to be seen, bounded enough not to be noise.
|
|
657
|
+
*
|
|
658
|
+
* Diagnostic only, and deliberately non-fatal: renewal is an operator action (`cotal down` then
|
|
659
|
+
* `cotal up --rotate-sys`, which needs a broker restart), so the manager must report it, never
|
|
660
|
+
* attempt it. An absent file is the unprovisioned space, reported by the daemon that needs it. */
|
|
661
|
+
warnOnSystemCredExpiry() {
|
|
662
|
+
for (const file of SYSTEM_CREDS_FILES) {
|
|
663
|
+
const path = join(this.workspaceRoot, ".cotal", file);
|
|
664
|
+
if (!existsSync(path))
|
|
665
|
+
continue;
|
|
666
|
+
let health;
|
|
667
|
+
try {
|
|
668
|
+
health = inspectCredHealth(readFileSync(path, "utf8"));
|
|
669
|
+
}
|
|
670
|
+
catch {
|
|
671
|
+
continue; // an unreadable $SYS file is the daemon's loud failure, not a renewal-pass crash
|
|
672
|
+
}
|
|
673
|
+
const when = health.exp ? new Date(health.exp * 1000).toISOString() : "an unknown date";
|
|
674
|
+
if (health.state === "expired")
|
|
675
|
+
console.error(`! $SYS credential ${file} EXPIRED ${when} - the broker denies it, and NOTHING renews it in place (it is rotation-renewed). Live eviction and the membership feed stay down until: \`cotal down\` then \`cotal up --rotate-sys\` (agents, creds and data survive)`);
|
|
676
|
+
else if (health.state === "near-expiry")
|
|
677
|
+
console.error(`! $SYS credential ${file} expires ${when} and nothing renews it in place (it is rotation-renewed) - schedule: \`cotal down\` then \`cotal up --rotate-sys\` (agents, creds and data survive)`);
|
|
678
|
+
}
|
|
679
|
+
}
|
|
375
680
|
/** Admit one lifecycle/control operation while active. The synchronous increment is the fence:
|
|
376
681
|
* preserveState flips state before its first await, so work is either counted or rejected. */
|
|
377
682
|
beginLifecycle(resumeOperation = false) {
|
|
@@ -679,19 +984,30 @@ export class Manager {
|
|
|
679
984
|
: { owner: DEV_OWNER, actor: a.id };
|
|
680
985
|
if (!principal)
|
|
681
986
|
throw new Error(`managed agent ${a.name} has an invalid principal ${a.id}`);
|
|
682
|
-
|
|
987
|
+
// The RECORDED secret-family paths (set at spawn or adoption) — never a re-derivation by name:
|
|
988
|
+
// under mixed generations (a name-keyed pre-split incarnation adopted by this manager) the
|
|
989
|
+
// recorded path is the only truth, and re-deriving would preserve a family that isn't there.
|
|
990
|
+
const files = a.secretPaths;
|
|
683
991
|
const identity = a.userOwner
|
|
684
|
-
? {
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
992
|
+
? (() => {
|
|
993
|
+
if (!files?.actorToken || !files.sentinelCreds || !files.health)
|
|
994
|
+
throw new Error(`managed agent ${a.name} is user-mode but its secret-family paths were not recorded`);
|
|
995
|
+
return {
|
|
996
|
+
mode: "user",
|
|
997
|
+
owner: principal.owner,
|
|
998
|
+
actor: principal.actor,
|
|
999
|
+
lifecycleUid: a.lifecycleUid,
|
|
1000
|
+
actorToken: { kind: "file", path: files.actorToken, sha256: this.fileDigestOrEmpty(files.actorToken) },
|
|
1001
|
+
sentinelCredential: { kind: "file", path: files.sentinelCreds, sha256: this.fileDigestOrEmpty(files.sentinelCreds) },
|
|
1002
|
+
health: { kind: "file", path: files.health },
|
|
1003
|
+
};
|
|
1004
|
+
})()
|
|
693
1005
|
: this.auth
|
|
694
|
-
?
|
|
1006
|
+
? (() => {
|
|
1007
|
+
if (!files?.creds)
|
|
1008
|
+
throw new Error(`managed agent ${a.name} is static-auth but its credential path was not recorded`);
|
|
1009
|
+
return { mode: "static", id: principal.actor, lifecycleUid: a.lifecycleUid, credential: { kind: "file", path: files.creds, sha256: this.fileDigestOrEmpty(files.creds) } };
|
|
1010
|
+
})()
|
|
695
1011
|
: { mode: "open", id: principal.actor, lifecycleUid: a.lifecycleUid };
|
|
696
1012
|
const dependencies = [a.launch.source.configPath];
|
|
697
1013
|
if (a.launch.source.kind === "manifest" && a.launch.source.runId)
|
|
@@ -779,14 +1095,42 @@ export class Manager {
|
|
|
779
1095
|
// A signal after a partial preservation must never fall back into destructive teardown.
|
|
780
1096
|
await this.stopRetainedAgentsOnExit();
|
|
781
1097
|
}
|
|
782
|
-
await this.ep.releaseManagerLease(this.leaseRevision);
|
|
1098
|
+
await this.ep.releaseManagerLease(this.managerInstanceId, this.leaseRevision);
|
|
1099
|
+
await this.stopServiceServe();
|
|
1100
|
+
await this.stopGoalWriter();
|
|
1101
|
+
await this.stopSessionPlane();
|
|
783
1102
|
await this.ep.stop();
|
|
784
1103
|
await this.attach.stop();
|
|
785
1104
|
}
|
|
786
|
-
/**
|
|
787
|
-
*
|
|
788
|
-
*
|
|
789
|
-
|
|
1105
|
+
/** Stop the v0.4 service-endpoint serve loop (drain subscriptions, await in-flight handlers)
|
|
1106
|
+
* and drop its dedicated connection. Best-effort by design — both exit paths (graceful stop,
|
|
1107
|
+
* lease-loss fail-close) must complete their remaining teardown even if the broker is gone. */
|
|
1108
|
+
async stopServiceServe() {
|
|
1109
|
+
const s = this.serviceServe;
|
|
1110
|
+
if (!s)
|
|
1111
|
+
return;
|
|
1112
|
+
this.serviceServe = undefined;
|
|
1113
|
+
try {
|
|
1114
|
+
await s.handle.stop();
|
|
1115
|
+
}
|
|
1116
|
+
catch { /* best effort */ }
|
|
1117
|
+
try {
|
|
1118
|
+
await s.nc.drain();
|
|
1119
|
+
}
|
|
1120
|
+
catch {
|
|
1121
|
+
try {
|
|
1122
|
+
s.nc.close();
|
|
1123
|
+
}
|
|
1124
|
+
catch { /* best effort */ }
|
|
1125
|
+
}
|
|
1126
|
+
}
|
|
1127
|
+
/** Refresh THIS instance's liveness lease before the bucket TTL expires it. On loss (missed the TTL —
|
|
1128
|
+
* this instance stalled past the renew window) FAIL CLOSED for THIS INSTANCE ONLY: stop serving +
|
|
1129
|
+
* tear down OUR managed agents + exit, so a stalled instance can't keep double-processing under a key
|
|
1130
|
+
* a same-id restart may re-acquire. Keyed per instance, so this NEVER frees or touches a sibling
|
|
1131
|
+
* manager's key and NEVER freezes the space (security pin 6) — the sibling keeps serving. We do NOT
|
|
1132
|
+
* re-acquire (a same-id restart may already be live) and do NOT release the key (it may be the
|
|
1133
|
+
* restart's). A DIFFERENT instance losing ITS key is a separate, independent event. */
|
|
790
1134
|
async renewLease() {
|
|
791
1135
|
try {
|
|
792
1136
|
if (!this.leaseInfo || this.leaseRevision === undefined)
|
|
@@ -794,7 +1138,7 @@ export class Manager {
|
|
|
794
1138
|
this.leaseRevision = await this.ep.renewManagerLease(this.leaseInfo, this.leaseRevision);
|
|
795
1139
|
}
|
|
796
1140
|
catch (e) {
|
|
797
|
-
console.error(`! manager lost its
|
|
1141
|
+
console.error(`! manager instance ${this.managerInstanceId} lost its liveness lease for space "${this.space}" (${e.message}) - shutting down THIS instance (its serving only; siblings keep the space)`);
|
|
798
1142
|
if (this.leaseTimer)
|
|
799
1143
|
clearInterval(this.leaseTimer);
|
|
800
1144
|
// Tear down our managed agents' footprints too (#159 B2) — this exit path leaks them otherwise. Do
|
|
@@ -806,6 +1150,9 @@ export class Manager {
|
|
|
806
1150
|
await this.stopRetainedAgentsOnExit();
|
|
807
1151
|
}
|
|
808
1152
|
catch { /* best effort */ }
|
|
1153
|
+
await this.stopServiceServe();
|
|
1154
|
+
await this.stopGoalWriter();
|
|
1155
|
+
await this.stopSessionPlane();
|
|
809
1156
|
try {
|
|
810
1157
|
await this.ep.stop();
|
|
811
1158
|
}
|
|
@@ -817,13 +1164,11 @@ export class Manager {
|
|
|
817
1164
|
process.exit(1);
|
|
818
1165
|
}
|
|
819
1166
|
}
|
|
820
|
-
async
|
|
821
|
-
|
|
822
|
-
if (tier !== CONTROL_ADMIN)
|
|
823
|
-
return { ok: false, error: "finalizeResume is admin-only; not allowed on this control subject" };
|
|
1167
|
+
async opFinalizeResume(rawArgs) {
|
|
1168
|
+
{
|
|
824
1169
|
let args;
|
|
825
1170
|
try {
|
|
826
|
-
args = parseResumeFinalizeArgs(
|
|
1171
|
+
args = parseResumeFinalizeArgs(rawArgs);
|
|
827
1172
|
}
|
|
828
1173
|
catch (e) {
|
|
829
1174
|
return { ok: false, error: e.message };
|
|
@@ -853,16 +1198,27 @@ export class Manager {
|
|
|
853
1198
|
if (managed)
|
|
854
1199
|
managed.suppressCleanup = false;
|
|
855
1200
|
}
|
|
1201
|
+
// Unit B (F3, distsys/security CONDITIONAL @ 9e13648): the boot sweep DEFERRED every active
|
|
1202
|
+
// slot while a resume was pending (it could not know which would be adopted). Now adoption
|
|
1203
|
+
// is complete and `this.agents` is EXACTLY the adopted set, so re-sweep to terminalize any
|
|
1204
|
+
// active slot the resume did NOT claim — a durable ACTIVE ORPHAN (crashed after slot->active
|
|
1205
|
+
// before agents.set, then not in the resumed inventory). This runs while `resumeRequired` is
|
|
1206
|
+
// still true, so no ordinary spawn can race it (beginLifecycle refuses non-resume ops), and
|
|
1207
|
+
// it closes both the alias wedge AND the F5(a) gap (the orphan's principal enters
|
|
1208
|
+
// retiredPrincipals, so a copied JWT is refused). Best-effort + loud: a sweep failure must
|
|
1209
|
+
// not fail the finalize (the next non-resume boot re-drives it), but it is never swallowed.
|
|
1210
|
+
if (this.auth && !this.userMode)
|
|
1211
|
+
await this.reconcileStaticLifecycles(true).catch((e) => console.error(`! post-resume static reconcile: ${e.message} - a durable active orphan may still wedge its alias until the next non-resume restart`));
|
|
856
1212
|
this.resumeFinalized = true;
|
|
857
1213
|
this.resumeRequired = false;
|
|
858
1214
|
return { ok: true, data: { attemptId: args.attemptId, state: "active" } };
|
|
859
1215
|
}
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
1216
|
+
}
|
|
1217
|
+
async opCommitResume(rawArgs) {
|
|
1218
|
+
{
|
|
863
1219
|
let attemptId;
|
|
864
1220
|
try {
|
|
865
|
-
attemptId = parseResumeCommitArgs(
|
|
1221
|
+
attemptId = parseResumeCommitArgs(rawArgs).attemptId;
|
|
866
1222
|
}
|
|
867
1223
|
catch (e) {
|
|
868
1224
|
return { ok: false, error: e.message };
|
|
@@ -890,11 +1246,11 @@ export class Manager {
|
|
|
890
1246
|
this.resumeCommitTask = undefined;
|
|
891
1247
|
}
|
|
892
1248
|
}
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
|
|
1249
|
+
}
|
|
1250
|
+
async opResumePreserved(rawArgs) {
|
|
1251
|
+
{
|
|
896
1252
|
try {
|
|
897
|
-
const args = parseResumeControlArgs(
|
|
1253
|
+
const args = parseResumeControlArgs(rawArgs);
|
|
898
1254
|
const inventoryDigest = createHash("sha256").update(JSON.stringify(args.inventory)).digest("hex");
|
|
899
1255
|
if (!this.resumeAttemptId)
|
|
900
1256
|
return { ok: false, error: "resumePreserved requires a manager started with --resume-attempt" };
|
|
@@ -925,44 +1281,220 @@ export class Manager {
|
|
|
925
1281
|
return { ok: false, error: e.message };
|
|
926
1282
|
}
|
|
927
1283
|
}
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
return { ok: true, data: { attemptId, state: "active" } };
|
|
940
|
-
}
|
|
941
|
-
const result = req.op === "preparePreservation"
|
|
942
|
-
? await this.preparePreservation(attemptId)
|
|
943
|
-
: await this.commitPreservation(attemptId);
|
|
944
|
-
return result.ok
|
|
945
|
-
? { ok: true, data: result }
|
|
946
|
-
: {
|
|
947
|
-
ok: false,
|
|
948
|
-
data: result,
|
|
949
|
-
error: `preservation incomplete: ${result.failures.map((f) => `${f.name}: ${f.error}`).join("; ")}`,
|
|
950
|
-
};
|
|
951
|
-
}
|
|
952
|
-
catch (e) {
|
|
953
|
-
return { ok: false, error: e.message };
|
|
1284
|
+
}
|
|
1285
|
+
async opPreservationCtl(op, rawArgs) {
|
|
1286
|
+
if (this.resumeRequired)
|
|
1287
|
+
return { ok: false, error: this.maintenanceError() };
|
|
1288
|
+
const attemptId = String(rawArgs?.attemptId ?? "").trim();
|
|
1289
|
+
if (!attemptId)
|
|
1290
|
+
return { ok: false, error: `${op} requires attemptId` };
|
|
1291
|
+
try {
|
|
1292
|
+
if (op === "abortPreservation") {
|
|
1293
|
+
this.abortPreservation(attemptId);
|
|
1294
|
+
return { ok: true, data: { attemptId, state: "active" } };
|
|
954
1295
|
}
|
|
1296
|
+
const result = op === "preparePreservation"
|
|
1297
|
+
? await this.preparePreservation(attemptId)
|
|
1298
|
+
: await this.commitPreservation(attemptId);
|
|
1299
|
+
return result.ok
|
|
1300
|
+
? { ok: true, data: result }
|
|
1301
|
+
: {
|
|
1302
|
+
ok: false,
|
|
1303
|
+
data: result,
|
|
1304
|
+
error: `preservation incomplete: ${result.failures.map((f) => `${f.name}: ${f.error}`).join("; ")}`,
|
|
1305
|
+
};
|
|
1306
|
+
}
|
|
1307
|
+
catch (e) {
|
|
1308
|
+
return { ok: false, error: e.message };
|
|
955
1309
|
}
|
|
1310
|
+
}
|
|
1311
|
+
/** The ONE shared control-admission chokepoint (P2 item 1, checklist 3/8) BOTH dispatch doors
|
|
1312
|
+
* run — the v0.3 `ctl` door ({@link handle}) and the v0.4 `ep` service handlers
|
|
1313
|
+
* ({@link serveGated}): the maintenance/resume fence (`beginLifecycle`: a resume-pending or
|
|
1314
|
+
* non-active manager accepts no ordinary control work) and then the F5(a) membership gate
|
|
1315
|
+
* ({@link lifecycleMembershipRefusal}: a retiring/terminalizing/retired managed incarnation's
|
|
1316
|
+
* AUTHENTICATED principal holds no control authority even with a valid JWT). Refusal carries
|
|
1317
|
+
* WHICH fence refused so the ep door can map onto the §13.3 catalog; admission returns the
|
|
1318
|
+
* accepted-work release. Never re-implemented per door — a fence on one door is a bypass. */
|
|
1319
|
+
admitControl(caller) {
|
|
956
1320
|
const release = this.beginLifecycle();
|
|
957
1321
|
if (!release)
|
|
958
|
-
return {
|
|
1322
|
+
return { refusal: this.maintenanceError(), fence: "maintenance" };
|
|
1323
|
+
const membership = this.lifecycleMembershipRefusal(caller);
|
|
1324
|
+
if (membership) {
|
|
1325
|
+
release();
|
|
1326
|
+
return { refusal: membership, fence: "membership" };
|
|
1327
|
+
}
|
|
1328
|
+
return { release };
|
|
1329
|
+
}
|
|
1330
|
+
/** Run one v0.4 service-command handler through the SHARED admission chokepoint
|
|
1331
|
+
* ({@link admitControl}) on the broker-authenticated caller principal, mapping the two fences
|
|
1332
|
+
* onto the §13.3 catalog: maintenance/resume → `unavailable`, F5(a) membership →
|
|
1333
|
+
* `permission-denied`. The serve boundary publishes the structured error reply. */
|
|
1334
|
+
async serveGated(ctx, fn) {
|
|
1335
|
+
const caller = principalKey(ctx.subject.caller.owner, ctx.subject.caller.actor).key;
|
|
1336
|
+
const admission = this.admitControl(caller);
|
|
1337
|
+
if (admission.refusal !== undefined)
|
|
1338
|
+
throw new EpEnvelopeError(admission.fence === "membership" ? "permission-denied" : "unavailable", admission.refusal);
|
|
959
1339
|
try {
|
|
960
|
-
return await
|
|
1340
|
+
return await fn();
|
|
961
1341
|
}
|
|
962
1342
|
finally {
|
|
963
|
-
release();
|
|
1343
|
+
admission.release();
|
|
1344
|
+
}
|
|
1345
|
+
}
|
|
1346
|
+
/** The ep door's ADMIN flag for a caller (the 1c tier refinement). Static mesh: `true` — the
|
|
1347
|
+
* admin-grade rows (any-mode despawn/attach, the `manager.admin` family, `launch`) are minted
|
|
1348
|
+
* only into operator instruments (§13.2: `any` is operator-policy-mintable; the agent/spawn
|
|
1349
|
+
* rollups never carry them), so REACHING the handler is holding the admin tier, exactly as
|
|
1350
|
+
* holding `ctl.<admin>` is today. User mesh: the caller's CURRENT ledger scope must carry
|
|
1351
|
+
* `admin` — the same fresh-read authority {@link psOwnerFilter} consults, so a revoked scope
|
|
1352
|
+
* demotes the very next call even on a still-valid bearer. Fail-closed: an unreadable ledger
|
|
1353
|
+
* authorizes nothing. NAMED RESIDUAL (critic, 1c.2b): the static `true` has no serve-time
|
|
1354
|
+
* re-check — a LEAKED static admin instrument keeps its reach until the credential's bounded
|
|
1355
|
+
* TTL (the one-shot 5-minute profile), the same static-revoke≠reconnect-death class ruled
|
|
1356
|
+
* across this campaign; static revocation is the TTL, not a ledger. */
|
|
1357
|
+
async epAdminReach(caller) {
|
|
1358
|
+
if (!this.userMode)
|
|
1359
|
+
return true;
|
|
1360
|
+
const key = parsePrincipalKey(caller);
|
|
1361
|
+
if (!key)
|
|
1362
|
+
return false;
|
|
1363
|
+
try {
|
|
1364
|
+
const scope = await resolveAuthProvider().actorScope({
|
|
1365
|
+
dir: userAuthStateDir(this.workspaceRoot, this.space),
|
|
1366
|
+
owner: key.owner,
|
|
1367
|
+
actor: key.actor,
|
|
1368
|
+
});
|
|
1369
|
+
return scope?.includes("admin") === true;
|
|
1370
|
+
}
|
|
1371
|
+
catch {
|
|
1372
|
+
return false;
|
|
964
1373
|
}
|
|
965
1374
|
}
|
|
1375
|
+
/** A targeted request's admin flag: mode `any` (the operator instrument's cross-agent form,
|
|
1376
|
+
* rev 3) resolves through {@link epAdminReach}; a user-mode any-mode caller whose CURRENT
|
|
1377
|
+
* ledger row lost `admin` since its rows were minted refuses loud rather than silently
|
|
1378
|
+
* downgrading to the owner path (the request's declared mode is honored or denied, never
|
|
1379
|
+
* reinterpreted). Owner mode is always the privileged (own-domain) path. */
|
|
1380
|
+
async epAnyModeAdmin(ctx) {
|
|
1381
|
+
if (ctx.subject.target?.mode !== "any")
|
|
1382
|
+
return false;
|
|
1383
|
+
if (!(await this.epAdminReach(principalKey(ctx.subject.caller.owner, ctx.subject.caller.actor).key)))
|
|
1384
|
+
throw new EpEnvelopeError("permission-denied", `an any-mode ${ctx.subject.command} is operator reach; the caller's current ledger grant does not carry "admin" (SPEC 13.2)`);
|
|
1385
|
+
return true;
|
|
1386
|
+
}
|
|
1387
|
+
/** The v0.4 typed command table (P2 item 1, slice 1b): every ordinary handler runs the SHARED
|
|
1388
|
+
* admission chokepoint ({@link serveGated}) and then delegates to the SAME op core the ctl
|
|
1389
|
+
* door dispatches (checklist 8: one core, two thin doors). The resume/preservation family
|
|
1390
|
+
* deliberately BYPASSES serveGated — exactly as it sits before {@link admitControl} on the ctl
|
|
1391
|
+
* door (those ops must run while `resumeRequired` fences ordinary work) — riding its own state
|
|
1392
|
+
* fences; its ep gate is the admin-grade `manager.admin` capability grant (the 1b rule: static
|
|
1393
|
+
* admin-class commands are capability-gated + untargeted, never a fabricated ledger mode).
|
|
1394
|
+
*
|
|
1395
|
+
* TIER SEMANTICS on the ep door (the 1c grant-migration table): the tier lives in the CALLER'S
|
|
1396
|
+
* GRANT, refined per-op exactly as the ctl doors refine their subject tier. Owner-mode
|
|
1397
|
+
* `despawn`/`attach` keep the privileged semantics (`admin=false`, own-domain via
|
|
1398
|
+
* {@link authorizeNamed}) — every spawn-capable agent holds those rows. ANY-mode requests are
|
|
1399
|
+
* the operator instrument's cross-agent reach (rev 3): the any-mode subject row is mintable
|
|
1400
|
+
* only under operator policy (§13.2), so on a static mesh holding it IS the admin tier, and in
|
|
1401
|
+
* user mode the caller's CURRENT ledger scope must still carry `admin`
|
|
1402
|
+
* ({@link epAdminReach}, the same fresh-read authority `psOwnerFilter` consults). The
|
|
1403
|
+
* `manager.admin` family (purge + the resume/preservation ops) is capability-gated at mint AND
|
|
1404
|
+
* re-checked at serve time via {@link epAdminReach} (the `adminGated` wrapper) so a user's
|
|
1405
|
+
* revoked scope demotes the next call. `launch` is OWNER-EQUALITY on this door for everyone
|
|
1406
|
+
* (freelance HIGH #2): the deploy path is its only consumer and stamps the caller's own owner,
|
|
1407
|
+
* so cross-owner launch was a ctl-tier incidental never exercised, and keying it on the actor's
|
|
1408
|
+
* ledger scope broke the deployer-view attenuation - uniform owner-equality is the safe tier.
|
|
1409
|
+
* TWO DELIBERATE NARROWINGS vs the ctl doors (NOT bit-exact parity, panel-accepted): (1)
|
|
1410
|
+
* `define-persona` is `admin=false` for everyone (own-persona discipline; no ep consumer needs
|
|
1411
|
+
* cross-owner persona writes - an operator redefines via config, not the wire), where the ctl
|
|
1412
|
+
* admin tier allowed operator cross-owner redefine; (2) launch is owner-equality-only, above.
|
|
1413
|
+
* Both are least-privilege reductions, never widenings. */
|
|
1414
|
+
managerServiceDefs() {
|
|
1415
|
+
const args = (ctx) => (ctx.request.args ?? {});
|
|
1416
|
+
const callerOf = (ctx) => principalKey(ctx.subject.caller.owner, ctx.subject.caller.actor).key;
|
|
1417
|
+
// A ctl-core failure reply becomes the §13.3 structured error the serve boundary publishes.
|
|
1418
|
+
// The data half of a failure reply (e.g. a degraded resume result) rides the error MESSAGE
|
|
1419
|
+
// only — the item-2 action model gives failures a typed channel.
|
|
1420
|
+
const unwrap = (r) => {
|
|
1421
|
+
if (!r.ok)
|
|
1422
|
+
throw new EpEnvelopeError("failed-precondition", r.error ?? "the operation failed");
|
|
1423
|
+
return r.data;
|
|
1424
|
+
};
|
|
1425
|
+
// The admin-family serve gate (1c.2c, security4's hardening): every `manager.admin`-class
|
|
1426
|
+
// command re-checks operator reach AT SERVE TIME - static: true (the mint boundary already
|
|
1427
|
+
// gates the rows to instruments); user mesh: the caller's CURRENT ledger scope must still
|
|
1428
|
+
// carry `admin` ({@link epAdminReach}'s fresh read), so a revoked scope demotes the very next
|
|
1429
|
+
// call instead of riding the bearer's remaining JWT-row lifetime. The resume family keeps its
|
|
1430
|
+
// serveGated BYPASS (those ops must run while the maintenance fence holds) but not the gate.
|
|
1431
|
+
const adminGated = async (ctx, fn) => {
|
|
1432
|
+
if (!(await this.epAdminReach(callerOf(ctx))))
|
|
1433
|
+
throw new EpEnvelopeError("permission-denied", `${ctx.subject.command} is operator reach; the caller's current ledger grant does not carry "admin" (SPEC 13.2)`);
|
|
1434
|
+
return fn();
|
|
1435
|
+
};
|
|
1436
|
+
const targetAgent = (ctx) => {
|
|
1437
|
+
const t = ctx.request.target; // targeted commands only: the serve boundary enforced body-target presence + fresh currency
|
|
1438
|
+
const a = this.findManagedByTarget(t);
|
|
1439
|
+
if (!a)
|
|
1440
|
+
throw new EpEnvelopeError("expired", `target ${t.owner}.${t.actor} (lifecycle ${t.lifecycleUid}) is not a live managed agent of this manager`);
|
|
1441
|
+
return a;
|
|
1442
|
+
};
|
|
1443
|
+
return managerCommandDefs({
|
|
1444
|
+
status: (ctx) => this.serveGated(ctx, () => this.managerStatusData()),
|
|
1445
|
+
ps: (ctx) => this.serveGated(ctx, async () => this.list(await this.psOwnerFilter(callerOf(ctx), false))),
|
|
1446
|
+
inspect: (ctx) => this.serveGated(ctx, async () => {
|
|
1447
|
+
const name = String(args(ctx).name ?? "").trim();
|
|
1448
|
+
const row = this.list(await this.psOwnerFilter(callerOf(ctx), false)).find((x) => x.name === name);
|
|
1449
|
+
if (!row)
|
|
1450
|
+
throw new EpEnvelopeError("not-found", `no agent "${name}"`);
|
|
1451
|
+
return row;
|
|
1452
|
+
}),
|
|
1453
|
+
models: (ctx) => this.serveGated(ctx, async () => {
|
|
1454
|
+
const data = unwrap(await this.opModels(args(ctx)));
|
|
1455
|
+
return { catalogs: Array.isArray(data) ? data : [data] };
|
|
1456
|
+
}),
|
|
1457
|
+
// P2 item 2: `spawn` is an ACTION - accept a goal + reply the acceptance floor payload, drive
|
|
1458
|
+
// progress + terminal off-handler (no ~30s block). The blocking reply path is gone (pin 8).
|
|
1459
|
+
spawn: (ctx) => this.serveGated(ctx, () => this.serveSpawnGoal(ctx, (h) => this.opStart(args(ctx), callerOf(ctx), h))),
|
|
1460
|
+
despawn: (ctx) => this.serveGated(ctx, async () => {
|
|
1461
|
+
const a = targetAgent(ctx);
|
|
1462
|
+
const denied = await this.authorizeNamed(a, callerOf(ctx), await this.epAnyModeAdmin(ctx));
|
|
1463
|
+
if (denied)
|
|
1464
|
+
throw new EpEnvelopeError("permission-denied", denied);
|
|
1465
|
+
return unwrap(this.despawnAuthorized(a, args(ctx).graceful !== false, true));
|
|
1466
|
+
}),
|
|
1467
|
+
attach: (ctx) => this.serveGated(ctx, async () => {
|
|
1468
|
+
const a = targetAgent(ctx);
|
|
1469
|
+
const denied = await this.authorizeNamed(a, callerOf(ctx), await this.epAnyModeAdmin(ctx));
|
|
1470
|
+
if (denied)
|
|
1471
|
+
throw new EpEnvelopeError("permission-denied", denied);
|
|
1472
|
+
return unwrap(await this.attachAuthorized(a, ctx.subject.caller));
|
|
1473
|
+
}),
|
|
1474
|
+
stopSelf: (ctx) => this.serveGated(ctx, () => unwrap(this.opStopSelf(callerOf(ctx), args(ctx)))),
|
|
1475
|
+
definePersona: (ctx) => this.serveGated(ctx, () => unwrap(this.opDefinePersona(args(ctx), callerOf(ctx), false))),
|
|
1476
|
+
purge: (ctx) => this.serveGated(ctx, () => adminGated(ctx, async () => unwrap(await this.opPurge(args(ctx), callerOf(ctx))))),
|
|
1477
|
+
// launch is OWNER-EQUALITY on the ep door for every caller (freelance HIGH #2): the deploy
|
|
1478
|
+
// path is the only launch consumer and its spec stamps the CALLER's own owner, so
|
|
1479
|
+
// owner-equality always holds for a legitimate deploy; cross-owner launch was a ctl
|
|
1480
|
+
// admin-tier INCIDENTAL never exercised by a real flow (static is single-owner, so the flag
|
|
1481
|
+
// is a no-op there). Keying admin on epAdminReach read the ACTOR's ledger scope, which does
|
|
1482
|
+
// NOT reflect the deployer VIEW's privileged-tier attenuation - an admin user's stolen
|
|
1483
|
+
// deployer bearer would then bypass owner-equality (operator launch) despite the view holding
|
|
1484
|
+
// no admin rows. Uniform owner-equality removes that divergence in the least-privilege
|
|
1485
|
+
// direction (consistent with the delta-(b) tier narrowing the panel endorsed).
|
|
1486
|
+
// P2 item 2 (ruling 3): manifest `launch` is an ACTION through the SAME chokepoint as spawn -
|
|
1487
|
+
// the manifest resolve + owner-equality authz run in opLaunch's accept path, then the goal
|
|
1488
|
+
// drives progress + terminal. The acceptance floor is the allocated identity + goal coords.
|
|
1489
|
+
launch: (ctx) => this.serveGated(ctx, () => this.serveSpawnGoal(ctx, (h) => this.opLaunch(args(ctx), callerOf(ctx), false, h))),
|
|
1490
|
+
resumePreserved: (ctx) => adminGated(ctx, async () => unwrap(await this.opResumePreserved(args(ctx)))),
|
|
1491
|
+
commitResume: (ctx) => adminGated(ctx, async () => unwrap(await this.opCommitResume(args(ctx)))),
|
|
1492
|
+
finalizeResume: (ctx) => adminGated(ctx, async () => unwrap(await this.opFinalizeResume(args(ctx)))),
|
|
1493
|
+
preparePreservation: (ctx) => adminGated(ctx, async () => unwrap(await this.opPreservationCtl("preparePreservation", args(ctx)))),
|
|
1494
|
+
commitPreservation: (ctx) => adminGated(ctx, async () => unwrap(await this.opPreservationCtl("commitPreservation", args(ctx)))),
|
|
1495
|
+
abortPreservation: (ctx) => adminGated(ctx, async () => unwrap(await this.opPreservationCtl("abortPreservation", args(ctx)))),
|
|
1496
|
+
});
|
|
1497
|
+
}
|
|
966
1498
|
async commitResumeActivation(attemptId) {
|
|
967
1499
|
if (!this.resumeAwaitingCommit || !this.resumeResult?.ok)
|
|
968
1500
|
return { ok: false, error: `resume attempt ${attemptId} has no successful activation to commit` };
|
|
@@ -998,83 +1530,12 @@ export class Manager {
|
|
|
998
1530
|
data: { attemptId, state: "awaitingFinalize", durableCommitToken: this.resumeDurableCommitToken },
|
|
999
1531
|
};
|
|
1000
1532
|
}
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
|
|
1005
|
-
|
|
1006
|
-
|
|
1007
|
-
// so authz (P2c) and the spawner ledger (P4b) can act on it.
|
|
1008
|
-
const caller = req.from.id;
|
|
1009
|
-
const name = String(args.name ?? "").trim();
|
|
1010
|
-
// Op↔tier binding — the real enforcement per the split. The cred gates WHO can reach each
|
|
1011
|
-
// subject; this gates WHAT each subject will honor, fail-closed. A privileged op arriving on
|
|
1012
|
-
// the self-service subject (publishable by all) must be rejected or the split does nothing.
|
|
1013
|
-
if (tier === CONTROL_SELF_SERVICE) {
|
|
1014
|
-
// Self-service honors self-ops only: a no-name stop (self-despawn). Durable join/leave/list moved
|
|
1015
|
-
// OFF the manager onto the server-side delivery daemon's `ctl.delivery` service (the manager is
|
|
1016
|
-
// lifecycle-only). A named stop (belongs on privileged/admin) or anything else is a misroute.
|
|
1017
|
-
if (req.op !== "stop")
|
|
1018
|
-
return { ok: false, error: `op "${req.op}" not allowed on self-service control subject` };
|
|
1019
|
-
if (name)
|
|
1020
|
-
return { ok: false, error: "named stop not allowed on self-service subject; send it on the privileged subject" };
|
|
1021
|
-
return this.opStopSelf(caller, args);
|
|
1022
|
-
}
|
|
1023
|
-
const admin = tier === CONTROL_ADMIN;
|
|
1024
|
-
// Privileged + admin tiers. A no-name stop is a self-op and belongs on the self-service subject.
|
|
1025
|
-
switch (req.op) {
|
|
1026
|
-
case "start":
|
|
1027
|
-
// Spawn is a privileged-tier op; reaching it via admin is fine (admin ⊇ privileged powers).
|
|
1028
|
-
return this.opStart(args, caller);
|
|
1029
|
-
case "launch":
|
|
1030
|
-
// SECURITY: on a STATIC mesh, manifest launch is operator-only (admin tier). It is
|
|
1031
|
-
// higher-power than `start` — it boots an operator-authored, coordinated policy set from a
|
|
1032
|
-
// run spec and underpins the ownership ledger — so a merely spawn-capable agent (which CAN
|
|
1033
|
-
// publish to the privileged subject) must not reach it. Gate at the handler like `purge`;
|
|
1034
|
-
// the subject alone isn't a boundary because `spawn` grants privileged-subject publish and
|
|
1035
|
-
// dispatch is by op here. On a USER mesh, a spawn-scoped operator deploys THEIR OWN team on
|
|
1036
|
-
// the privileged tier: opLaunch enforces owner-equality (the spec's apply-time stamped
|
|
1037
|
-
// owner === the subject-pinned caller's owner) BEFORE any side effect.
|
|
1038
|
-
if (!admin && !this.userMode)
|
|
1039
|
-
return { ok: false, error: "launch is admin-only; not allowed on the privileged subject" };
|
|
1040
|
-
return this.opLaunch(args, caller, admin);
|
|
1041
|
-
case "stop": {
|
|
1042
|
-
if (!name)
|
|
1043
|
-
return { ok: false, error: "self-stop not allowed on privileged subject; send it on the self-service subject" };
|
|
1044
|
-
return this.opStop(args, caller, admin);
|
|
1045
|
-
}
|
|
1046
|
-
case "definePersona":
|
|
1047
|
-
return this.opDefinePersona(args, caller, admin);
|
|
1048
|
-
case "purge":
|
|
1049
|
-
// SECURITY: purge clears space history incl. DMs — admin-only. On the privileged tier any
|
|
1050
|
-
// spawn-capable agent could wipe the space, so it must not be honored there.
|
|
1051
|
-
if (!admin)
|
|
1052
|
-
return { ok: false, error: "purge is admin-only; not allowed on the privileged subject" };
|
|
1053
|
-
return this.opPurge(args, caller);
|
|
1054
|
-
case "attach":
|
|
1055
|
-
return this.opAttach(args, caller, admin);
|
|
1056
|
-
case "ps":
|
|
1057
|
-
// USER mesh, privileged tier: `ps` lists only the CALLER's own owner-domain (the admin tier
|
|
1058
|
-
// OR a fresh ledger `admin` scope sees all) — cross-owner agent metadata (principals,
|
|
1059
|
-
// personas, auth health) is operator-grade. Fail-closed: an unparseable caller sees nothing.
|
|
1060
|
-
// Static meshes are unchanged.
|
|
1061
|
-
return { ok: true, data: this.list(await this.psOwnerFilter(caller, admin)) };
|
|
1062
|
-
case "models":
|
|
1063
|
-
return this.opModels(args);
|
|
1064
|
-
case "status": {
|
|
1065
|
-
// Same owner-domain bound as `ps`: a cross-owner target reads as absent, never as metadata.
|
|
1066
|
-
const a = this.list(await this.psOwnerFilter(caller, admin)).find((x) => x.name === name);
|
|
1067
|
-
return a ? { ok: true, data: a } : { ok: false, error: `no agent "${name}"` };
|
|
1068
|
-
}
|
|
1069
|
-
default:
|
|
1070
|
-
return { ok: false, error: `unknown op: ${req.op}` };
|
|
1071
|
-
}
|
|
1072
|
-
}
|
|
1073
|
-
/** Collapsed despawn/attach authorization (P4b). The caller already reached the privileged or
|
|
1074
|
-
* admin tier (cred-gated). On the admin tier any named target is allowed (operator). On the
|
|
1075
|
-
* privileged tier a named target is allowed if it's the caller's OWN child (`spawner ==
|
|
1076
|
-
* caller`) — and, on a user mesh, if it runs under the CALLER'S OWNER (owner-domain) or the
|
|
1077
|
-
* caller's ledger row holds `admin`, read fresh. The policy is the pure
|
|
1533
|
+
/** Collapsed despawn/attach authorization (P4b). The caller already reached the command's ep
|
|
1534
|
+
* row (cred-gated: owner-mode rows via the spawn capability, any-mode rows only in admin
|
|
1535
|
+
* instruments). With admin=true (any-mode) any named target is allowed (operator). Otherwise
|
|
1536
|
+
* a named target is allowed if it's the caller's OWN child (`spawner == caller`) — and, on a
|
|
1537
|
+
* user mesh, if it runs under the CALLER'S OWNER (owner-domain) or the caller's ledger row
|
|
1538
|
+
* holds `admin`, read fresh. The policy is the pure
|
|
1078
1539
|
* {@link authorizeNamedControl}; this wrapper only binds the manager's state (the mode flag +
|
|
1079
1540
|
* the provider-backed ledger read — a build with no provider authorizes nothing extra,
|
|
1080
1541
|
* fail-closed via the policy's catch). Error string when denied, `undefined` when allowed. */
|
|
@@ -1174,6 +1635,9 @@ export class Manager {
|
|
|
1174
1635
|
* footprint, nor (in `reapChildrenOf`) abort the reap of later siblings. The failure is logged loudly,
|
|
1175
1636
|
* never swallowed silently. Being the single stop chokepoint, guarding here covers all callers at once. */
|
|
1176
1637
|
stopHandle(a, graceful) {
|
|
1638
|
+
// The F5 TERMINALIZING latch (Unit B): flipped SYNCHRONOUSLY, before any await anywhere on
|
|
1639
|
+
// this stop path — from here this principal's control ops refuse and no credential renews.
|
|
1640
|
+
a.terminalizing = true;
|
|
1177
1641
|
try {
|
|
1178
1642
|
if (graceful && process.platform === "win32" && a.control)
|
|
1179
1643
|
controlShutdown(a.control);
|
|
@@ -1258,7 +1722,9 @@ export class Manager {
|
|
|
1258
1722
|
// user-mode spawn reads the callout material from it — the same store the auth-store kinds
|
|
1259
1723
|
// (callout/issuer/…) were migrated onto — so this is no longer a local-only path.
|
|
1260
1724
|
const secrets = this.secrets;
|
|
1261
|
-
|
|
1725
|
+
// LIFECYCLE-KEYED family (SPEC 13.1 name-disjointness on the FS): this incarnation's files
|
|
1726
|
+
// embed its uid, so no teardown addressed to another incarnation can ever reach them.
|
|
1727
|
+
const files = agentLifecycleSecretFilePaths(this.workspaceRoot, name, opts.lifecycleUid);
|
|
1262
1728
|
const { actorToken: tokenPath, sentinelCreds: sentinelPath, health: healthPath } = files;
|
|
1263
1729
|
try {
|
|
1264
1730
|
// The GRANT first — it is the envelope-rule enforcement point (a delegation must sit within
|
|
@@ -1289,10 +1755,10 @@ export class Manager {
|
|
|
1289
1755
|
// The store holds the source of truth; the bearer re-exec (`--token-file`) and the launch's
|
|
1290
1756
|
// sentinel handoff read FILES, so materialize both at the canonical paths (under the local
|
|
1291
1757
|
// FS composition, a byte-identical rewrite of the keys' own locations).
|
|
1292
|
-
await secrets.put(
|
|
1293
|
-
await secrets.put(
|
|
1294
|
-
await materializeSecretToFile(secrets,
|
|
1295
|
-
await materializeSecretToFile(secrets,
|
|
1758
|
+
await secrets.put(agentSecretKeyForFile(tokenPath), grant.actorToken);
|
|
1759
|
+
await secrets.put(agentSecretKeyForFile(sentinelPath), grant.sentinelCreds);
|
|
1760
|
+
await materializeSecretToFile(secrets, agentSecretKeyForFile(tokenPath), tokenPath);
|
|
1761
|
+
await materializeSecretToFile(secrets, agentSecretKeyForFile(sentinelPath), sentinelPath);
|
|
1296
1762
|
rmSync(healthPath, { force: true }); // a fresh start opens a fresh health window
|
|
1297
1763
|
const bearerCmd = [
|
|
1298
1764
|
// The manager's own invocation prefix (node + loader flags + the cotal entry) — the agent
|
|
@@ -1311,7 +1777,7 @@ export class Manager {
|
|
|
1311
1777
|
"--health-file", healthPath,
|
|
1312
1778
|
];
|
|
1313
1779
|
await execBearerPreflight(bearerCmd);
|
|
1314
|
-
return { owner, launch: { owner, actor: name, sentinelCredsPath: sentinelPath, bearerCmd } };
|
|
1780
|
+
return { owner, files, launch: { owner, actor: name, sentinelCredsPath: sentinelPath, bearerCmd } };
|
|
1315
1781
|
}
|
|
1316
1782
|
catch (e) {
|
|
1317
1783
|
// Roll back everything this attempt materialized — a refused spawn must leave no standing
|
|
@@ -1319,12 +1785,12 @@ export class Manager {
|
|
|
1319
1785
|
// may respawn the moment it reads the refusal, and a detached teardown would race (and
|
|
1320
1786
|
// delete) that fresh spawn's just-provisioned durables.
|
|
1321
1787
|
await provider.revokeAgent({ dir, owner, actor: name }).catch(() => { });
|
|
1322
|
-
await secrets.delete(
|
|
1323
|
-
await secrets.delete(
|
|
1788
|
+
await secrets.delete(agentSecretKeyForFile(tokenPath)).catch(() => { });
|
|
1789
|
+
await secrets.delete(agentSecretKeyForFile(sentinelPath)).catch(() => { });
|
|
1324
1790
|
rmSync(tokenPath, { force: true });
|
|
1325
1791
|
rmSync(sentinelPath, { force: true });
|
|
1326
1792
|
rmSync(healthPath, { force: true });
|
|
1327
|
-
await this.deprovision({ id: principalKey(owner, name).key, name, lifecycleUid: opts.lifecycleUid, userOwner: owner }).catch((err) => console.error(`rollback deprovision ${name}: ${err.message}`));
|
|
1793
|
+
await this.deprovision({ id: principalKey(owner, name).key, name, lifecycleUid: opts.lifecycleUid, userOwner: owner, secretPaths: files }).catch((err) => console.error(`rollback deprovision ${name}: ${err.message}`));
|
|
1328
1794
|
return { error: `agent auth preflight failed for "${name}": ${e.message}` };
|
|
1329
1795
|
}
|
|
1330
1796
|
}
|
|
@@ -1336,7 +1802,12 @@ export class Manager {
|
|
|
1336
1802
|
freeSlot(a, floor, acceptedBeforeFence = false) {
|
|
1337
1803
|
if (this.agents.get(a.name) !== a)
|
|
1338
1804
|
return; // already freed (exit raced despawn, etc.)
|
|
1805
|
+
a.terminalizing = true; // F5 latch (Unit B): also covers exit/reap paths that never rode stopHandle
|
|
1339
1806
|
this.agents.delete(a.name);
|
|
1807
|
+
// P2 item 6 (pin 4): end any live §13.6 attach session bound to THIS incarnation with the honest
|
|
1808
|
+
// `target-despawn` reason. Fires once per agent on every free path (despawn / self-stop / reap /
|
|
1809
|
+
// exit) via the `agents` guard above; a no-op when no plane or no live session for the target.
|
|
1810
|
+
this.sessionPlane?.endForTarget(a.name, a.lifecycleUid, "target-despawn");
|
|
1340
1811
|
if (floor && Date.now() - a.startedAt < MIN_LIFETIME)
|
|
1341
1812
|
this.cooling.push(a.startedAt + MIN_LIFETIME);
|
|
1342
1813
|
// #29 piece 3: on a USER mesh the name is RESERVED PENDING RETIREMENT — despawn started this
|
|
@@ -1350,7 +1821,14 @@ export class Manager {
|
|
|
1350
1821
|
if (this.userMode) {
|
|
1351
1822
|
const p = parsePrincipalKey(a.id);
|
|
1352
1823
|
if (p)
|
|
1353
|
-
this.retiring.set(a.name, { opId: retireOpId(a.lifecycleUid), lifecycleUid: a.lifecycleUid, owner: p.owner, actor: p.actor, agentId: a.id, userOwner: a.userOwner, startedAt: Date.now() });
|
|
1824
|
+
this.retiring.set(a.name, { opId: retireOpId(a.lifecycleUid), lifecycleUid: a.lifecycleUid, owner: p.owner, actor: p.actor, agentId: a.id, userOwner: a.userOwner, secretPaths: a.secretPaths, startedAt: Date.now() });
|
|
1825
|
+
}
|
|
1826
|
+
else if (this.auth) {
|
|
1827
|
+
// Unit B: a STATIC lifecycle now also holds its name pending its own terminal (the F1
|
|
1828
|
+
// static retirement the detached deprovision below drives) — the alias frees only when the
|
|
1829
|
+
// gate+head terminal completes, exactly the user-mode discipline. The wire principal is the
|
|
1830
|
+
// incarnation-unique nkey (F5-bind); owner is the dev owner.
|
|
1831
|
+
this.retiring.set(a.name, { opId: retireOpId(a.lifecycleUid), lifecycleUid: a.lifecycleUid, owner: DEV_OWNER, actor: a.id, agentId: a.id, secretPaths: a.secretPaths, startedAt: Date.now() });
|
|
1354
1832
|
}
|
|
1355
1833
|
// Auth mode: tear down the departed agent's minted broker footprint + creds file (#159 B2). The
|
|
1356
1834
|
// process is already gone, so this must never block the slot free or throw into the caller — it runs
|
|
@@ -1392,25 +1870,44 @@ export class Manager {
|
|
|
1392
1870
|
async driveDeprovision(a) {
|
|
1393
1871
|
if (!this.auth)
|
|
1394
1872
|
return; // guaranteed by deprovision; re-checked for the deprovisionBroker narrowing
|
|
1873
|
+
if (!this.userMode && !a.userOwner) {
|
|
1874
|
+
// Unit B: a STATIC lifecycle retires through the F1 terminal barrier — freeze → head
|
|
1875
|
+
// retiring → B1 ledger revoke → footprint cleanup (creds file + broker durables/ACL, INSIDE
|
|
1876
|
+
// the barrier) → gate retired → head retired → alias free. The eviction step is the process
|
|
1877
|
+
// kill the stop path already performed (static's best-effort eviction).
|
|
1878
|
+
return this.driveStaticRetirement(a);
|
|
1879
|
+
}
|
|
1395
1880
|
// Drop the local creds file FIRST + unconditionally — it is a usable identity on disk, useless for a
|
|
1396
1881
|
// departed agent, so it must not survive even if the broker teardown below fails or times out. The
|
|
1397
1882
|
// teardown mints its OWN deprovisioner cred (not this file), so removing it early is independent.
|
|
1398
1883
|
// Migrated kinds: the store delete is the authoritative removal; the rmSync clears the FS
|
|
1399
|
-
// materialization (a byte-identical no-op under the local composition, real once the manager
|
|
1884
|
+
// materialization (a byte-identical no-op under the local composition, real once the manager
|
|
1400
1885
|
// `secretStore` is a non-FS store).
|
|
1886
|
+
//
|
|
1887
|
+
// LIFECYCLE-OWNED (SPEC 13.1, the manager-local half): the family deleted here is the RECORDED
|
|
1888
|
+
// one (spawn/adoption), else the lifecycle-keyed derivation for THIS uid — never a name-only
|
|
1889
|
+
// derivation, so a stale/replayed teardown addresses only names a same-alias successor never
|
|
1890
|
+
// uses. It deliberately CANNOT remove a name-keyed family it holds no record of (an operator's
|
|
1891
|
+
// standing `cotal mint` cred, a seeded workstation cred, a pre-split leftover): deleting an
|
|
1892
|
+
// unowned same-name file is the exact successor-clobber this ownership discipline removes.
|
|
1401
1893
|
const secrets = this.secrets;
|
|
1402
|
-
const files =
|
|
1403
|
-
|
|
1404
|
-
|
|
1894
|
+
const files = a.secretPaths ?? agentLifecycleSecretFilePaths(this.workspaceRoot, a.name, a.lifecycleUid);
|
|
1895
|
+
if (files.creds) {
|
|
1896
|
+
await secrets.delete(agentSecretKeyForFile(files.creds));
|
|
1897
|
+
rmSync(files.creds, { force: true });
|
|
1898
|
+
}
|
|
1405
1899
|
if (a.userOwner) {
|
|
1406
1900
|
// USER MODE: this teardown IS revocation, not just footprint reduction — the ledger row is
|
|
1407
1901
|
// the agent's standing mint authority, so delete it (next exchange refused, next connect
|
|
1408
1902
|
// denied) and shred the secret/sentinel/health files. A copied actor token dies here; a
|
|
1409
1903
|
// still-LIVE connection ends at its bearer-bound JWT expiry (≤ the agent TTL).
|
|
1410
|
-
|
|
1411
|
-
|
|
1904
|
+
if (files.actorToken)
|
|
1905
|
+
await secrets.delete(agentSecretKeyForFile(files.actorToken));
|
|
1906
|
+
if (files.sentinelCreds)
|
|
1907
|
+
await secrets.delete(agentSecretKeyForFile(files.sentinelCreds));
|
|
1412
1908
|
for (const f of [files.actorToken, files.sentinelCreds, files.health])
|
|
1413
|
-
|
|
1909
|
+
if (f)
|
|
1910
|
+
rmSync(f, { force: true });
|
|
1414
1911
|
// The ledger row IS the agent's STANDING mint authority (a different store from the auth-plane
|
|
1415
1912
|
// cred ledger the rail retirement covers): while it lives, a copied actor token can still mint a
|
|
1416
1913
|
// fresh connect credential. So a FAILED revoke must NOT be swallowed into a clean terminal (INT-2):
|
|
@@ -1440,8 +1937,10 @@ export class Manager {
|
|
|
1440
1937
|
}
|
|
1441
1938
|
await this.deprovisionBroker(a);
|
|
1442
1939
|
// #29 piece 3: after the footprint teardown, ask the AUTH plane to RETIRE the lifecycle over
|
|
1443
|
-
// the auth
|
|
1444
|
-
//
|
|
1940
|
+
// the auth endpoint rail. The rail re-checks the SERVE-ISSUANCE GATE at serve time (not the
|
|
1941
|
+
// space-manager lease - that check was replaced in 02794b2f) and refuses unless the registration
|
|
1942
|
+
// this request names belongs to our own principal; the terminal (or an already-retired answer)
|
|
1943
|
+
// clears the name reservation. Failures keep the hold with
|
|
1445
1944
|
// their operator copy — legible, retryable, never a silent half-state.
|
|
1446
1945
|
await this.requestRetirement(a);
|
|
1447
1946
|
}
|
|
@@ -1486,12 +1985,44 @@ export class Manager {
|
|
|
1486
1985
|
}
|
|
1487
1986
|
const uncertain = (why) => `the despawn stopped "${a.name}", but the retirement's completion could NOT be confirmed (${why}). The name stays held - not failed, not done - and a same-name spawn re-drives the same teardown; the auth service also finishes any started retirement on its next boot. NEXT: if the auth rail stays unreachable, recover the stack (\`cotal supervise\`), then re-attempt the same-name spawn.`;
|
|
1488
1987
|
try {
|
|
1489
|
-
|
|
1988
|
+
// The caller triple and the TARGET are both grant-pinned now (#350): the `handle` target
|
|
1989
|
+
// rides the subject, so this ephemeral credential can ask to retire exactly this
|
|
1990
|
+
// incarnation and nothing else.
|
|
1991
|
+
const caller = { owner: me.owner, actor: me.actor, uid: this.managerLifecycleUid };
|
|
1992
|
+
const creds = await mintCreds(this.auth, newIdentity(), "retirement-requester", {
|
|
1993
|
+
retirementRequester: { ...caller, target: { owner: target.owner, actor: target.actor, lifecycleUid: a.lifecycleUid } },
|
|
1994
|
+
});
|
|
1490
1995
|
const nc = await connect({ servers: this.servers ?? DEFAULT_SERVER, authenticator: credsAuthenticator(new TextEncoder().encode(creds)), maxReconnectAttempts: 0 });
|
|
1491
1996
|
try {
|
|
1492
|
-
|
|
1493
|
-
const
|
|
1494
|
-
|
|
1997
|
+
// §13.2 nonce: >=128 bits of CSPRNG entropy, base64url (the `endpoint-invoke` idiom).
|
|
1998
|
+
const nonce = randomBytes(24).toString("base64url");
|
|
1999
|
+
// The caller-chosen request id the reply MUST echo (the minimal correctness guard from the
|
|
2000
|
+
// not-yet-migrated endpoint envelope - see the residual in the auth listener).
|
|
2001
|
+
const requestId = randomBytes(16).toString("base64url");
|
|
2002
|
+
const subject = epRequestSubject(this.space, {
|
|
2003
|
+
route: { mode: "one" },
|
|
2004
|
+
endpoint: AUTH_ENDPOINT,
|
|
2005
|
+
command: EP_CMD_RETIRE_LIFECYCLE,
|
|
2006
|
+
// The TARGET rides the subject (authz mode `handle`, arity 3) — broker-enforced, and
|
|
2007
|
+
// pinned by the grant minted above, so it is not a body claim the caller can vary.
|
|
2008
|
+
target: { mode: "handle", tOwner: target.owner, tActor: target.actor, tUid: a.lifecycleUid },
|
|
2009
|
+
caller,
|
|
2010
|
+
nonce,
|
|
2011
|
+
});
|
|
2012
|
+
// The responder derives its OWN reply subject from this request (caller triple + nonce,
|
|
2013
|
+
// prefixed with the RESPONDER's instance identity), so a caller-supplied `reply` header
|
|
2014
|
+
// would be ignored — that is the confused-deputy boundary being structural. The caller
|
|
2015
|
+
// therefore reads its own reply-plane filter and binds the answer off the reply SUBJECT.
|
|
2016
|
+
const m = await epAwaitReply(nc, this.space, caller, nonce, requestId, subject,
|
|
2017
|
+
// Declare THIS manager instance's serve identity. Since #350 these SELECT the gate row;
|
|
2018
|
+
// they no longer authorize — the rail refuses unless the row they name belongs to this
|
|
2019
|
+
// caller's own subject-derived principal. A superseded predecessor (same instanceId, OLD
|
|
2020
|
+
// epoch after a restart) is still refused by the epoch comparison.
|
|
2021
|
+
JSON.stringify({ id: requestId, op: "retireLifecycle", args: {
|
|
2022
|
+
opId: retireOpId(a.lifecycleUid),
|
|
2023
|
+
serveEndpoint: MANAGER_ENDPOINT, serveInstanceId: this.managerInstanceId, serveEpoch: this.serviceServe?.grant.epoch ?? 0,
|
|
2024
|
+
} }), 20_000);
|
|
2025
|
+
const r = m;
|
|
1495
2026
|
if (r.ok) {
|
|
1496
2027
|
// CAS the hold clear (audit #1 ABA): free the alias ONLY if the current hold is still THIS
|
|
1497
2028
|
// lifecycle's - a late reply for a retired predecessor must never clear a successor's newer hold.
|
|
@@ -1584,11 +2115,37 @@ export class Manager {
|
|
|
1584
2115
|
? undefined
|
|
1585
2116
|
: `unsafe name ${JSON.stringify(name)} (allowed: letters, digits, _ -)`;
|
|
1586
2117
|
}
|
|
1587
|
-
/** First free name in the series `base`, `base-2`, `base-3`, … — checked against
|
|
1588
|
-
* in-flight (reserved) slots
|
|
1589
|
-
*
|
|
2118
|
+
/** First free name in the series `base`, `base-2`, `base-3`, … — checked against live slots,
|
|
2119
|
+
* in-flight (reserved) slots, names held pending retirement, AND the live mesh roster. The
|
|
2120
|
+
* roster check covers occupants this manager does not manage (a foreground `cotal spawn`, a
|
|
2121
|
+
* connector session, another manager's agent): allocating their name would mint a sibling the
|
|
2122
|
+
* broker/auth then refuses to admit, surfacing as a 30s launch-uncertain black hole instead of
|
|
2123
|
+
* the auto-number the join path gives. Presence is ADVISORY (SPEC §6) — this is an availability
|
|
2124
|
+
* choice at allocation, never an authority check (the broker still enforces): a stale
|
|
2125
|
+
* still-live-looking row only costs a numbered suffix, and a missed freshly-joined occupant is
|
|
2126
|
+
* still refused downstream exactly as before. Offline rows do NOT occupy — a properly retired
|
|
2127
|
+
* name stays reusable. */
|
|
2128
|
+
/** The roster's LIVE occupant names (status !== offline) — occupants this manager may NOT manage
|
|
2129
|
+
* (a foreground `cotal spawn`, a connector session, ANOTHER manager's agent). Allocating over any
|
|
2130
|
+
* of them mints a sibling the broker/auth then refuses to admit, surfacing as the 30s launch-
|
|
2131
|
+
* uncertain black hole. */
|
|
2132
|
+
liveRosterNames() {
|
|
2133
|
+
const live = new Set();
|
|
2134
|
+
for (const p of this.ep.getRoster())
|
|
2135
|
+
if (p.status !== "offline")
|
|
2136
|
+
live.add(p.card.name);
|
|
2137
|
+
return live;
|
|
2138
|
+
}
|
|
2139
|
+
/** THE single name-liveness predicate both the hard-pinned collision refuse (M6, P2 item 2) and
|
|
2140
|
+
* uniqueName's numbering consult, so they can never drift: a name is taken if this manager
|
|
2141
|
+
* reserves/manages/retires it OR a roster-live occupant already holds it. Pass a pre-built
|
|
2142
|
+
* {@link liveRosterNames} set when checking many names in one allocation. */
|
|
2143
|
+
nameInUse(name, live = this.liveRosterNames()) {
|
|
2144
|
+
return this.agents.has(name) || this.reserved.has(name) || this.retiring.has(name) || live.has(name);
|
|
2145
|
+
}
|
|
1590
2146
|
uniqueName(base) {
|
|
1591
|
-
|
|
2147
|
+
const live = this.liveRosterNames();
|
|
2148
|
+
return firstFreeName(base, (n) => this.nameInUse(n, live));
|
|
1592
2149
|
}
|
|
1593
2150
|
/** Spawn a teammate by persona ref (`name` loads `.cotal/agents/<name>.md`; the peer presents
|
|
1594
2151
|
* under that file's own `name:`), as if a peer asked via the control plane. Used to pre-spawn the
|
|
@@ -1611,7 +2168,7 @@ export class Manager {
|
|
|
1611
2168
|
return false;
|
|
1612
2169
|
}
|
|
1613
2170
|
/** Parse an untyped control-plane `start` request into {@link StartAgentOpts}. */
|
|
1614
|
-
opStart(args, caller) {
|
|
2171
|
+
opStart(args, caller, hooks) {
|
|
1615
2172
|
// `resume`, when present, must be a non-empty session id. An empty/whitespace value is a
|
|
1616
2173
|
// malformed request, not an implicit "spawn fresh" (no fallbacks). The CLI surfaces reject it,
|
|
1617
2174
|
// but a raw control message could otherwise slip an empty value through and silently start fresh.
|
|
@@ -1658,7 +2215,7 @@ export class Manager {
|
|
|
1658
2215
|
allowSubscribe,
|
|
1659
2216
|
allowPublish,
|
|
1660
2217
|
shareTools: args.shareTools !== undefined ? String(args.shareTools) : undefined,
|
|
1661
|
-
}, caller);
|
|
2218
|
+
}, caller, hooks);
|
|
1662
2219
|
}
|
|
1663
2220
|
/** Resolve a connector by agent type. Library composition (installedExtensions off) → a registry
|
|
1664
2221
|
* hit, exactly as before (the composition root imported what it wants). The published binary gates
|
|
@@ -1768,7 +2325,7 @@ export class Manager {
|
|
|
1768
2325
|
* (collision-numbered) name + nkey id creds are filed under, plus the manifest `requested` name,
|
|
1769
2326
|
* `runId`, and resolved `hash`. USER mesh: a privileged-tier launch is owner-equality-authorized
|
|
1770
2327
|
* (spec owner === caller owner) before any side effect; the admin tier keeps operator behavior. */
|
|
1771
|
-
async opLaunch(args, caller, admin) {
|
|
2328
|
+
async opLaunch(args, caller, admin, hooks) {
|
|
1772
2329
|
const runId = String(args.runId ?? "").trim();
|
|
1773
2330
|
const name = String(args.name ?? "").trim();
|
|
1774
2331
|
if (!runId || !name)
|
|
@@ -1820,7 +2377,7 @@ export class Manager {
|
|
|
1820
2377
|
catch (e) {
|
|
1821
2378
|
return { ok: false, error: e.message };
|
|
1822
2379
|
}
|
|
1823
|
-
const reply = await this.startAgent(launchAgentToStartOpts(la, configPath, spec.owner, runId), caller);
|
|
2380
|
+
const reply = await this.startAgent(launchAgentToStartOpts(la, configPath, spec.owner, runId), caller, hooks);
|
|
1824
2381
|
if (reply.ok)
|
|
1825
2382
|
// `data.name` stays the spawned (numbered) identity — what creds are filed under and the ledger
|
|
1826
2383
|
// keys on; `requested`/`runId`/`hash` give the CLI the manifest name + drift hash for the ledger.
|
|
@@ -1833,18 +2390,18 @@ export class Manager {
|
|
|
1833
2390
|
* `spawner` is the authenticated id of the peer that requested the spawn (`req.from.id`),
|
|
1834
2391
|
* defaulting to the manager's own id for roster/pre-spawn — recorded for the spawner
|
|
1835
2392
|
* ledger (own-children despawn + reap-on-parent-exit). */
|
|
1836
|
-
async startAgent(opts, spawner) {
|
|
2393
|
+
async startAgent(opts, spawner, hooks) {
|
|
1837
2394
|
const release = this.beginLifecycle();
|
|
1838
2395
|
if (!release)
|
|
1839
2396
|
return { ok: false, error: this.maintenanceError() };
|
|
1840
2397
|
try {
|
|
1841
|
-
return await this.startAgentActive(opts, spawner);
|
|
2398
|
+
return await this.startAgentActive(opts, spawner, hooks);
|
|
1842
2399
|
}
|
|
1843
2400
|
finally {
|
|
1844
2401
|
release();
|
|
1845
2402
|
}
|
|
1846
2403
|
}
|
|
1847
|
-
async startAgentActive(opts, spawner) {
|
|
2404
|
+
async startAgentActive(opts, spawner, hooks) {
|
|
1848
2405
|
// The spawn argument is a persona REF — a filename in `.cotal/agents` (the unique spawn KEY), or
|
|
1849
2406
|
// a path via `--config`. It is NOT the mesh identity: the identity comes from inside the file
|
|
1850
2407
|
// (`name:`), so a persona can be filed descriptively (review-critic.md) yet present under a
|
|
@@ -1974,7 +2531,7 @@ export class Manager {
|
|
|
1974
2531
|
// outstanding. All the teardown ops are idempotent, and the rail request is single-flighted.
|
|
1975
2532
|
const held = this.retiring.get(identityName);
|
|
1976
2533
|
if (held !== undefined) {
|
|
1977
|
-
void this.deprovision({ id: held.agentId, name: identityName, lifecycleUid: held.lifecycleUid, userOwner: held.userOwner }).catch(() => { });
|
|
2534
|
+
void this.deprovision({ id: held.agentId, name: identityName, lifecycleUid: held.lifecycleUid, userOwner: held.userOwner, secretPaths: held.secretPaths }).catch(() => { });
|
|
1978
2535
|
return {
|
|
1979
2536
|
ok: false,
|
|
1980
2537
|
error: `the name "${identityName}" is reserved pending retirement: its previous agent's despawn started that lifecycle's teardown (footprint + standing-authority revoke + auth-side retirement), and the name frees only when all of it completes${held.lastError !== undefined ? ` (last attempt: ${held.lastError})` : ""}. NEXT: wait a moment and retry this spawn (retrying re-drives the whole teardown), or pick another name.`,
|
|
@@ -1982,7 +2539,34 @@ export class Manager {
|
|
|
1982
2539
|
}
|
|
1983
2540
|
if (variant && !connector.supportsModelVariant)
|
|
1984
2541
|
return { ok: false, error: `${agent} connector does not support model variants (variant)` };
|
|
1985
|
-
|
|
2542
|
+
// #4 A4 (panel): the roster the allocation consults must reflect the initial presence snapshot,
|
|
2543
|
+
// or a spawn immediately after manager boot races an already-live unmanaged peer and re-opens the
|
|
2544
|
+
// very collision black-hole this closes. Await the snapshot (bounded internally, fail-safe on an
|
|
2545
|
+
// empty mesh) before allocating; the broker/auth remain the authority downstream. Deliberately
|
|
2546
|
+
// unconditional: a half-wired endpoint without the seam must fail loud here, not silently
|
|
2547
|
+
// allocate off a pre-snapshot roster.
|
|
2548
|
+
await this.ep.waitForPresenceSnapshot();
|
|
2549
|
+
// M6 (P2 item 2 spawn-as-action): a HARD-PINNED name — an imperative `--name`/identity override
|
|
2550
|
+
// or a manifest-declared name (opts.resolved) — that collides with a LIVE/provisioning/reserved
|
|
2551
|
+
// incarnation REFUSES loud at accept, BEFORE any reserve/mint/bind (pin 1), never a silent `-2`
|
|
2552
|
+
// suffix (so an address-by-triple caller's pinned name can't be re-pointed). A PERSONA-DERIVED
|
|
2553
|
+
// base name (no pin) keeps uniqueName's collision numbering, so multi-peer `spawn reviewer` twice
|
|
2554
|
+
// still yields reviewer + reviewer-2. The retiring-hold refuse (~2472) is orthogonal and already fired.
|
|
2555
|
+
const hardPinned = opts.identity !== undefined || opts.resolved !== undefined;
|
|
2556
|
+
let name;
|
|
2557
|
+
if (hardPinned) {
|
|
2558
|
+
// The collision check consults THE SAME liveness source uniqueName uses ({@link nameInUse}:
|
|
2559
|
+
// this manager's agents/reserved/retiring PLUS the roster-live set) - a hard-pinned name
|
|
2560
|
+
// colliding with ANY live incarnation (managed, unmanaged foreground/connector, or another
|
|
2561
|
+
// manager's agent) refuses cleanly at accept, rather than minting the collision and black-
|
|
2562
|
+
// holing on the broker/auth refusal (item 3: a pinned name live under another manager MUST refuse).
|
|
2563
|
+
if (this.nameInUse(identityName))
|
|
2564
|
+
return { ok: false, error: `the name "${identityName}" is hard-pinned (${opts.resolved ? "manifest-declared" : "--name/identity override"}) but is already held by a live incarnation (managed here, an unmanaged foreground/connector session, or another manager's agent); a pinned same-name collision refuses at accept - pick another name or despawn the existing one` };
|
|
2565
|
+
name = identityName;
|
|
2566
|
+
}
|
|
2567
|
+
else {
|
|
2568
|
+
name = this.uniqueName(identityName);
|
|
2569
|
+
}
|
|
1986
2570
|
this.reserved.add(name);
|
|
1987
2571
|
// Transcript mirroring (opt-in: `--transcript` / COTAL_TRANSCRIPT_DEFAULT=1) → grant the agent pub
|
|
1988
2572
|
// on its OWN transcript channel; auth-mode publish is default-deny, so without the grant the mirror's
|
|
@@ -1999,6 +2583,18 @@ export class Manager {
|
|
|
1999
2583
|
}
|
|
2000
2584
|
allowPublish = [...(allowPublish ?? []), connector.transcriptChannel(name)];
|
|
2001
2585
|
}
|
|
2586
|
+
// F2 (Unit B): a STATIC managed spawn REFUSES endpoint capabilities, fail-closed IN CODE (not
|
|
2587
|
+
// a doc note): the static terminal has no obligation-drain/frontier steps yet, so an accepted-
|
|
2588
|
+
// but-uncompleted endpoint obligation could execute AFTER its uid is declared retired. The
|
|
2589
|
+
// refusal sits at spawn-accept, before any provisioning, over the same records a persona or
|
|
2590
|
+
// manifest self-claim would ride in on — capabilities cannot slip past it into the grant path.
|
|
2591
|
+
if (this.auth && !this.userMode) {
|
|
2592
|
+
const claims = [opts, (opts.resolved ?? {})];
|
|
2593
|
+
if (claims.some((c) => c.endpointCapabilities !== undefined)) {
|
|
2594
|
+
this.reserved.delete(name);
|
|
2595
|
+
return { ok: false, error: "a static managed spawn refuses endpointCapabilities (Unit B F2): the static lifecycle terminal carries no obligation-drain/frontier steps, so endpoint-rail grants are not containable in static mode" };
|
|
2596
|
+
}
|
|
2597
|
+
}
|
|
2002
2598
|
// Set once the agent's creds + durables are minted; cleared the moment a live slot takes ownership
|
|
2003
2599
|
// (`agents.set`, after which freeSlot deprovisions on exit). If it survives to `finally`, the spawn
|
|
2004
2600
|
// threw AFTER minting (buildLaunch / runtime.spawn) — tear the orphan down so no footprint leaks (#159 B).
|
|
@@ -2016,6 +2612,19 @@ export class Manager {
|
|
|
2016
2612
|
// broker resource (dm_/dlv_/chathist_ durables, ACL row, memberships) and the teardown
|
|
2017
2613
|
// credential carry it, so a same-name successor's footprint is name-disjoint by construction.
|
|
2018
2614
|
const lifecycleUid = mintLifecycleUid();
|
|
2615
|
+
// ACCEPT SEAM (P2 item 2 spawn-as-action): the incarnation identity is minted and NOTHING has
|
|
2616
|
+
// been provisioned yet — the action serve path binds the goal + replies the acceptance HERE. A
|
|
2617
|
+
// throw (bind conflict / duplicate goalId) aborts the spawn before provisioning: the catch below
|
|
2618
|
+
// returns the failure and the finally releases the reserve, so a refused accept leaves zero
|
|
2619
|
+
// footprint (pin 1). Blocking callers (roster boot) pass no hooks and this is a no-op.
|
|
2620
|
+
// The ALLOCATED agent's addressing triple (the acceptance floor names what was actually
|
|
2621
|
+
// allocated, never the requested-but-unallocated name). Static/open key on DEV_OWNER + the
|
|
2622
|
+
// freshly-minted nkey; user mode keys on the derived owner (opts.owner, else a u_-owner spawner)
|
|
2623
|
+
// + the alias — derived HERE where the mode and owner source are in scope.
|
|
2624
|
+
const agentTriple = this.userMode
|
|
2625
|
+
? { owner: opts.owner ?? (spawner && parsePrincipalKey(spawner)?.owner.startsWith("u_") ? parsePrincipalKey(spawner).owner : DEV_OWNER), actor: name, uid: lifecycleUid }
|
|
2626
|
+
: { owner: DEV_OWNER, actor: identity.id, uid: lifecycleUid };
|
|
2627
|
+
await hooks?.onAccepted?.({ name, identity, lifecycleUid, agentTriple });
|
|
2019
2628
|
// In auth mode, mint the agent's creds from the space signing key and write them where the
|
|
2020
2629
|
// spawned session reads them (COTAL_CREDS path). Open mesh → no creds. Scope = the resolved
|
|
2021
2630
|
// subscribe/allowSubscribe (read) + allowPublish (post, default-deny).
|
|
@@ -2040,13 +2649,25 @@ export class Manager {
|
|
|
2040
2649
|
}
|
|
2041
2650
|
userLaunch = prep.launch;
|
|
2042
2651
|
userOwner = prep.owner;
|
|
2043
|
-
provisioned = { id: principalKey(prep.owner, name).key, name, lifecycleUid, userOwner: prep.owner };
|
|
2652
|
+
provisioned = { id: principalKey(prep.owner, name).key, name, lifecycleUid, userOwner: prep.owner, secretPaths: prep.files };
|
|
2044
2653
|
}
|
|
2045
2654
|
else if (this.auth) {
|
|
2655
|
+
// Unit B (§13.1): reserve + activate this incarnation's DURABLE identity BEFORE any
|
|
2656
|
+
// broker footprint — the F3 outer spawn intent first (slot row, phase `provisioning`),
|
|
2657
|
+
// then the SHARED core activation saga (reserve uid -> gate frozen -> head CAS -> reopen
|
|
2658
|
+
// LAST) over the key-pinned executor. The wire AUTHORITY principal is the incarnation-
|
|
2659
|
+
// unique nkey (F5-bind); the alias is protected by the name-keyed slot + freeSlot hold.
|
|
2660
|
+
await this.withLifecycleExecutor({ owner: DEV_OWNER, actor: identity.id, lifecycleUid, alias: name }, (t) => activateStaticLifecycle(t, { owner: DEV_OWNER, alias: name, actor: identity.id, lifecycleUid, managerInstance: this.managerLifecycleUid, ownerInstanceId: this.managerInstanceId }));
|
|
2661
|
+
// From here the DURABLE registration exists: arm the rollback BEFORE minting, so a throw
|
|
2662
|
+
// between activation and provisioning still drives the exact-op static terminal (the
|
|
2663
|
+
// finally's deprovision tolerates absent files; the broker teardown is idempotent).
|
|
2664
|
+
provisioned = { id: identity.id, name, lifecycleUid };
|
|
2046
2665
|
// Pre-create the agent's bind-only chat (+ DM + role TASK) durables and mint its scoped creds
|
|
2047
2666
|
// — the shared onboarding step (provisionAgent). It runs on a short-lived PROVISIONER connection
|
|
2048
2667
|
// (NOT the supervisor's long-lived endpoint), so the DM/DLV consumer-create surface exists only
|
|
2049
2668
|
// for the provisioning window, never as a standing grant on the always-on daemon (residual 2).
|
|
2669
|
+
// F5(b): the credential is BOUNDED (`expiresAt`) — the manager push-renews it ahead of expiry.
|
|
2670
|
+
const exp = Math.floor(Date.now() / 1000) + MANAGED_STATIC_TTL_SEC;
|
|
2050
2671
|
const creds = await this.withProvisioner((prov) => provisionAgent(prov, this.auth, identity, {
|
|
2051
2672
|
subscribe,
|
|
2052
2673
|
allowSubscribe,
|
|
@@ -2054,15 +2675,26 @@ export class Manager {
|
|
|
2054
2675
|
role,
|
|
2055
2676
|
capabilities,
|
|
2056
2677
|
lifecycleUid,
|
|
2678
|
+
expiresAt: exp,
|
|
2057
2679
|
}));
|
|
2680
|
+
// Ledger BEFORE materialization (§13.1): record the credentialId on the slot, append the
|
|
2681
|
+
// `cred.<uid>.<credId>` row, and only then write the credential where anything can read
|
|
2682
|
+
// it — a credential is never materialized before its ledger row exists.
|
|
2683
|
+
const credentialId = rawDigest(creds).replace("sha256:", "sha256-");
|
|
2684
|
+
await this.withLifecycleExecutor({ owner: DEV_OWNER, actor: identity.id, lifecycleUid, alias: name }, async (t) => {
|
|
2685
|
+
await recordSlotCredential(t, DEV_OWNER, name, lifecycleUid, credentialId);
|
|
2686
|
+
await appendStaticCredentialRow(t, { lifecycleUid, credentialId, holderPrincipal: principalKey(DEV_OWNER, identity.id).key, exp });
|
|
2687
|
+
});
|
|
2058
2688
|
// Store first (the source of truth), then materialize: `buildLaunch` hands the CHILD this
|
|
2059
2689
|
// file path, so the cred must exist as a file regardless of the store behind the seam. The
|
|
2060
2690
|
// manager's ONE store (injected for hosted, workstation FS locally).
|
|
2061
2691
|
const secrets = this.secrets;
|
|
2062
|
-
|
|
2063
|
-
|
|
2064
|
-
|
|
2065
|
-
|
|
2692
|
+
// LIFECYCLE-KEYED (SPEC 13.1 on the FS): the incarnation's cred file embeds its uid, so a
|
|
2693
|
+
// replayed/stale teardown can never address a same-name successor's credential.
|
|
2694
|
+
credsPath = agentLifecycleSecretFilePaths(this.workspaceRoot, name, lifecycleUid).creds;
|
|
2695
|
+
await secrets.put(agentSecretKeyForFile(credsPath), creds);
|
|
2696
|
+
await materializeSecretToFile(secrets, agentSecretKeyForFile(credsPath), credsPath);
|
|
2697
|
+
provisioned = { id: identity.id, name, lifecycleUid, secretPaths: { creds: credsPath } }; // footprint now exists — the finally rolls it back if the spawn throws
|
|
2066
2698
|
}
|
|
2067
2699
|
// Personal MCP servers the operator opted to share with manager-spawned agents of this type
|
|
2068
2700
|
// (cotal config; default none → isolated, the memory-safe default this guards), narrowed by
|
|
@@ -2116,12 +2748,16 @@ export class Manager {
|
|
|
2116
2748
|
workspaceRoot: this.workspaceRoot,
|
|
2117
2749
|
});
|
|
2118
2750
|
const handle = this.runtime.spawn(name, spec, cwd);
|
|
2751
|
+
hooks?.onLaunched?.(); // P2 item 2: the "launched" progress edge (process spawned, pre-presence)
|
|
2119
2752
|
const managed = {
|
|
2120
2753
|
name,
|
|
2121
2754
|
role,
|
|
2122
2755
|
agent,
|
|
2123
2756
|
id: userLaunch ? principalKey(userLaunch.owner, name).key : identity.id,
|
|
2124
2757
|
lifecycleUid,
|
|
2758
|
+
// The lifecycle-keyed family this spawn just materialized (absent on an open mesh) — the
|
|
2759
|
+
// recorded truth teardown/preservation/health consume, never re-derived by name.
|
|
2760
|
+
secretPaths: provisioned?.secretPaths,
|
|
2125
2761
|
...(userLaunch ? { userOwner } : { seed: identity.seed }),
|
|
2126
2762
|
spawner: spawner ?? this.ep.ref().id,
|
|
2127
2763
|
authorityParent: userLaunch && spawner && parsePrincipalKey(spawner) ? spawner : undefined,
|
|
@@ -2157,6 +2793,18 @@ export class Manager {
|
|
|
2157
2793
|
: undefined,
|
|
2158
2794
|
},
|
|
2159
2795
|
};
|
|
2796
|
+
// Unit B: the DURABLE slot takes the `active` phase before the in-memory row takes the
|
|
2797
|
+
// name — a crash between the two leaves an active-but-unadopted slot the boot sweep
|
|
2798
|
+
// terminalizes (never an untracked orphan). Static auth only; a failed CAS fails the spawn
|
|
2799
|
+
// (the finally's rollback then drives the exact-op terminal).
|
|
2800
|
+
if (this.auth && !this.userMode) {
|
|
2801
|
+
await this.withLifecycleExecutor({ owner: DEV_OWNER, actor: managed.id, lifecycleUid, alias: name }, async (t) => {
|
|
2802
|
+
const slot = await readStaticSlot(t, DEV_OWNER, name);
|
|
2803
|
+
if (slot === undefined || slot.row.lifecycleUid !== lifecycleUid || slot.row.phase !== "provisioning")
|
|
2804
|
+
throw new Error(`the static slot for "${name}" is ${slot === undefined ? "absent" : `${slot.row.phase} at uid ${slot.row.lifecycleUid}`}, not this spawn's provisioning intent; refusing to take the slot`);
|
|
2805
|
+
await casStaticSlot(t, { ...slot.row, phase: "active" }, slot.revision);
|
|
2806
|
+
});
|
|
2807
|
+
}
|
|
2160
2808
|
this.agents.set(name, managed);
|
|
2161
2809
|
// The live slot now owns teardown — freeSlot deprovisions this identity on exit — so the
|
|
2162
2810
|
// orphan-rollback in `finally` no longer applies to it.
|
|
@@ -2166,17 +2814,36 @@ export class Manager {
|
|
|
2166
2814
|
// neither in time → uncertain. `✓ started` therefore means "it joined", never just "a process
|
|
2167
2815
|
// launched".
|
|
2168
2816
|
const readiness = await this.awaitReadiness(managed);
|
|
2169
|
-
|
|
2170
|
-
|
|
2817
|
+
// Deliberately stopped mid-launch: reaped by onExit, and the despawn/stop path owns the
|
|
2818
|
+
// goal terminal. Return BEFORE the failed/uncertain arms so this emits no competing
|
|
2819
|
+
// outcome and does not re-arm an exit watcher on an agent already gone.
|
|
2820
|
+
if (!readiness.ok && readiness.deliberate) {
|
|
2821
|
+
hooks?.onTerminalDeferred?.();
|
|
2822
|
+
return { ok: false, error: readiness.detail };
|
|
2823
|
+
}
|
|
2824
|
+
if (!readiness.ok && !readiness.uncertain) {
|
|
2825
|
+
await hooks?.onOutcome?.({ kind: "failed", data: { error: readiness.detail } });
|
|
2826
|
+
return { ok: false, error: readiness.detail };
|
|
2827
|
+
} // failed → already reaped
|
|
2171
2828
|
// Started OR uncertain: the agent stays managed, so wire the ongoing exit reaper (it reaps a later
|
|
2172
2829
|
// death — including one that follows an `uncertain` verdict, which deliberately does NOT deprovision).
|
|
2173
2830
|
this.watchExit(managed);
|
|
2174
|
-
if (!readiness.ok)
|
|
2175
|
-
|
|
2831
|
+
if (!readiness.ok) {
|
|
2832
|
+
await hooks?.onOutcome?.({ kind: "uncertain" });
|
|
2833
|
+
return { ok: false, error: readiness.detail };
|
|
2834
|
+
} // uncertain — non-success, but kept
|
|
2176
2835
|
// Reply with the id the slot actually carries (user-mode: the owner.actor principal —
|
|
2177
2836
|
// presence, ps, and the manifest ownership ledger all key on it; the throwaway static nkey
|
|
2178
2837
|
// would never match and down -f would treat the agent as foreign).
|
|
2179
|
-
|
|
2838
|
+
// `lifecycleUid` rides the reply so callers that record this spawn (the manifest ledger) can
|
|
2839
|
+
// later address the incarnation's lifecycle-keyed artifacts without re-deriving by name.
|
|
2840
|
+
// OMIT an absent role: the goal terminal commits this data through the strict
|
|
2841
|
+
// canonicalJson (undefined never coerces to null, SPEC 13.6), so a role-less spawn would
|
|
2842
|
+
// otherwise fail its succeeded terminal. The CLI/connector already render an absent role as
|
|
2843
|
+
// "no role", so dropping the key preserves the reply (P2 item 2, surfaced by readiness:live).
|
|
2844
|
+
const okData = { name, agent, id: managed.id, mode: handle.kind, lifecycleUid, ...(role !== undefined ? { role } : {}) };
|
|
2845
|
+
await hooks?.onOutcome?.({ kind: "succeeded", data: okData });
|
|
2846
|
+
return { ok: true, data: okData };
|
|
2180
2847
|
}
|
|
2181
2848
|
catch (e) {
|
|
2182
2849
|
// Failure after reserve (provision / launch threw): the slot was never live, so no cold-start
|
|
@@ -2287,9 +2954,16 @@ export class Manager {
|
|
|
2287
2954
|
if (entry.identity.mode === "static") {
|
|
2288
2955
|
if (!this.auth || this.userMode)
|
|
2289
2956
|
throw new Error(`retained agent ${entry.name} is static-auth but the current manager is not`);
|
|
2290
|
-
|
|
2291
|
-
|
|
2292
|
-
|
|
2957
|
+
// CLOSED candidate set, not an open path: the lifecycle-keyed derivation (this generation's
|
|
2958
|
+
// layout) or the name-keyed one (a pre-split inventory being carried across the upgrade).
|
|
2959
|
+
// Anything else is a foreign path and refused exactly as before.
|
|
2960
|
+
const candidates = [
|
|
2961
|
+
resolve(agentLifecycleSecretFilePaths(this.workspaceRoot, entry.name, entry.identity.lifecycleUid).creds),
|
|
2962
|
+
resolve(agentSecretFilePaths(this.workspaceRoot, entry.name).creds),
|
|
2963
|
+
];
|
|
2964
|
+
const expected = resolve(entry.identity.credential.path);
|
|
2965
|
+
if (!candidates.includes(expected))
|
|
2966
|
+
throw new Error(`retained credential reference for ${entry.name} is not a manager-owned path (expected ${candidates.join(" or ")})`);
|
|
2293
2967
|
let credentialText;
|
|
2294
2968
|
try {
|
|
2295
2969
|
// The lstat guards the FS MATERIALIZATION the child will read at launch; the identity check
|
|
@@ -2298,7 +2972,7 @@ export class Manager {
|
|
|
2298
2972
|
const st = lstatSync(expected);
|
|
2299
2973
|
if (!st.isFile() || st.isSymbolicLink())
|
|
2300
2974
|
throw new Error("not a regular non-symlink file");
|
|
2301
|
-
const stored = await this.secrets.get(
|
|
2975
|
+
const stored = await this.secrets.get(agentSecretKeyForFile(expected));
|
|
2302
2976
|
if (stored === undefined)
|
|
2303
2977
|
throw new Error("the credential is not in the secret store");
|
|
2304
2978
|
credentialText = stored;
|
|
@@ -2318,16 +2992,26 @@ export class Manager {
|
|
|
2318
2992
|
throw new Error(`retained agent ${entry.name} is user-auth but the current manager is not`);
|
|
2319
2993
|
try {
|
|
2320
2994
|
const provider = resolveAuthProvider();
|
|
2321
|
-
// Mirror the static branch's expected-path
|
|
2322
|
-
//
|
|
2323
|
-
//
|
|
2324
|
-
|
|
2325
|
-
|
|
2326
|
-
|
|
2327
|
-
|
|
2995
|
+
// Mirror the static branch's expected-path discipline, but pin the WHOLE secret FAMILY as ONE
|
|
2996
|
+
// unit: all three of {actorToken, sentinelCreds, health} must equal the lifecycle-keyed triple
|
|
2997
|
+
// (this generation) OR the name-keyed triple (a pre-split inventory carried across the upgrade).
|
|
2998
|
+
// A per-file OR-pin let a corrupt inventory MIX families (lifecycle token + legacy sentinel)
|
|
2999
|
+
// and, worse, left `health` UNPINNED entirely — an arbitrary recorded health path flowed into
|
|
3000
|
+
// the bearer argv and was `rmSync`'d at terminal teardown (inventory-as-delete-gadget). Pinning
|
|
3001
|
+
// the atomic family closes both: `health` is pinned by PATH EQUALITY (never by file existence,
|
|
3002
|
+
// so a transiently-absent health file still validates), and the store reads below key off the
|
|
3003
|
+
// RECORDED path, so a foreign path can neither pass the pin nor address a different row.
|
|
3004
|
+
const lifecycleFiles = agentLifecycleSecretFilePaths(this.workspaceRoot, entry.name, entry.identity.lifecycleUid);
|
|
3005
|
+
const legacyFiles = agentSecretFilePaths(this.workspaceRoot, entry.name);
|
|
3006
|
+
const recordedToken = resolve(entry.identity.actorToken.path);
|
|
3007
|
+
const recordedSentinel = resolve(entry.identity.sentinelCredential.path);
|
|
3008
|
+
const recordedHealth = resolve(entry.identity.health.path);
|
|
3009
|
+
const matchesFamily = (f) => recordedToken === resolve(f.actorToken) && recordedSentinel === resolve(f.sentinelCreds) && recordedHealth === resolve(f.health);
|
|
3010
|
+
if (!matchesFamily(lifecycleFiles) && !matchesFamily(legacyFiles))
|
|
3011
|
+
throw new Error(`retained identity references for ${entry.name} are not one manager-owned secret family: all of actor-token, sentinel, and health must be the lifecycle-<uid> triple or the legacy name-keyed triple under ${agentCredsDir(this.workspaceRoot)} (no mixed families, no foreign health path)`);
|
|
2328
3012
|
const secrets = this.secrets;
|
|
2329
|
-
const actorToken = await secrets.get(
|
|
2330
|
-
const sentinelCreds = await secrets.get(
|
|
3013
|
+
const actorToken = await secrets.get(agentSecretKeyForFile(recordedToken));
|
|
3014
|
+
const sentinelCreds = await secrets.get(agentSecretKeyForFile(recordedSentinel));
|
|
2331
3015
|
if (actorToken === undefined || sentinelCreds === undefined)
|
|
2332
3016
|
throw new Error("the retained actor token / sentinel credential is not in the secret store");
|
|
2333
3017
|
const adopted = await provider.validateRetainedAgent({
|
|
@@ -2414,7 +3098,14 @@ export class Manager {
|
|
|
2414
3098
|
}
|
|
2415
3099
|
let connector;
|
|
2416
3100
|
try {
|
|
2417
|
-
|
|
3101
|
+
// The SAME resolver the spawn path uses. A bare `registry.resolve` here made preserve→resume
|
|
3102
|
+
// fail for EVERY retained agent on the published binary: the resuming manager is a fresh
|
|
3103
|
+
// process whose registry is empty (its supervise child runs with the connector seed
|
|
3104
|
+
// skipped), so nothing is registered until something materializes it from the ext manifest.
|
|
3105
|
+
// That is precisely what `resolveConnector` does and what every spawn path already calls.
|
|
3106
|
+
// Resolving bare meant a preserved mesh could never come back — first-party connectors
|
|
3107
|
+
// included, since the asymmetry is about materialization, not about which connector it is.
|
|
3108
|
+
connector = await this.resolveConnector(entry.launch.connector);
|
|
2418
3109
|
}
|
|
2419
3110
|
catch (e) {
|
|
2420
3111
|
return { ok: false, error: e.message };
|
|
@@ -2505,14 +3196,33 @@ export class Manager {
|
|
|
2505
3196
|
if (!batchReserved)
|
|
2506
3197
|
this.reserved.add(entry.name);
|
|
2507
3198
|
try {
|
|
3199
|
+
// Unit B F5(b): recover the STATIC identity's nkey seed from the adopted credential (the
|
|
3200
|
+
// creds file embeds it) so the manager stays this incarnation's RENEWAL OWNER across a
|
|
3201
|
+
// preserve/resume — without it the adopted cred would die loud at its TTL with no remint.
|
|
3202
|
+
let adoptedSeed;
|
|
3203
|
+
if (entry.identity.mode === "static") {
|
|
3204
|
+
const stored = await this.secrets.get(agentSecretKeyForFile(resolve(entry.identity.credential.path)));
|
|
3205
|
+
adoptedSeed = stored === undefined ? undefined : /-----BEGIN USER NKEY SEED-----\s*([A-Z0-9]+)\s*-----END USER NKEY SEED-----/.exec(stored)?.[1];
|
|
3206
|
+
if (adoptedSeed === undefined)
|
|
3207
|
+
console.error(`! resume ${entry.name}: the adopted credential carries no readable nkey seed - the manager cannot renew it (it dies loud at its exp)`);
|
|
3208
|
+
}
|
|
2508
3209
|
const handle = this.runtime.spawn(entry.name, prepared.spec, entry.launch.cwd);
|
|
2509
3210
|
const managed = {
|
|
2510
3211
|
name: entry.name,
|
|
2511
3212
|
role: entry.role,
|
|
2512
3213
|
agent: entry.launch.connector,
|
|
2513
3214
|
id: entry.identity.mode === "user" ? principalKey(entry.identity.owner, entry.identity.actor).key : entry.identity.id,
|
|
3215
|
+
seed: adoptedSeed,
|
|
2514
3216
|
// Recover the ORIGINAL incarnation uid the durables are keyed by (never a fresh mint on resume).
|
|
2515
3217
|
lifecycleUid: entry.identity.lifecycleUid,
|
|
3218
|
+
// Adopt the INVENTORY's recorded family (possibly a pre-split name-keyed layout) — the
|
|
3219
|
+
// validated paths above, so this incarnation's later teardown addresses exactly what its
|
|
3220
|
+
// spawn materialized, never a re-derivation.
|
|
3221
|
+
secretPaths: entry.identity.mode === "user"
|
|
3222
|
+
? { actorToken: entry.identity.actorToken.path, sentinelCreds: entry.identity.sentinelCredential.path, health: entry.identity.health.path }
|
|
3223
|
+
: entry.identity.mode === "static"
|
|
3224
|
+
? { creds: entry.identity.credential.path }
|
|
3225
|
+
: undefined,
|
|
2516
3226
|
userOwner: entry.identity.mode === "user" ? entry.identity.owner : undefined,
|
|
2517
3227
|
spawner: entry.spawner,
|
|
2518
3228
|
authorityParent: entry.authorityParent,
|
|
@@ -2645,9 +3355,46 @@ export class Manager {
|
|
|
2645
3355
|
if (done || !s)
|
|
2646
3356
|
return;
|
|
2647
3357
|
clearTimeout(timer);
|
|
3358
|
+
// ┌─ DO NOT MOVE THIS READ. Its POSITION is the fix; its value is not. ──────────────────┐
|
|
3359
|
+
// MOVING IT BELOW `onAgentExit` MAKES READINESS STOP REPORTING GENUINE LAUNCH FAILURES.
|
|
3360
|
+
// That is what breaks — not a style regression, a silent loss of every `failed` terminal
|
|
3361
|
+
// for an agent that really did die on launch. `onAgentExit` reaches `freeSlot`, which sets
|
|
3362
|
+
// this SAME latch on its way through (see its "also covers exit/reap paths" comment), so a
|
|
3363
|
+
// read taken after that call is true for EVERY exit, deliberate or not.
|
|
3364
|
+
//
|
|
3365
|
+
// Read HERE — first statement, before the await and before `onAgentExit` — a set latch can
|
|
3366
|
+
// only have been set by someone else, and the only other setter on this path is
|
|
3367
|
+
// `stopHandle`, which latches SYNCHRONOUSLY and then kills with no suspension between. So a
|
|
3368
|
+
// despawn-caused exit is guaranteed observed with the latch UP and a natural exit with it
|
|
3369
|
+
// DOWN: the distinction holds by program order, never by winning a race.
|
|
3370
|
+
//
|
|
3371
|
+
// The same latch, read one function call apart, answers two different questions.
|
|
3372
|
+
//
|
|
3373
|
+
// TO RE-VERIFY (this is the mutation that proves it, and it is the exact regression a tidy
|
|
3374
|
+
// refactor produces): move this capture below `onAgentExit` and run
|
|
3375
|
+
// `pnpm smoke:manager-spawn-action`. M3 `process exit -> failed` must FAIL. Note that M4 —
|
|
3376
|
+
// the case this fix exists for — still PASSES under that mutation, so the suite this fix
|
|
3377
|
+
// was written against cannot catch its own regression. M3 catches it only because it
|
|
3378
|
+
// happens to share a file.
|
|
3379
|
+
// └──────────────────────────────────────────────────────────────────────────────────────┘
|
|
3380
|
+
const deliberate = a.terminalizing === true;
|
|
2648
3381
|
void (async () => {
|
|
2649
3382
|
const tail = this.tail(await s.backlog());
|
|
2650
3383
|
this.onAgentExit(a);
|
|
3384
|
+
// A DELIBERATE STOP IS NOT A LAUNCH FAILURE. The despawn path owns this goal's terminal
|
|
3385
|
+
// and commits `cancel`; reporting `failed` here races it and, when it wins, tells the
|
|
3386
|
+
// caller the agent died on launch when in fact an operator cancelled it. The process
|
|
3387
|
+
// teardown above still runs — only the goal's OUTCOME is left to the path that caused it.
|
|
3388
|
+
// A deliberate stop still has to SETTLE this promise. `clearTimeout` above already
|
|
3389
|
+
// removed the only other resolver, so returning here leaves it pending forever and the
|
|
3390
|
+
// spawn's lifecycle ticket is never released — which permanently wedges every drain
|
|
3391
|
+
// (preparePreservation, and through it a preserving `down`). Settle it as its own
|
|
3392
|
+
// variant: not `failed` and not `uncertain`, so the caller emits NO terminal and the
|
|
3393
|
+
// despawn path keeps sole ownership of this goal's `cancel`.
|
|
3394
|
+
if (deliberate) {
|
|
3395
|
+
finish({ ok: false, deliberate: true, detail: `${a.name} was stopped before it reported ready` });
|
|
3396
|
+
return;
|
|
3397
|
+
}
|
|
2651
3398
|
finish({ ok: false, detail: `${a.name} exited on launch${tail ? ` - last output: ${tail}` : ""}` });
|
|
2652
3399
|
})();
|
|
2653
3400
|
};
|
|
@@ -2712,19 +3459,1221 @@ export class Manager {
|
|
|
2712
3459
|
const a = this.agents.get(name);
|
|
2713
3460
|
if (!a)
|
|
2714
3461
|
return { ok: false, error: `no agent "${name}"` };
|
|
3462
|
+
return this.despawnCore(a, caller, admin, args.graceful !== false);
|
|
3463
|
+
}
|
|
3464
|
+
/** The ONE named-terminal core both doors share (P2 item 1, checklist 8): the ctl named `stop`
|
|
3465
|
+
* and the v0.4 targeted `despawn` are the same terminal — authorize by the shared policy
|
|
3466
|
+
* ({@link authorizeNamed}: own-child / owner-domain on privileged, any on admin), stop, track.
|
|
3467
|
+
* The ep door runs the SAME two pieces separately so a policy denial surfaces as the §13.3
|
|
3468
|
+
* `permission-denied` (never a generic failure). */
|
|
3469
|
+
async despawnCore(a, caller, admin, graceful) {
|
|
2715
3470
|
const denied = await this.authorizeNamed(a, caller, admin);
|
|
2716
3471
|
if (denied)
|
|
2717
3472
|
return { ok: false, error: denied };
|
|
2718
|
-
|
|
3473
|
+
return this.despawnAuthorized(a, graceful, !admin);
|
|
3474
|
+
}
|
|
3475
|
+
/** The post-authorization terminal effect (both doors). `trackNonAdmin` mirrors the ctl door's
|
|
3476
|
+
* `trackStoppedHandle(a, !admin)` disposition. */
|
|
3477
|
+
despawnAuthorized(a, graceful, trackNonAdmin) {
|
|
2719
3478
|
this.stopHandle(a, graceful);
|
|
2720
|
-
this.trackStoppedHandle(a,
|
|
2721
|
-
|
|
3479
|
+
this.trackStoppedHandle(a, trackNonAdmin);
|
|
3480
|
+
void this.cancelAgentGoal(a.name, graceful ? "graceful" : "terminate"); // M4: cancel a live spawn goal
|
|
3481
|
+
return { ok: true, data: { name: a.name, stopped: true, graceful } };
|
|
3482
|
+
}
|
|
3483
|
+
/** Resolve a v0.4 TARGET triple (owner, actor, lifecycleUid — broker-validated subject/body
|
|
3484
|
+
* agreement, currency re-checked by the serve boundary's resolver) to the live managed agent it
|
|
3485
|
+
* names. Static agents key `(DEV_OWNER, nkey)`; user-mode agents store the principal dot-form
|
|
3486
|
+
* in `id`. A uid mismatch is a superseded incarnation — never resolved to its successor. */
|
|
3487
|
+
findManagedByTarget(t) {
|
|
3488
|
+
for (const a of this.agents.values()) {
|
|
3489
|
+
const matches = a.userOwner ? a.id === principalKey(t.owner, t.actor).key : t.owner === DEV_OWNER && a.id === t.actor;
|
|
3490
|
+
if (matches && a.lifecycleUid === t.lifecycleUid)
|
|
3491
|
+
return a;
|
|
3492
|
+
}
|
|
3493
|
+
return undefined;
|
|
2722
3494
|
}
|
|
2723
3495
|
/** Open a short-lived PROVISIONER connection, run the onboarding ops on it, and drain it (closure (ii),
|
|
2724
3496
|
* residual 2). The DM/DLV consumer-create surface — the irreducible onboarding power — lives only for
|
|
2725
3497
|
* this window, never as a standing grant on the long-lived supervisor. A provision-only endpoint
|
|
2726
3498
|
* (no presence/consume/channel-watch) connected with memory-only `provisioner` creds; it sets its own
|
|
2727
3499
|
* `inboxPrefix` so JS-API replies land on the `_INBOX_<id>.>` the provisioner cred subscribes. */
|
|
3500
|
+
/** Run one static §13.1 lifecycle OPERATION over an ephemeral, key-pinned `lifecycle-executor`
|
|
3501
|
+
* connection (Unit B): the credential's grants name exactly ONE incarnation's head/uid/gate/
|
|
3502
|
+
* cred-family/slot keys, so the write authority exists only for this operation's window and
|
|
3503
|
+
* can move nothing else. The transport is the direct-KV binding the shared core saga drives. */
|
|
3504
|
+
async withLifecycleExecutor(pin, fn) {
|
|
3505
|
+
if (!this.auth)
|
|
3506
|
+
throw new Error("withLifecycleExecutor: no space auth (an open mesh has no lifecycle registry)");
|
|
3507
|
+
const identity = newIdentity();
|
|
3508
|
+
const creds = await mintCreds(this.auth, identity, "lifecycle-executor", {
|
|
3509
|
+
lifecycleExecutor: { owner: pin.owner, actor: pin.actor, lifecycleUid: pin.lifecycleUid, alias: pin.alias },
|
|
3510
|
+
});
|
|
3511
|
+
const nc = await connect({ servers: this.servers ?? DEFAULT_SERVER, ...standaloneConnectOpts({ creds, /* not yet wired to a recorded transport */ tls: false }), maxReconnectAttempts: 0 });
|
|
3512
|
+
try {
|
|
3513
|
+
const kvm = new Kvm(nc);
|
|
3514
|
+
const recordsKv = await kvm.open(recordsBucket(this.space));
|
|
3515
|
+
const authKv = await kvm.open(epAuthBucket(this.space));
|
|
3516
|
+
return await fn(staticLifecycleTransport(recordsKv, authKv));
|
|
3517
|
+
}
|
|
3518
|
+
finally {
|
|
3519
|
+
await nc.drain().catch(() => nc.close());
|
|
3520
|
+
}
|
|
3521
|
+
}
|
|
3522
|
+
/** Run one §13.1 ENDPOINT-SERVE credential operation (P2 item 1, 1a-serve) over an ephemeral,
|
|
3523
|
+
* key-pinned `endpoint-serve-executor` connection: the credential's grants name exactly the
|
|
3524
|
+
* manager instance's `epgate`/`epcred` keys plus its registration's two records keys, so the
|
|
3525
|
+
* gate CAS, the mint fence, and the spec/governance writes ride a one-shot scoped authority —
|
|
3526
|
+
* NEVER the manager's standing seed/supervisor connection (the panel's "no seed shortcut"). */
|
|
3527
|
+
async withEndpointServeExecutor(fn) {
|
|
3528
|
+
if (!this.auth)
|
|
3529
|
+
throw new Error("withEndpointServeExecutor: no space auth (an open mesh has no service registry)");
|
|
3530
|
+
const identity = newIdentity();
|
|
3531
|
+
const creds = await mintCreds(this.auth, identity, "endpoint-serve-executor", {
|
|
3532
|
+
endpointServeExecutor: { endpoint: MANAGER_ENDPOINT, instanceId: this.managerInstanceId },
|
|
3533
|
+
});
|
|
3534
|
+
const nc = await connect({ servers: this.servers ?? DEFAULT_SERVER, ...standaloneConnectOpts({ creds, /* not yet wired to a recorded transport */ tls: false }), maxReconnectAttempts: 0 });
|
|
3535
|
+
try {
|
|
3536
|
+
const kvm = new Kvm(nc);
|
|
3537
|
+
return await fn({ recordsKv: await kvm.open(recordsBucket(this.space)), authKv: await kvm.open(epAuthBucket(this.space)), nc });
|
|
3538
|
+
}
|
|
3539
|
+
finally {
|
|
3540
|
+
await nc.drain().catch(() => nc.close());
|
|
3541
|
+
}
|
|
3542
|
+
}
|
|
3543
|
+
/** 1d open-mesh counterpart of {@link withEndpointServeExecutor}: an OPEN mesh has no
|
|
3544
|
+
* credential system, so there is no scoped executor to mint - the same §13.1 gate/records
|
|
3545
|
+
* writes ride a bare one-shot connection (the broker enforces nothing on an open mesh; the
|
|
3546
|
+
* ceremony still produces the real gate, epoch, and registration the serve rails run on). */
|
|
3547
|
+
async withOpenServeConnection(fn) {
|
|
3548
|
+
if (this.auth)
|
|
3549
|
+
throw new Error("withOpenServeConnection: an auth mesh must use the scoped endpoint-serve executor");
|
|
3550
|
+
const nc = await connect({ servers: this.servers ?? DEFAULT_SERVER, maxReconnectAttempts: 0 });
|
|
3551
|
+
try {
|
|
3552
|
+
const kvm = new Kvm(nc);
|
|
3553
|
+
// An open mesh may be a RAW broker (no `cotal up` provisioning ran), and `Kvm.open` binds
|
|
3554
|
+
// lazily without checking the stream exists — create-or-verify the §13.12 authority stores
|
|
3555
|
+
// first (the same mode-neutral treatment {@link registerManagerService} gives the contract
|
|
3556
|
+
// store), or the first gate write dies "stream not found".
|
|
3557
|
+
await ensureAuthorityStores(await jetstreamManager(nc), kvm, this.space);
|
|
3558
|
+
return await fn({ recordsKv: await kvm.open(recordsBucket(this.space)), authKv: await kvm.open(epAuthBucket(this.space)), nc });
|
|
3559
|
+
}
|
|
3560
|
+
finally {
|
|
3561
|
+
await nc.drain().catch(() => nc.close());
|
|
3562
|
+
}
|
|
3563
|
+
}
|
|
3564
|
+
/** The served manager-level health summary (1a's one read-only command). */
|
|
3565
|
+
managerStatusData() {
|
|
3566
|
+
return {
|
|
3567
|
+
instanceId: this.managerInstanceId,
|
|
3568
|
+
runtime: this.runtime.kind,
|
|
3569
|
+
agentCount: this.agents.size,
|
|
3570
|
+
uptimeMs: Date.now() - this.startedAtMs,
|
|
3571
|
+
};
|
|
3572
|
+
}
|
|
3573
|
+
/** P2 item 1: register the manager as an ordinary v0.4 `service` endpoint and serve its typed
|
|
3574
|
+
* command surface on the ep rails - since 1d the manager's ONLY control door. On an AUTH mesh
|
|
3575
|
+
* the whole credential path is the SAME one an ordinary endpoint traverses (the enforcement
|
|
3576
|
+
* test that keeps "ordinary" honest): provision the §13.1 issuance gate, drive the
|
|
3577
|
+
* registration BARRIER's gate CAS, then release the serve credential only on the mint FENCE's
|
|
3578
|
+
* revision-pinned CAS win — all over the scoped one-shot executor ({@link
|
|
3579
|
+
* withEndpointServeExecutor}), never a seed-signed shortcut. Holding the signing seed only
|
|
3580
|
+
* AUTHORIZES the reserved single-label name (`manager`, operator name authority, DEV_OWNER).
|
|
3581
|
+
* On an OPEN mesh the same gate/registration/serve-grant ceremony runs over bare one-shot
|
|
3582
|
+
* connections and NO credential is ever minted: there is no credential system to issue from,
|
|
3583
|
+
* so the gate legitimately keeps an empty `epcred` family (the §13.1 fence is issuance-only)
|
|
3584
|
+
* and the serve connection is bare — the broker enforces nothing on an open mesh, exactly the
|
|
3585
|
+
* old open-mesh ctl trust ("open = single-trusted-host"). */
|
|
3586
|
+
async registerManagerService() {
|
|
3587
|
+
const auth = this.auth;
|
|
3588
|
+
// The §13.7 contract store is REGISTRATION's dependency, ensured here MODE-NEUTRALLY (1c.2c):
|
|
3589
|
+
// it used to ride the static-only lifecycle reconcile, so a USER-mode manager registered
|
|
3590
|
+
// against an absent stream and its artifact publish died no-responders (live-repro'd). A
|
|
3591
|
+
// provisioner one-shot creates-or-verifies it (config-B immutability incl. the shadowed-legacy
|
|
3592
|
+
// refuse) before the executor publishes a single artifact.
|
|
3593
|
+
{
|
|
3594
|
+
// Open mesh: the bare connection holds the rights (there is no credential system to mint from).
|
|
3595
|
+
const provCreds = auth ? await mintCreds(auth, newIdentity(), "provisioner") : undefined;
|
|
3596
|
+
const provNc = await connect({ servers: this.servers ?? DEFAULT_SERVER, ...standaloneConnectOpts({ creds: provCreds, /* not yet wired to a recorded transport */ tls: false }), maxReconnectAttempts: 0 });
|
|
3597
|
+
try {
|
|
3598
|
+
// P2 item 2: the manager now WRITES goal facts (EPF) + progress events (EPE), so the §13.12
|
|
3599
|
+
// endpoint streams must exist. Nothing provisioned them before spawn-as-action (no endpoint
|
|
3600
|
+
// wrote to EPF/EPE), so the manager ensures the full set here over the provisioner (whose
|
|
3601
|
+
// STREAM.CREATE now covers them), idempotently - createEndpointStreams is a superset of
|
|
3602
|
+
// ensureContractStore + ensureAuthorityStores, fail-loud on drift. Auth + open both run this.
|
|
3603
|
+
await createEndpointStreams(await jetstreamManager(provNc), new Kvm(provNc), this.space);
|
|
3604
|
+
}
|
|
3605
|
+
finally {
|
|
3606
|
+
await provNc.drain().catch(() => provNc.close());
|
|
3607
|
+
}
|
|
3608
|
+
}
|
|
3609
|
+
const iid = this.managerInstanceId;
|
|
3610
|
+
const artifacts = managerClusterArtifacts();
|
|
3611
|
+
// In-memory §13.7 content store: the manager is this document's AUTHOR, so registration and
|
|
3612
|
+
// serve authorization verify against the exact artifacts it publishes from memory. The DURABLE
|
|
3613
|
+
// `epc` contract-store publication (for third-party digest fetches) runs below inside the same
|
|
3614
|
+
// executor, BEFORE the registration that advertises the digests.
|
|
3615
|
+
const store = new Map([
|
|
3616
|
+
[artifacts.rootDigest, artifacts.document],
|
|
3617
|
+
[artifacts.closureDigest, artifacts.manifest],
|
|
3618
|
+
]);
|
|
3619
|
+
const readClusterArtifact = (digest) => store.get(digest);
|
|
3620
|
+
// §13.9 name authority, static mode: `manager` is a core single-label name requiring OPERATOR
|
|
3621
|
+
// authority — the manager holds the space signing seed, so it self-authorizes exactly its own
|
|
3622
|
+
// name for exactly DEV_OWNER (never a general authority; any other (name, owner) refuses).
|
|
3623
|
+
const authority = {
|
|
3624
|
+
authorize: (name, owner) => ({ authorized: name === MANAGER_ENDPOINT && owner === DEV_OWNER, revision: 0 }),
|
|
3625
|
+
};
|
|
3626
|
+
// The STABLE serve identity (P2 item 3): the PERSISTED serve nkey, reused across restart so the
|
|
3627
|
+
// gate binds the SAME principal (§13.1 serving-principal binding) - provisionEndpointGateOpen
|
|
3628
|
+
// stays idempotent and verified eviction has a stable target. Renewals re-mint the same nkey
|
|
3629
|
+
// with a fresh bounded exp; a restart re-provisions the same (idempotent) gate + re-registers.
|
|
3630
|
+
const serveIdentity = this.managerServeIdentity;
|
|
3631
|
+
const servePrincipal = principalKey(DEV_OWNER, serveIdentity.id).key;
|
|
3632
|
+
// must-5 (b): the STABLE goal-writer identity — a SIBLING credential in the same §13.1 family
|
|
3633
|
+
// (not the gate's bound serving principal), minted here so the run block can family-stage it.
|
|
3634
|
+
this.goalWriterIdentity = newIdentity();
|
|
3635
|
+
// P2 item 6: the STABLE session-LEDGER identity — another SIBLING in the SAME §13.1 family, so
|
|
3636
|
+
// the takeover barrier revokes a deposed manager's ledger cred alongside its goal-writer. The
|
|
3637
|
+
// per-session serving creds join the same family, each with its own fresh identity.
|
|
3638
|
+
this.sessionLedgerIdentity = newIdentity();
|
|
3639
|
+
const run = async ({ recordsKv, authKv, nc: execNc }) => {
|
|
3640
|
+
// §13.7 contract-artifact publication (1c): every schema root + its closure manifest, plus
|
|
3641
|
+
// the cluster document + ITS manifest, land in the EPC store BEFORE the registration that
|
|
3642
|
+
// advertises their digests — so a caller can always fetch-verify-compile a registered
|
|
3643
|
+
// digest (the item-5 generic-invoke read path). Create-only + content-addressed: a retry
|
|
3644
|
+
// or a same-artifact republish is an idempotent lost-CAS. The registration itself still
|
|
3645
|
+
// verifies against the in-memory copies (the manager is the author).
|
|
3646
|
+
const storeCtx = await contractStoreContext(execNc, this.space);
|
|
3647
|
+
for (const value of [...managerContractArtifactValues(), artifacts.document, artifacts.manifest])
|
|
3648
|
+
await publishContractArtifact(storeCtx, contractArtifactCanonicalBytes(value));
|
|
3649
|
+
// §13.1 pre-registration (checklist 1): the issuance gate, born open@gen0 bound to the serve
|
|
3650
|
+
// principal — provisioned ONCE, on the FIRST registration. On a RESTART (P2 item 3) the gate
|
|
3651
|
+
// already EXISTS (advanced past gen0 by the prior registration), so re-provisioning the gen0
|
|
3652
|
+
// row would conflict "foreign content"; the persisted instanceId makes this a TAKEOVER, and
|
|
3653
|
+
// registerServiceInstance below freezes + re-registers the EXISTING gate (advancing the epoch
|
|
3654
|
+
// after it verify-evicts the superseded family). The serve principal is the SAME persisted one,
|
|
3655
|
+
// so the gate's principal binding is unchanged either way.
|
|
3656
|
+
if ((await serveIssuanceGateKv(authKv, this.space, { endpoint: MANAGER_ENDPOINT, instanceId: iid }).observe()) === null)
|
|
3657
|
+
await provisionEndpointGateOpen(authKv, { endpoint: MANAGER_ENDPOINT, instanceId: iid, principal: servePrincipal });
|
|
3658
|
+
// P2 item 3 (slice 3a): on an AUTH mesh a RE-registration (restart of the persisted instanceId)
|
|
3659
|
+
// must VERIFY-EVICT the superseded serve family BEFORE the epoch advances (§13.1 "old authority
|
|
3660
|
+
// dies before new authority is visible"). Inject the SCOPED delivery-admin evictor; the OPEN
|
|
3661
|
+
// mesh mints no serve family, so no evictor (the empty-family path never consults it and a
|
|
3662
|
+
// restart there works evictor-free). NO-ORACLE = LOUD: with no reachable delivery daemon the
|
|
3663
|
+
// evictor THROWS naming the cure, so PHASE 2 fails closed with the delivery-daemon fix in the
|
|
3664
|
+
// error text — a crash-restart never silently skips eviction (no-fallbacks).
|
|
3665
|
+
const barrier = endpointRegistrationBarrier(authKv, this.space, {
|
|
3666
|
+
endpoint: MANAGER_ENDPOINT, instanceId: iid, opId: mintLifecycleUid(),
|
|
3667
|
+
...(auth ? { evict: makeManagerEndpointEvictor({ space: this.space, servers: this.servers ?? DEFAULT_SERVER, auth, log: (line) => console.error(line) }) } : {}),
|
|
3668
|
+
});
|
|
3669
|
+
const spec = { endpoint: MANAGER_ENDPOINT, owner: DEV_OWNER, clusterDigests: [artifacts.closureDigest], protocol: { v: 1 } };
|
|
3670
|
+
const { registrationRevision } = await registerServiceInstance(recordsKv, {
|
|
3671
|
+
space: this.space, spec, instanceId: iid, registrant: { owner: DEV_OWNER }, authority, barrier, readClusterArtifact,
|
|
3672
|
+
});
|
|
3673
|
+
// processEpoch comes from the GATE (checklist 4: never derived from the uid string); the
|
|
3674
|
+
// fence below is also the mint's §13.1 release CAS.
|
|
3675
|
+
const fence = serveIssuanceGateKv(authKv, this.space, { endpoint: MANAGER_ENDPOINT, instanceId: iid });
|
|
3676
|
+
const observed = await fence.observe();
|
|
3677
|
+
if (observed === null)
|
|
3678
|
+
throw new Error(`the issuance gate for ${MANAGER_ENDPOINT}/${iid} vanished after registration`);
|
|
3679
|
+
const grant = await authorizeServeGrant(recordsKv, {
|
|
3680
|
+
space: this.space, endpoint: MANAGER_ENDPOINT, instanceId: iid, epoch: observed.processEpoch,
|
|
3681
|
+
holder: { owner: DEV_OWNER }, authority, readClusterArtifact,
|
|
3682
|
+
readProcessEpoch: async () => {
|
|
3683
|
+
const g = await fence.observe();
|
|
3684
|
+
if (g === null)
|
|
3685
|
+
throw new Error(`no issuance gate for ${MANAGER_ENDPOINT}/${iid}`);
|
|
3686
|
+
return g.processEpoch;
|
|
3687
|
+
},
|
|
3688
|
+
});
|
|
3689
|
+
// P2 item 3 (class scatter): write this instance's CONVERGED svc status so it is a §13.5
|
|
3690
|
+
// scatter member — `freezeExpectedSet` skips any instance whose status is absent or lags the
|
|
3691
|
+
// current registration. Instance-side `ready` at the just-registered spec revision, epoch-fenced
|
|
3692
|
+
// to the gate's processEpoch (the same leader-served reader `authorizeServeGrant` used); on a
|
|
3693
|
+
// restart it CAS-updates the predecessor's status forward (the advanced epoch supersedes the old
|
|
3694
|
+
// one). Key-pinned to this instance's own status key on the SAME executor.
|
|
3695
|
+
await writeServiceStatus(recordsKv, {
|
|
3696
|
+
endpoint: MANAGER_ENDPOINT, instanceId: iid, epoch: observed.processEpoch,
|
|
3697
|
+
status: { state: SERVICE_READY, epoch: observed.processEpoch, observedSpecRevision: registrationRevision },
|
|
3698
|
+
readProcessEpoch: async () => {
|
|
3699
|
+
const g = await fence.observe();
|
|
3700
|
+
if (g === null)
|
|
3701
|
+
throw new Error(`no issuance gate for ${MANAGER_ENDPOINT}/${iid}`);
|
|
3702
|
+
return g.processEpoch;
|
|
3703
|
+
},
|
|
3704
|
+
});
|
|
3705
|
+
// Open mesh: NO mint - the §13.1 fence is issuance-only and nothing is ever issued, so the
|
|
3706
|
+
// gate keeps an empty `epcred` family; the serve connection below stays bare.
|
|
3707
|
+
const creds = auth ? await mintCreds(auth, serveIdentity, "endpoint-serve", { serveIssuance: fence, endpointServe: grant }) : undefined;
|
|
3708
|
+
// must-5 (b): mint + family-stage the goal-writer credential HERE, over this executor's
|
|
3709
|
+
// authKv (the fence is live), so its credId lands in `epcred.<e>.<iid>` and the takeover
|
|
3710
|
+
// barrier revokes it. Open mesh: no mint (no credential system; the goal-writer conn is bare).
|
|
3711
|
+
const goalWriterCreds = auth ? await this.mintAndStageGoalWriter(authKv) : undefined;
|
|
3712
|
+
// P2 item 6: mint + family-stage the session-LEDGER credential HERE too (same executor, same
|
|
3713
|
+
// fence, same §13.1 family), so takeover revokes a deposed manager's ledger connection. The
|
|
3714
|
+
// per-session SERVING credentials are minted later, one per redemption, into the SAME family.
|
|
3715
|
+
// Open mesh: no mint (no credential system; the ledger connection stays bare).
|
|
3716
|
+
const sessionLedgerCreds = auth ? await this.mintAndStageSessionLedger(authKv) : undefined;
|
|
3717
|
+
return { grant, creds, goalWriterCreds, sessionLedgerCreds };
|
|
3718
|
+
};
|
|
3719
|
+
const { grant, creds, goalWriterCreds, sessionLedgerCreds } = await (auth ? this.withEndpointServeExecutor(run) : this.withOpenServeConnection(run));
|
|
3720
|
+
this.goalWriterCreds = goalWriterCreds;
|
|
3721
|
+
this.sessionLedgerCreds = sessionLedgerCreds;
|
|
3722
|
+
// The serve connection presents the CURRENT credential on every (re)connect (the state object
|
|
3723
|
+
// is captured by the authenticator), so a fence-traversing renewal is adopted by reconnect
|
|
3724
|
+
// without re-registration. Reconnects stay unbounded: the serve rails are this instance's
|
|
3725
|
+
// registered surface for its whole incarnation.
|
|
3726
|
+
const state = { handle: undefined, nc: undefined, identity: serveIdentity, grant, creds };
|
|
3727
|
+
const enc = new TextEncoder();
|
|
3728
|
+
const nc = await connect({
|
|
3729
|
+
servers: this.servers ?? DEFAULT_SERVER,
|
|
3730
|
+
// Open mesh: a bare serve connection (no credential exists; the broker enforces nothing).
|
|
3731
|
+
...(creds !== undefined ? { authenticator: (nonce) => credsAuthenticator(enc.encode(state.creds))(nonce) } : {}),
|
|
3732
|
+
inboxPrefix: `_INBOX_${serveIdentity.id}`,
|
|
3733
|
+
maxReconnectAttempts: -1,
|
|
3734
|
+
});
|
|
3735
|
+
nc.closed().then((err) => { if (err)
|
|
3736
|
+
console.error(`! manager service endpoint connection closed: ${err.message}`); });
|
|
3737
|
+
try {
|
|
3738
|
+
// The 1b typed surface + the derived `describe`. The descriptor stays PUBLIC in static
|
|
3739
|
+
// mode: the broker grant (who holds each command's request-publish row) is the
|
|
3740
|
+
// load-bearing authority tier, and a static single-operator mesh leaks nothing by listing
|
|
3741
|
+
// command names; the trusted per-caller `view(caller)` scoping joins the user-mode
|
|
3742
|
+
// registration follow-up (where actorScope is the trusted source). Every ordinary handler
|
|
3743
|
+
// runs the SHARED admission chokepoint ({@link serveGated}).
|
|
3744
|
+
state.handle = serveEndpoint(nc, this.space, grant, this.managerServiceDefs(), { public: true }, {
|
|
3745
|
+
// The FRESH target resolver (§13.3) for the targeted commands (`despawn`/`attach`): the
|
|
3746
|
+
// manager's live managed set IS the current-mapping authority for its own agents (the
|
|
3747
|
+
// durable slot rows mirror it). Static mode carries no mapping-revision dimension, so
|
|
3748
|
+
// the revision is the constant 0 — a caller that pins a revision pins 0.
|
|
3749
|
+
resolveTarget: (t) => {
|
|
3750
|
+
if (t.owner === DEV_OWNER) {
|
|
3751
|
+
for (const a of this.agents.values())
|
|
3752
|
+
if (!a.userOwner && a.id === t.actor)
|
|
3753
|
+
return { lifecycleUid: a.lifecycleUid, mappingRevision: 0 };
|
|
3754
|
+
return undefined;
|
|
3755
|
+
}
|
|
3756
|
+
const key = principalKey(t.owner, t.actor).key;
|
|
3757
|
+
for (const a of this.agents.values())
|
|
3758
|
+
if (a.userOwner && a.id === key)
|
|
3759
|
+
return { lifecycleUid: a.lifecycleUid, mappingRevision: 0 };
|
|
3760
|
+
return undefined;
|
|
3761
|
+
},
|
|
3762
|
+
});
|
|
3763
|
+
}
|
|
3764
|
+
catch (e) {
|
|
3765
|
+
await nc.drain().catch(() => nc.close());
|
|
3766
|
+
throw e;
|
|
3767
|
+
}
|
|
3768
|
+
state.nc = nc;
|
|
3769
|
+
this.serviceServe = state;
|
|
3770
|
+
console.error(`manager service endpoint registered: ${MANAGER_ENDPOINT}/${iid} (epoch ${grant.epoch}, registrationRevision ${grant.registrationRevision})`);
|
|
3771
|
+
}
|
|
3772
|
+
/** P2 item 2 must-5 (b): mint the standing `goal-writer` credential and STAGE it into this
|
|
3773
|
+
* instance's §13.1 revocation family (`epcred.<e>.<iid>`), over the passed executor's `authKv`
|
|
3774
|
+
* (the scoped `endpoint-serve-executor`, which holds the epcred write grant). The GRANT profile
|
|
3775
|
+
* stays goal-writer-only (Q2 — disjoint from the serve credential); only the FAMILY membership
|
|
3776
|
+
* is shared, so the registration barrier's existing enumerate+revoke+evict catches the
|
|
3777
|
+
* goal-writer on takeover/retire with NO barrier code change. Used both at registration (the run
|
|
3778
|
+
* block's `authKv`) and at renewal (a fresh executor's), re-minting the SAME stable nkey with a
|
|
3779
|
+
* fresh bounded exp — each issuance writes a DISTINCT ledger row (per-JWT credentialId digest). */
|
|
3780
|
+
async mintAndStageGoalWriter(authKv) {
|
|
3781
|
+
const auth = this.auth;
|
|
3782
|
+
const identity = this.goalWriterIdentity;
|
|
3783
|
+
// The issuance gate + §13.1 revocation family are keyed by the REGISTRATION instanceId
|
|
3784
|
+
// (the persisted logical id, item 3's split), NOT the per-process lifecycleUid — the barrier
|
|
3785
|
+
// enumerates `epcred.<e>.<managerInstanceId>`, so the goal-writer must stage into that family.
|
|
3786
|
+
const iid = this.managerInstanceId;
|
|
3787
|
+
const creds = await mintCreds(auth, identity, "goal-writer", { goalWriter: { endpoint: MANAGER_ENDPOINT } });
|
|
3788
|
+
const exp = inspectCredHealth(creds).exp;
|
|
3789
|
+
if (exp === undefined)
|
|
3790
|
+
throw new Error(`the goal-writer credential for ${MANAGER_ENDPOINT}/${iid} is unbounded; the §13.1 ledger row requires an expiry`);
|
|
3791
|
+
const fence = serveIssuanceGateKv(authKv, this.space, { endpoint: MANAGER_ENDPOINT, instanceId: iid });
|
|
3792
|
+
const observed = await fence.observe();
|
|
3793
|
+
if (observed === null)
|
|
3794
|
+
throw new Error(`the issuance gate for ${MANAGER_ENDPOINT}/${iid} vanished; the goal-writer cannot join its §13.1 revocation family`);
|
|
3795
|
+
// The §13.1 open-and-commit fence, the SAME one the serve credential's own mint runs: a frozen
|
|
3796
|
+
// gate refuses, and a gate that moved under us loses the CAS and revokes the staged row. Without
|
|
3797
|
+
// it a barrier mid-takeover (already past its enumeration) would let this credential land
|
|
3798
|
+
// ACTIVE in a family nothing revokes again.
|
|
3799
|
+
await commitSiblingIssuance(fence, observed, {
|
|
3800
|
+
credentialId: rawDigest(creds).replace("sha256:", "sha256-"),
|
|
3801
|
+
credentialKey: identity.id,
|
|
3802
|
+
holderPrincipal: principalKey(DEV_OWNER, identity.id).key,
|
|
3803
|
+
endpoint: MANAGER_ENDPOINT, lifecycleUid: iid, sourceChain: ["root"], state: "active", exp,
|
|
3804
|
+
generation: observed.generation, processEpoch: observed.processEpoch,
|
|
3805
|
+
registrationRevision: observed.registrationRevision, nameAuthorityRevision: observed.nameAuthorityRevision,
|
|
3806
|
+
});
|
|
3807
|
+
return creds;
|
|
3808
|
+
}
|
|
3809
|
+
/** P2 item 2 (spawn-as-action): stand up the standing self-mediated goal-writer connection +
|
|
3810
|
+
* ActionContext. Mode-dual, mirroring {@link registerManagerService}: an AUTH mesh uses the
|
|
3811
|
+
* scoped `goal-writer` credential already minted + family-STAGED inside registration's run block
|
|
3812
|
+
* ({@link mintAndStageGoalWriter} — DISJOINT grant from the serve cred, SHARED §13.1 revocation
|
|
3813
|
+
* family); an OPEN mesh uses a bare connection (no credential system to mint from - the broker
|
|
3814
|
+
* enforces nothing). The connection presents the CURRENT credential on every (re)connect (a
|
|
3815
|
+
* renewal is adopted without reconnecting the whole endpoint); the ActionContext bonds its
|
|
3816
|
+
* KV + JS + JSM to this one connection and space (SPEC 13.4), so a composition mixup cannot splice
|
|
3817
|
+
* goal state across brokers. */
|
|
3818
|
+
async startGoalWriter() {
|
|
3819
|
+
const identity = this.auth ? this.goalWriterIdentity : newIdentity();
|
|
3820
|
+
const enc = new TextEncoder();
|
|
3821
|
+
// The mutable holder captured by the authenticator (mirrors the serve connection): a half-TTL
|
|
3822
|
+
// renewal updates `gw.creds` and the next (re)connect presents the refreshed credential.
|
|
3823
|
+
const gw = { nc: undefined, ctx: undefined, creds: this.auth ? this.goalWriterCreds : undefined, identity };
|
|
3824
|
+
const nc = await connect({
|
|
3825
|
+
servers: this.servers ?? DEFAULT_SERVER,
|
|
3826
|
+
...(this.auth ? { authenticator: (nonce) => credsAuthenticator(enc.encode(gw.creds))(nonce) } : {}),
|
|
3827
|
+
inboxPrefix: `_INBOX_${identity.id}`,
|
|
3828
|
+
maxReconnectAttempts: -1,
|
|
3829
|
+
});
|
|
3830
|
+
nc.closed().then((err) => { if (err)
|
|
3831
|
+
console.error(`! manager goal-writer connection closed: ${err.message}`); });
|
|
3832
|
+
gw.nc = nc;
|
|
3833
|
+
// The (i) fence resolver (SPEC 13.6 P2 item 3): resolve an executing instance's CURRENT gate
|
|
3834
|
+
// epoch. This manager reconciles only ITS OWN goals (security pin 4), so it resolves its own
|
|
3835
|
+
// registration instanceId to this incarnation's serve-grant epoch; a foreign/retired instance
|
|
3836
|
+
// is `null` (no current terminal to surface). A successor incarnation carries an ADVANCED epoch
|
|
3837
|
+
// here, so its reads pick the current-epoch subject and the predecessor's terminal is fenced out.
|
|
3838
|
+
gw.ctx = await actionContext(nc, this.space);
|
|
3839
|
+
// The own-issuance-gate READER for the currency belt, on BOTH mesh modes. Reads
|
|
3840
|
+
// `epgate.<e>.<iid>` over this connection; observe-only (stage/commit/revoke are never called
|
|
3841
|
+
// through it — the goal-writer holds no gate/epcred WRITE grant, so a mis-call would
|
|
3842
|
+
// broker-deny anyway).
|
|
3843
|
+
//
|
|
3844
|
+
// OPEN MESH IS NOT EXEMPT, and this is load-bearing. An open-mesh registration mints no
|
|
3845
|
+
// credentials, so the §13.1 takeover barrier's revoke-and-evict loop enumerates an EMPTY family
|
|
3846
|
+
// and is VACUOUS — yet it still advances processEpoch. The gate row itself DOES exist there
|
|
3847
|
+
// (`provisionEndpointGateOpen`, over the authority stores the open serve path ensures), and a
|
|
3848
|
+
// bare connection CAN read it (executed probe). So this belt is the ONLY thing standing between
|
|
3849
|
+
// a deposed open-mesh incarnation and a wrong terminal on the one create-only result subject.
|
|
3850
|
+
//
|
|
3851
|
+
// NAME IT HONESTLY: on an open mesh this is a COOPERATIVE fence. The broker enforces nothing, so
|
|
3852
|
+
// a hostile or non-conformant process can simply not run the check — the same guarantee class as
|
|
3853
|
+
// every other open-mesh property. It is also a read-then-CAS, so it NARROWS the commit window
|
|
3854
|
+
// rather than closing it. An AUTH mesh gets durable closure from the §13.1 barrier (revoke +
|
|
3855
|
+
// cluster-verified eviction BEFORE the epoch advance); an open mesh gets none.
|
|
3856
|
+
gw.gate = serveIssuanceGateKv(await new Kvm(nc).open(epAuthBucket(this.space)), this.space, { endpoint: MANAGER_ENDPOINT, instanceId: this.managerInstanceId });
|
|
3857
|
+
this.goalWriter = gw;
|
|
3858
|
+
console.error(`manager goal-writer standing (endpoint ${MANAGER_ENDPOINT}, ${this.auth ? "scoped cred, §13.1 family-staged" : "open/bare"})`);
|
|
3859
|
+
}
|
|
3860
|
+
/** Drain the goal-writer connection (best-effort, both exit paths). */
|
|
3861
|
+
async stopGoalWriter() {
|
|
3862
|
+
const gw = this.goalWriter;
|
|
3863
|
+
if (!gw)
|
|
3864
|
+
return;
|
|
3865
|
+
this.goalWriter = undefined;
|
|
3866
|
+
try {
|
|
3867
|
+
await gw.nc.drain();
|
|
3868
|
+
}
|
|
3869
|
+
catch {
|
|
3870
|
+
try {
|
|
3871
|
+
gw.nc.close();
|
|
3872
|
+
}
|
|
3873
|
+
catch { /* best effort */ }
|
|
3874
|
+
}
|
|
3875
|
+
}
|
|
3876
|
+
/** P2 item 6: mint the standing `session-ledger` credential and STAGE it into this instance's
|
|
3877
|
+
* §13.1 revocation family (`epcred.<e>.<iid>`), over the passed executor's `authKv` — EXACTLY the
|
|
3878
|
+
* {@link mintAndStageGoalWriter} pattern, under the same open-and-commit fence.
|
|
3879
|
+
*
|
|
3880
|
+
* This credential carries the DEDICATED sessions-bucket ledger rows and no session rail at all.
|
|
3881
|
+
* It therefore takes no epoch pin: §13.6 makes it the durable revocation authority that must
|
|
3882
|
+
* outlive the serving endpoint, so scoping it to one serving epoch would defeat its purpose. The
|
|
3883
|
+
* epoch lives where it belongs, on the per-session serving credentials, which are minted per
|
|
3884
|
+
* redemption into this same family. Re-minting the SAME nkey on renewal writes a DISTINCT ledger
|
|
3885
|
+
* row (per-JWT credentialId digest). */
|
|
3886
|
+
async mintAndStageSessionLedger(authKv) {
|
|
3887
|
+
const auth = this.auth;
|
|
3888
|
+
const identity = this.sessionLedgerIdentity;
|
|
3889
|
+
// Registration instanceId (item 3's persisted logical id), not the per-process lifecycleUid:
|
|
3890
|
+
// the barrier enumerates `epcred.<e>.<managerInstanceId>`, so the ledger cred joins that family.
|
|
3891
|
+
const iid = this.managerInstanceId;
|
|
3892
|
+
const fence = serveIssuanceGateKv(authKv, this.space, { endpoint: MANAGER_ENDPOINT, instanceId: iid });
|
|
3893
|
+
const observed = await fence.observe();
|
|
3894
|
+
if (observed === null)
|
|
3895
|
+
throw new Error(`the issuance gate for ${MANAGER_ENDPOINT}/${iid} vanished; the session-ledger cred cannot join its §13.1 revocation family`);
|
|
3896
|
+
const creds = await mintCreds(auth, identity, "session-ledger");
|
|
3897
|
+
const exp = inspectCredHealth(creds).exp;
|
|
3898
|
+
if (exp === undefined)
|
|
3899
|
+
throw new Error(`the session-ledger credential for ${MANAGER_ENDPOINT}/${iid} is unbounded; the §13.1 ledger row requires an expiry`);
|
|
3900
|
+
// The §13.1 open-and-commit fence (see {@link mintAndStageGoalWriter}): stage + revision-pinned
|
|
3901
|
+
// commit against the gate this mint's grant was scoped from, releasing only on the win.
|
|
3902
|
+
await commitSiblingIssuance(fence, observed, {
|
|
3903
|
+
credentialId: rawDigest(creds).replace("sha256:", "sha256-"),
|
|
3904
|
+
credentialKey: identity.id,
|
|
3905
|
+
holderPrincipal: principalKey(DEV_OWNER, identity.id).key,
|
|
3906
|
+
endpoint: MANAGER_ENDPOINT, lifecycleUid: iid, sourceChain: ["root"], state: "active", exp,
|
|
3907
|
+
generation: observed.generation, processEpoch: observed.processEpoch,
|
|
3908
|
+
registrationRevision: observed.registrationRevision, nameAuthorityRevision: observed.nameAuthorityRevision,
|
|
3909
|
+
});
|
|
3910
|
+
return creds;
|
|
3911
|
+
}
|
|
3912
|
+
/** P2 item 6: stand up the ONE §13.6 session plane on its own standing connection. Mode-dual,
|
|
3913
|
+
* mirroring {@link startGoalWriter}: an AUTH mesh presents the scoped `session-ledger` cred
|
|
3914
|
+
* already minted + family-staged inside registration's run block ({@link mintAndStageSessionLedger});
|
|
3915
|
+
* an OPEN mesh uses a bare connection (no credential system to mint from). The connection presents
|
|
3916
|
+
* the CURRENT credential on every (re)connect, so a half-TTL renewal is adopted without reconnecting.
|
|
3917
|
+
*
|
|
3918
|
+
* The offer SIGNER is a per-incarnation in-memory keypair: the static collapsed path mints AND
|
|
3919
|
+
* redeems the offer in one call ({@link ManagerSessionPlane.establishAttach}), so the manager
|
|
3920
|
+
* self-signs and self-verifies its own §13.6 grants and the keypair never leaves the process — a
|
|
3921
|
+
* holder never verifies the signature (it presents the grant back over the rail; the broker's
|
|
3922
|
+
* per-session caller cred is the holder's real fence). The plane's ledger lives in the DEDICATED
|
|
3923
|
+
* sessions bucket (createEndpointStreams provisioned it at registration). */
|
|
3924
|
+
async startSessionPlane() {
|
|
3925
|
+
const identity = this.auth ? this.sessionLedgerIdentity : newIdentity();
|
|
3926
|
+
const enc = new TextEncoder();
|
|
3927
|
+
// The mutable holder captured by the authenticator (mirrors the goal-writer): a half-TTL renewal
|
|
3928
|
+
// updates `sw.creds` and the next (re)connect presents the refreshed credential.
|
|
3929
|
+
const sw = { nc: undefined, creds: this.auth ? this.sessionLedgerCreds : undefined };
|
|
3930
|
+
const nc = await connect({
|
|
3931
|
+
servers: this.servers ?? DEFAULT_SERVER,
|
|
3932
|
+
...(this.auth ? { authenticator: (nonce) => credsAuthenticator(enc.encode(sw.creds))(nonce) } : {}),
|
|
3933
|
+
inboxPrefix: `_INBOX_${identity.id}`,
|
|
3934
|
+
maxReconnectAttempts: -1,
|
|
3935
|
+
});
|
|
3936
|
+
nc.closed().then((err) => { if (err)
|
|
3937
|
+
console.error(`! manager session-ledger connection closed: ${err.message}`); });
|
|
3938
|
+
sw.nc = nc;
|
|
3939
|
+
const serveEpoch = this.serviceServe?.grant.epoch;
|
|
3940
|
+
if (serveEpoch === undefined)
|
|
3941
|
+
throw new Error("the manager session plane needs the serve grant epoch; registerManagerService must run first");
|
|
3942
|
+
const ledgerKv = await openSessionLedgerKv(nc, sessionsBucket(this.space));
|
|
3943
|
+
const signer = newArtifactSigner();
|
|
3944
|
+
const keyId = `mgr-sessions-${identity.id.slice(0, 12)}`;
|
|
3945
|
+
const anchor = {
|
|
3946
|
+
keyId, publicKey: signer.publicKey, owner: MANAGER_ENDPOINT, roles: ["sessions"],
|
|
3947
|
+
scope: { sessions: [MANAGER_ENDPOINT] }, validFrom: Date.now() - 60_000, validTo: Date.now() + SESSION_GRANT_MAX_TTL_MS,
|
|
3948
|
+
};
|
|
3949
|
+
this.sessionPlane = new ManagerSessionPlane({
|
|
3950
|
+
space: this.space,
|
|
3951
|
+
// The session's serving identity is the persisted REGISTRATION instanceId (item 3), not the
|
|
3952
|
+
// per-process lifecycleUid: a restarted manager re-registers the SAME logical instanceId with
|
|
3953
|
+
// an ADVANCED epoch, so a client re-attaches by the same instance while the epoch fences the
|
|
3954
|
+
// old incarnation's sessions (item 6's restart-refusal composed with item 3's addressing).
|
|
3955
|
+
serving: { instanceId: this.managerInstanceId, epoch: serveEpoch },
|
|
3956
|
+
signer: { keyId, keyPair: signer }, resolveAnchor: (id) => (id === keyId ? anchor : undefined),
|
|
3957
|
+
ledgerKv, ttlMs: SESSION_GRANT_MAX_TTL_MS,
|
|
3958
|
+
servingCredential: this.sessionServingCredentials(),
|
|
3959
|
+
...(this.maxSessions !== undefined ? { maxSessions: this.maxSessions } : {}),
|
|
3960
|
+
});
|
|
3961
|
+
this.sessionLedgerConn = sw;
|
|
3962
|
+
console.error(`manager session plane standing (endpoint ${MANAGER_ENDPOINT}, epoch ${serveEpoch}, ${this.auth ? "scoped session-ledger cred, §13.1 family-staged" : "open/bare"})`);
|
|
3963
|
+
}
|
|
3964
|
+
/**
|
|
3965
|
+
* The per-session SERVING credential seam (P2 item 6, SPEC 13.6): the manager mints, gate-stages,
|
|
3966
|
+
* connects and revokes ONE credential per live session, replacing a standing credential that held
|
|
3967
|
+
* `eps.manager.*.<epoch>.{in,out}` and so reached every live session's bytes at its epoch.
|
|
3968
|
+
*
|
|
3969
|
+
* Each session gets its OWN nkey identity, so the §13.1 barrier's evict-by-holderPrincipal reaches
|
|
3970
|
+
* it individually, and each is staged into `epcred.manager.<instanceId>` — the SAME family the
|
|
3971
|
+
* ledger and goal-writer creds join. That is how manager takeover still kills a deposed manager's
|
|
3972
|
+
* sessions: the barrier enumerates the family, revokes every row, and evicts every holder, so the
|
|
3973
|
+
* per-session creds die with the incarnation exactly as the standing one did, with the blast
|
|
3974
|
+
* radius of a leaked credential cut from "every session at this epoch" to "one dead session".
|
|
3975
|
+
*
|
|
3976
|
+
* OPEN MESH: no credential system exists to mint from, so the seam mints nothing and opens a bare
|
|
3977
|
+
* connection. That is not a degraded auth path — an open mesh has no broker enforcement at all —
|
|
3978
|
+
* and it is still per-session: the connection and the ledger row are still one-per-session, so
|
|
3979
|
+
* teardown behaves identically in both modes.
|
|
3980
|
+
*/
|
|
3981
|
+
sessionServingCredentials() {
|
|
3982
|
+
const iid = this.managerInstanceId;
|
|
3983
|
+
const gate = async (fn) => this.withEndpointServeExecutor(({ authKv }) => fn(authKv));
|
|
3984
|
+
return {
|
|
3985
|
+
mint: async (grant) => {
|
|
3986
|
+
// Open mesh: no auth to mint from. The id still names the session so the ledger row and the
|
|
3987
|
+
// teardown path are identical in both modes.
|
|
3988
|
+
if (!this.auth)
|
|
3989
|
+
return { id: `${grant.sessionId}.s`, creds: "", exp: grant.exp };
|
|
3990
|
+
const identity = newIdentity();
|
|
3991
|
+
const creds = await mintCreds(this.auth, identity, "session-serving", {
|
|
3992
|
+
sessionServing: { endpoint: grant.endpoint, sessionId: grant.sessionId, epoch: grant.serving.epoch },
|
|
3993
|
+
expiresAt: Math.floor(grant.exp / 1000), // grant.exp is ms; the JWT exp is seconds
|
|
3994
|
+
});
|
|
3995
|
+
const health = inspectCredHealth(creds);
|
|
3996
|
+
if (health.exp === undefined)
|
|
3997
|
+
throw new Error(`the session-serving credential for ${grant.sessionId} is unbounded; a per-session credential never outlives its session (SPEC 13.6)`);
|
|
3998
|
+
this.sessionServingKeys.set(rawDigest(creds).replace("sha256:", "sha256-"), identity.id);
|
|
3999
|
+
return { id: rawDigest(creds).replace("sha256:", "sha256-"), creds, exp: grant.exp };
|
|
4000
|
+
},
|
|
4001
|
+
observeGate: async (_endpoint, instanceId) => {
|
|
4002
|
+
// Open mesh: nothing is minted and nothing is staged, so there is no gate to pin (see
|
|
4003
|
+
// ServingGatePin.gate — the stage refuses loudly if this is ever missing on an auth mesh).
|
|
4004
|
+
if (!this.auth)
|
|
4005
|
+
return { key: epgateKey(MANAGER_ENDPOINT, instanceId), revision: 0 };
|
|
4006
|
+
return gate(async (authKv) => {
|
|
4007
|
+
const observed = await serveIssuanceGateKv(authKv, this.space, { endpoint: MANAGER_ENDPOINT, instanceId }).observe();
|
|
4008
|
+
if (observed === null)
|
|
4009
|
+
throw new Error(`the issuance gate for ${MANAGER_ENDPOINT}/${instanceId} vanished; a session credential never stages against a missing gate (SPEC 13.1)`);
|
|
4010
|
+
// The WHOLE observation rides the pin: the stage's fence compares every field of it, which
|
|
4011
|
+
// is what lets a lost CAS be classified rather than blanket-refused.
|
|
4012
|
+
return { key: epgateKey(MANAGER_ENDPOINT, instanceId), revision: observed.revision, gate: observed };
|
|
4013
|
+
});
|
|
4014
|
+
},
|
|
4015
|
+
stage: async (grant, cred, pin) => {
|
|
4016
|
+
if (!this.auth)
|
|
4017
|
+
return; // open mesh: nothing minted, so nothing to make revocable
|
|
4018
|
+
const key = this.sessionServingKeys.get(cred.id);
|
|
4019
|
+
if (key === undefined)
|
|
4020
|
+
throw new Error(`no minted identity for session credential ${cred.id}; the stage cannot record a holder it did not mint (SPEC 13.1)`);
|
|
4021
|
+
// THE REDEMPTION'S OWN PIN IS THE FENCE, and it is never re-read into something newer here:
|
|
4022
|
+
// `commitSiblingIssuance` CASes on this observation's revision and, on a loss, refuses unless
|
|
4023
|
+
// the gate is still identical to it in every field but the revision. A barrier
|
|
4024
|
+
// (freeze, or reopen at a successor coordinate) is therefore always a refusal, while another
|
|
4025
|
+
// session's identical-bytes commit touch is not — per-session credentials all serialize on
|
|
4026
|
+
// this one gate key, so refusing on that would fail live sessions for contention rather than
|
|
4027
|
+
// for a barrier. A read is never a fence (SPEC 13.1); the pinned CAS is.
|
|
4028
|
+
const observed = pin.gate;
|
|
4029
|
+
if (observed === undefined)
|
|
4030
|
+
throw new Error(`the redemption of session ${grant.sessionId} carries no gate observation for ${MANAGER_ENDPOINT}/${iid}; a session credential never stages unfenced (SPEC 13.1)`);
|
|
4031
|
+
await gate(async (authKv) => {
|
|
4032
|
+
const fence = serveIssuanceGateKv(authKv, this.space, { endpoint: MANAGER_ENDPOINT, instanceId: iid });
|
|
4033
|
+
await commitSiblingIssuance(fence, observed, {
|
|
4034
|
+
credentialId: cred.id,
|
|
4035
|
+
credentialKey: key,
|
|
4036
|
+
holderPrincipal: principalKey(DEV_OWNER, key).key,
|
|
4037
|
+
endpoint: MANAGER_ENDPOINT, lifecycleUid: iid,
|
|
4038
|
+
// The lineage records that this credential exists because a session was redeemed, so a
|
|
4039
|
+
// ledger reader can tell a per-session row from a standing one (SPEC 13.6 sourceChain).
|
|
4040
|
+
sourceChain: [`session.${grant.sessionId}`], state: "active",
|
|
4041
|
+
exp: Math.floor(cred.exp / 1000),
|
|
4042
|
+
generation: observed.generation, processEpoch: observed.processEpoch,
|
|
4043
|
+
registrationRevision: observed.registrationRevision, nameAuthorityRevision: observed.nameAuthorityRevision,
|
|
4044
|
+
});
|
|
4045
|
+
});
|
|
4046
|
+
},
|
|
4047
|
+
open: async (cred) => {
|
|
4048
|
+
// FAIL LOUD: there is deliberately no shared connection to fall back to. Serving a session
|
|
4049
|
+
// without its own credential is exactly the standing-writer shape this design removes.
|
|
4050
|
+
const opts = this.auth ? standaloneConnectOpts({ creds: cred.creds, /* not yet wired to a recorded transport */ tls: false }) : {};
|
|
4051
|
+
return connect({ servers: this.servers ?? DEFAULT_SERVER, ...opts, maxReconnectAttempts: -1 });
|
|
4052
|
+
},
|
|
4053
|
+
revoke: async (credentialId) => {
|
|
4054
|
+
if (!this.auth)
|
|
4055
|
+
return; // open mesh: nothing was minted
|
|
4056
|
+
await gate(async (authKv) => {
|
|
4057
|
+
await markLedgerRowRevoked(authKv, epcredRowKey(MANAGER_ENDPOINT, iid, credentialId));
|
|
4058
|
+
});
|
|
4059
|
+
this.sessionServingKeys.delete(credentialId);
|
|
4060
|
+
},
|
|
4061
|
+
};
|
|
4062
|
+
}
|
|
4063
|
+
/** Tear the session plane down (best-effort, both exit paths): end every live bridge with the
|
|
4064
|
+
* honest `manager-restart` reason (this incarnation is going away; any successor takes a new epoch
|
|
4065
|
+
* and refuses these grants), then drain each session's own connection and the ledger connection. */
|
|
4066
|
+
async stopSessionPlane() {
|
|
4067
|
+
const plane = this.sessionPlane;
|
|
4068
|
+
const sw = this.sessionLedgerConn;
|
|
4069
|
+
this.sessionPlane = undefined;
|
|
4070
|
+
this.sessionLedgerConn = undefined;
|
|
4071
|
+
// `drain` awaits each session's teardown (connection close, terminal row, credential revoke);
|
|
4072
|
+
// `endAll` alone would let the process exit with per-session connections still open.
|
|
4073
|
+
try {
|
|
4074
|
+
await plane?.drain("manager-restart");
|
|
4075
|
+
}
|
|
4076
|
+
catch { /* best effort */ }
|
|
4077
|
+
if (sw) {
|
|
4078
|
+
try {
|
|
4079
|
+
await sw.nc.drain();
|
|
4080
|
+
}
|
|
4081
|
+
catch {
|
|
4082
|
+
try {
|
|
4083
|
+
sw.nc.close();
|
|
4084
|
+
}
|
|
4085
|
+
catch { /* best effort */ }
|
|
4086
|
+
}
|
|
4087
|
+
}
|
|
4088
|
+
}
|
|
4089
|
+
/** P2 item 2 must-5 Q-B — the boot reconcile: a fresh incarnation (a manager restart takes a NEW
|
|
4090
|
+
* instanceId, so the in-memory acceptance map starts empty) inherits the endpoint's accepted-but-
|
|
4091
|
+
* unterminal goals from any predecessor. Enumerate the durable index over a scoped PROVISIONER
|
|
4092
|
+
* (records CONSUMER.CREATE; the goal-writer holds NO enumeration grant, exactly the ruling) and
|
|
4093
|
+
* settle each orphan so an accepted goal is NEVER dropped across a restart. Open mesh: a bare
|
|
4094
|
+
* connection (the broker enforces nothing). Runs ONCE at start, BEFORE spawn-as-action begins
|
|
4095
|
+
* accepting (the `goalReconcileDone` gate), so it never races a live goal's acceptance. Never
|
|
4096
|
+
* fatal — a reconcile failure is logged and the gate opens either way. */
|
|
4097
|
+
async reconcileGoalIndex() {
|
|
4098
|
+
const gw = this.goalWriter;
|
|
4099
|
+
if (!gw) {
|
|
4100
|
+
this.goalReconcileDone = true;
|
|
4101
|
+
return;
|
|
4102
|
+
}
|
|
4103
|
+
try {
|
|
4104
|
+
let entries = [];
|
|
4105
|
+
const nc = this.auth
|
|
4106
|
+
? await connect({ servers: this.servers ?? DEFAULT_SERVER, ...standaloneConnectOpts({ creds: await mintCreds(this.auth, newIdentity(), "provisioner"), /* not yet wired to a recorded transport */ tls: false }), maxReconnectAttempts: 0 })
|
|
4107
|
+
: await connect({ servers: this.servers ?? DEFAULT_SERVER, maxReconnectAttempts: 0 });
|
|
4108
|
+
try {
|
|
4109
|
+
const kvm = new Kvm(nc);
|
|
4110
|
+
await ensureAuthorityStores(await jetstreamManager(nc), kvm, this.space);
|
|
4111
|
+
entries = await listGoalIndex(await kvm.open(recordsBucket(this.space)), MANAGER_ENDPOINT);
|
|
4112
|
+
}
|
|
4113
|
+
finally {
|
|
4114
|
+
await nc.drain().catch(() => nc.close());
|
|
4115
|
+
}
|
|
4116
|
+
// Single-manager item 2: EVERY inherited entry belongs to a DEAD predecessor (only one manager
|
|
4117
|
+
// at a time), so all are reconciled. The `iid` field is the hook item-3's multi-instance sweep
|
|
4118
|
+
// filters on (skip a goal whose accepting `iid` is a still-LIVE sibling — never settle its goal).
|
|
4119
|
+
for (const { ref, iid } of entries) {
|
|
4120
|
+
if (this.goalAcceptances.has(ref.goalId))
|
|
4121
|
+
continue; // never settle a goal THIS incarnation drives
|
|
4122
|
+
try {
|
|
4123
|
+
await this.reconcileOneGoal(ref, iid);
|
|
4124
|
+
}
|
|
4125
|
+
catch (e) {
|
|
4126
|
+
console.error(`! goal reconcile for ${ref.goalId}: ${e.message}`);
|
|
4127
|
+
}
|
|
4128
|
+
}
|
|
4129
|
+
if (entries.length)
|
|
4130
|
+
console.error(`goal-index boot reconcile: swept ${entries.length} inherited goal(s)`);
|
|
4131
|
+
}
|
|
4132
|
+
catch (e) {
|
|
4133
|
+
console.error(`! goal-index boot reconcile failed: ${e.message} - accepted goals from a predecessor may stay unsettled until the next restart`);
|
|
4134
|
+
}
|
|
4135
|
+
finally {
|
|
4136
|
+
this.goalReconcileDone = true;
|
|
4137
|
+
}
|
|
4138
|
+
}
|
|
4139
|
+
/** Settle ONE inherited goal by evidence: no goal record (a crash between the index write and the
|
|
4140
|
+
* goal-record create) leaves the pointer untouched (never settle a goal that was never accepted,
|
|
4141
|
+
* and clearing it would race a live goal mid-creation); a TERMINAL goal clears the index
|
|
4142
|
+
* (converged — the predecessor committed but died before clearing); a NON-TERMINAL goal settles
|
|
4143
|
+
* `uncertain` (the accepting incarnation is gone, so the success signal will never reach us — the
|
|
4144
|
+
* bounded readiness outcome the plan maps the window to). Within the readiness window it arms a
|
|
4145
|
+
* bounded, unref'd timer to settle at the deadline (an early uncertain would steal a still-possible
|
|
4146
|
+
* success the substrate guards against). */
|
|
4147
|
+
async reconcileOneGoal(ref, acceptedByIid) {
|
|
4148
|
+
const gw = this.goalWriter;
|
|
4149
|
+
if (!gw)
|
|
4150
|
+
return;
|
|
4151
|
+
const status = await readGoalStatus(gw.ctx, ref);
|
|
4152
|
+
if (status === undefined)
|
|
4153
|
+
return; // index points at no goal record: a dead pointer, left for honesty
|
|
4154
|
+
if (GOAL_TERMINAL_STATES.includes(status.value.state)) {
|
|
4155
|
+
await clearGoalIndex(gw.ctx, ref);
|
|
4156
|
+
return;
|
|
4157
|
+
}
|
|
4158
|
+
const spec = await readGoalSpec(gw.ctx, ref);
|
|
4159
|
+
if (spec === undefined)
|
|
4160
|
+
return; // a status without its spec is garbled — leave for the next boot
|
|
4161
|
+
// NEVER A CROSS-INSTANCE SETTLE. The accepting incarnation is recorded on the INDEX ENTRY
|
|
4162
|
+
// itself (`iid`, written at accept), which is the honest coordinate: a same-instanceId restart
|
|
4163
|
+
// inherits its predecessor's orphans, and a goal accepted by a DIFFERENT (possibly still-live)
|
|
4164
|
+
// sibling is left for its owner. This replaces the goal spec's `executor` pin, which existed
|
|
4165
|
+
// only to epoch-scope the terminal subject and is gone with it (SPEC:1394).
|
|
4166
|
+
if (acceptedByIid !== this.managerInstanceId) {
|
|
4167
|
+
console.error(`goal reconcile ${ref.goalId}: accepted by instance "${acceptedByIid}", not this incarnation "${this.managerInstanceId}"; left for its owner (never a cross-instance settle)`);
|
|
4168
|
+
return;
|
|
4169
|
+
}
|
|
4170
|
+
const settle = async () => {
|
|
4171
|
+
// A SUCCESSOR settling work it inherited: the committer is THIS incarnation at its CURRENT
|
|
4172
|
+
// serve epoch, which is strictly greater than the goal's acceptedEpoch — the `committed >
|
|
4173
|
+
// accepted` arm of the attribution rule, and the reason that arm has to exist.
|
|
4174
|
+
await settleGoalUncertain(gw.ctx, { ref, now: Date.now(), committer: { instanceId: this.managerInstanceId, epoch: this.serviceServe?.grant.epoch ?? 0 } }); // first-terminal-wins: a racing terminal returns the winner, no throw
|
|
4175
|
+
await clearGoalIndex(gw.ctx, ref);
|
|
4176
|
+
};
|
|
4177
|
+
const remaining = spec.value.acceptedAt + (spec.value.readinessDeadlineMs ?? this.readinessTimeoutMs) - Date.now();
|
|
4178
|
+
if (remaining <= 0) {
|
|
4179
|
+
await settle();
|
|
4180
|
+
return;
|
|
4181
|
+
}
|
|
4182
|
+
console.error(`goal reconcile ${ref.goalId}: within the readiness window (${remaining}ms) - arming a bounded settle`);
|
|
4183
|
+
const t = setTimeout(() => { settle().catch((e) => console.error(`! goal reconcile settle ${ref.goalId}: ${e.message}`)); }, remaining + 100);
|
|
4184
|
+
t.unref?.();
|
|
4185
|
+
}
|
|
4186
|
+
/** P2 item 2: publish a goal PROGRESS event on the caller-scoped epe subtree, over the SERVE
|
|
4187
|
+
* connection (which holds the `epe.<e>.<iid>.<epoch>.>` egress grant; the goal-writer deliberately
|
|
4188
|
+
* does not). The terminal rides a final event `phase:"terminal"` (Q1 — the caller follows epe to
|
|
4189
|
+
* the terminal; the durable result fact + inspect/ps are the reconcile authority). A dropped event
|
|
4190
|
+
* is non-fatal (the terminal is authoritative in the journal). */
|
|
4191
|
+
emitGoalProgress(ref, epoch, event) {
|
|
4192
|
+
const nc = this.serviceServe?.nc;
|
|
4193
|
+
if (!nc)
|
|
4194
|
+
return;
|
|
4195
|
+
try {
|
|
4196
|
+
nc.publish(epeSubject(this.space, MANAGER_ENDPOINT, this.managerInstanceId, epoch, goalProgressTopic(ref)), new TextEncoder().encode(JSON.stringify({ v: 1, goalId: ref.goalId, ...event })));
|
|
4197
|
+
}
|
|
4198
|
+
catch (e) {
|
|
4199
|
+
console.error(`! goal progress emit for ${ref.goalId} failed: ${e.message}`);
|
|
4200
|
+
}
|
|
4201
|
+
}
|
|
4202
|
+
/** The own-gate currency belt: before the goal-writer commits a terminal fact, the manager reads
|
|
4203
|
+
* its OWN issuance gate epoch and REFUSES the commit if superseded. This NARROWS the window; it
|
|
4204
|
+
* is not the fence. Layer 1 below is closed by the sibling-mint fence; layers 2 and 3 are not
|
|
4205
|
+
* closed by any planned slice and must not be described as temporary.
|
|
4206
|
+
*
|
|
4207
|
+
* THE RESIDUAL, STACKED:
|
|
4208
|
+
* 1. SIBLING-MINT INJECTION. A §13.1 barrier's revoke/evict loop closes only over the family
|
|
4209
|
+
* SNAPSHOT IT ENUMERATED, so a sibling mint that observes the gate and stages a ledger row
|
|
4210
|
+
* WITHOUT the observe/open/commit fence can be staged and released AFTER that enumerate and
|
|
4211
|
+
* never be revoked. This layer is closed exactly where BOTH sibling mint sites
|
|
4212
|
+
* ({@link mintAndStageGoalWriter}, {@link mintAndStageSessionLedger}) route their stage
|
|
4213
|
+
* through `commitSiblingIssuance` (the revision-pinned CAS that makes a losing mint release
|
|
4214
|
+
* nothing), and open exactly where they do not — state the mechanism, never the branch.
|
|
4215
|
+
* 2. THE BARRIER WINDOW. A gate FREEZE neither kills this connection nor advances the epoch,
|
|
4216
|
+
* and this belt compares `processEpoch` alone, so it still PASSES from barrier start until
|
|
4217
|
+
* the reopen. A ledger revoke marks a row; it does not re-check a live JWT mid-publish. The
|
|
4218
|
+
* durable kill is the CLUSTER-VERIFIED EVICTION, so a deposed manager can INITIATE new
|
|
4219
|
+
* terminal publishes from barrier start until eviction is verified — not merely finish bytes
|
|
4220
|
+
* already on the wire. Successor EXISTENCE and corpse DEATH are different phases, so the
|
|
4221
|
+
* barrier's ordering licenses no conclusion about when the corpse stops being able to write.
|
|
4222
|
+
* 3. OPEN MESH. No credential family exists, so the revoke/evict loop is vacuous and this belt
|
|
4223
|
+
* is COOPERATIVE only: a non-conformant process simply does not run it.
|
|
4224
|
+
* The named follow-up that would close 2 and 3 is the gate-linearized commit (routing the
|
|
4225
|
+
* terminal through the issuance gate's own CAS), deliberately deferred as substrate territory.
|
|
4226
|
+
* An earlier revision of this comment claimed the residual was "closed by item-3 slice 3.0,
|
|
4227
|
+
* never a permanent residual". That asserted a closure that does not exist. */
|
|
4228
|
+
async assertGoalWriterEpochCurrent(epoch) {
|
|
4229
|
+
const gate = this.goalWriter?.gate;
|
|
4230
|
+
if (!gate)
|
|
4231
|
+
return; // no goal-writer standing yet
|
|
4232
|
+
const observed = await gate.observe();
|
|
4233
|
+
if (observed === null)
|
|
4234
|
+
throw new EpEnvelopeError("expired", `the manager's issuance gate for ${MANAGER_ENDPOINT}/${this.managerInstanceId} is gone; a retired incarnation never commits a goal terminal (SPEC 13.1/13.6)`);
|
|
4235
|
+
if (observed.processEpoch !== epoch)
|
|
4236
|
+
throw new EpEnvelopeError("expired", `the manager's issuance gate epoch is ${observed.processEpoch} but this goal was accepted under epoch ${epoch}; a superseded incarnation never commits a goal terminal (must-5 (a) own-gate belt, SPEC 13.6)`);
|
|
4237
|
+
}
|
|
4238
|
+
/** Serve `spawn`/`launch` as an ACTION (P2 item 2). Authz already ran in {@link serveGated}. The
|
|
4239
|
+
* accept path runs INLINE on the handler ({@link startAgent} with hooks): the goal binds + the
|
|
4240
|
+
* acceptance replies the moment the identity is minted, BEFORE any provision (pin 1); progress and
|
|
4241
|
+
* the terminal are driven OFF-handler, so the ~30s readiness wait no longer blocks the reply.
|
|
4242
|
+
* Returns the acceptance floor payload {name, owner, actor, uid, goalId, fingerprint, executor}
|
|
4243
|
+
* (the ALLOCATED identity). goalId = the request id (env.id, Q3). */
|
|
4244
|
+
async serveSpawnGoal(ctx, run) {
|
|
4245
|
+
const gw = this.goalWriter;
|
|
4246
|
+
if (!gw)
|
|
4247
|
+
throw new EpEnvelopeError("unavailable", "the manager goal-writer connection is not standing; spawn-as-action cannot accept (SPEC 13.6)");
|
|
4248
|
+
// must-5 Q-B: refuse to accept until the boot reconcile of inherited goals completes, so a fresh
|
|
4249
|
+
// acceptance never races the sweep (settling a live goal mid-flight would steal its real terminal).
|
|
4250
|
+
if (!this.goalReconcileDone)
|
|
4251
|
+
throw new EpEnvelopeError("unavailable", "the manager is still reconciling accepted goals at boot; retry shortly (SPEC 13.6)");
|
|
4252
|
+
const goalId = ctx.request.id;
|
|
4253
|
+
const { fingerprint } = submissionFingerprint(ctx.request, ctx.subject);
|
|
4254
|
+
const ref = goalRefOf(ctx.subject, goalId);
|
|
4255
|
+
const executor = { lifecycleUid: this.managerInstanceId, epoch: this.serviceServe?.grant.epoch ?? 0 };
|
|
4256
|
+
const epoch = executor.epoch;
|
|
4257
|
+
const acceptedAt = Date.now();
|
|
4258
|
+
// Idempotent same-goalId retry (a client re-send): serve the IDENTICAL acceptance without
|
|
4259
|
+
// re-running the accept path — so a HARD-PINNED retry does not trip the M6 same-name refuse and no
|
|
4260
|
+
// name is re-allocated. Same-incarnation rides the live map; the create-only bindGoal in onAccepted
|
|
4261
|
+
// still fences a CONCURRENT same-goalId race (the loser aborts and serves the winner's acceptance).
|
|
4262
|
+
const prior = this.goalAcceptances.get(goalId);
|
|
4263
|
+
if (prior !== undefined) {
|
|
4264
|
+
if (prior.fingerprint !== fingerprint)
|
|
4265
|
+
throw new EpEnvelopeError("failed-precondition", `goal "${goalId}" was accepted under a different submission; one goalId never carries two specs (SPEC 13.6)`);
|
|
4266
|
+
return prior;
|
|
4267
|
+
}
|
|
4268
|
+
let resolveAccept;
|
|
4269
|
+
let rejectAccept;
|
|
4270
|
+
const acceptP = new Promise((res, rej) => { resolveAccept = res; rejectAccept = rej; });
|
|
4271
|
+
let acceptance;
|
|
4272
|
+
// H1: set the instant the terminal path is ENTERED, not when it succeeds — the post-accept
|
|
4273
|
+
// fallback below must fire only when `onOutcome` never ran at all, never as a second attempt
|
|
4274
|
+
// behind a commit that threw.
|
|
4275
|
+
let terminalEntered = false;
|
|
4276
|
+
// TWO DIFFERENT QUESTIONS, DELIBERATELY NOT ONE FLAG.
|
|
4277
|
+
//
|
|
4278
|
+
// `terminalEntered` HAS this goal already been settled (or claimed) by someone?
|
|
4279
|
+
// `ownsGoal` MAY THIS ATTEMPT settle it at all?
|
|
4280
|
+
//
|
|
4281
|
+
// The second is the authority, and it is default-CLOSED: an attempt earns it by WINNING the
|
|
4282
|
+
// create-only `bindGoal` CAS below, and nothing else grants it. That matters because the
|
|
4283
|
+
// post-accept fallback exists to settle a goal nobody answered, so every new way of leaving
|
|
4284
|
+
// this function is opted INTO committing a terminal unless something stops it. Deriving the
|
|
4285
|
+
// right from the claim instead of from a running record means a losing attempt CANNOT commit
|
|
4286
|
+
// down any unwind path, including ones added later that never thought about this. Collapsing
|
|
4287
|
+
// the two into one boolean is what let a duplicate-goal loser steal the winner's terminal
|
|
4288
|
+
// (#357): it had entered no terminal, so the fallback wrote one for it, using its own abort
|
|
4289
|
+
// message as the caller-visible outcome.
|
|
4290
|
+
let ownsGoal = false;
|
|
4291
|
+
// The terminal commits OFF-handler on the goal-writer connection (manager-only authority; a
|
|
4292
|
+
// caller cannot publish it). TWO COMPOSED FENCES (defense in depth): must-5 (a) reads THIS
|
|
4293
|
+
// incarnation's OWN gate epoch and REFUSES a superseded commit (the currency belt), and (b)
|
|
4294
|
+
// barrier-revoke evicts this connection on takeover. The terminal lands on the ONE subject
|
|
4295
|
+
// SPEC:1394 reserves; first-terminal-fact-wins is global, so a committed outcome is visible
|
|
4296
|
+
// to every reader in every incarnation. On success the reconcile-index entry is cleared.
|
|
4297
|
+
//
|
|
4298
|
+
// Named rather than inlined into the hooks below so the H1 post-accept fallback drives THIS
|
|
4299
|
+
// path — one commit site, so the progress event, the index clear and the `agentGoals` cleanup
|
|
4300
|
+
// cannot drift between a normal outcome and a recovered one.
|
|
4301
|
+
const onOutcome = async (o) => {
|
|
4302
|
+
// THE AUTHORITY CHECK, and the only one. An attempt that never won the bind provisioned
|
|
4303
|
+
// nothing and has no outcome to report: the goal belongs to whoever won it, in this
|
|
4304
|
+
// incarnation or a sibling. Refusing here rather than at each unwind site is the point --
|
|
4305
|
+
// there is exactly one commit path, so this fences every route into it, present and future.
|
|
4306
|
+
if (!ownsGoal)
|
|
4307
|
+
return;
|
|
4308
|
+
terminalEntered = true; // entered, not succeeded — see the catch below
|
|
4309
|
+
try {
|
|
4310
|
+
await this.assertGoalWriterEpochCurrent(epoch); // must-5 (a): a superseded corpse never commits
|
|
4311
|
+
let fact;
|
|
4312
|
+
if (o.kind === "succeeded") {
|
|
4313
|
+
this.emitGoalProgress(ref, epoch, { phase: "presence" });
|
|
4314
|
+
({ fact } = await commitGoalResult(gw.ctx, { ref, now: Date.now(), cause: "complete", state: "succeeded", data: o.data, committer: { instanceId: this.managerInstanceId, epoch } }));
|
|
4315
|
+
}
|
|
4316
|
+
else if (o.kind === "failed") {
|
|
4317
|
+
({ fact } = await commitGoalResult(gw.ctx, { ref, now: Date.now(), cause: "complete", state: "failed", data: o.data, committer: { instanceId: this.managerInstanceId, epoch } }));
|
|
4318
|
+
}
|
|
4319
|
+
else {
|
|
4320
|
+
({ fact } = await settleGoalUncertain(gw.ctx, { ref, now: Date.now(), committer: { instanceId: this.managerInstanceId, epoch } }));
|
|
4321
|
+
}
|
|
4322
|
+
this.emitGoalProgress(ref, epoch, { phase: "terminal", state: fact.state, ...(fact.data !== undefined ? { data: fact.data } : {}) });
|
|
4323
|
+
await clearGoalIndex(gw.ctx, ref); // must-5 Q-B: terminal reached - the successor never reconciles it
|
|
4324
|
+
if (acceptance)
|
|
4325
|
+
this.agentGoals.delete(acceptance.name); // goal terminal - no cancel path left
|
|
4326
|
+
}
|
|
4327
|
+
catch (e) {
|
|
4328
|
+
// THE NARROWER LEG, LEFT OPEN DELIBERATELY. If the COMMIT ITSELF throws (the currency belt
|
|
4329
|
+
// refused a superseded commit, or the broker failed) there is genuinely no terminal, and the
|
|
4330
|
+
// H1 fallback must NOT retry it: a retry either loses the same way or overwrites a real
|
|
4331
|
+
// supersession refusal with a manufactured outcome. That is why `terminalEntered` is set on
|
|
4332
|
+
// ENTRY. This leg is infrastructure-class and converges through the reconcile index at the
|
|
4333
|
+
// next boot, which is why the index is NOT cleared above on this path. Do not "close" it
|
|
4334
|
+
// here with a retry loop.
|
|
4335
|
+
console.error(`! goal terminal commit for ${goalId} failed: ${e.message}`);
|
|
4336
|
+
}
|
|
4337
|
+
};
|
|
4338
|
+
const bg = run({
|
|
4339
|
+
onAccepted: async ({ name, agentTriple }) => {
|
|
4340
|
+
// must-5 Q-B: record the goal in the reconcile index BEFORE the bind (index-CAS-before-bind),
|
|
4341
|
+
// so a successor incarnation finds + settles this goal if we crash before its terminal. A
|
|
4342
|
+
// crash between this write and the bind leaves an index entry whose goal status is absent —
|
|
4343
|
+
// the sweep clears it as a no-goal; a crash before it leaves no entry (never durable). A
|
|
4344
|
+
// Carry THIS incarnation's instanceId (the executor coord) so a multi-instance sweep can skip a live sibling's goal.
|
|
4345
|
+
//
|
|
4346
|
+
// H2, THE ACCEPTANCE FLOOR: the allocated identity is written HERE, before the bind and so
|
|
4347
|
+
// before the acceptance is acked, because this entry is the only durable record of it that
|
|
4348
|
+
// exists that early — the goal spec does not carry it and the terminal does not exist yet.
|
|
4349
|
+
// A same-goalId attempt that loses the bind while the winner is still in flight can then
|
|
4350
|
+
// serve what the winner actually allocated instead of inventing an empty identity.
|
|
4351
|
+
const idx = await recordGoalIndex(gw.ctx, ref, executor.lifecycleUid, { name, actor: agentTriple.actor, uid: agentTriple.uid });
|
|
4352
|
+
// A create loss is an idempotent retry ONLY for the same incarnation. A FOREIGN iid means a
|
|
4353
|
+
// sibling instance accepted this goalId (the live vector is a client retry over ANYCAST, not
|
|
4354
|
+
// a journal consumer): this attempt provisions nothing and answers with the winner's floor,
|
|
4355
|
+
// or refuses if that instance never persisted one.
|
|
4356
|
+
if (!idx.recorded && idx.existing.iid !== executor.lifecycleUid) {
|
|
4357
|
+
acceptance = this.acceptanceFromIndex(idx.existing, goalId, fingerprint, executor);
|
|
4358
|
+
resolveAccept(acceptance);
|
|
4359
|
+
// No claim needed here: `ownsGoal` is still false, so the commit path refuses this attempt.
|
|
4360
|
+
throw new EpEnvelopeError("failed-precondition", `goal "${goalId}" was accepted by instance "${idx.existing.iid}"; that instance's acceptance is served and this attempt provisions nothing (SPEC 13.6)`);
|
|
4361
|
+
}
|
|
4362
|
+
// Bind AFTER the accept-path checks (M6/capacity/persona) + identity mint, BEFORE any provision:
|
|
4363
|
+
// a create-only CAS per goalId (pin 1 — a refused accept above left zero bind, zero reserve).
|
|
4364
|
+
const b = await bindGoal(gw.ctx, ref, fingerprint);
|
|
4365
|
+
if (!b.bound) {
|
|
4366
|
+
if (b.existing.fingerprint !== fingerprint)
|
|
4367
|
+
throw new EpEnvelopeError("failed-precondition", `goal "${goalId}" is already bound to a different submission; one goalId never carries two specs (SPEC 13.6)`);
|
|
4368
|
+
// Lost a concurrent same-goalId race: serve the winner's acceptance and abort THIS provision
|
|
4369
|
+
// (no second spawn). The throw unwinds to the finally, which releases this attempt's reserve.
|
|
4370
|
+
acceptance = this.goalAcceptances.get(goalId) ?? await this.cachedSpawnAcceptance(ref, goalId, fingerprint, executor);
|
|
4371
|
+
resolveAccept(acceptance);
|
|
4372
|
+
// No claim needed here either. This attempt lost the CAS, so `ownsGoal` is still false and
|
|
4373
|
+
// the single check in `onOutcome` refuses it. Before that check existed this site had to
|
|
4374
|
+
// remember to claim the terminal by hand, and the one place that forgot is how a loser
|
|
4375
|
+
// came to commit `failed` on the winner's goal, using its own abort text as the outcome.
|
|
4376
|
+
// Unwind the provision (the acceptance is already served); the code is discarded by the
|
|
4377
|
+
// caller (acceptance !== undefined), so it just aborts this attempt's side-effects.
|
|
4378
|
+
throw new EpEnvelopeError("failed-precondition", `goal "${goalId}" is already accepted; the cached acceptance is served`);
|
|
4379
|
+
}
|
|
4380
|
+
// WON the claim: this attempt, and only this attempt, may settle this goal.
|
|
4381
|
+
ownsGoal = true;
|
|
4382
|
+
await createGoal(gw.ctx, ref, {
|
|
4383
|
+
fingerprint,
|
|
4384
|
+
command: ctx.subject.command,
|
|
4385
|
+
caller: { id: `${ctx.subject.caller.owner}.${ctx.subject.caller.actor}`, lifecycleUid: ctx.subject.caller.uid },
|
|
4386
|
+
// The ACCEPTING incarnation's epoch: half of the terminal's attribution pair (§13.6).
|
|
4387
|
+
acceptedEpoch: epoch,
|
|
4388
|
+
requestId: goalId,
|
|
4389
|
+
sourceSeq: 0,
|
|
4390
|
+
acceptedAt,
|
|
4391
|
+
readinessDeadlineMs: this.readinessTimeoutMs,
|
|
4392
|
+
});
|
|
4393
|
+
acceptance = { name, owner: agentTriple.owner, actor: agentTriple.actor, uid: agentTriple.uid, goalId, fingerprint, executor };
|
|
4394
|
+
this.goalAcceptances.set(goalId, acceptance);
|
|
4395
|
+
this.agentGoals.set(name, ref); // M4: a despawn of this name mid-goal drives the cancel path
|
|
4396
|
+
resolveAccept(acceptance);
|
|
4397
|
+
this.emitGoalProgress(ref, epoch, { phase: "handoff" });
|
|
4398
|
+
},
|
|
4399
|
+
onLaunched: () => this.emitGoalProgress(ref, epoch, { phase: "launched" }),
|
|
4400
|
+
onOutcome,
|
|
4401
|
+
// Claim the terminal WITHOUT committing one: the despawn/stop that ended this launch owns
|
|
4402
|
+
// it and commits `cancel`. This only stops the non-ok reply below from manufacturing a
|
|
4403
|
+
// `failed` that would race that `cancel` (first-terminal-fact-wins).
|
|
4404
|
+
onTerminalDeferred: () => { terminalEntered = true; },
|
|
4405
|
+
});
|
|
4406
|
+
bg.then((reply) => {
|
|
4407
|
+
// Refused BEFORE onAccepted (M6 hard-pin collision, capacity, persona-not-found) — no goal bound.
|
|
4408
|
+
if (acceptance === undefined) {
|
|
4409
|
+
rejectAccept(new EpEnvelopeError("failed-precondition", reply.error ?? "spawn refused at accept"));
|
|
4410
|
+
return;
|
|
4411
|
+
}
|
|
4412
|
+
// H1: `run` CATCHES its own body (a throw in buildLaunch/runtime.spawn) and RESOLVES `{ok:false}`
|
|
4413
|
+
// rather than rejecting, so a post-accept failure arrives here, not in the catch below, and
|
|
4414
|
+
// reaches none of the onOutcome sites. Without this the goal stays accepted-but-unanswered:
|
|
4415
|
+
// the caller follows epe to a terminal that never comes, and the reconcile index that would
|
|
4416
|
+
// settle it is only swept at BOOT, so a manager that stays up never converges it.
|
|
4417
|
+
if (reply.ok === false && !terminalEntered)
|
|
4418
|
+
return onOutcome({ kind: "failed", data: { error: reply.error ?? "spawn failed after accept" } });
|
|
4419
|
+
}).catch((e) => {
|
|
4420
|
+
if (acceptance === undefined) {
|
|
4421
|
+
rejectAccept(e);
|
|
4422
|
+
return;
|
|
4423
|
+
}
|
|
4424
|
+
// Same obligation for a genuine rejection (one that escaped `run`'s own catch).
|
|
4425
|
+
if (!terminalEntered)
|
|
4426
|
+
return onOutcome({ kind: "failed", data: { error: e?.message ?? String(e) } });
|
|
4427
|
+
console.error(`! spawn-as-action async body for ${goalId}: ${e?.message ?? String(e)}`);
|
|
4428
|
+
}).catch((e) => console.error(`! goal terminal fallback for ${goalId}: ${e?.message ?? String(e)}`));
|
|
4429
|
+
return acceptP;
|
|
4430
|
+
}
|
|
4431
|
+
/** H2: an acceptance served from a WINNER'S durable acceptance floor (the goal-index entry it
|
|
4432
|
+
* wrote before its own ack). Refuses `unavailable` rather than inventing one — see
|
|
4433
|
+
* {@link cachedSpawnAcceptance} for why an empty identity is never an acceptable answer. */
|
|
4434
|
+
acceptanceFromIndex(entry, goalId, fingerprint, executor) {
|
|
4435
|
+
if (entry.allocated === undefined)
|
|
4436
|
+
throw new EpEnvelopeError("unavailable", `goal "${goalId}" was accepted by instance "${entry.iid}", which persisted no acceptance floor; its allocated identity is not readable from here (SPEC 13.6)`);
|
|
4437
|
+
return { name: entry.allocated.name, owner: DEV_OWNER, actor: entry.allocated.actor, uid: entry.allocated.uid, goalId, fingerprint, executor };
|
|
4438
|
+
}
|
|
4439
|
+
/** Reconstruct a cached acceptance for a same-goalId retry NOT in the live map (a prior incarnation
|
|
4440
|
+
* accepted it, or a concurrent winner whose map write this reader has not yet observed).
|
|
4441
|
+
*
|
|
4442
|
+
* H2 — WHY THIS PREFERS THE INDEX OVER THE TERMINAL. It used to read only the committed terminal
|
|
4443
|
+
* and fall back to `{name:"", actor:"", uid:""}` when there was none. That is the common case,
|
|
4444
|
+
* not a corner: a client retry over ANYCAST reaches a sibling while the winner is still
|
|
4445
|
+
* provisioning, so no terminal exists yet, and the caller was handed an ACCEPTED reply naming an
|
|
4446
|
+
* empty agent it can never address. The acceptance floor in the goal index exists from the moment
|
|
4447
|
+
* of acceptance, so it answers precisely the window the terminal cannot. Where neither is
|
|
4448
|
+
* readable the honest answer is a REFUSAL: an accepted goal whose identity nobody can name is
|
|
4449
|
+
* `unavailable`, never a hollow success. */
|
|
4450
|
+
async cachedSpawnAcceptance(ref, goalId, fingerprint, executor) {
|
|
4451
|
+
const entry = await readGoalIndex(this.goalWriter.ctx, ref);
|
|
4452
|
+
if (entry?.allocated !== undefined)
|
|
4453
|
+
return this.acceptanceFromIndex(entry, goalId, fingerprint, executor);
|
|
4454
|
+
// The index is CLEARED at terminal, so a settled goal legitimately has no entry: fall back to
|
|
4455
|
+
// the terminal's data, which carries the same identity for exactly that case.
|
|
4456
|
+
const result = await readGoalResult(this.goalWriter.ctx, ref);
|
|
4457
|
+
const d = (result?.data ?? {});
|
|
4458
|
+
if (typeof d.name === "string" && d.name.length > 0 && typeof d.id === "string" && d.id.length > 0 && typeof d.lifecycleUid === "string" && d.lifecycleUid.length > 0)
|
|
4459
|
+
return { name: d.name, owner: DEV_OWNER, actor: d.id, uid: d.lifecycleUid, goalId, fingerprint, executor };
|
|
4460
|
+
throw new EpEnvelopeError("unavailable", `goal "${goalId}" is already accepted but its allocated identity is not readable (no acceptance floor, and no terminal carrying one); retry (SPEC 13.6)`);
|
|
4461
|
+
}
|
|
4462
|
+
/** M4 (settle race): a despawn MID-GOAL drives the goal's cancel terminal - transition to
|
|
4463
|
+
* `cancelling`, then commit the `cancel` cause on the goal-writer connection. First-terminal-fact
|
|
4464
|
+
* wins: if the readiness outcome already committed (succeeded/failed/uncertain) the transition or
|
|
4465
|
+
* the create-only commit loses gracefully and the readiness terminal stands. Fire-and-forget from
|
|
4466
|
+
* despawn (the process teardown is authoritative for the agent; this settles the GOAL honestly).
|
|
4467
|
+
* Cancel rides the despawn's own authorizeNamed reach (pin 5) - there is no cancel-by-goalId. */
|
|
4468
|
+
async cancelAgentGoal(name, mode) {
|
|
4469
|
+
const gw = this.goalWriter;
|
|
4470
|
+
const ref = this.agentGoals.get(name);
|
|
4471
|
+
if (!gw || !ref)
|
|
4472
|
+
return;
|
|
4473
|
+
this.agentGoals.delete(name);
|
|
4474
|
+
const epoch = this.serviceServe?.grant.epoch ?? 0;
|
|
4475
|
+
try {
|
|
4476
|
+
await this.assertGoalWriterEpochCurrent(epoch); // must-5 (a): a superseded corpse never commits a cancel terminal either
|
|
4477
|
+
await transitionGoal(gw.ctx, ref, "cancelling", { fields: { cancelMode: mode } });
|
|
4478
|
+
const r = await commitGoalResult(gw.ctx, { ref, now: Date.now(), cause: "cancel", data: { cancelledBy: "despawn" }, committer: { instanceId: this.managerInstanceId, epoch } });
|
|
4479
|
+
this.emitGoalProgress(ref, epoch, { phase: "terminal", state: r.fact.state, ...(r.fact.data !== undefined ? { data: r.fact.data } : {}) });
|
|
4480
|
+
await clearGoalIndex(gw.ctx, ref); // must-5 Q-B: terminal reached - the successor never reconciles it
|
|
4481
|
+
}
|
|
4482
|
+
catch {
|
|
4483
|
+
// the goal already terminalized (the readiness outcome won the settle race) - nothing to cancel.
|
|
4484
|
+
}
|
|
4485
|
+
}
|
|
4486
|
+
/** The static F1 terminal for one departed incarnation (Unit B): delegates the gate/head CAS
|
|
4487
|
+
* sequence to the shared core saga over the executor transport; the footprint teardown (creds
|
|
4488
|
+
* file + broker durables/ACL) runs INSIDE the barrier as its cleanup step. On completion the
|
|
4489
|
+
* wire principal joins {@link retiredPrincipals} (the F5 refusal index) and the name hold
|
|
4490
|
+
* clears (ABA-guarded by uid). A PRE-UNIT-B lifecycle (no slot row — spawned before the
|
|
4491
|
+
* durable registry existed) has nothing to terminalize: its footprint teardown runs directly
|
|
4492
|
+
* and the hold clears, the honest upgrade path. */
|
|
4493
|
+
async driveStaticRetirement(a) {
|
|
4494
|
+
const opId = retireOpId(a.lifecycleUid);
|
|
4495
|
+
const cleanup = async () => {
|
|
4496
|
+
const secrets = this.secrets;
|
|
4497
|
+
const files = a.secretPaths ?? agentLifecycleSecretFilePaths(this.workspaceRoot, a.name, a.lifecycleUid);
|
|
4498
|
+
if (files.creds) {
|
|
4499
|
+
await secrets.delete(agentSecretKeyForFile(files.creds));
|
|
4500
|
+
rmSync(files.creds, { force: true });
|
|
4501
|
+
}
|
|
4502
|
+
await this.deprovisionBroker(a);
|
|
4503
|
+
};
|
|
4504
|
+
try {
|
|
4505
|
+
await this.withLifecycleExecutor({ owner: DEV_OWNER, actor: a.id, lifecycleUid: a.lifecycleUid, alias: a.name }, async (t) => {
|
|
4506
|
+
const slot = await readStaticSlot(t, DEV_OWNER, a.name);
|
|
4507
|
+
if (slot === undefined || slot.row.lifecycleUid !== a.lifecycleUid) {
|
|
4508
|
+
// No durable registration for THIS incarnation: a pre-Unit-B spawn (or a slot already
|
|
4509
|
+
// replaced by a successor — then this stale teardown must not touch the registry at all).
|
|
4510
|
+
await cleanup();
|
|
4511
|
+
return;
|
|
4512
|
+
}
|
|
4513
|
+
await runStaticTerminal(t, { owner: DEV_OWNER, alias: a.name, actor: a.id, lifecycleUid: a.lifecycleUid, opId }, { cleanup, log: (line) => console.error(`static retirement ${a.name}: ${line}`) });
|
|
4514
|
+
});
|
|
4515
|
+
this.retiredPrincipals.add(principalKey(DEV_OWNER, a.id).key);
|
|
4516
|
+
const cur = this.retiring.get(a.name);
|
|
4517
|
+
if (cur && cur.lifecycleUid === a.lifecycleUid)
|
|
4518
|
+
this.retiring.delete(a.name); // ABA-guarded hold clear
|
|
4519
|
+
}
|
|
4520
|
+
catch (e) {
|
|
4521
|
+
const h = this.retiring.get(a.name);
|
|
4522
|
+
if (h && h.lifecycleUid === a.lifecycleUid)
|
|
4523
|
+
h.lastError = `the static retirement did not complete (${e.message}); the name stays held - a same-name spawn retries the same terminal (op ${opId})`;
|
|
4524
|
+
console.error(`static retirement ${a.name} (${a.id}): ${e.message}`);
|
|
4525
|
+
}
|
|
4526
|
+
}
|
|
4527
|
+
/** F5(b) push renewal of ONE live managed-static credential (Unit B): re-mint the SAME nkey
|
|
4528
|
+
* identity with the SAME scope (recorded on the managed row at spawn) and a fresh bounded
|
|
4529
|
+
* exp, ledger the new credentialId (slot record first, then the row, then the file — a
|
|
4530
|
+
* credential is never materialized before its ledger row exists), and re-sign the SAME
|
|
4531
|
+
* lifecycle-keyed file the agent endpoint's source seam re-reads. Never advances the epoch,
|
|
4532
|
+
* never routes through any barrier (renewal is the THIRD transition). */
|
|
4533
|
+
async renewManagedStaticCred(a) {
|
|
4534
|
+
if (!this.auth || !a.seed || !a.secretPaths?.creds)
|
|
4535
|
+
throw new Error("renewManagedStaticCred: not a renewable managed-static agent");
|
|
4536
|
+
// THIS CHECK IS THE AUTHORITATIVE ONE. The renewal sweep's own `a.terminalizing` filter is an
|
|
4537
|
+
// optimisation that has already-awaited by the time it matters; removing or weakening this line
|
|
4538
|
+
// promotes that filter into the whole guard, with nothing failing at the moment of the change.
|
|
4539
|
+
//
|
|
4540
|
+
// CONFIRMED, OPEN, AND UNGATED. This check runs at ENTRY and there are FOUR awaits before the
|
|
4541
|
+
// two writes below (`secrets.put` and `materializeSecretToFile`). A despawn landing in that
|
|
4542
|
+
// window latches `terminalizing` and the retirement cleanup deletes exactly those two things —
|
|
4543
|
+
// same secret key, same path — so an in-flight renewal RE-CREATES a valid bounded credential
|
|
4544
|
+
// after teardown removed it, and `appendStaticCredentialRow` lands in the window too, which is
|
|
4545
|
+
// the worse half: a stale file is recoverable by re-running cleanup, a durable credential row
|
|
4546
|
+
// is the journal asserting the credential is legitimate.
|
|
4547
|
+
//
|
|
4548
|
+
// Reproduced by `smoke:renewal-terminal-race` (`renewal-terminal-race.smoke.ts`), which asserts
|
|
4549
|
+
// the DURABLE ROW rather than the file — the file is timing-dependent, the row is a KV read.
|
|
4550
|
+
// That suite is deliberately NOT in `smoke:ci`: it is expected RED until this is fixed, and
|
|
4551
|
+
// gating a known red trains readers to treat the gate as noisy. So the absence of a red here
|
|
4552
|
+
// is not evidence this is closed; run that suite.
|
|
4553
|
+
//
|
|
4554
|
+
// Reproduced on the FIRST attempt that reached the race, and that suite cannot produce a second:
|
|
4555
|
+
// the alias frees only when teardown completes, and the defect is that teardown does not, so
|
|
4556
|
+
// every later attempt is refused at spawn. That is a limit of the probe, NOT of the world — a
|
|
4557
|
+
// FRESH ALIAS PER ATTEMPT makes a rate measurable. Do not read hits-over-attempts off that file
|
|
4558
|
+
// as written; it is a number that is not a count.
|
|
4559
|
+
//
|
|
4560
|
+
// The fix is to make the WRITES conditional on the same latch the teardown orders against,
|
|
4561
|
+
// never to retry: the correct outcome is "no credential", never "a credential minted later".
|
|
4562
|
+
if (a.terminalizing)
|
|
4563
|
+
throw new Error("renewManagedStaticCred: the lifecycle is terminalizing; no credential is minted after the terminal begins");
|
|
4564
|
+
const exp = Math.floor(Date.now() / 1000) + MANAGED_STATIC_TTL_SEC;
|
|
4565
|
+
// The SAME permission scope the spawn minted (recorded on the managed row): allowSubscribe/
|
|
4566
|
+
// allowPublish/role/capabilities are the JWT-shaping inputs; `subscribe` (the active read
|
|
4567
|
+
// set) shapes durable membership only and is not a mint input.
|
|
4568
|
+
const creds = await mintCreds(this.auth, { id: a.id, seed: a.seed }, "agent", {
|
|
4569
|
+
allowSubscribe: a.launch.allowSubscribe,
|
|
4570
|
+
allowPublish: a.launch.allowPublish,
|
|
4571
|
+
role: a.role,
|
|
4572
|
+
capabilities: a.launch.capabilities,
|
|
4573
|
+
lifecycleUid: a.lifecycleUid,
|
|
4574
|
+
expiresAt: exp,
|
|
4575
|
+
});
|
|
4576
|
+
const credentialId = rawDigest(creds).replace("sha256:", "sha256-");
|
|
4577
|
+
await this.withLifecycleExecutor({ owner: DEV_OWNER, actor: a.id, lifecycleUid: a.lifecycleUid, alias: a.name }, async (t) => {
|
|
4578
|
+
await recordSlotCredential(t, DEV_OWNER, a.name, a.lifecycleUid, credentialId);
|
|
4579
|
+
await appendStaticCredentialRow(t, { lifecycleUid: a.lifecycleUid, credentialId, holderPrincipal: principalKey(DEV_OWNER, a.id).key, exp });
|
|
4580
|
+
});
|
|
4581
|
+
const secrets = this.secrets;
|
|
4582
|
+
await secrets.put(agentSecretKeyForFile(a.secretPaths.creds), creds);
|
|
4583
|
+
await materializeSecretToFile(secrets, agentSecretKeyForFile(a.secretPaths.creds), a.secretPaths.creds);
|
|
4584
|
+
console.error(`managed cred renewal ${a.name}: re-signed for the same identity (exp +${MANAGED_STATIC_TTL_SEC}s); the agent endpoint's source re-read adopts it`);
|
|
4585
|
+
}
|
|
4586
|
+
/** The Unit B reconciliation (F3 "no active orphan"): ensure the authority stores, then sweep
|
|
4587
|
+
* every durable slot row and act by the TOTAL resume table — `provisioning`/`terminalizing`
|
|
4588
|
+
* re-drive the exact-op terminal; an `active` row survives ONLY when a LIVE managed agent this
|
|
4589
|
+
* process owns backs it at the same uid (`adopted`), else its process is gone and it
|
|
4590
|
+
* terminalizes; `retired` rows seed the F5 refusal index. Two call sites: the BOOT sweep
|
|
4591
|
+
* (`postAdoption=false`, under the lease before control serving) DEFERS active-non-adopted
|
|
4592
|
+
* slots while a resume is still pending (adoption runs after it); the POST-ADOPTION sweep
|
|
4593
|
+
* (`postAdoption=true`, inside finalizeResume while `resumeRequired` still fences ordinary
|
|
4594
|
+
* spawns) terminalizes any active slot the resume did not claim. */
|
|
4595
|
+
async reconcileStaticLifecycles(postAdoption = false) {
|
|
4596
|
+
if (!this.auth)
|
|
4597
|
+
return;
|
|
4598
|
+
const identity = newIdentity();
|
|
4599
|
+
const creds = await mintCreds(this.auth, identity, "provisioner");
|
|
4600
|
+
const nc = await connect({ servers: this.servers ?? DEFAULT_SERVER, ...standaloneConnectOpts({ creds, /* not yet wired to a recorded transport */ tls: false }), maxReconnectAttempts: 0 });
|
|
4601
|
+
const slotRows = [];
|
|
4602
|
+
try {
|
|
4603
|
+
const jsm = await jetstreamManager(nc);
|
|
4604
|
+
const kvm = new Kvm(nc);
|
|
4605
|
+
await ensureAuthorityStores(jsm, kvm, this.space);
|
|
4606
|
+
const recordsKv = await kvm.open(recordsBucket(this.space));
|
|
4607
|
+
const t = staticLifecycleTransport(recordsKv, recordsKv /* auth reads unused in the sweep */);
|
|
4608
|
+
const keys = await recordsKv.keys(`${STATIC_SLOT_PREFIX}.${DEV_OWNER}.>`);
|
|
4609
|
+
const aliases = [];
|
|
4610
|
+
for await (const k of keys)
|
|
4611
|
+
aliases.push(k.split(".").slice(2).join("."));
|
|
4612
|
+
for (const alias of aliases) {
|
|
4613
|
+
const slot = await readStaticSlot(t, DEV_OWNER, alias);
|
|
4614
|
+
if (slot !== undefined)
|
|
4615
|
+
slotRows.push(slot.row);
|
|
4616
|
+
}
|
|
4617
|
+
}
|
|
4618
|
+
finally {
|
|
4619
|
+
await nc.drain().catch(() => nc.close());
|
|
4620
|
+
}
|
|
4621
|
+
for (const row of slotRows) {
|
|
4622
|
+
if (row.phase === "retired") {
|
|
4623
|
+
// A retirement is a GLOBAL refusal fact — seed the F5 index for EVERY retired incarnation
|
|
4624
|
+
// regardless of which instance owned it, so a sibling-retired incarnation's copied credential
|
|
4625
|
+
// is refused at this control surface too. Ownership gates only the DESTRUCTIVE sweep below.
|
|
4626
|
+
this.retiredPrincipals.add(principalKey(row.owner, row.actor).key);
|
|
4627
|
+
continue;
|
|
4628
|
+
}
|
|
4629
|
+
// 3b-2 RECONCILE OWNERSHIP (multi-manager-per-space): a manager adjudicates ONLY the non-retired
|
|
4630
|
+
// rows THIS logical instance owns. A SIBLING manager's active/provisioning row is LEFT UNTOUCHED —
|
|
4631
|
+
// sweep-terminalizing it would destroy the sibling's live agent (the historical all-agents-kill
|
|
4632
|
+
// hazard, now cross-instance). A legacy row (pre-3b-2, no owner recorded) predates multi-manager,
|
|
4633
|
+
// so this manager is its legitimate single-manager-past successor and reconciles it. An orphaned
|
|
4634
|
+
// sibling row is reclaimed only by an explicit operator CAS takeover (ruling 1), never here.
|
|
4635
|
+
if (row.ownerInstanceId !== undefined && row.ownerInstanceId !== this.managerInstanceId)
|
|
4636
|
+
continue;
|
|
4637
|
+
// ADOPTION is genuine membership: a slot backed by a live managed agent THIS process owns
|
|
4638
|
+
// at the SAME uid is never an orphan (empty at boot; exactly the adopted set at the
|
|
4639
|
+
// post-adoption sweep — the fix for the F3 resume hole).
|
|
4640
|
+
const live = this.agents.get(row.alias);
|
|
4641
|
+
const adopted = live !== undefined && live.lifecycleUid === row.lifecycleUid;
|
|
4642
|
+
// Boot sweep with a resume PENDING: an active slot may yet be adopted (the resume path runs
|
|
4643
|
+
// AFTER this boot sweep), so DEFER it — the post-adoption sweep terminalizes any the resume
|
|
4644
|
+
// did not claim. provisioning/terminalizing NEVER defer (they are crashed operations, never
|
|
4645
|
+
// an agent to adopt). At `postAdoption` (or a non-resume boot) nothing defers.
|
|
4646
|
+
if (!postAdoption && row.phase === "active" && !adopted && this.resumeRequired)
|
|
4647
|
+
continue;
|
|
4648
|
+
const action = planStaticSlotResume(row, adopted);
|
|
4649
|
+
if (action === "none")
|
|
4650
|
+
continue;
|
|
4651
|
+
console.error(`static reconcile ${row.alias}: slot is ${row.phase} with no live managed owner${postAdoption ? " after resume adoption" : ""} - driving its exact-op terminal (uid ${row.lifecycleUid})`);
|
|
4652
|
+
await this.driveStaticRetirement({ id: row.actor, name: row.alias, lifecycleUid: row.lifecycleUid });
|
|
4653
|
+
}
|
|
4654
|
+
}
|
|
4655
|
+
/** The F5(a) membership gate (Unit B, the F5-bind design): decide a control caller by its
|
|
4656
|
+
* AUTHENTICATED wire principal. A LIVE managed slot passes (unless terminalizing); a RETIRING
|
|
4657
|
+
* hold or a RETIRED static incarnation refuses even with a tier-valid JWT (the
|
|
4658
|
+
* copied-credential vector — its subject can never collide with a successor's, so this match
|
|
4659
|
+
* is non-forgeable); any OTHER principal is not a managed lifecycle (an operator instrument:
|
|
4660
|
+
* the credential tier governs, exactly as before). Never name alone, never a payload field. */
|
|
4661
|
+
lifecycleMembershipRefusal(caller) {
|
|
4662
|
+
for (const a of this.agents.values()) {
|
|
4663
|
+
if (this.managedPrincipal(a) === caller)
|
|
4664
|
+
return a.terminalizing
|
|
4665
|
+
? `the caller's lifecycle ${a.lifecycleUid} is terminalizing; control is refused from the first terminal step (F5)`
|
|
4666
|
+
: undefined;
|
|
4667
|
+
}
|
|
4668
|
+
for (const [name, hold] of this.retiring) {
|
|
4669
|
+
const held = hold.userOwner ? hold.agentId : principalKey(DEV_OWNER, hold.agentId).key;
|
|
4670
|
+
if (held === caller)
|
|
4671
|
+
return `the caller's lifecycle ${hold.lifecycleUid} (name "${name}") is retiring; a retiring incarnation's credential holds no control authority (F5)`;
|
|
4672
|
+
}
|
|
4673
|
+
if (this.retiredPrincipals.has(caller))
|
|
4674
|
+
return "the caller's lifecycle is retired; a retired incarnation's credential holds no control authority (F5)";
|
|
4675
|
+
return undefined;
|
|
4676
|
+
}
|
|
2728
4677
|
async withProvisioner(fn) {
|
|
2729
4678
|
if (!this.auth)
|
|
2730
4679
|
throw new Error("withProvisioner: no space auth (an open mesh has no scoped creds)");
|
|
@@ -2821,33 +4770,92 @@ export class Manager {
|
|
|
2821
4770
|
}
|
|
2822
4771
|
return { ok: true, data: { name, path } };
|
|
2823
4772
|
}
|
|
2824
|
-
|
|
2825
|
-
|
|
2826
|
-
|
|
2827
|
-
|
|
2828
|
-
|
|
2829
|
-
|
|
2830
|
-
|
|
2831
|
-
|
|
2832
|
-
if (denied)
|
|
2833
|
-
return { ok: false, error: denied };
|
|
4773
|
+
/** The post-authorization attach effect (P2 item 6): mint the holder-bound §13.6 offer, redeem it
|
|
4774
|
+
* through the ONE session plane (one-use CAS + presenter-equality), and stand up the PTY bridge —
|
|
4775
|
+
* atomically. The reply is the SIGNED grant (no ws:// URL, non-bearer, never logged); the caller
|
|
4776
|
+
* redeems it over the mesh with a per-session rails-only cred it mints itself. Only streamable
|
|
4777
|
+
* backends (pty/host) attach; an external runtime's attach() throws with per-runtime guidance. */
|
|
4778
|
+
async attachAuthorized(a, caller) {
|
|
4779
|
+
if (!this.sessionPlane)
|
|
4780
|
+
return { ok: false, error: "the manager session plane is not available (the manager is not fully started)" };
|
|
2834
4781
|
// A name is a reusable slot and the authorization above can await (user mode reads the ledger).
|
|
2835
|
-
//
|
|
2836
|
-
//
|
|
2837
|
-
//
|
|
2838
|
-
if (this.agents.get(name) !== a)
|
|
2839
|
-
return { ok: false, error: `agent "${name}" was replaced during authorization - retry` };
|
|
2840
|
-
//
|
|
2841
|
-
//
|
|
2842
|
-
if (a.handle.
|
|
2843
|
-
|
|
2844
|
-
|
|
2845
|
-
|
|
2846
|
-
|
|
2847
|
-
|
|
2848
|
-
|
|
4782
|
+
// The ep target is incarnation-pinned, so `a` cannot BE a successor — but the slot it names can
|
|
4783
|
+
// have been stopped and refilled while we waited, and everything below must act on the
|
|
4784
|
+
// incarnation this caller was actually authorized for, never on whoever holds the name now.
|
|
4785
|
+
if (this.agents.get(a.name) !== a)
|
|
4786
|
+
return { ok: false, error: `agent "${a.name}" was replaced during authorization - retry` };
|
|
4787
|
+
// Never establish a session over a dead agent — a doomed session (the caller would get an
|
|
4788
|
+
// immediate process-exit at best, a confusing empty terminal at worst). Refuse honestly.
|
|
4789
|
+
if (a.handle.status() !== "running")
|
|
4790
|
+
return { ok: false, error: `agent "${a.name}" is not running (${a.handle.status()}); nothing to attach` };
|
|
4791
|
+
// CLAIM the session slot BEFORE attaching the target's PTY, matching the console door: this door
|
|
4792
|
+
// used to attach first and check capacity inside establishAttach, so a `resource-exhausted`
|
|
4793
|
+
// refusal landed AFTER the attach. The claim is carried INTO establishAttach, so one reservation
|
|
4794
|
+
// spans the attach and the establishment, and it is released on every refusal below.
|
|
4795
|
+
//
|
|
4796
|
+
// RESIDUAL, NAMED: no shipped runtime acquires anything at attach time — pty's `attach()` returns
|
|
4797
|
+
// a pure view (it registers a data subscriber only when `onData` is called) and tmux/cmux/orca
|
|
4798
|
+
// throw — so an attach nobody bridges is a garbage-collectible object, not a held resource, and
|
|
4799
|
+
// ordering alone suffices. A future runtime that DOES acquire something in `attach()` reopens
|
|
4800
|
+
// this: it would need a release on the failure paths, and `AttachSession` has no close today.
|
|
4801
|
+
const slot = this.sessionPlane.claimSlot();
|
|
4802
|
+
let session;
|
|
4803
|
+
try {
|
|
4804
|
+
session = a.handle.attach();
|
|
4805
|
+
}
|
|
4806
|
+
catch (e) {
|
|
4807
|
+
slot.release();
|
|
4808
|
+
return { ok: false, error: e.message };
|
|
4809
|
+
}
|
|
4810
|
+
// establishAttach releases the claim it was handed on every exit; nothing to unwind here.
|
|
4811
|
+
const { grant } = await this.sessionPlane.establishAttach(caller, { name: a.name, lifecycleUid: a.lifecycleUid }, session, slot);
|
|
4812
|
+
return { ok: true, data: { grant } };
|
|
4813
|
+
}
|
|
4814
|
+
/** P2 item 6: the console's mesh §13.6 session establisher (backing `POST /session/<name>` on the
|
|
4815
|
+
* loopback face). Drives THE ONE plane — same establishAttach as the ep `attach` command — with
|
|
4816
|
+
* the loopback OPERATOR as holder (same-host trust boundary), then hands the browser everything it
|
|
4817
|
+
* needs to open the caller rail over the broker ws listener: the holder-bound grant, a per-session
|
|
4818
|
+
* RAILS-ONLY caller cred (static mints from the seed, TTL-bound to the session; an open mesh has no
|
|
4819
|
+
* credential system so the browser connects bare), and the ws URL. NO 127.0.0.1 terminal transport
|
|
4820
|
+
* — the terminal rides the mesh session. Injected only when a wsPort exists (see the constructor). */
|
|
4821
|
+
async establishConsoleSession(name) {
|
|
4822
|
+
if (!this.sessionPlane)
|
|
4823
|
+
throw new Error("the manager session plane is not available (the manager is not fully started)");
|
|
4824
|
+
if (this.wsPort === undefined)
|
|
4825
|
+
throw new Error("the broker websocket port is not configured; the console cannot open a mesh session");
|
|
4826
|
+
// CLAIM the session slot here, before the PTY attach and before this establisher goes on to
|
|
4827
|
+
// mint a seed-signed `session-caller` credential: a capacity refusal must land before anything
|
|
4828
|
+
// with a side effect or a cost. The claim is carried INTO establishAttach, so the reservation
|
|
4829
|
+
// spans the whole establishment rather than being a check that a concurrent caller can race.
|
|
4830
|
+
// (See attachAuthorized for why the attach itself needs no unwind on any shipped runtime.)
|
|
4831
|
+
const slot = this.sessionPlane.claimSlot();
|
|
4832
|
+
try {
|
|
4833
|
+
const a = this.agents.get(name);
|
|
4834
|
+
if (!a)
|
|
4835
|
+
throw new Error(`no managed agent "${name}"`);
|
|
4836
|
+
if (a.handle.status() !== "running")
|
|
4837
|
+
throw new Error(`agent "${name}" is not running (${a.handle.status()}); nothing to attach`);
|
|
4838
|
+
const session = a.handle.attach(); // throws for non-streamable runtimes — surfaced to the browser as a 500
|
|
4839
|
+
// The loopback operator is the console's holder (same-host trust boundary).
|
|
4840
|
+
const caller = { owner: DEV_OWNER, actor: "console", uid: this.managerLifecycleUid };
|
|
4841
|
+
const { grant } = await this.sessionPlane.establishAttach(caller, { name: a.name, lifecycleUid: a.lifecycleUid }, session, slot);
|
|
4842
|
+
// The caller credential is minted ONLY after a session is really live, so a refused or failed
|
|
4843
|
+
// establishment never yields a seed-signed JWT. (A failure HERE would leave a live session the
|
|
4844
|
+
// browser never reaches, holding its slot until the grant expires — unreachable in practice,
|
|
4845
|
+
// because the SERVING mint above signs from this same `this.auth` first and would have failed
|
|
4846
|
+
// before any session existed.)
|
|
4847
|
+
const creds = this.auth
|
|
4848
|
+
? await mintCreds(this.auth, newIdentity(), "session-caller", {
|
|
4849
|
+
sessionCaller: { endpoint: MANAGER_ENDPOINT, sessionId: grant.sessionId, epoch: grant.serving.epoch },
|
|
4850
|
+
expiresAt: Math.floor(grant.exp / 1000), // grant.exp is ms (now+ttlMs); the JWT exp is seconds
|
|
4851
|
+
})
|
|
4852
|
+
: "";
|
|
4853
|
+
return { grant, wsUrl: `ws://127.0.0.1:${this.wsPort}`, creds };
|
|
4854
|
+
}
|
|
4855
|
+
catch (e) {
|
|
4856
|
+
slot.release(); // idempotent with establishAttach's own release
|
|
4857
|
+
throw e;
|
|
2849
4858
|
}
|
|
2850
|
-
return { ok: true, data: { ws: this.attach.url(name, a.handle) } };
|
|
2851
4859
|
}
|
|
2852
4860
|
/** Managed agents cross-referenced with live presence (the manager sees the roster). */
|
|
2853
4861
|
/** `ownerFilter`: restrict to agents whose spawn-time stored `userOwner` equals it (the ps/status
|
|
@@ -2860,7 +4868,7 @@ export class Manager {
|
|
|
2860
4868
|
// FAIL-CLOSED: a failed record is the failure + repair sentence; a missing/malformed or
|
|
2861
4869
|
// stale record on a live agent is auth-unknown/auth-stale, NEVER silently healthy.
|
|
2862
4870
|
const health = a.userOwner
|
|
2863
|
-
? agentAuthState(
|
|
4871
|
+
? agentAuthState(a.secretPaths?.health ?? agentLifecycleSecretFilePaths(this.workspaceRoot, a.name, a.lifecycleUid).health)
|
|
2864
4872
|
: undefined;
|
|
2865
4873
|
return {
|
|
2866
4874
|
name: a.name,
|
|
@@ -2875,6 +4883,9 @@ export class Manager {
|
|
|
2875
4883
|
status: a.handle.status(),
|
|
2876
4884
|
uptimeMs: Date.now() - a.startedAt,
|
|
2877
4885
|
mesh: roster.get(a.name)?.status ?? "absent",
|
|
4886
|
+
// The incarnation coordinate (SPEC 13.1) — with `id`, exactly what a v0.4 caller needs to
|
|
4887
|
+
// build a targeted (`despawn`/`attach`) request against THIS incarnation.
|
|
4888
|
+
lifecycleUid: a.lifecycleUid,
|
|
2878
4889
|
...(health && health.state !== "ok" ? { authHealth: health.state, authReason: health.reason } : {}),
|
|
2879
4890
|
};
|
|
2880
4891
|
});
|