@cotal-ai/manager 0.57.0 → 0.59.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +13 -0
- package/dist/commands.d.ts.map +1 -1
- package/dist/commands.js +98 -16
- package/dist/commands.js.map +1 -1
- package/dist/console/session-bundle.js +1 -1
- package/dist/endpoint-evict.d.ts +9 -4
- package/dist/endpoint-evict.d.ts.map +1 -1
- package/dist/endpoint-evict.js +82 -26
- package/dist/endpoint-evict.js.map +1 -1
- package/dist/holder-liveness.d.ts +4 -0
- package/dist/holder-liveness.d.ts.map +1 -1
- package/dist/holder-liveness.js +95 -3
- package/dist/holder-liveness.js.map +1 -1
- package/dist/index.d.ts +5 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3 -0
- package/dist/index.js.map +1 -1
- package/dist/manager-service-contract.d.ts +15 -6
- package/dist/manager-service-contract.d.ts.map +1 -1
- package/dist/manager-service-contract.js +84 -13
- package/dist/manager-service-contract.js.map +1 -1
- package/dist/manager.d.ts +165 -67
- package/dist/manager.d.ts.map +1 -1
- package/dist/manager.js +706 -233
- package/dist/manager.js.map +1 -1
- package/dist/remote-authority.d.ts +54 -3
- package/dist/remote-authority.d.ts.map +1 -1
- package/dist/remote-authority.js +168 -2
- package/dist/remote-authority.js.map +1 -1
- package/dist/resume.d.ts.map +1 -1
- package/dist/resume.js +3 -0
- package/dist/resume.js.map +1 -1
- package/dist/run-hosting.d.ts +66 -1
- package/dist/run-hosting.d.ts.map +1 -1
- package/dist/run-hosting.js +126 -14
- package/dist/run-hosting.js.map +1 -1
- package/dist/runtime/custodial-pty.d.ts +12 -9
- package/dist/runtime/custodial-pty.d.ts.map +1 -1
- package/dist/runtime/custodial-pty.js +27 -16
- package/dist/runtime/custodial-pty.js.map +1 -1
- package/dist/runtime/index.d.ts +2 -0
- package/dist/runtime/index.d.ts.map +1 -1
- package/dist/runtime/index.js +148 -5
- package/dist/runtime/index.js.map +1 -1
- package/dist/runtime/pty.d.ts +3 -4
- package/dist/runtime/pty.d.ts.map +1 -1
- package/dist/runtime/pty.js +13 -6
- package/dist/runtime/pty.js.map +1 -1
- package/package.json +15 -13
- package/dist/ui.d.ts +0 -10
- package/dist/ui.d.ts.map +0 -1
- package/dist/ui.js +0 -10
- package/dist/ui.js.map +0 -1
package/dist/manager.js
CHANGED
|
@@ -4,11 +4,11 @@ import { hostname } from "node:os";
|
|
|
4
4
|
import { credsAuthenticator } from "@nats-io/transport-node";
|
|
5
5
|
import { existsSync, lstatSync, readFileSync, realpathSync, rmSync, statSync } from "node:fs";
|
|
6
6
|
import { isAbsolute, join, dirname, resolve } from "node:path";
|
|
7
|
-
import { CotalEndpoint, DEFAULT_SERVER, DEV_OWNER, MANAGER_LEASE_RENEW_MS, STANDING_RENEWABLE_TTL_SEC, assertLifecycleToken, divergentSecretStoreNotice, parseDaemonStoreAnswer, parseSecretStoreIdentity, sameSecretStoreIdentity, agentFilePath, clearSpaceHistory, dialerFor, connectorServers, spawnEnvAllow, deprovisionAgent, firstFreeName, spawnNameError, idFromCreds, inspectCredHealth, credsRenewalDelayMs, composeWirePersona, loadAgentFile, listPersonaCatalog, personaCatalogDescription, personaCatalogReadable, loadCotalConfig, mintCreds, mintGeneration, mintAcceptedToken, openIssuedStore, acceptedBucket, issuedBucket, mintLifecycleUid, managedRetirementOpId, mkSecretDir, newIdentity, actionContext, parsePrincipalKey, parseShareSelection, principalKey, probeConnect, provisionAgent, provisionAgentDurables, registry, resolveAuthProvider, saveAgentFile, subjectMatches, AUTH_ENDPOINT, EP_CMD_RETIRE_LIFECYCLE,
|
|
7
|
+
import { CotalEndpoint, accountFromCreds, credsClaims, DEFAULT_SERVER, DEV_OWNER, MANAGER_LEASE_RENEW_MS, STANDING_RENEWABLE_TTL_SEC, assertLifecycleToken, discardLaunchArtifacts, divergentSecretStoreNotice, parseDaemonStoreAnswer, parseSecretStoreIdentity, sameSecretStoreIdentity, agentFilePath, clearSpaceHistory, dialerFor, connectorServers, spawnEnvAllow, deprovisionAgent, DeprovisionError, isConcreteChannel, deprovisionTargetPrincipal, firstFreeName, spawnNameError, idFromCreds, inspectCredHealth, credsRenewalDelayMs, composeWirePersona, loadAgentFile, listPersonaCatalog, personaCatalogDescription, personaCatalogReadable, loadCotalConfig, mintCreds, mintGeneration, mintAcceptedToken, openIssuedStore, acceptedBucket, issuedBucket, mintLifecycleUid, managedRetirementOpId, mkSecretDir, newIdentity, actionContext, parsePrincipalKey, parseShareSelection, principalKey, probeConnect, provisionAgent, provisionAgentDurables, registry, resolveAuthProvider, saveAgentFile, subjectMatches, AUTH_ENDPOINT, EP_CMD_RETIRE_LIFECYCLE, controlServiceSubject, eventChannelPrincipal, resolveService, invokeCommand, } from "@cotal-ai/core";
|
|
8
8
|
import { agentAuthState, agentCredsDir, agentLifecycleSecretFilePaths, agentSecretFilePaths, agentSecretKeyForFile, authDir, connectorInstallHint, DEFAULT_CONNECTOR, defaultAgentType, DELIVERY_CREDS_KIND, extensionConnectors, findCotalRoot, getSpaceAuth, hasUserAuthState, loadExtensionsManifest, loadManagerInstanceIdentity, loadMeshes, localTrustOfSpace, manifestExtensionNames, materializeFromManifest, materializeSecretToFile, MEMBERSHIP_RW_CREDS_KIND, mergeLaunchOptions, remintDaemonCreds, resolveOnPath, createManagerInstanceIdentity, spaceMaterialKey, SYSTEM_CREDS_FILES, userAuthStateDir, workspaceSecretStore, writeRenewalRecord } from "@cotal-ai/workspace";
|
|
9
9
|
import { createRuntime, isCustodialRuntime, requireRuntimeAdopt, requireRuntimeReap, RuntimeReapUnproven, } from "./runtime/index.js";
|
|
10
10
|
import { AttachEndpoint } from "./attach-endpoint.js";
|
|
11
|
-
import { makeManagerEndpointEvictionEvidence, makeManagerEndpointEvictor } from "./endpoint-evict.js";
|
|
11
|
+
import { makeManagerEndpointEvictionEvidence, makeManagerEndpointEvictor, makeManagerEndpointHolderEvictor } from "./endpoint-evict.js";
|
|
12
12
|
import { makeManagerHolderLivenessProbe } from "./holder-liveness.js";
|
|
13
13
|
import { GateReconcileRefused, reconcileEndpointGate } from "./reconcile-gate.js";
|
|
14
14
|
import { launchSpecForRun, materializePersona, launchAgentToStartOpts, parseLaunchSpec, persistLaunchSpec } from "./launch.js";
|
|
@@ -197,61 +197,6 @@ function parseTurnNote(raw) {
|
|
|
197
197
|
function turnHoldToken(goalId) {
|
|
198
198
|
return createHash("sha256").update(`${goalId}:turn-deadline`, "utf8").digest("base64url").slice(0, 43);
|
|
199
199
|
}
|
|
200
|
-
/** One ep request/reply round-trip on the caller's OWN reply-plane filter (§13.2). The responder
|
|
201
|
-
* derives the reply subject from the authenticated request, so there is no caller-selected reply
|
|
202
|
-
* target to honour; the caller binds the answer off the reply SUBJECT — endpoint and nonce, both
|
|
203
|
-
* broker-pinned by the responder's serve publish grant. A reply on another nonce belongs to a
|
|
204
|
-
* different request (the rail is shared) and is ignored rather than allowed to fail this one. */
|
|
205
|
-
export async function epAwaitReply(nc, space, caller, nonce, requestId, requestSubject, body, timeoutMs) {
|
|
206
|
-
let sub;
|
|
207
|
-
let timer;
|
|
208
|
-
try {
|
|
209
|
-
const got = new Promise((resolve, reject) => {
|
|
210
|
-
sub = nc.subscribe(epCallerReplyFilter(space, caller), {
|
|
211
|
-
callback: (err, msg) => {
|
|
212
|
-
if (err) {
|
|
213
|
-
reject(new Error(`the retirement reply subscription failed: ${err.message}`));
|
|
214
|
-
return;
|
|
215
|
-
}
|
|
216
|
-
const parsed = parseEpSubject(msg.subject);
|
|
217
|
-
if (!parsed || parsed.plane !== "reply" || parsed.endpoint !== AUTH_ENDPOINT || parsed.nonce !== nonce)
|
|
218
|
-
return;
|
|
219
|
-
let body;
|
|
220
|
-
try {
|
|
221
|
-
body = JSON.parse(new TextDecoder().decode(msg.data));
|
|
222
|
-
}
|
|
223
|
-
catch {
|
|
224
|
-
return;
|
|
225
|
-
} // a malformed body on our nonce is not an answer; keep waiting
|
|
226
|
-
// REQUIRE THE ID ECHO. Binding on (endpoint, nonce) alone would accept a malformed or
|
|
227
|
-
// WRONG-ID `{ok:true}` and clear a retirement hold on it. Ignore rather than fail: the
|
|
228
|
-
// rail is shared across concurrent requests and a responder may publish at any nonce, so
|
|
229
|
-
// an unmatched reply is someone else's answer, never grounds to fail the honest one.
|
|
230
|
-
if (body.id !== requestId)
|
|
231
|
-
return;
|
|
232
|
-
resolve(body);
|
|
233
|
-
},
|
|
234
|
-
});
|
|
235
|
-
timer = setTimeout(() => reject(new Error("timeout")), timeoutMs);
|
|
236
|
-
nc.publish(requestSubject, new TextEncoder().encode(body));
|
|
237
|
-
});
|
|
238
|
-
return await got;
|
|
239
|
-
}
|
|
240
|
-
finally {
|
|
241
|
-
if (timer)
|
|
242
|
-
clearTimeout(timer);
|
|
243
|
-
try {
|
|
244
|
-
sub?.unsubscribe();
|
|
245
|
-
}
|
|
246
|
-
catch { /* connection already down */ }
|
|
247
|
-
}
|
|
248
|
-
}
|
|
249
|
-
/**
|
|
250
|
-
* The agent supervisor: a long-lived mesh node that owns agent process lifecycle.
|
|
251
|
-
* It serves control requests on the "manager" service and spawns/kills agents
|
|
252
|
-
* through a pluggable {@link Runtime} (pty by default). It does NOT proxy agent
|
|
253
|
-
* mesh traffic — terminal I/O streams over its own attach endpoint instead.
|
|
254
|
-
*/
|
|
255
200
|
/**
|
|
256
201
|
* Event channels in `channels` that do NOT belong to `{owner, actor}`.
|
|
257
202
|
*
|
|
@@ -292,6 +237,12 @@ function isAbsentDeliveryAdmin(msg) {
|
|
|
292
237
|
return false;
|
|
293
238
|
return /no responders|\b503\b/i.test(msg);
|
|
294
239
|
}
|
|
240
|
+
/**
|
|
241
|
+
* The agent supervisor: a long-lived mesh node that owns agent process lifecycle.
|
|
242
|
+
* It serves control requests on the "manager" service and spawns/kills agents
|
|
243
|
+
* through a pluggable {@link Runtime} (pty by default). It does NOT proxy agent
|
|
244
|
+
* mesh traffic — terminal I/O streams over its own attach endpoint instead.
|
|
245
|
+
*/
|
|
295
246
|
export class Manager {
|
|
296
247
|
space;
|
|
297
248
|
servers;
|
|
@@ -317,8 +268,13 @@ export class Manager {
|
|
|
317
268
|
endpointServeExecutorExpiresInSeconds;
|
|
318
269
|
agents = new Map();
|
|
319
270
|
/** Names whose spawn is in flight (reserved synchronously before the provision await) — counted
|
|
320
|
-
* toward the ceiling
|
|
271
|
+
* toward the ceiling (once per name, see {@link Manager.occupancy}) so two concurrent same-name
|
|
272
|
+
* spawns can't both pass the gate (P4a). */
|
|
321
273
|
reserved = new Set();
|
|
274
|
+
/** Reserved names whose launch already put its seat in `agents`. The name stays reserved until
|
|
275
|
+
* the launch settles, so no other spawn can take it, but the ceiling counts that seat through
|
|
276
|
+
* `agents` and, if it frees young, its cooling stamp, never through the reservation (#906). */
|
|
277
|
+
reservedLive = new Set();
|
|
322
278
|
/** Expiry stamps (`startedAt + MIN_LIFETIME`) for slots that freed while still young — a
|
|
323
279
|
* count-only, lazily-pruned recycle floor (P4c). Pruned + summed into the ceiling gate. */
|
|
324
280
|
cooling = [];
|
|
@@ -370,6 +326,11 @@ export class Manager {
|
|
|
370
326
|
* every (re)connect, so a renewal is adopted without re-registration). Absent on open meshes,
|
|
371
327
|
* in user mode (the named 1a follow-up), and before registration completes. */
|
|
372
328
|
serviceServe;
|
|
329
|
+
/** #1577: serve connections whose transport is down while the client reconnects them. Such a
|
|
330
|
+
* connection is open (`isClosed()` is false under unbounded reconnects) yet the broker holds
|
|
331
|
+
* none of its rails, so the liveness responder must not grade it `bound`. Keyed by the
|
|
332
|
+
* connection, so a re-dial that swaps in a fresh one starts from its own state. */
|
|
333
|
+
serveTransportDown = new WeakSet();
|
|
373
334
|
/** P2 item 2 (spawn-as-action): the SELF-MEDIATED goal-writer connection + ActionContext — a
|
|
374
335
|
* standing connection DISJOINT from the serve credential (Q2), scoped to exactly this endpoint's
|
|
375
336
|
* goal bind/terminal facts + goal-record writes ({@link goalWriterGrants}). Auth mode mints the
|
|
@@ -424,10 +385,11 @@ export class Manager {
|
|
|
424
385
|
/** P2 item 2 (M4): the live spawn goal ref for each managed agent name, so a despawn MID-GOAL
|
|
425
386
|
* drives the cancel path (transition -> cancel terminal). Cleared when the goal terminalizes. */
|
|
426
387
|
agentGoals = new Map();
|
|
427
|
-
/** The turn relay's same-incarnation idempotency map ({@link goalAcceptances}'s twin): a
|
|
428
|
-
* same-goalId retry serves the identical acceptance; cross-incarnation retries rebuild from
|
|
429
|
-
* the goal-index entry (its acceptance floor + note). */
|
|
430
388
|
/**
|
|
389
|
+
* The turn relay's same-incarnation idempotency map ({@link goalAcceptances}'s twin): a
|
|
390
|
+
* same-goalId retry serves the identical acceptance; cross-incarnation retries rebuild from
|
|
391
|
+
* the goal-index entry (its acceptance floor + note).
|
|
392
|
+
*
|
|
431
393
|
* One entry per turn this incarnation accepted: the acceptance a duplicate submission is served
|
|
432
394
|
* from, and — once the turn settles — the answer a RETRIED yield is served from.
|
|
433
395
|
*
|
|
@@ -467,6 +429,8 @@ export class Manager {
|
|
|
467
429
|
* lifecycle authority: every re-drive re-reads broker KV and re-runs planStaticSlotResume. */
|
|
468
430
|
staticReconcileItems = new Map();
|
|
469
431
|
staticReconcileLastSweep;
|
|
432
|
+
staticReconcileLastSweepResources;
|
|
433
|
+
lastDeprovisionResources;
|
|
470
434
|
staticReconcileSweepsInFlight = 0;
|
|
471
435
|
staticReconcileDrainWaiters = [];
|
|
472
436
|
staticReconcileStopping = false;
|
|
@@ -549,6 +513,11 @@ export class Manager {
|
|
|
549
513
|
resumedAgentNames = new Set();
|
|
550
514
|
remoteAuthority;
|
|
551
515
|
remoteExecutorCreds;
|
|
516
|
+
remoteSupervisorCreds;
|
|
517
|
+
remoteBundleRenewal;
|
|
518
|
+
/** A refused or failed standing candidate is cleanup debt, never a silent loss: the last-good
|
|
519
|
+
* family stays live and this records what the host may have signed but the manager never adopted. */
|
|
520
|
+
remoteRenewalDebt;
|
|
552
521
|
/** Every broker dial this manager makes, through ONE transport decision.
|
|
553
522
|
*
|
|
554
523
|
* `this.servers` is whatever the mesh record held (`cotal supervise` reads it from there), so it
|
|
@@ -561,6 +530,10 @@ export class Manager {
|
|
|
561
530
|
return dialerFor(servers)({ ...opts, servers });
|
|
562
531
|
}
|
|
563
532
|
constructor(opts) {
|
|
533
|
+
if (opts.pooled && (!opts.remoteAuthority || opts.runtime === undefined || opts.runtime === "auto" || opts.runtime === "pty"))
|
|
534
|
+
throw new Error("pooled control requires signerless remote authority and an explicit non-PTY runtime; local custodial execution is forbidden");
|
|
535
|
+
if (opts.pooled && !opts.remoteAuthority?.renewStandingBundle)
|
|
536
|
+
throw new Error("pooled control requires a closed host-issued all-duty renewal callback before construction");
|
|
564
537
|
this.space = opts.space;
|
|
565
538
|
this.servers = opts.servers;
|
|
566
539
|
this.name = opts.name ?? "manager";
|
|
@@ -569,6 +542,10 @@ export class Manager {
|
|
|
569
542
|
this.maxSessions = opts.maxSessions;
|
|
570
543
|
this.remoteAuthority = opts.remoteAuthority;
|
|
571
544
|
this.remoteExecutorCreds = opts.remoteAuthority?.executorCreds;
|
|
545
|
+
this.remoteSupervisorCreds = opts.remoteAuthority?.supervisorCreds;
|
|
546
|
+
if (opts.pooled && (!opts.remoteAuthority?.accountPublicKey ||
|
|
547
|
+
accountFromCreds(opts.remoteAuthority.supervisorCreds) !== opts.remoteAuthority.accountPublicKey))
|
|
548
|
+
throw new Error("pooled control requires a proved assigned account for its initial supervisor credential");
|
|
572
549
|
if (opts.remoteAuthority)
|
|
573
550
|
this.managerLifecycleUid = opts.remoteAuthority.lifecycleUid;
|
|
574
551
|
this.secrets = opts.secretStore ?? workspaceSecretStore(this.workspaceRoot);
|
|
@@ -577,6 +554,8 @@ export class Manager {
|
|
|
577
554
|
: { kind: "fs", root: resolve(this.workspaceRoot) };
|
|
578
555
|
this.installedExtensions = opts.installedExtensions ?? false;
|
|
579
556
|
this.runtime = createRuntime(opts.runtime ?? "auto", `cotal-${this.space}`);
|
|
557
|
+
if (opts.pooled && isCustodialRuntime(this.runtime))
|
|
558
|
+
throw new Error(`pooled control refuses custodial runtime "${this.runtime.kind}" before starting agents`);
|
|
580
559
|
this.preserveStopTimeoutMs = opts.preserveStopTimeoutMs ?? PRESERVE_STOP_TIMEOUT_MS;
|
|
581
560
|
this.endpointServeExecutorExpiresInSeconds = opts.endpointServeExecutorExpiresInSeconds;
|
|
582
561
|
if (opts.resumeAttemptId && !/^[A-Za-z0-9_-]{1,128}$/.test(opts.resumeAttemptId))
|
|
@@ -688,8 +667,8 @@ export class Manager {
|
|
|
688
667
|
case "user":
|
|
689
668
|
return (`\`cotal actor grant <reader> --owner ${owner} --scope '' --allow-subscribe '${channel}' ` +
|
|
690
669
|
`--allow-publish ''\`, with every field spelled out: \`actor grant\` is an upsert of the ` +
|
|
691
|
-
`WHOLE row and an omitted flag
|
|
692
|
-
`scope)
|
|
670
|
+
`WHOLE row, and an omitted flag is refused, or with \`--full\` is the WIDE default (\`>\` read, \`>\` post, \`spawn,role:default\` ` +
|
|
671
|
+
`scope).`);
|
|
693
672
|
case "static":
|
|
694
673
|
return (`\`cotal mint <reader> --profile agent --allow-subscribe ${channel} --provision\`, where there ` +
|
|
695
674
|
`is no actor ledger for \`actor grant\` to write to. The AGENT profile, not the observer ` +
|
|
@@ -792,7 +771,7 @@ export class Manager {
|
|
|
792
771
|
if (this.remoteAuthority) {
|
|
793
772
|
const remote = this.remoteAuthority;
|
|
794
773
|
id = remote.identities.supervisor.id;
|
|
795
|
-
creds = async () =>
|
|
774
|
+
creds = async () => this.remoteSupervisorCreds;
|
|
796
775
|
}
|
|
797
776
|
else if (this.auth) {
|
|
798
777
|
const identity = newIdentity();
|
|
@@ -836,6 +815,10 @@ export class Manager {
|
|
|
836
815
|
const reportEndpoint = (e) => console.error(`! manager endpoint: ${e.message}`);
|
|
837
816
|
this.ep.on("error", reportEndpoint);
|
|
838
817
|
this.ep.on("warning", reportEndpoint);
|
|
818
|
+
// Every roster change and every view transition, so a seat that is offline in a fresh snapshot
|
|
819
|
+
// (recorded with no presence event) is logged the moment the view can support the verdict.
|
|
820
|
+
this.ep.on("roster", () => this.logSeatMeshTransitions());
|
|
821
|
+
this.ep.on("presence-view", () => this.logSeatMeshTransitions());
|
|
839
822
|
await this.ep.start();
|
|
840
823
|
await this.ep.setActivity(`supervisor (${this.runtime.kind})`);
|
|
841
824
|
// Per-instance liveness lease (P2 item 3 — the old per-space singleton is DEMOTED per D9). Acquire
|
|
@@ -861,6 +844,31 @@ export class Manager {
|
|
|
861
844
|
}
|
|
862
845
|
this.leaseTimer = setInterval(() => { void this.renewLease(); }, MANAGER_LEASE_RENEW_MS);
|
|
863
846
|
this.leaseTimer.unref?.();
|
|
847
|
+
// THE PEER-READABLE LIVENESS RESPONDER for the manager plane (#1577). Bound HERE — after the
|
|
848
|
+
// lease, before registration — precisely so it is up during the window in which the manager is
|
|
849
|
+
// NOT yet serving. A responder that only appeared once everything worked could never report the
|
|
850
|
+
// broken state it exists to report, which is the reported defect in a new place.
|
|
851
|
+
//
|
|
852
|
+
// WHAT IT GRADES is `serviceServe`, the v0.4 service endpoint's serve handle, which is assigned
|
|
853
|
+
// only after `serveEndpoint` has bound this instance's rails, AND that handle's connection. A
|
|
854
|
+
// closed serve connection holds no subscriptions, while `onServeConnectionClosed` keeps the
|
|
855
|
+
// handle through its re-dial waits, so the handle alone would answer `bound` with every rail
|
|
856
|
+
// gone. The re-dial swaps in the fresh connection and handle together, so `bound` returns only
|
|
857
|
+
// once the rails are served again. A DISCONNECTED serve connection is the same fault without the
|
|
858
|
+
// close: it reconnects without limit, so it stays open while the broker holds none of its rails,
|
|
859
|
+
// and this probe arrives on the separate supervisor connection that is still up. So the grade
|
|
860
|
+
// also requires the serve transport to be connected. That is a RESPONDER fact. It is
|
|
861
|
+
// deliberately NOT "this process is alive": this process is trivially alive whenever it answers
|
|
862
|
+
// at all, so grading on that would make the surface a `pgrep` that also lies in both directions
|
|
863
|
+
// — the exact instrument the reporter's evidence table indicts. Between lease acquisition and a
|
|
864
|
+
// successful registration the honest answer is `unbound`, and that is what a peer is told.
|
|
865
|
+
//
|
|
866
|
+
// The lease row is never consulted and never sent: presence is the whole ask, and the row
|
|
867
|
+
// carries the operator's workspace root and pid.
|
|
868
|
+
this.ep.serveLiveness("manager", () => {
|
|
869
|
+
const s = this.serviceServe;
|
|
870
|
+
return s?.handle && !s.nc.isClosed() && !this.serveTransportDown.has(s.nc) ? "bound" : "unbound";
|
|
871
|
+
});
|
|
864
872
|
// Unit B (static §13.1): after this instance holds its lease, collect the durable static rows
|
|
865
873
|
// now, but do not let their exact-op terminals make the whole space unreachable. The service
|
|
866
874
|
// comes up below, then the sweep overlaps the remaining registration work. Two-window
|
|
@@ -895,8 +903,38 @@ export class Manager {
|
|
|
895
903
|
this.credRenewTimer.unref?.();
|
|
896
904
|
}
|
|
897
905
|
else if (this.remoteAuthority) {
|
|
898
|
-
|
|
899
|
-
|
|
906
|
+
if (this.remoteAuthority.renewStandingBundle)
|
|
907
|
+
await this.renewRemoteStandingBundle();
|
|
908
|
+
else
|
|
909
|
+
await this.renewRemoteExecutor();
|
|
910
|
+
let standingTtl = STANDING_RENEWABLE_TTL_SEC;
|
|
911
|
+
if (this.remoteSupervisorCreds) {
|
|
912
|
+
try {
|
|
913
|
+
const claims = credsClaims(this.remoteSupervisorCreds);
|
|
914
|
+
if (typeof claims.exp === "number" && typeof claims.iat === "number" && claims.exp > claims.iat) {
|
|
915
|
+
standingTtl = claims.exp - claims.iat;
|
|
916
|
+
}
|
|
917
|
+
}
|
|
918
|
+
catch { }
|
|
919
|
+
}
|
|
920
|
+
const intervalMs = Math.min(credRenewIntervalMs(5 * 60), credRenewIntervalMs(standingTtl));
|
|
921
|
+
this.credRenewTimer = setInterval(async () => {
|
|
922
|
+
try {
|
|
923
|
+
if (this.remoteAuthority?.renewStandingBundle)
|
|
924
|
+
await this.renewRemoteStandingBundle();
|
|
925
|
+
else
|
|
926
|
+
await this.renewRemoteExecutor();
|
|
927
|
+
}
|
|
928
|
+
catch (e) {
|
|
929
|
+
console.error(`! remote manager renewal pass failed: ${e.message}`);
|
|
930
|
+
}
|
|
931
|
+
try {
|
|
932
|
+
await this.runHosting?.renew();
|
|
933
|
+
}
|
|
934
|
+
catch (e) {
|
|
935
|
+
console.error(`! remote run-hosting renewal pass failed: ${e.message}`);
|
|
936
|
+
}
|
|
937
|
+
}, intervalMs);
|
|
900
938
|
this.credRenewTimer.unref?.();
|
|
901
939
|
}
|
|
902
940
|
// stop() fences before it waits for an accepted startup reconciliation terminal. Once that
|
|
@@ -929,9 +967,26 @@ export class Manager {
|
|
|
929
967
|
// incarnation takes back every run a dead predecessor was driving before it accepts new ones.
|
|
930
968
|
// The serve surface is already live by here, so the family itself holds the gate: `runHost()`
|
|
931
969
|
// refuses `run-start`/`run-resume` as `unavailable` until the host exists and `RunHosting`
|
|
932
|
-
// refuses them until its reconcile has returned. A user-auth
|
|
933
|
-
//
|
|
934
|
-
|
|
970
|
+
// refuses them until its reconcile has returned. A remote user-auth manager uses its four
|
|
971
|
+
// registered issuer callbacks instead of holding a signer; a local user mesh stands none up.
|
|
972
|
+
const remoteRuns = this.remoteAuthority?.runHosting;
|
|
973
|
+
if (this.remoteAuthority && remoteRuns) {
|
|
974
|
+
this.runHosting = new RunHosting({
|
|
975
|
+
space: this.space,
|
|
976
|
+
servers: this.servers,
|
|
977
|
+
endpoint: MANAGER_ENDPOINT,
|
|
978
|
+
instanceId: this.managerInstanceId,
|
|
979
|
+
holder: { id: this.remoteAuthority.identities.supervisor.id, lifecycleUid: this.managerLifecycleUid },
|
|
980
|
+
auth: undefined,
|
|
981
|
+
log: (line) => console.error(line),
|
|
982
|
+
admitRun: remoteRuns.admitRun,
|
|
983
|
+
issueAttempt: remoteRuns.issueAttempt,
|
|
984
|
+
issueOperator: remoteRuns.issueOperator,
|
|
985
|
+
renewRun: remoteRuns.renewRun,
|
|
986
|
+
});
|
|
987
|
+
await this.runHosting.reconcile();
|
|
988
|
+
}
|
|
989
|
+
else if (!this.remoteAuthority && !this.userMode) {
|
|
935
990
|
this.runHosting = new RunHosting({
|
|
936
991
|
space: this.space,
|
|
937
992
|
servers: this.servers,
|
|
@@ -1339,6 +1394,94 @@ export class Manager {
|
|
|
1339
1394
|
throw e;
|
|
1340
1395
|
}
|
|
1341
1396
|
}
|
|
1397
|
+
/** An active signerless manager renews its standing duties together. The issuer stages the closed
|
|
1398
|
+
* credential family first; broker preflight proves every JWT and existing caller-held nkey
|
|
1399
|
+
* before any mutable holder changes. A failed candidate leaves all current creds intact. */
|
|
1400
|
+
async renewRemoteStandingBundle(force = false) {
|
|
1401
|
+
if (this.remoteBundleRenewal)
|
|
1402
|
+
return this.remoteBundleRenewal;
|
|
1403
|
+
const renewal = this.renewRemoteStandingBundleOnce(force);
|
|
1404
|
+
this.remoteBundleRenewal = renewal;
|
|
1405
|
+
try {
|
|
1406
|
+
await renewal;
|
|
1407
|
+
}
|
|
1408
|
+
finally {
|
|
1409
|
+
if (this.remoteBundleRenewal === renewal)
|
|
1410
|
+
this.remoteBundleRenewal = undefined;
|
|
1411
|
+
}
|
|
1412
|
+
}
|
|
1413
|
+
/** A signerless manager's closed standing connection re-dials through the all-duty family only.
|
|
1414
|
+
* A healthy holder re-dials as is. An unhealthy one waits for the closed renewal and adoption,
|
|
1415
|
+
* and a refusal fails this attempt with last-good and debt kept. Nothing is minted locally. */
|
|
1416
|
+
async renewRemoteBeforeRedial(label, read) {
|
|
1417
|
+
if (inspectCredHealth(read() ?? "").state === "healthy")
|
|
1418
|
+
return;
|
|
1419
|
+
await this.renewRemoteStandingBundle(true);
|
|
1420
|
+
if (inspectCredHealth(read() ?? "").state !== "healthy")
|
|
1421
|
+
throw new Error(`the ${label} credential is not healthy after all-duty renewal; last-good is kept and nothing is minted locally`);
|
|
1422
|
+
}
|
|
1423
|
+
async renewRemoteStandingBundleOnce(force) {
|
|
1424
|
+
const remote = this.remoteAuthority;
|
|
1425
|
+
if (!remote?.renewStandingBundle || !this.serviceServe)
|
|
1426
|
+
return;
|
|
1427
|
+
const current = [this.remoteSupervisorCreds, this.remoteExecutorCreds, this.serviceServe.creds,
|
|
1428
|
+
this.goalWriterCreds, this.sessionLedgerCreds];
|
|
1429
|
+
if (!force && current.every((cred) => cred && inspectCredHealth(cred).state === "healthy"))
|
|
1430
|
+
return;
|
|
1431
|
+
const processEpoch = this.serviceServe.grant.epoch;
|
|
1432
|
+
let candidate;
|
|
1433
|
+
try {
|
|
1434
|
+
candidate = await remote.renewStandingBundle(processEpoch);
|
|
1435
|
+
const names = ["supervisor", "executor", "serve", "goalWriter", "sessionLedger"];
|
|
1436
|
+
if (Object.keys(candidate).sort().join(",") !== [...names].sort().join(","))
|
|
1437
|
+
throw new Error("host returned an incomplete or widened standing credential family");
|
|
1438
|
+
for (const name of names) {
|
|
1439
|
+
if (typeof candidate[name] !== "string" || inspectCredHealth(candidate[name]).state === "unreadable")
|
|
1440
|
+
throw new Error(`host returned an unreadable ${name} credential`);
|
|
1441
|
+
if (!remote.accountPublicKey || accountFromCreds(candidate[name]) !== remote.accountPublicKey)
|
|
1442
|
+
throw new Error(`host returned a ${name} credential for another account`);
|
|
1443
|
+
if (credsClaims(candidate[name]).sub !== remote.identities[name].id)
|
|
1444
|
+
throw new Error(`host returned a ${name} credential for a different nkey than the held identity`);
|
|
1445
|
+
const staged = candidate[name];
|
|
1446
|
+
const nc = await this.dial({ authenticator: (nonce) => credsAuthenticator(new TextEncoder().encode(staged))(nonce), maxReconnectAttempts: 0 });
|
|
1447
|
+
await nc.close();
|
|
1448
|
+
}
|
|
1449
|
+
if (this.leaseStopping || !this.serviceServe || this.serviceServe.grant.epoch !== processEpoch)
|
|
1450
|
+
throw new Error("manager changed its active serve epoch while standing renewal was in flight");
|
|
1451
|
+
this.remoteSupervisorCreds = candidate.supervisor;
|
|
1452
|
+
this.remoteExecutorCreds = candidate.executor;
|
|
1453
|
+
this.serviceServe.creds = candidate.serve;
|
|
1454
|
+
this.goalWriterCreds = candidate.goalWriter;
|
|
1455
|
+
this.sessionLedgerCreds = candidate.sessionLedger;
|
|
1456
|
+
if (this.goalWriter)
|
|
1457
|
+
this.goalWriter.creds = candidate.goalWriter;
|
|
1458
|
+
if (this.sessionLedgerConn)
|
|
1459
|
+
this.sessionLedgerConn.creds = candidate.sessionLedger;
|
|
1460
|
+
// Adoption is committed above; a connection that cannot come back yet is a push failure the
|
|
1461
|
+
// endpoint's own reconnect loop retries with the adopted credential, not unadopted debt.
|
|
1462
|
+
this.remoteRenewalDebt = undefined;
|
|
1463
|
+
await Promise.all([
|
|
1464
|
+
this.ep.reconnect().catch((e) => console.error(`! remote manager supervisor reconnect after renewal: ${e.message}`)),
|
|
1465
|
+
this.pushRenewedCreds(this.serviceServe.nc),
|
|
1466
|
+
...(this.goalWriter ? [this.pushRenewedCreds(this.goalWriter.nc)] : []),
|
|
1467
|
+
...(this.sessionLedgerConn ? [this.pushRenewedCreds(this.sessionLedgerConn.nc)] : []),
|
|
1468
|
+
]);
|
|
1469
|
+
}
|
|
1470
|
+
catch (e) {
|
|
1471
|
+
const unadoptedSubjects = candidate && typeof candidate === "object"
|
|
1472
|
+
? Object.values(candidate).flatMap((cred) => { try {
|
|
1473
|
+
return typeof cred === "string" && credsClaims(cred).sub ? [credsClaims(cred).sub] : [];
|
|
1474
|
+
}
|
|
1475
|
+
catch {
|
|
1476
|
+
return [];
|
|
1477
|
+
} })
|
|
1478
|
+
: [];
|
|
1479
|
+
this.remoteRenewalDebt = { at: Date.now(), processEpoch, reason: e.message, unadoptedSubjects };
|
|
1480
|
+
console.error(`! remote manager all-duty renewal: ${e.message} - no unsigned or local replacement is available`);
|
|
1481
|
+
if (force)
|
|
1482
|
+
throw e;
|
|
1483
|
+
}
|
|
1484
|
+
}
|
|
1342
1485
|
/** Warn, on every renewal pass, when a $SYS credential is at or past its renewal point.
|
|
1343
1486
|
*
|
|
1344
1487
|
* The manager is the renewal owner for every credential it CAN re-sign, and these two are the ones
|
|
@@ -1736,6 +1879,7 @@ export class Manager {
|
|
|
1736
1879
|
events: a.launch.events,
|
|
1737
1880
|
shareTools: a.launch.shareTools,
|
|
1738
1881
|
forkSource: a.launch.forkSource,
|
|
1882
|
+
...(this.forkProvenance(a) ? { resumed: this.forkProvenance(a) } : {}),
|
|
1739
1883
|
sessionId: a.restart?.armed ? this.readManagedSession(a) : a.launch.sessionId,
|
|
1740
1884
|
// Additive and only when the connector supplied one. A seat with no pointer records no
|
|
1741
1885
|
// field, which is what keeps an older inventory and a fresh one the same document shape.
|
|
@@ -1861,38 +2005,64 @@ export class Manager {
|
|
|
1861
2005
|
// revision it has since moved is a CAS the broker correctly refuses, silently, forever.
|
|
1862
2006
|
await this.leaseRenewTask?.catch(() => { });
|
|
1863
2007
|
let releaseFailures = [];
|
|
1864
|
-
|
|
1865
|
-
|
|
1866
|
-
|
|
1867
|
-
|
|
1868
|
-
|
|
2008
|
+
let seatFailure;
|
|
2009
|
+
try {
|
|
2010
|
+
if (this.maintenanceState === "active" && !this.resumeRequired) {
|
|
2011
|
+
if (options.withAgents)
|
|
2012
|
+
await this.teardownManagedAgents();
|
|
2013
|
+
else
|
|
2014
|
+
releaseFailures = this.releaseManagedAgents();
|
|
2015
|
+
}
|
|
2016
|
+
else {
|
|
2017
|
+
// A signal after a partial preservation must never fall back into destructive teardown.
|
|
2018
|
+
await this.stopRetainedAgentsOnExit();
|
|
2019
|
+
}
|
|
1869
2020
|
}
|
|
1870
|
-
|
|
1871
|
-
|
|
1872
|
-
|
|
2021
|
+
catch (e) {
|
|
2022
|
+
seatFailure = e;
|
|
2023
|
+
}
|
|
2024
|
+
// Every step below runs even when an earlier one fails, so a failed stop still closes this
|
|
2025
|
+
// process's connections and listener (#2290). Failures are collected and thrown at the end.
|
|
2026
|
+
const failures = [];
|
|
2027
|
+
const step = async (name, run) => {
|
|
2028
|
+
try {
|
|
2029
|
+
await run();
|
|
2030
|
+
}
|
|
2031
|
+
catch (e) {
|
|
2032
|
+
failures.push(`${name}: ${e.message}`);
|
|
2033
|
+
}
|
|
2034
|
+
};
|
|
2035
|
+
// A seat that may still be alive keeps this instance's leases and registration (see
|
|
2036
|
+
// teardownManagedAgents): they lapse on their TTL instead of being handed to a successor.
|
|
2037
|
+
if (!seatFailure) {
|
|
2038
|
+
await step("release manager lease", () => this.releaseOwnManagerLease());
|
|
2039
|
+
// #1634: hand the renewal lease back on a clean stop so a sibling picks it up on its next pass
|
|
2040
|
+
// rather than waiting out the bucket TTL. A non-owner holds no revision and this is a no-op.
|
|
2041
|
+
await step("release renewal lease", () => this.ep.releaseDaemonRenewalLease());
|
|
1873
2042
|
}
|
|
1874
|
-
await this.releaseOwnManagerLease();
|
|
1875
|
-
// #1634: hand the renewal lease back on a clean stop so a sibling picks it up on its next pass
|
|
1876
|
-
// rather than waiting out the bucket TTL. A non-owner holds no revision and this is a no-op.
|
|
1877
|
-
await this.ep.releaseDaemonRenewalLease();
|
|
1878
2043
|
// Capture BEFORE the serve loop is torn down: `stopServiceServe` clears the state, and what is
|
|
1879
2044
|
// being asked here is "did this process register a service instance", which only the state
|
|
1880
2045
|
// before teardown can answer. Deregistration runs AFTER the serve loop has drained, so no
|
|
1881
2046
|
// in-flight command can write a status back onto the record it just removed.
|
|
1882
2047
|
const registered = this.serviceServe !== undefined;
|
|
1883
|
-
await this.stopServiceServe();
|
|
2048
|
+
await step("stop service serve", () => this.stopServiceServe());
|
|
1884
2049
|
// The drives AFTER the serve loop (no new `run-start` can land) and BEFORE deregistration: a
|
|
1885
2050
|
// released run's status write is the last thing this incarnation says about it.
|
|
1886
|
-
await this.runHosting?.stop();
|
|
2051
|
+
await step("stop run hosting", () => this.runHosting?.stop());
|
|
1887
2052
|
this.runHosting = undefined;
|
|
1888
|
-
if (registered)
|
|
1889
|
-
await this.deregisterServiceOnStop();
|
|
1890
|
-
await this.stopGoalWriter();
|
|
1891
|
-
await this.stopSessionPlane();
|
|
1892
|
-
await this.ep.stop();
|
|
1893
|
-
await this.attach.stop();
|
|
2053
|
+
if (registered && !seatFailure)
|
|
2054
|
+
await step("deregister service", () => this.deregisterServiceOnStop());
|
|
2055
|
+
await step("stop goal writer", () => this.stopGoalWriter());
|
|
2056
|
+
await step("stop session plane", () => this.stopSessionPlane());
|
|
2057
|
+
await step("stop endpoint", () => this.ep.stop());
|
|
2058
|
+
await step("stop attach endpoint", () => this.attach.stop());
|
|
2059
|
+
const also = failures.length ? `; teardown also failed: ${failures.join("; ")}` : "";
|
|
2060
|
+
if (seatFailure)
|
|
2061
|
+
throw new Error(`${seatFailure.message}${also}`);
|
|
1894
2062
|
if (releaseFailures.length)
|
|
1895
|
-
throw new Error(`manager shutdown could not release every detachable seat: ${releaseFailures.join("; ")}`);
|
|
2063
|
+
throw new Error(`manager shutdown could not release every detachable seat: ${releaseFailures.join("; ")}${also}`);
|
|
2064
|
+
if (failures.length)
|
|
2065
|
+
throw new Error(`manager shutdown incomplete: ${failures.join("; ")}`);
|
|
1896
2066
|
}
|
|
1897
2067
|
/** Release this instance's liveness lease key on a clean stop, at the broker's own revision
|
|
1898
2068
|
* rather than a cached one that a renew still in flight moments ago may have already moved past.
|
|
@@ -2329,37 +2499,9 @@ export class Manager {
|
|
|
2329
2499
|
throw new EpEnvelopeError("permission-denied", `an any-mode ${ctx.subject.command} is operator reach; the caller's current ledger grant does not carry "admin" (SPEC 13.2)`);
|
|
2330
2500
|
return true;
|
|
2331
2501
|
}
|
|
2332
|
-
/** The v0.4 typed command table (P2 item 1, slice 1b): every ordinary handler runs the SHARED
|
|
2333
|
-
* admission chokepoint ({@link serveGated}) and then delegates to the SAME op core the ctl
|
|
2334
|
-
* door dispatches (checklist 8: one core, two thin doors). The resume/preservation family
|
|
2335
|
-
* deliberately BYPASSES serveGated — exactly as it sits before {@link admitControl} on the ctl
|
|
2336
|
-
* door (those ops must run while `resumeRequired` fences ordinary work) — riding its own state
|
|
2337
|
-
* fences; its ep gate is the admin-grade `manager.admin` capability grant (the 1b rule: static
|
|
2338
|
-
* admin-class commands are capability-gated + untargeted, never a fabricated ledger mode).
|
|
2339
|
-
*
|
|
2340
|
-
* TIER SEMANTICS on the ep door (the 1c grant-migration table): the tier lives in the CALLER'S
|
|
2341
|
-
* GRANT, refined per-op exactly as the ctl doors refine their subject tier. Owner-mode
|
|
2342
|
-
* `despawn`/`attach` keep the privileged semantics (`admin=false`, own-domain via
|
|
2343
|
-
* {@link authorizeNamed}) — every spawn-capable agent holds those rows. ANY-mode requests are
|
|
2344
|
-
* the operator instrument's cross-agent reach (rev 3): the any-mode subject row is mintable
|
|
2345
|
-
* only under operator policy (§13.2), so on a static mesh holding it IS the admin tier, and in
|
|
2346
|
-
* user mode the caller's CURRENT ledger scope must still carry `admin`
|
|
2347
|
-
* ({@link epAdminReach}, the same fresh-read authority `psOwnerFilter` consults). The
|
|
2348
|
-
* `manager.admin` family (purge + the resume/preservation ops) is capability-gated at mint AND
|
|
2349
|
-
* re-checked at serve time via {@link epAdminReach} (the `adminGated` wrapper) so a user's
|
|
2350
|
-
* revoked scope demotes the next call. `launch` is OWNER-EQUALITY on this door for everyone
|
|
2351
|
-
* (freelance HIGH #2): the deploy path is its only consumer and stamps the caller's own owner,
|
|
2352
|
-
* so cross-owner launch was a ctl-tier incidental never exercised, and keying it on the actor's
|
|
2353
|
-
* ledger scope broke the deployer-view attenuation - uniform owner-equality is the safe tier.
|
|
2354
|
-
* TWO DELIBERATE NARROWINGS vs the ctl doors (NOT bit-exact parity, panel-accepted): (1)
|
|
2355
|
-
* `define-persona` is `admin=false` for everyone (own-persona discipline; no ep consumer needs
|
|
2356
|
-
* cross-owner persona writes - an operator redefines via config, not the wire), where the ctl
|
|
2357
|
-
* admin tier allowed operator cross-owner redefine; (2) launch is owner-equality-only, above.
|
|
2358
|
-
* Both are least-privilege reductions, never widenings. */
|
|
2359
2502
|
/** The run host, or one of three refusals. A remote-authority manager holds no space signer, so
|
|
2360
|
-
* it cannot mint the per-run driver credential SPEC 14.6 requires
|
|
2361
|
-
*
|
|
2362
|
-
* user-auth mesh is `unimplemented` too, for a different reason it names: a hosted run's seats
|
|
2503
|
+
* it cannot mint the per-run driver credential SPEC 14.6 requires without all four closed
|
|
2504
|
+
* host callbacks. A local user-auth mesh is `unimplemented`: a hosted run's seats
|
|
2363
2505
|
* are spawned, turned and despawned by a caller derived from the run id under the static
|
|
2364
2506
|
* owner, which the user-mode spawn door refuses (no `u_` owner), so a program would fail at
|
|
2365
2507
|
* its first seat; no path to `--local` is offered there since a user bearer holds no run rows
|
|
@@ -2369,9 +2511,9 @@ export class Manager {
|
|
|
2369
2511
|
runHost() {
|
|
2370
2512
|
if (this.runHosting)
|
|
2371
2513
|
return this.runHosting;
|
|
2372
|
-
if (this.remoteAuthority)
|
|
2373
|
-
throw new EpEnvelopeError("unimplemented", "this manager does not host workflow runs: a remote-authority manager mints no run-driver credentials (SPEC 14.6); drive the run from a terminal with `cotal run start --local --file <program>`");
|
|
2374
|
-
if (this.userMode)
|
|
2514
|
+
if (this.remoteAuthority && !this.remoteAuthority.runHosting)
|
|
2515
|
+
throw new EpEnvelopeError("unimplemented", "this manager does not host workflow runs: its issuing host supplied no closed run callbacks, and a remote-authority manager mints no run-driver credentials (SPEC 14.6); drive the run from a terminal with `cotal run start --local --file <program>`");
|
|
2516
|
+
if (this.userMode && !this.remoteAuthority?.runHosting)
|
|
2375
2517
|
throw new EpEnvelopeError("unimplemented", `user-auth space "${this.space}" hosts no workflow runs yet: a hosted run's seats would be spawned under the static owner, which a user mesh refuses; run programs on a static-auth mesh`);
|
|
2376
2518
|
throw new EpEnvelopeError("unavailable", "the manager is still booting its workflow-run host; retry shortly (SPEC 14.3)");
|
|
2377
2519
|
}
|
|
@@ -2413,6 +2555,44 @@ export class Manager {
|
|
|
2413
2555
|
if (p.run !== args.runId || p.step !== args.stepKey || addressedToken !== open.token)
|
|
2414
2556
|
throw new EpEnvelopeError("permission-denied", `run-answer is allowed to ${agent.name} only for the open pause named by its pending ask or escalation relay; ${args.runId} ${args.stepKey} is not that pause`);
|
|
2415
2557
|
}
|
|
2558
|
+
/** A baseline seat may amend only an answer recorded under its own name: it changes its own
|
|
2559
|
+
* position, never another answerer's. Explicit `run` holders and unmanaged operator instruments
|
|
2560
|
+
* keep the reach they have to answer, as {@link authorizeRunAnswer} does. */
|
|
2561
|
+
authorizeRunAmend(ctx, accepted) {
|
|
2562
|
+
const caller = principalKey(ctx.subject.caller.owner, ctx.subject.caller.actor).key;
|
|
2563
|
+
const agent = [...this.agents.values()].find((a) => this.managedPrincipal(a) === caller && a.lifecycleUid === ctx.subject.caller.uid);
|
|
2564
|
+
if (agent === undefined || agent.launch.capabilities?.includes("run"))
|
|
2565
|
+
return;
|
|
2566
|
+
if (accepted.by !== agent.name)
|
|
2567
|
+
throw new EpEnvelopeError("permission-denied", `run-answer amend is allowed to ${agent.name} only for an answer recorded under its own name; this step's accepted answer is ${JSON.stringify(accepted.by)}'s`);
|
|
2568
|
+
}
|
|
2569
|
+
/** The v0.4 typed command table (P2 item 1, slice 1b): every ordinary handler runs the SHARED
|
|
2570
|
+
* admission chokepoint ({@link serveGated}) and then delegates to the SAME op core the ctl
|
|
2571
|
+
* door dispatches (checklist 8: one core, two thin doors). The resume/preservation family
|
|
2572
|
+
* deliberately BYPASSES serveGated — exactly as it sits before {@link admitControl} on the ctl
|
|
2573
|
+
* door (those ops must run while `resumeRequired` fences ordinary work) — riding its own state
|
|
2574
|
+
* fences; its ep gate is the admin-grade `manager.admin` capability grant (the 1b rule: static
|
|
2575
|
+
* admin-class commands are capability-gated + untargeted, never a fabricated ledger mode).
|
|
2576
|
+
*
|
|
2577
|
+
* TIER SEMANTICS on the ep door (the 1c grant-migration table): the tier lives in the CALLER'S
|
|
2578
|
+
* GRANT, refined per-op exactly as the ctl doors refine their subject tier. Owner-mode
|
|
2579
|
+
* `despawn`/`attach` keep the privileged semantics (`admin=false`, own-domain via
|
|
2580
|
+
* {@link authorizeNamed}) — every spawn-capable agent holds those rows. ANY-mode requests are
|
|
2581
|
+
* the operator instrument's cross-agent reach (rev 3): the any-mode subject row is mintable
|
|
2582
|
+
* only under operator policy (§13.2), so on a static mesh holding it IS the admin tier, and in
|
|
2583
|
+
* user mode the caller's CURRENT ledger scope must still carry `admin`
|
|
2584
|
+
* ({@link epAdminReach}, the same fresh-read authority `psOwnerFilter` consults). The
|
|
2585
|
+
* `manager.admin` family (purge + the resume/preservation ops) is capability-gated at mint AND
|
|
2586
|
+
* re-checked at serve time via {@link epAdminReach} (the `adminGated` wrapper) so a user's
|
|
2587
|
+
* revoked scope demotes the next call. `launch` is OWNER-EQUALITY on this door for everyone
|
|
2588
|
+
* (freelance HIGH #2): the deploy path is its only consumer and stamps the caller's own owner,
|
|
2589
|
+
* so cross-owner launch was a ctl-tier incidental never exercised, and keying it on the actor's
|
|
2590
|
+
* ledger scope broke the deployer-view attenuation - uniform owner-equality is the safe tier.
|
|
2591
|
+
* TWO DELIBERATE NARROWINGS vs the ctl doors (NOT bit-exact parity, panel-accepted): (1)
|
|
2592
|
+
* `define-persona` is `admin=false` for everyone (own-persona discipline; no ep consumer needs
|
|
2593
|
+
* cross-owner persona writes - an operator redefines via config, not the wire), where the ctl
|
|
2594
|
+
* admin tier allowed operator cross-owner redefine; (2) launch is owner-equality-only, above.
|
|
2595
|
+
* Both are least-privilege reductions, never widenings. */
|
|
2416
2596
|
managerServiceDefs() {
|
|
2417
2597
|
const args = (ctx) => (ctx.request.args ?? {});
|
|
2418
2598
|
const callerOf = (ctx) => principalKey(ctx.subject.caller.owner, ctx.subject.caller.actor).key;
|
|
@@ -2601,6 +2781,8 @@ export class Manager {
|
|
|
2601
2781
|
runResume: (ctx) => this.serveGated(ctx, () => this.runHost().resume(args(ctx))),
|
|
2602
2782
|
runAnswer: (ctx) => this.serveGated(ctx, () => {
|
|
2603
2783
|
const input = args(ctx);
|
|
2784
|
+
if (input.amend === true)
|
|
2785
|
+
return this.runHost().amend(input, this.runAnswerer(ctx), (accepted) => this.authorizeRunAmend(ctx, accepted));
|
|
2604
2786
|
return this.runHost().answer(input, this.runAnswerer(ctx), (open) => this.authorizeRunAnswer(ctx, input, open));
|
|
2605
2787
|
}),
|
|
2606
2788
|
runStatus: (ctx) => this.serveGated(ctx, () => this.runHost().status(args(ctx))),
|
|
@@ -3035,11 +3217,6 @@ export class Manager {
|
|
|
3035
3217
|
return { error: `agent auth preflight failed for "${name}": ${e.message}` };
|
|
3036
3218
|
}
|
|
3037
3219
|
}
|
|
3038
|
-
/** Drop a live agent's slot. When `floor` is set and the agent died young (lived less than
|
|
3039
|
-
* MIN_LIFETIME), push a cooling stamp so the freed slot still counts toward the ceiling until it
|
|
3040
|
-
* expires — flooring the RECYCLE, not the call, so both free paths (despawn + exit/reap) are
|
|
3041
|
-
* covered (P4c). Floor self + own-child despawn and natural exit; NEVER admin despawn (operator
|
|
3042
|
-
* emergency-kill stays unthrottled) and NEVER the reserved-rollback path (no cold-start paid). */
|
|
3043
3220
|
/**
|
|
3044
3221
|
* WHY A SEAT LEFT — one line, at the one place every free path passes through.
|
|
3045
3222
|
*
|
|
@@ -3062,7 +3239,7 @@ export class Manager {
|
|
|
3062
3239
|
const info = a.handle.exitInfo?.();
|
|
3063
3240
|
detail = info === undefined
|
|
3064
3241
|
? `exit detail unavailable from runtime "${a.handle.kind}"`
|
|
3065
|
-
: `exit code ${info.code ?? "unknown"}${info.signal === undefined ? "" : `, signal ${info.signal}`}`;
|
|
3242
|
+
: `exit code ${info.code ?? "unknown"}${info.signal === undefined ? "" : `, signal ${info.signal}`}${info.diagnostic ? `; last connector diagnostic: ${info.diagnostic}` : ""}`;
|
|
3066
3243
|
}
|
|
3067
3244
|
catch (e) {
|
|
3068
3245
|
// A runtime that throws while being asked has told us something real; it must not take the
|
|
@@ -3071,6 +3248,49 @@ export class Manager {
|
|
|
3071
3248
|
}
|
|
3072
3249
|
console.error(`seat reaped: ${a.name} (${a.id}, uid ${a.lifecycleUid}${a.handle.pid !== undefined ? `, pid ${a.handle.pid}` : ""}) - ${causeText}; ${detail}`);
|
|
3073
3250
|
}
|
|
3251
|
+
/**
|
|
3252
|
+
* A SEAT THAT LEAVES THE MESH WHILE ITS SLOT IS HELD GETS A LINE, AND SO DOES ITS RETURN.
|
|
3253
|
+
*
|
|
3254
|
+
* Seats once read `running · mesh offline` for many hours with no manager log line, so a watchdog
|
|
3255
|
+
* keyed on process liveness saw nothing wrong and nobody could say when each seat dropped (#1208).
|
|
3256
|
+
* The line follows the same state `ps` reads, checked on every roster change and view transition,
|
|
3257
|
+
* because not every offline record arrives as a presence event: after a reconnect the fresh snapshot
|
|
3258
|
+
* records an already-offline seat quietly. Only a current view is a verdict, so a stale or
|
|
3259
|
+
* unpopulated one logs nothing. The match is {@link isOwnPresence}. A seat being stopped is left to
|
|
3260
|
+
* the reap line. The heartbeat is the seat's own record and nothing validated it, so an unreadable
|
|
3261
|
+
* one prints as unknown instead of throwing inside the watch.
|
|
3262
|
+
*/
|
|
3263
|
+
logSeatMeshTransitions() {
|
|
3264
|
+
if (typeof this.ep.presenceView === "function" && this.ep.presenceView().state !== "current")
|
|
3265
|
+
return;
|
|
3266
|
+
const roster = new Map(this.ep.getRoster().map((p) => [p.card.id, p]));
|
|
3267
|
+
for (const a of this.agents.values()) {
|
|
3268
|
+
const seen = roster.get(this.managedPrincipal(a));
|
|
3269
|
+
if (a.terminalizing || !this.isOwnPresence(a, seen))
|
|
3270
|
+
continue;
|
|
3271
|
+
const who = `${a.name} (${a.id}, uid ${a.lifecycleUid}${a.handle.pid !== undefined ? `, pid ${a.handle.pid}` : ""})`;
|
|
3272
|
+
if (seen.status === "offline" && !a.offMesh) {
|
|
3273
|
+
a.offMesh = true;
|
|
3274
|
+
const ts = seen.ts;
|
|
3275
|
+
const last = typeof ts === "number" && !Number.isNaN(new Date(ts).getTime()) ? new Date(ts).toISOString() : "unknown";
|
|
3276
|
+
console.error(`seat offline on the mesh: ${who} - last heartbeat ${last}; process ${a.handle.status()}`);
|
|
3277
|
+
}
|
|
3278
|
+
else if (seen.status !== "offline" && a.offMesh) {
|
|
3279
|
+
a.offMesh = false;
|
|
3280
|
+
console.error(`seat back on the mesh: ${who}`);
|
|
3281
|
+
}
|
|
3282
|
+
}
|
|
3283
|
+
}
|
|
3284
|
+
/** Whether a roster record is this seat's own: the readiness fence's exact principal and
|
|
3285
|
+
* incarnation, so a same-named foreign peer or an older incarnation never speaks for it (#1208). */
|
|
3286
|
+
isOwnPresence(a, p) {
|
|
3287
|
+
return p !== undefined && p.card.id === this.managedPrincipal(a) && p.lifecycleUid === a.lifecycleUid;
|
|
3288
|
+
}
|
|
3289
|
+
/** Drop a live agent's slot. When `floor` is set and the agent died young (lived less than
|
|
3290
|
+
* MIN_LIFETIME), push a cooling stamp so the freed slot still counts toward the ceiling until it
|
|
3291
|
+
* expires — flooring the RECYCLE, not the call, so both free paths (despawn + exit/reap) are
|
|
3292
|
+
* covered (P4c). Floor self + own-child despawn and natural exit; NEVER admin despawn (operator
|
|
3293
|
+
* emergency-kill stays unthrottled) and NEVER the reserved-rollback path (no cold-start paid). */
|
|
3074
3294
|
freeSlot(a, floor, cause, acceptedBeforeFence = false) {
|
|
3075
3295
|
if (this.agents.get(a.name) !== a)
|
|
3076
3296
|
return; // already freed (exit raced despawn, etc.)
|
|
@@ -3101,14 +3321,14 @@ export class Manager {
|
|
|
3101
3321
|
if (this.userMode) {
|
|
3102
3322
|
const p = parsePrincipalKey(a.id);
|
|
3103
3323
|
if (p)
|
|
3104
|
-
this.retiring.set(a.name, { opId: retireOpId(a.lifecycleUid), lifecycleUid: a.lifecycleUid, owner: p.owner, actor: p.actor, agentId: a.id, userOwner: a.userOwner, secretPaths: a.secretPaths, startedAt: Date.now() });
|
|
3324
|
+
this.retiring.set(a.name, { opId: retireOpId(a.lifecycleUid), lifecycleUid: a.lifecycleUid, owner: p.owner, actor: p.actor, agentId: a.id, userOwner: a.userOwner, secretPaths: a.secretPaths, launch: { allowSubscribe: a.launch.allowSubscribe }, startedAt: Date.now() });
|
|
3105
3325
|
}
|
|
3106
3326
|
else if (this.auth) {
|
|
3107
3327
|
// Unit B: a STATIC lifecycle now also holds its name pending its own terminal (the F1
|
|
3108
3328
|
// static retirement the detached deprovision below drives) — the alias frees only when the
|
|
3109
3329
|
// gate+head terminal completes, exactly the user-mode discipline. The wire principal is the
|
|
3110
3330
|
// incarnation-unique nkey (F5-bind); owner is the dev owner.
|
|
3111
|
-
this.retiring.set(a.name, { opId: retireOpId(a.lifecycleUid), lifecycleUid: a.lifecycleUid, owner: DEV_OWNER, actor: a.id, agentId: a.id, secretPaths: a.secretPaths, runtime: a.handle?.reference, startedAt: Date.now() });
|
|
3331
|
+
this.retiring.set(a.name, { opId: retireOpId(a.lifecycleUid), lifecycleUid: a.lifecycleUid, owner: DEV_OWNER, actor: a.id, agentId: a.id, secretPaths: a.secretPaths, launch: { allowSubscribe: a.launch.allowSubscribe }, runtime: a.handle?.reference, startedAt: Date.now() });
|
|
3112
3332
|
}
|
|
3113
3333
|
// Auth mode: tear down the departed agent's minted broker footprint + creds file (#159 B2). The
|
|
3114
3334
|
// process is already gone, so this must never block the slot free or throw into the caller — it runs
|
|
@@ -3232,7 +3452,21 @@ export class Manager {
|
|
|
3232
3452
|
console.error(`revoke agent grant ${a.name}: ${e.message}`);
|
|
3233
3453
|
}
|
|
3234
3454
|
}
|
|
3235
|
-
|
|
3455
|
+
try {
|
|
3456
|
+
const res = await this.deprovisionBroker(a);
|
|
3457
|
+
const h = this.retiring.get(a.name);
|
|
3458
|
+
if (h && h.lifecycleUid === a.lifecycleUid)
|
|
3459
|
+
h.lastResources = res;
|
|
3460
|
+
}
|
|
3461
|
+
catch (e) {
|
|
3462
|
+
const h = this.retiring.get(a.name);
|
|
3463
|
+
if (h && h.lifecycleUid === a.lifecycleUid) {
|
|
3464
|
+
h.lastError = e.message;
|
|
3465
|
+
if (e instanceof DeprovisionError)
|
|
3466
|
+
h.lastResources = e.accounting;
|
|
3467
|
+
}
|
|
3468
|
+
throw e;
|
|
3469
|
+
}
|
|
3236
3470
|
// #29 piece 3: after the footprint teardown, ask the AUTH plane to RETIRE the lifecycle over
|
|
3237
3471
|
// the auth endpoint rail. The rail re-checks the SERVE-ISSUANCE GATE at serve time (not the
|
|
3238
3472
|
// space-manager lease - that check was replaced in 02794b2f) and refuses unless the registration
|
|
@@ -3324,46 +3558,44 @@ export class Manager {
|
|
|
3324
3558
|
: await mintCreds(this.auth, requestIdentity, "retirement-requester", {
|
|
3325
3559
|
retirementRequester: { ...caller, target: retirementTarget },
|
|
3326
3560
|
});
|
|
3327
|
-
|
|
3561
|
+
// The requester grant subscribes `_INBOX_<id>.>` only (provision.ts): the endpoint path's
|
|
3562
|
+
// JetStream API requests (the contract-store read inside resolveService) must reply under it, not under the client default `_INBOX.`.
|
|
3563
|
+
const nc = await this.dial({ authenticator: credsAuthenticator(new TextEncoder().encode(creds)), inboxPrefix: `_INBOX_${requestIdentity.id}`, maxReconnectAttempts: 0 });
|
|
3328
3564
|
try {
|
|
3329
|
-
//
|
|
3330
|
-
|
|
3331
|
-
//
|
|
3332
|
-
|
|
3333
|
-
|
|
3334
|
-
|
|
3335
|
-
|
|
3336
|
-
|
|
3337
|
-
|
|
3338
|
-
|
|
3339
|
-
|
|
3340
|
-
|
|
3341
|
-
caller,
|
|
3342
|
-
nonce,
|
|
3565
|
+
// The service resolves FRESH on every attempt (never cached on `this`): a plane restart
|
|
3566
|
+
// mid-despawn then lands on the new epoch through an ordinary resolve, rather than a
|
|
3567
|
+
// caller pinned to a now-superseded incarnation.
|
|
3568
|
+
const service = await resolveService(nc, this.space, AUTH_ENDPOINT, caller, { deadlineMs: 10_000 });
|
|
3569
|
+
// The TARGET rides the invoke's own target (authz mode `exact`, arity 3) — broker-enforced,
|
|
3570
|
+
// and pinned by the grant minted above, so it is not a body claim the caller can vary.
|
|
3571
|
+
const attributed = await invokeCommand(nc, this.space, service, EP_CMD_RETIRE_LIFECYCLE, {
|
|
3572
|
+
opId,
|
|
3573
|
+
serveEndpoint: MANAGER_ENDPOINT, serveInstanceId: this.managerInstanceId, serveEpoch,
|
|
3574
|
+
}, {
|
|
3575
|
+
target: { mode: "exact", owner: target.owner, actor: target.actor, lifecycleUid: a.lifecycleUid },
|
|
3576
|
+
deadlineMs: 20_000,
|
|
3343
3577
|
});
|
|
3344
|
-
|
|
3345
|
-
// prefixed with the RESPONDER's instance identity), so a caller-supplied `reply` header
|
|
3346
|
-
// would be ignored — that is the confused-deputy boundary being structural. The caller
|
|
3347
|
-
// therefore reads its own reply-plane filter and binds the answer off the reply SUBJECT.
|
|
3348
|
-
const m = await epAwaitReply(nc, this.space, caller, nonce, requestId, subject,
|
|
3349
|
-
// Declare THIS manager instance's serve identity. Since #350 these SELECT the gate row;
|
|
3350
|
-
// they no longer authorize — the rail refuses unless the row they name belongs to this
|
|
3351
|
-
// caller's own subject-derived principal. A superseded predecessor (same instanceId, OLD
|
|
3352
|
-
// epoch after a restart) is still refused by the epoch comparison.
|
|
3353
|
-
JSON.stringify({ id: requestId, op: "retireLifecycle", args: {
|
|
3354
|
-
opId,
|
|
3355
|
-
serveEndpoint: MANAGER_ENDPOINT, serveInstanceId: this.managerInstanceId, serveEpoch,
|
|
3356
|
-
} }), 20_000);
|
|
3357
|
-
const r = m;
|
|
3578
|
+
const r = attributed.reply;
|
|
3358
3579
|
if (r.ok) {
|
|
3359
3580
|
this.confirmRetirement(a);
|
|
3360
3581
|
}
|
|
3582
|
+
else if (r.error?.code === "expired") {
|
|
3583
|
+
// A stale-epoch refusal: the responder answered, but at another epoch than this resolve
|
|
3584
|
+
// bound (a plane restart mid-despawn landed on the new epoch) - the retry copy, never a
|
|
3585
|
+
// hard despawn failure line, since a same-name spawn (or the auth service's own boot
|
|
3586
|
+
// resume) re-drives the same teardown.
|
|
3587
|
+
const copy = uncertain(r.error.message);
|
|
3588
|
+
if (held)
|
|
3589
|
+
held.lastError = copy;
|
|
3590
|
+
console.error(`despawn ${a.name}: ${copy}`);
|
|
3591
|
+
}
|
|
3361
3592
|
else {
|
|
3362
3593
|
// The rail's refusal is already the operator copy (lease-loss/stale/foreign-op faces,
|
|
3363
3594
|
// full-no-op statements included) - surface it INTACT, never flattened.
|
|
3595
|
+
const message = r.error?.message ?? "the auth service refused the retirement without a reason";
|
|
3364
3596
|
if (held)
|
|
3365
|
-
held.lastError =
|
|
3366
|
-
console.error(`despawn ${a.name}: ${
|
|
3597
|
+
held.lastError = message;
|
|
3598
|
+
console.error(`despawn ${a.name}: ${message}`);
|
|
3367
3599
|
}
|
|
3368
3600
|
}
|
|
3369
3601
|
finally {
|
|
@@ -3414,13 +3646,59 @@ export class Manager {
|
|
|
3414
3646
|
// LIFECYCLE-PINNED (SPEC 13.1): both the credential's exact-name grants and the delete names
|
|
3415
3647
|
// carry a.lifecycleUid, so a stale/replayed teardown for this retired incarnation is broker-denied
|
|
3416
3648
|
// against a same-name successor's footprint (its names embed a different uid).
|
|
3649
|
+
// Durable membership rows are lifecycle-keyed per concrete channel. Start with the launch's
|
|
3650
|
+
// concrete read channels, then include wildcard-covered and unnamed channels from the inventory.
|
|
3651
|
+
const memberChannels = new Set((a.launch?.allowSubscribe ?? []).filter(isConcreteChannel));
|
|
3652
|
+
// Complete the set from the delivery daemon's lifecycle-exact inventory, which covers rows on
|
|
3653
|
+
// wildcard-granted channels and teardowns that carry no launch spec. An inventory that cannot be
|
|
3654
|
+
// read leaves those rows RETAINED and says so; it never widens the delete beyond exact keys.
|
|
3655
|
+
const inventory = await this.lifecycleMembershipInventory(a);
|
|
3656
|
+
for (const ch of inventory.channels)
|
|
3657
|
+
memberChannels.add(ch);
|
|
3658
|
+
if (!inventory.complete)
|
|
3659
|
+
console.error(`deprovision ${a.name} (${a.id}): membership inventory incomplete (${inventory.reason}); rows on channels outside its launch policy stay RETAINED for uid ${a.lifecycleUid}`);
|
|
3417
3660
|
const creds = await mintCreds(this.auth, newIdentity(), "deprovisioner", {
|
|
3418
|
-
deprovisionTarget: { principal: a.id, lifecycleUid: a.lifecycleUid },
|
|
3661
|
+
deprovisionTarget: { principal: a.id, lifecycleUid: a.lifecycleUid, memberChannels: [...memberChannels] },
|
|
3419
3662
|
});
|
|
3420
3663
|
// Bound the detached broker teardown so a wedged broker can't leave the deprovision promise pending
|
|
3421
3664
|
// forever with no log — the timeout rejects into freeSlot's fail-loud `.catch` (paired with the
|
|
3422
3665
|
// helper's own fail-fast connect). The durables/ACL row still fall to space teardown as a backstop.
|
|
3423
|
-
|
|
3666
|
+
let accounting;
|
|
3667
|
+
try {
|
|
3668
|
+
accounting = await withTimeout(deprovisionAgent({
|
|
3669
|
+
servers: this.servers ?? DEFAULT_SERVER,
|
|
3670
|
+
space: this.space,
|
|
3671
|
+
targetId: a.id,
|
|
3672
|
+
lifecycleUid: a.lifecycleUid,
|
|
3673
|
+
memberChannels: [...memberChannels],
|
|
3674
|
+
creds,
|
|
3675
|
+
}), DEPROVISION_TIMEOUT_MS, `deprovision ${a.name} (${a.id}): broker teardown timed out`);
|
|
3676
|
+
this.lastDeprovisionResources = accounting;
|
|
3677
|
+
}
|
|
3678
|
+
catch (e) {
|
|
3679
|
+
if (e instanceof DeprovisionError) {
|
|
3680
|
+
this.lastDeprovisionResources = e.accounting;
|
|
3681
|
+
}
|
|
3682
|
+
throw e;
|
|
3683
|
+
}
|
|
3684
|
+
if (!inventory.complete)
|
|
3685
|
+
throw new Error(`membership inventory incomplete (${inventory.reason}); rows on channels outside its launch policy stay RETAINED for uid ${a.lifecycleUid}`);
|
|
3686
|
+
return accounting;
|
|
3687
|
+
}
|
|
3688
|
+
/** The retiring lifecycle's durable membership channels, read from the delivery daemon's
|
|
3689
|
+
* `lifecycleMemberships` admin verb (read-only, lifecycle-exact). Incomplete on any refusal. */
|
|
3690
|
+
async lifecycleMembershipInventory(a) {
|
|
3691
|
+
const t = deprovisionTargetPrincipal({ principal: a.id, lifecycleUid: a.lifecycleUid });
|
|
3692
|
+
try {
|
|
3693
|
+
const r = await this.ep.requestDeliveryAdmin("lifecycleMemberships", { principal: principalKey(t.owner, t.actor).key, lifecycleUid: t.lifecycleUid }, 5_000);
|
|
3694
|
+
const data = r.data;
|
|
3695
|
+
if (!r.ok || data?.complete !== true || !Array.isArray(data.channels) || !data.channels.every((c) => typeof c === "string" && isConcreteChannel(c)))
|
|
3696
|
+
return { complete: false, channels: [], reason: r.ok ? "malformed inventory reply" : (r.error ?? "refused") };
|
|
3697
|
+
return { complete: true, channels: data.channels };
|
|
3698
|
+
}
|
|
3699
|
+
catch (e) {
|
|
3700
|
+
return { complete: false, channels: [], reason: e.message };
|
|
3701
|
+
}
|
|
3424
3702
|
}
|
|
3425
3703
|
/** Reap a parent's children on its exit (P4b). Every descendant remains managed until the runtime's
|
|
3426
3704
|
* authoritative wait proves exit; the wait participates in the lifecycle drain, so preservation can
|
|
@@ -3471,6 +3749,28 @@ export class Manager {
|
|
|
3471
3749
|
`(${error.message}). Reload the live Pi seat before the preservation cut; refusing to resume fresh and lose its context.`);
|
|
3472
3750
|
}
|
|
3473
3751
|
}
|
|
3752
|
+
/** A resumed seat's fork provenance (#1500). The seat forks after it launches, so this reads the
|
|
3753
|
+
* record its connector named once the seat has written it, and keeps it on the launch so the
|
|
3754
|
+
* resume document carries it across a manager restart. Until the seat has written it, or when the
|
|
3755
|
+
* record cannot be read or does not name this seat's source, it is absent: `ps` then shows the
|
|
3756
|
+
* source id alone rather than failing the listing for every other seat. */
|
|
3757
|
+
forkProvenance(a) {
|
|
3758
|
+
if (a.launch.resumed || !a.launch.resumeRecordPath || !a.launch.forkSource)
|
|
3759
|
+
return a.launch.resumed;
|
|
3760
|
+
let record;
|
|
3761
|
+
try {
|
|
3762
|
+
record = JSON.parse(readFileSync(a.launch.resumeRecordPath, "utf8"));
|
|
3763
|
+
}
|
|
3764
|
+
catch {
|
|
3765
|
+
return undefined;
|
|
3766
|
+
}
|
|
3767
|
+
const r = (typeof record === "object" && record !== null ? record : {});
|
|
3768
|
+
if (r.source !== a.launch.forkSource || typeof r.transcriptSha256 !== "string" || !/^[a-f0-9]{64}$/.test(r.transcriptSha256) ||
|
|
3769
|
+
!(r.title === undefined || r.title === null || (typeof r.title === "string" && r.title.length <= 1024)))
|
|
3770
|
+
return undefined;
|
|
3771
|
+
a.launch.resumed = { source: r.source, ...(typeof r.title === "string" && r.title ? { title: r.title } : {}), transcriptSha256: r.transcriptSha256 };
|
|
3772
|
+
return a.launch.resumed;
|
|
3773
|
+
}
|
|
3474
3774
|
readManagedSession(a) {
|
|
3475
3775
|
return this.readManagedSessionState(a).sessionId;
|
|
3476
3776
|
}
|
|
@@ -4181,9 +4481,9 @@ export class Manager {
|
|
|
4181
4481
|
// SYNCHRONOUS (existsSync / registry / accessSync / readFileSync — no await), so the gate stays
|
|
4182
4482
|
// atomic: the capacity snapshot and the reserve land in one tick (P4a/P4c), and two concurrent
|
|
4183
4483
|
// spawns can't overshoot the ceiling or pick the same name.
|
|
4184
|
-
const
|
|
4185
|
-
if (
|
|
4186
|
-
return { ok: false, error:
|
|
4484
|
+
const occupancy = this.occupancy(); // prunes expired cooling stamps; counts each name once
|
|
4485
|
+
if (occupancy.used >= MAX_AGENTS)
|
|
4486
|
+
return { ok: false, error: this.capacityRefusal(occupancy) };
|
|
4187
4487
|
// Harness preflight before reserving a slot or minting — a missing `claude`/`opencode` binary
|
|
4188
4488
|
// fails here with a clear name, not obscurely at process spawn. No fallback. All synchronous, so
|
|
4189
4489
|
// the reserve gate stays atomic. (The connector itself was resolved up top, before the capacity gate.)
|
|
@@ -4328,7 +4628,7 @@ export class Manager {
|
|
|
4328
4628
|
return { ok: false, error: `the name "${identityName}" is still reconciling at manager startup; its prior lifecycle terminal owns this alias until it completes. Retry shortly.` };
|
|
4329
4629
|
const held = this.retiring.get(identityName);
|
|
4330
4630
|
if (held !== undefined) {
|
|
4331
|
-
void this.deprovision({ id: held.agentId, name: identityName, lifecycleUid: held.lifecycleUid, userOwner: held.userOwner, secretPaths: held.secretPaths }).catch(() => { });
|
|
4631
|
+
void this.deprovision({ id: held.agentId, name: identityName, lifecycleUid: held.lifecycleUid, userOwner: held.userOwner, secretPaths: held.secretPaths, launch: held.launch }).catch(() => { });
|
|
4332
4632
|
const err = lifecycleBlocked("failed-precondition", `the name "${identityName}" is reserved pending retirement: its previous agent's despawn started that lifecycle's teardown (footprint + standing-authority revoke + auth-side retirement), and the name frees only when all of it completes${held.lastError !== undefined ? ` (last attempt: ${held.lastError})` : ""}. NEXT: wait a moment and retry this spawn (retrying re-drives the whole teardown), or pick another name.`, { blockedOp: "retirement", headState: "retiring", opId: held.opId, remedy: "retry" });
|
|
4333
4633
|
return { ok: false, error: renderLifecycleBlocked(err.message, err), details: err.details };
|
|
4334
4634
|
}
|
|
@@ -4673,6 +4973,7 @@ export class Manager {
|
|
|
4673
4973
|
events,
|
|
4674
4974
|
shareTools: opts.shareTools,
|
|
4675
4975
|
forkSource: opts.resume,
|
|
4976
|
+
...(opts.resume !== undefined && spec.resumeRecordPath ? { resumeRecordPath: spec.resumeRecordPath } : {}),
|
|
4676
4977
|
// Opaque values may contain secrets. Preserve only their keys and require the referenced
|
|
4677
4978
|
// persona/manifest to resolve the values again; imperative overrides have no safe payload.
|
|
4678
4979
|
unresolvedLaunchOptionKeys: opts.launchOptions && Object.keys(opts.launchOptions).length
|
|
@@ -4707,6 +5008,7 @@ export class Manager {
|
|
|
4707
5008
|
});
|
|
4708
5009
|
}
|
|
4709
5010
|
this.agents.set(name, managed);
|
|
5011
|
+
this.reservedLive.add(name);
|
|
4710
5012
|
// The live slot now owns teardown — freeSlot deprovisions this identity on exit — so the
|
|
4711
5013
|
// orphan-rollback in `finally` no longer applies to it.
|
|
4712
5014
|
provisioned = undefined;
|
|
@@ -4776,6 +5078,7 @@ export class Manager {
|
|
|
4776
5078
|
}
|
|
4777
5079
|
finally {
|
|
4778
5080
|
this.reserved.delete(name);
|
|
5081
|
+
this.reservedLive.delete(name);
|
|
4779
5082
|
// Minted but never handed to a live slot (buildLaunch / runtime.spawn threw after mint) → tear the
|
|
4780
5083
|
// orphan down (detached, fail-loud) so a failed spawn leaves no creds/durables behind (#159 B).
|
|
4781
5084
|
if (provisioned) {
|
|
@@ -4792,6 +5095,7 @@ export class Manager {
|
|
|
4792
5095
|
if (!release)
|
|
4793
5096
|
return { ok: false, agents: [], error: this.maintenanceError() };
|
|
4794
5097
|
const batchReservations = [];
|
|
5098
|
+
const prepared = new Map();
|
|
4795
5099
|
try {
|
|
4796
5100
|
if (inventory.version !== "cotal-manager-resume/v1")
|
|
4797
5101
|
return { ok: false, agents: [], error: `unsupported manager resume inventory version ${String(inventory.version)}` };
|
|
@@ -4801,8 +5105,9 @@ export class Manager {
|
|
|
4801
5105
|
const principals = new Set();
|
|
4802
5106
|
await this.ep.waitForPresenceSnapshot();
|
|
4803
5107
|
const liveRoster = this.ep.getRoster().filter((presence) => presence.status !== "offline");
|
|
4804
|
-
|
|
4805
|
-
|
|
5108
|
+
const occupancy = this.occupancy();
|
|
5109
|
+
if (occupancy.used + inventory.agents.length > MAX_AGENTS)
|
|
5110
|
+
return { ok: false, agents: [], error: `resume inventory of ${inventory.agents.length} would exceed manager capacity (${this.occupancyText(occupancy)})` };
|
|
4806
5111
|
for (const entry of inventory.agents) {
|
|
4807
5112
|
if (seen.has(entry.name))
|
|
4808
5113
|
return { ok: false, agents: [], error: `resume inventory contains duplicate agent name "${entry.name}"` };
|
|
@@ -4858,7 +5163,6 @@ export class Manager {
|
|
|
4858
5163
|
this.reserved.add(entry.name);
|
|
4859
5164
|
batchReservations.push(entry.name);
|
|
4860
5165
|
}
|
|
4861
|
-
const prepared = new Map();
|
|
4862
5166
|
const preflight = [];
|
|
4863
5167
|
for (const entry of inventory.agents) {
|
|
4864
5168
|
const reply = await this.resumePreservedAgent(entry, true, true, prepared);
|
|
@@ -4887,8 +5191,22 @@ export class Manager {
|
|
|
4887
5191
|
return { ok: true, agents };
|
|
4888
5192
|
}
|
|
4889
5193
|
finally {
|
|
4890
|
-
for (const name of batchReservations)
|
|
5194
|
+
for (const name of batchReservations) {
|
|
4891
5195
|
this.reserved.delete(name);
|
|
5196
|
+
this.reservedLive.delete(name);
|
|
5197
|
+
}
|
|
5198
|
+
// Preflight builds every launch before any child starts, so a refused batch, a failed launch,
|
|
5199
|
+
// or the agents skipped after it leave built specs no runtime ever received.
|
|
5200
|
+
for (const { spec, spawned } of prepared.values()) {
|
|
5201
|
+
if (spawned)
|
|
5202
|
+
continue;
|
|
5203
|
+
try {
|
|
5204
|
+
discardLaunchArtifacts(spec.artifacts);
|
|
5205
|
+
}
|
|
5206
|
+
catch (e) {
|
|
5207
|
+
console.error(`! resume: ${e.message}`);
|
|
5208
|
+
}
|
|
5209
|
+
}
|
|
4892
5210
|
release();
|
|
4893
5211
|
}
|
|
4894
5212
|
}
|
|
@@ -5072,8 +5390,9 @@ export class Manager {
|
|
|
5072
5390
|
return { ok: false, error: nameErr };
|
|
5073
5391
|
if (this.agents.has(entry.name) || (!batchReserved && this.reserved.has(entry.name)))
|
|
5074
5392
|
return { ok: false, error: `retained agent "${entry.name}" is already managed or reserved; same-principal resume never auto-numbers` };
|
|
5075
|
-
|
|
5076
|
-
|
|
5393
|
+
const occupancy = batchReserved ? undefined : this.occupancy();
|
|
5394
|
+
if (occupancy && occupancy.used >= MAX_AGENTS)
|
|
5395
|
+
return { ok: false, error: `${this.capacityRefusal(occupancy)}; same-principal resume refused` };
|
|
5077
5396
|
const cached = prepared?.get(entry.name);
|
|
5078
5397
|
if (!preflightOnly && cached) {
|
|
5079
5398
|
try {
|
|
@@ -5232,6 +5551,7 @@ export class Manager {
|
|
|
5232
5551
|
const custody = this.reserveCustody();
|
|
5233
5552
|
if (custody)
|
|
5234
5553
|
await this.recordSlotCustody({ name: entry.name, id: resumeId, lifecycleUid: entry.identity.lifecycleUid }, custody);
|
|
5554
|
+
prepared.spawned = true;
|
|
5235
5555
|
const handle = await this.spawnCustodied(entry.name, prepared.spec, entry.launch.cwd, custody);
|
|
5236
5556
|
const managed = {
|
|
5237
5557
|
name: entry.name,
|
|
@@ -5268,6 +5588,8 @@ export class Manager {
|
|
|
5268
5588
|
events: entry.launch.events,
|
|
5269
5589
|
shareTools: entry.launch.shareTools,
|
|
5270
5590
|
forkSource: entry.launch.forkSource,
|
|
5591
|
+
...(entry.launch.resumed ? { resumed: entry.launch.resumed } : {}),
|
|
5592
|
+
...(prepared.launchOpts.resume !== undefined && prepared.spec.resumeRecordPath ? { resumeRecordPath: prepared.spec.resumeRecordPath } : {}),
|
|
5271
5593
|
sessionId: entry.launch.sessionId,
|
|
5272
5594
|
},
|
|
5273
5595
|
...(prepared.spec.sessionStatePath
|
|
@@ -5276,6 +5598,7 @@ export class Manager {
|
|
|
5276
5598
|
suppressCleanup: true,
|
|
5277
5599
|
};
|
|
5278
5600
|
this.agents.set(entry.name, managed);
|
|
5601
|
+
this.reservedLive.add(entry.name);
|
|
5279
5602
|
if (this.resumeAttemptId)
|
|
5280
5603
|
this.resumedAgentNames.add(entry.name);
|
|
5281
5604
|
await this.recordSlotRuntime(managed);
|
|
@@ -5312,8 +5635,10 @@ export class Manager {
|
|
|
5312
5635
|
return { ok: false, error: e.message };
|
|
5313
5636
|
}
|
|
5314
5637
|
finally {
|
|
5315
|
-
if (!batchReserved)
|
|
5638
|
+
if (!batchReserved) {
|
|
5316
5639
|
this.reserved.delete(entry.name);
|
|
5640
|
+
this.reservedLive.delete(entry.name);
|
|
5641
|
+
}
|
|
5317
5642
|
}
|
|
5318
5643
|
}
|
|
5319
5644
|
probeStaticCredential(creds) {
|
|
@@ -5517,6 +5842,48 @@ export class Manager {
|
|
|
5517
5842
|
this.cooling = this.cooling.filter((stamp) => stamp > now);
|
|
5518
5843
|
return this.cooling.length;
|
|
5519
5844
|
}
|
|
5845
|
+
/** What holds the ceiling's slots, one count per seat (#906). A launching seat sits in `reserved`
|
|
5846
|
+
* from accept until its launch settles, in `agents` once its process exists, and in `cooling`
|
|
5847
|
+
* if it then frees young, so a reservation stops counting once it reaches `agents`. `managed` is
|
|
5848
|
+
* what this manager's `ps` lists, `reserved` is the accepted launches that have no process yet,
|
|
5849
|
+
* and `launching` is every unsettled launch that still holds a slot through `reserved` or
|
|
5850
|
+
* `agents`. A launch whose seat already freed holds no slot but its cooling stamp, so its
|
|
5851
|
+
* failure can free nothing. */
|
|
5852
|
+
occupancy() {
|
|
5853
|
+
const cooling = this.coolingCount();
|
|
5854
|
+
let reserved = 0;
|
|
5855
|
+
let launching = 0;
|
|
5856
|
+
for (const name of this.reserved) {
|
|
5857
|
+
const placed = this.reservedLive.has(name);
|
|
5858
|
+
if (!placed)
|
|
5859
|
+
reserved++;
|
|
5860
|
+
if (!placed || this.agents.has(name))
|
|
5861
|
+
launching++;
|
|
5862
|
+
}
|
|
5863
|
+
const managed = this.agents.size;
|
|
5864
|
+
return {
|
|
5865
|
+
managed,
|
|
5866
|
+
reserved,
|
|
5867
|
+
cooling,
|
|
5868
|
+
launching,
|
|
5869
|
+
used: managed + reserved + cooling,
|
|
5870
|
+
nextCoolingMs: cooling ? Math.min(...this.cooling) - Date.now() : undefined,
|
|
5871
|
+
};
|
|
5872
|
+
}
|
|
5873
|
+
occupancyText(o) {
|
|
5874
|
+
return `${o.used} of ${MAX_AGENTS} slots: ${o.managed} managed, ${o.reserved} reserved, ${o.cooling} cooling`;
|
|
5875
|
+
}
|
|
5876
|
+
/** The capacity refusal states the split it enforces and whether waiting can free a slot: a
|
|
5877
|
+
* cooling slot frees at a known time, an unsettled launch frees one only if it fails, and a
|
|
5878
|
+
* managed seat frees one only when it is stopped. */
|
|
5879
|
+
capacityRefusal(o) {
|
|
5880
|
+
const remedy = o.nextCoolingMs !== undefined
|
|
5881
|
+
? `waiting frees a cooling slot in ${Math.ceil(o.nextCoolingMs / 1000)}s, or despawn one`
|
|
5882
|
+
: o.launching
|
|
5883
|
+
? `waiting frees a slot only if an unsettled launch fails (${o.launching} pending); otherwise despawn one`
|
|
5884
|
+
: "waiting frees no slot; despawn one";
|
|
5885
|
+
return `at capacity (${this.occupancyText(o)}); ${remedy}`;
|
|
5886
|
+
}
|
|
5520
5887
|
async opStop(args, caller, admin) {
|
|
5521
5888
|
const name = String(args.name ?? "").trim();
|
|
5522
5889
|
const a = this.agents.get(name);
|
|
@@ -5623,8 +5990,14 @@ export class Manager {
|
|
|
5623
5990
|
* This is NEVER the manager's standing seed/supervisor connection. */
|
|
5624
5991
|
async withEndpointServeExecutor(fn) {
|
|
5625
5992
|
const identity = this.remoteAuthority?.identities.executor ?? newIdentity();
|
|
5626
|
-
if (this.remoteAuthority && (!this.remoteExecutorCreds || inspectCredHealth(this.remoteExecutorCreds).state !== "healthy"))
|
|
5627
|
-
|
|
5993
|
+
if (this.remoteAuthority && (!this.remoteExecutorCreds || inspectCredHealth(this.remoteExecutorCreds).state !== "healthy")) {
|
|
5994
|
+
// Shutdown has drained and discarded the standing serve context. Renew only its scoped
|
|
5995
|
+
// maintenance executor through the existing host operation; never resurrect standing duties.
|
|
5996
|
+
if (this.leaseStopping)
|
|
5997
|
+
await this.renewRemoteExecutor(true);
|
|
5998
|
+
else
|
|
5999
|
+
await (this.remoteAuthority.renewStandingBundle ? this.renewRemoteStandingBundle(true) : this.renewRemoteExecutor(true));
|
|
6000
|
+
}
|
|
5628
6001
|
const creds = this.remoteExecutorCreds ?? (this.auth
|
|
5629
6002
|
? await mintCreds(this.auth, identity, "endpoint-serve-executor", {
|
|
5630
6003
|
endpointServeExecutor: { endpoint: MANAGER_ENDPOINT, instanceId: this.managerInstanceId },
|
|
@@ -5670,7 +6043,7 @@ export class Manager {
|
|
|
5670
6043
|
return {
|
|
5671
6044
|
instanceId: this.managerInstanceId,
|
|
5672
6045
|
runtime: this.runtime.kind,
|
|
5673
|
-
custody:
|
|
6046
|
+
custody: this.runtime.kind === "pty" && this.runtime.supportsRelease === true ? "custodied" : "legacy",
|
|
5674
6047
|
agentCount: this.agents.size,
|
|
5675
6048
|
uptimeMs: Date.now() - this.startedAtMs,
|
|
5676
6049
|
connectors: this.connectorStatuses.map((row) => ({ ...row, binaries: { ...row.binaries } })),
|
|
@@ -5721,7 +6094,7 @@ export class Manager {
|
|
|
5721
6094
|
report = await reconcileEndpointGate({
|
|
5722
6095
|
kv: authKv, space: this.space, endpoint: MANAGER_ENDPOINT, instanceId,
|
|
5723
6096
|
probeHolder: makeManagerHolderLivenessProbe({ space: this.space, servers, auth, log }),
|
|
5724
|
-
|
|
6097
|
+
evictHolders: makeManagerEndpointHolderEvictor({ space: this.space, servers, auth, log }),
|
|
5725
6098
|
log,
|
|
5726
6099
|
recordsKv,
|
|
5727
6100
|
});
|
|
@@ -5755,7 +6128,17 @@ export class Manager {
|
|
|
5755
6128
|
inboxPrefix: `_INBOX_${state.identity.id}`,
|
|
5756
6129
|
maxReconnectAttempts: -1,
|
|
5757
6130
|
});
|
|
5758
|
-
|
|
6131
|
+
// Track the transport from the first moment: nats.js reports `disconnect` when the socket
|
|
6132
|
+
// drops and `reconnect` only after it has re-sent this connection's subscriptions. If the
|
|
6133
|
+
// status stream itself fails, the transport can no longer be observed, so it counts as down.
|
|
6134
|
+
void (async () => {
|
|
6135
|
+
for await (const st of nc.status()) {
|
|
6136
|
+
if (st.type === "disconnect")
|
|
6137
|
+
this.serveTransportDown.add(nc);
|
|
6138
|
+
else if (st.type === "reconnect")
|
|
6139
|
+
this.serveTransportDown.delete(nc);
|
|
6140
|
+
}
|
|
6141
|
+
})().catch(() => this.serveTransportDown.add(nc));
|
|
5759
6142
|
let handle;
|
|
5760
6143
|
try {
|
|
5761
6144
|
// The 1b typed surface + the derived `describe`. The descriptor stays PUBLIC in static
|
|
@@ -5770,6 +6153,13 @@ export class Manager {
|
|
|
5770
6153
|
// durable slot rows mirror it). Static mode carries no mapping-revision dimension, so
|
|
5771
6154
|
// the revision is the constant 0 — a caller that pins a revision pins 0.
|
|
5772
6155
|
resolveTarget: (t) => {
|
|
6156
|
+
if (this.remoteAuthority) {
|
|
6157
|
+
const key = principalKey(t.owner, t.actor).key;
|
|
6158
|
+
for (const a of this.agents.values())
|
|
6159
|
+
if (a.id === key)
|
|
6160
|
+
return { lifecycleUid: a.lifecycleUid, mappingRevision: 0 };
|
|
6161
|
+
return undefined;
|
|
6162
|
+
}
|
|
5773
6163
|
if (t.owner === DEV_OWNER) {
|
|
5774
6164
|
for (const a of this.agents.values())
|
|
5775
6165
|
if (!a.userOwner && a.id === t.actor)
|
|
@@ -5789,6 +6179,9 @@ export class Manager {
|
|
|
5789
6179
|
await nc.drain().catch(() => nc.close());
|
|
5790
6180
|
throw e;
|
|
5791
6181
|
}
|
|
6182
|
+
// Wired only once the connection serves: a connection this method drains itself after a
|
|
6183
|
+
// refused serve is not a fault to recover, and the caller already sees the refusal.
|
|
6184
|
+
nc.closed().then((err) => { void this.onServeConnectionClosed(err ?? undefined); });
|
|
5792
6185
|
return { nc, handle };
|
|
5793
6186
|
}
|
|
5794
6187
|
/** #2073: re-mint the SAME `endpoint-serve` identity through the SAME scoped one-shot executor and
|
|
@@ -5823,7 +6216,9 @@ export class Manager {
|
|
|
5823
6216
|
const recovered = await this.boundedReconnect("service endpoint", async () => {
|
|
5824
6217
|
if (!s)
|
|
5825
6218
|
return false;
|
|
5826
|
-
if (
|
|
6219
|
+
if (this.remoteAuthority)
|
|
6220
|
+
await this.renewRemoteBeforeRedial("serve", () => s.creds);
|
|
6221
|
+
else if (inspectCredHealth(s.creds ?? "").state !== "healthy" && s.creds !== undefined) {
|
|
5827
6222
|
s.creds = await this.remintServeCreds(s);
|
|
5828
6223
|
this.scheduleServeRenewal(s.creds);
|
|
5829
6224
|
}
|
|
@@ -5867,28 +6262,9 @@ export class Manager {
|
|
|
5867
6262
|
grant: remote.serveGrant,
|
|
5868
6263
|
creds: remote.serveCreds,
|
|
5869
6264
|
};
|
|
5870
|
-
|
|
5871
|
-
const nc = await this.
|
|
5872
|
-
|
|
5873
|
-
inboxPrefix: `_INBOX_${state.identity.id}`,
|
|
5874
|
-
maxReconnectAttempts: -1,
|
|
5875
|
-
});
|
|
5876
|
-
try {
|
|
5877
|
-
state.handle = serveEndpoint(nc, this.space, state.grant, this.managerServiceDefs(), { public: true }, {
|
|
5878
|
-
resolveTarget: (t) => {
|
|
5879
|
-
const key = principalKey(t.owner, t.actor).key;
|
|
5880
|
-
for (const a of this.agents.values())
|
|
5881
|
-
if (a.id === key)
|
|
5882
|
-
return { lifecycleUid: a.lifecycleUid, mappingRevision: 0 };
|
|
5883
|
-
return undefined;
|
|
5884
|
-
},
|
|
5885
|
-
...(this.omitClassCommands() ? { omitClassCommands: this.omitClassCommands() } : {}),
|
|
5886
|
-
});
|
|
5887
|
-
}
|
|
5888
|
-
catch (e) {
|
|
5889
|
-
await nc.drain().catch(() => nc.close());
|
|
5890
|
-
throw e;
|
|
5891
|
-
}
|
|
6265
|
+
// The shared dialer wires the closed-connection handler, which re-dials through the all-duty family.
|
|
6266
|
+
const { nc, handle } = await this.dialServeConnection(state);
|
|
6267
|
+
state.handle = handle;
|
|
5892
6268
|
state.nc = nc;
|
|
5893
6269
|
this.serviceServe = state;
|
|
5894
6270
|
console.error(`remote manager service endpoint activated: ${MANAGER_ENDPOINT}/${this.managerInstanceId} (epoch ${state.grant.epoch})`);
|
|
@@ -6136,15 +6512,6 @@ export class Manager {
|
|
|
6136
6512
|
});
|
|
6137
6513
|
return creds;
|
|
6138
6514
|
}
|
|
6139
|
-
/** P2 item 2 (spawn-as-action): stand up the standing self-mediated goal-writer connection +
|
|
6140
|
-
* ActionContext. Mode-dual, mirroring {@link registerManagerService}: an AUTH mesh uses the
|
|
6141
|
-
* scoped `goal-writer` credential already minted + family-STAGED inside registration's run block
|
|
6142
|
-
* ({@link mintAndStageGoalWriter} — DISJOINT grant from the serve cred, SHARED §13.1 revocation
|
|
6143
|
-
* family); an OPEN mesh uses a bare connection (no credential system to mint from - the broker
|
|
6144
|
-
* enforces nothing). The connection presents the CURRENT credential on every (re)connect (a
|
|
6145
|
-
* renewal is adopted without reconnecting the whole endpoint); the ActionContext bonds its
|
|
6146
|
-
* KV + JS + JSM to this one connection and space (SPEC 13.4), so a composition mixup cannot splice
|
|
6147
|
-
* goal state across brokers. */
|
|
6148
6515
|
/** #2073: dial the goal-writer's ONE standing connection and rebuild everything derived from it
|
|
6149
6516
|
* (the fence-resolving action context, the issuance-gate reader) — shared by the initial start
|
|
6150
6517
|
* and the closed-connection recovery ({@link onGoalWriterConnectionClosed}), so a redial never
|
|
@@ -6198,7 +6565,9 @@ export class Manager {
|
|
|
6198
6565
|
const recovered = await this.boundedReconnect("goal-writer", async () => {
|
|
6199
6566
|
if (!gw)
|
|
6200
6567
|
return false;
|
|
6201
|
-
if (
|
|
6568
|
+
if (this.remoteAuthority)
|
|
6569
|
+
await this.renewRemoteBeforeRedial("goal-writer", () => gw.creds);
|
|
6570
|
+
else if (gw.creds !== undefined && inspectCredHealth(gw.creds).state !== "healthy") {
|
|
6202
6571
|
gw.creds = await this.withEndpointServeExecutor(({ authKv }) => this.mintAndStageGoalWriter(authKv));
|
|
6203
6572
|
this.goalWriterCreds = gw.creds;
|
|
6204
6573
|
}
|
|
@@ -6210,6 +6579,15 @@ export class Manager {
|
|
|
6210
6579
|
return;
|
|
6211
6580
|
console.error("! manager goal-writer could not be re-dialed; leaving the credential to the next renewal pass");
|
|
6212
6581
|
}
|
|
6582
|
+
/** P2 item 2 (spawn-as-action): stand up the standing self-mediated goal-writer connection +
|
|
6583
|
+
* ActionContext. Mode-dual, mirroring {@link registerManagerService}: an AUTH mesh uses the
|
|
6584
|
+
* scoped `goal-writer` credential already minted + family-STAGED inside registration's run block
|
|
6585
|
+
* ({@link mintAndStageGoalWriter} — DISJOINT grant from the serve cred, SHARED §13.1 revocation
|
|
6586
|
+
* family); an OPEN mesh uses a bare connection (no credential system to mint from - the broker
|
|
6587
|
+
* enforces nothing). The connection presents the CURRENT credential on every (re)connect (a
|
|
6588
|
+
* renewal is adopted without reconnecting the whole endpoint); the ActionContext bonds its
|
|
6589
|
+
* KV + JS + JSM to this one connection and space (SPEC 13.4), so a composition mixup cannot splice
|
|
6590
|
+
* goal state across brokers. */
|
|
6213
6591
|
async startGoalWriter() {
|
|
6214
6592
|
const identity = (this.auth || this.remoteAuthority) ? this.goalWriterIdentity : newIdentity();
|
|
6215
6593
|
// The mutable holder captured by the authenticator (mirrors the serve connection): a half-TTL
|
|
@@ -6271,18 +6649,6 @@ export class Manager {
|
|
|
6271
6649
|
});
|
|
6272
6650
|
return creds;
|
|
6273
6651
|
}
|
|
6274
|
-
/** P2 item 6: stand up the ONE §13.6 session plane on its own standing connection. Mode-dual,
|
|
6275
|
-
* mirroring {@link startGoalWriter}: an AUTH mesh presents the scoped `session-ledger` cred
|
|
6276
|
-
* already minted + family-staged inside registration's run block ({@link mintAndStageSessionLedger});
|
|
6277
|
-
* an OPEN mesh uses a bare connection (no credential system to mint from). The connection presents
|
|
6278
|
-
* the CURRENT credential on every (re)connect, so a half-TTL renewal is adopted without reconnecting.
|
|
6279
|
-
*
|
|
6280
|
-
* The offer SIGNER is a per-incarnation in-memory keypair: the static collapsed path mints AND
|
|
6281
|
-
* redeems the offer in one call ({@link ManagerSessionPlane.establishAttach}), so the manager
|
|
6282
|
-
* self-signs and self-verifies its own §13.6 grants and the keypair never leaves the process — a
|
|
6283
|
-
* holder never verifies the signature (it presents the grant back over the rail; the broker's
|
|
6284
|
-
* per-session caller cred is the holder's real fence). The plane's ledger lives in the DEDICATED
|
|
6285
|
-
* sessions bucket (createEndpointStreams provisioned it at registration). */
|
|
6286
6652
|
/** #2073: dial the session-ledger's ONE standing connection — shared by the initial start and the
|
|
6287
6653
|
* closed-connection recovery ({@link onSessionLedgerConnectionClosed}), the same shape {@link
|
|
6288
6654
|
* dialGoalWriter} uses for its rail. The KV/JetStream clients the session plane opened from the
|
|
@@ -6313,7 +6679,9 @@ export class Manager {
|
|
|
6313
6679
|
const recovered = await this.boundedReconnect("session-ledger", async () => {
|
|
6314
6680
|
if (!sw || !identity)
|
|
6315
6681
|
return false;
|
|
6316
|
-
if (
|
|
6682
|
+
if (this.remoteAuthority)
|
|
6683
|
+
await this.renewRemoteBeforeRedial("session-ledger", () => sw.creds);
|
|
6684
|
+
else if (sw.creds !== undefined && inspectCredHealth(sw.creds).state !== "healthy") {
|
|
6317
6685
|
sw.creds = await this.withEndpointServeExecutor(({ authKv }) => this.mintAndStageSessionLedger(authKv));
|
|
6318
6686
|
this.sessionLedgerCreds = sw.creds;
|
|
6319
6687
|
}
|
|
@@ -6325,6 +6693,18 @@ export class Manager {
|
|
|
6325
6693
|
return;
|
|
6326
6694
|
console.error("! manager session-ledger could not be re-dialed; leaving the credential to the next renewal pass");
|
|
6327
6695
|
}
|
|
6696
|
+
/** P2 item 6: stand up the ONE §13.6 session plane on its own standing connection. Mode-dual,
|
|
6697
|
+
* mirroring {@link startGoalWriter}: an AUTH mesh presents the scoped `session-ledger` cred
|
|
6698
|
+
* already minted + family-staged inside registration's run block ({@link mintAndStageSessionLedger});
|
|
6699
|
+
* an OPEN mesh uses a bare connection (no credential system to mint from). The connection presents
|
|
6700
|
+
* the CURRENT credential on every (re)connect, so a half-TTL renewal is adopted without reconnecting.
|
|
6701
|
+
*
|
|
6702
|
+
* The offer SIGNER is a per-incarnation in-memory keypair: the static collapsed path mints AND
|
|
6703
|
+
* redeems the offer in one call ({@link ManagerSessionPlane.establishAttach}), so the manager
|
|
6704
|
+
* self-signs and self-verifies its own §13.6 grants and the keypair never leaves the process — a
|
|
6705
|
+
* holder never verifies the signature (it presents the grant back over the rail; the broker's
|
|
6706
|
+
* per-session caller cred is the holder's real fence). The plane's ledger lives in the DEDICATED
|
|
6707
|
+
* sessions bucket (createEndpointStreams provisioned it at registration). */
|
|
6328
6708
|
async startSessionPlane() {
|
|
6329
6709
|
const identity = (this.auth || this.remoteAuthority) ? this.sessionLedgerIdentity : newIdentity();
|
|
6330
6710
|
// The mutable holder captured by the authenticator (mirrors the goal-writer): a half-TTL renewal
|
|
@@ -7921,6 +8301,10 @@ export class Manager {
|
|
|
7921
8301
|
finally {
|
|
7922
8302
|
await nc.drain().catch(() => nc.close());
|
|
7923
8303
|
}
|
|
8304
|
+
let retiredSlots = 0;
|
|
8305
|
+
let preservedForeign = 0;
|
|
8306
|
+
let preservedAdopted = 0;
|
|
8307
|
+
let deferredSlots = 0;
|
|
7924
8308
|
const terminalRows = [];
|
|
7925
8309
|
for (const row of slotRows) {
|
|
7926
8310
|
if (row.phase === "retired") {
|
|
@@ -7928,6 +8312,7 @@ export class Manager {
|
|
|
7928
8312
|
// regardless of which instance owned it, so a sibling-retired incarnation's copied credential
|
|
7929
8313
|
// is refused at this control surface too. Ownership gates only the DESTRUCTIVE sweep below.
|
|
7930
8314
|
this.retiredPrincipals.add(principalKey(row.owner, row.actor).key);
|
|
8315
|
+
retiredSlots++;
|
|
7931
8316
|
continue;
|
|
7932
8317
|
}
|
|
7933
8318
|
// 3b-2 RECONCILE OWNERSHIP (multi-manager-per-space): a manager adjudicates ONLY the non-retired
|
|
@@ -7936,19 +8321,26 @@ export class Manager {
|
|
|
7936
8321
|
// hazard, now cross-instance). A legacy row (pre-3b-2, no owner recorded) predates multi-manager,
|
|
7937
8322
|
// so this manager is its legitimate single-manager-past successor and reconciles it. An orphaned
|
|
7938
8323
|
// sibling row is reclaimed only by an explicit operator CAS takeover (ruling 1), never here.
|
|
7939
|
-
if (row.ownerInstanceId !== undefined && row.ownerInstanceId !== this.managerInstanceId)
|
|
8324
|
+
if (row.ownerInstanceId !== undefined && row.ownerInstanceId !== this.managerInstanceId) {
|
|
8325
|
+
preservedForeign++;
|
|
7940
8326
|
continue;
|
|
8327
|
+
}
|
|
7941
8328
|
// ADOPTION is genuine membership: a slot backed by a live managed agent THIS process owns
|
|
7942
8329
|
// at the SAME uid is never an orphan (empty at boot; exactly the adopted set at the
|
|
7943
8330
|
// post-adoption sweep — the fix for the F3 resume hole).
|
|
7944
8331
|
const live = this.agents.get(row.alias);
|
|
7945
8332
|
const adopted = live !== undefined && live.lifecycleUid === row.lifecycleUid;
|
|
8333
|
+
if (adopted) {
|
|
8334
|
+
preservedAdopted++;
|
|
8335
|
+
}
|
|
7946
8336
|
// Boot sweep with a resume PENDING: an active slot may yet be adopted (the resume path runs
|
|
7947
8337
|
// AFTER this boot sweep), so DEFER it — the post-adoption sweep terminalizes any the resume
|
|
7948
8338
|
// did not claim. provisioning/terminalizing NEVER defer (they are crashed operations, never
|
|
7949
8339
|
// an agent to adopt). At `postAdoption` (or a non-resume boot) nothing defers.
|
|
7950
|
-
if (!postAdoption && row.phase === "active" && !adopted && this.resumeRequired)
|
|
8340
|
+
if (!postAdoption && row.phase === "active" && !adopted && this.resumeRequired) {
|
|
8341
|
+
deferredSlots++;
|
|
7951
8342
|
continue;
|
|
8343
|
+
}
|
|
7952
8344
|
if (planStaticSlotResume(row, adopted) !== "none")
|
|
7953
8345
|
terminalRows.push(row);
|
|
7954
8346
|
}
|
|
@@ -7957,6 +8349,17 @@ export class Manager {
|
|
|
7957
8349
|
// alias, but the initial serial sweep must close every discovery-to-await gap up front.
|
|
7958
8350
|
for (const row of terminalRows)
|
|
7959
8351
|
this.reconcilingAliases.add(row.alias);
|
|
8352
|
+
const sweepDeprovision = {
|
|
8353
|
+
examined: 0,
|
|
8354
|
+
deleted: 0,
|
|
8355
|
+
absent: 0,
|
|
8356
|
+
refused: 0,
|
|
8357
|
+
acknowledged: 0,
|
|
8358
|
+
disappeared: 0,
|
|
8359
|
+
consumers: { examined: 0, deleted: 0, absent: 0, refused: 0, acknowledged: 0, disappeared: 0 },
|
|
8360
|
+
acls: { examined: 0, deleted: 0, absent: 0, refused: 0, disappeared: 0 },
|
|
8361
|
+
members: { examined: 0, deleted: 0, absent: 0, refused: 0, disappeared: 0 },
|
|
8362
|
+
};
|
|
7960
8363
|
for (const row of terminalRows) {
|
|
7961
8364
|
if (this.staticReconcileStopping)
|
|
7962
8365
|
break;
|
|
@@ -7976,7 +8379,33 @@ export class Manager {
|
|
|
7976
8379
|
};
|
|
7977
8380
|
this.staticReconcileItems.set(key, item);
|
|
7978
8381
|
}
|
|
8382
|
+
this.lastDeprovisionResources = undefined;
|
|
7979
8383
|
const succeeded = await this.attemptStaticReconcile(key, item, row);
|
|
8384
|
+
const acc = this.lastDeprovisionResources;
|
|
8385
|
+
if (acc) {
|
|
8386
|
+
sweepDeprovision.examined += acc.examined;
|
|
8387
|
+
sweepDeprovision.deleted = sweepDeprovision.deleted === null || acc.deleted === null ? null : sweepDeprovision.deleted + acc.deleted;
|
|
8388
|
+
sweepDeprovision.absent += acc.absent;
|
|
8389
|
+
sweepDeprovision.refused += acc.refused;
|
|
8390
|
+
sweepDeprovision.acknowledged += acc.acknowledged;
|
|
8391
|
+
sweepDeprovision.disappeared += acc.disappeared;
|
|
8392
|
+
sweepDeprovision.consumers.examined += acc.consumers.examined;
|
|
8393
|
+
sweepDeprovision.consumers.deleted = sweepDeprovision.consumers.deleted === null || acc.consumers.deleted === null ? null : sweepDeprovision.consumers.deleted + acc.consumers.deleted;
|
|
8394
|
+
sweepDeprovision.consumers.absent += acc.consumers.absent;
|
|
8395
|
+
sweepDeprovision.consumers.refused += acc.consumers.refused;
|
|
8396
|
+
sweepDeprovision.consumers.acknowledged += acc.consumers.acknowledged;
|
|
8397
|
+
sweepDeprovision.consumers.disappeared += acc.consumers.disappeared;
|
|
8398
|
+
sweepDeprovision.acls.examined += acc.acls.examined;
|
|
8399
|
+
sweepDeprovision.acls.deleted += acc.acls.deleted;
|
|
8400
|
+
sweepDeprovision.acls.absent += acc.acls.absent;
|
|
8401
|
+
sweepDeprovision.acls.refused += acc.acls.refused;
|
|
8402
|
+
sweepDeprovision.acls.disappeared += acc.acls.disappeared;
|
|
8403
|
+
sweepDeprovision.members.examined += acc.members.examined;
|
|
8404
|
+
sweepDeprovision.members.deleted += acc.members.deleted;
|
|
8405
|
+
sweepDeprovision.members.absent += acc.members.absent;
|
|
8406
|
+
sweepDeprovision.members.refused += acc.members.refused;
|
|
8407
|
+
sweepDeprovision.members.disappeared += acc.members.disappeared;
|
|
8408
|
+
}
|
|
7980
8409
|
if (succeeded)
|
|
7981
8410
|
sweep.succeeded++;
|
|
7982
8411
|
else
|
|
@@ -7984,6 +8413,18 @@ export class Manager {
|
|
|
7984
8413
|
this.reconcilingAliases.delete(row.alias);
|
|
7985
8414
|
}
|
|
7986
8415
|
sweep.completedAt = new Date().toISOString();
|
|
8416
|
+
this.staticReconcileLastSweepResources = {
|
|
8417
|
+
slots: {
|
|
8418
|
+
examined: slotRows.length,
|
|
8419
|
+
retired: retiredSlots,
|
|
8420
|
+
preservedForeign,
|
|
8421
|
+
preservedAdopted,
|
|
8422
|
+
deferred: deferredSlots,
|
|
8423
|
+
terminalized: sweep.succeeded,
|
|
8424
|
+
failed: sweep.failed,
|
|
8425
|
+
},
|
|
8426
|
+
deprovision: sweepDeprovision,
|
|
8427
|
+
};
|
|
7987
8428
|
for (const row of terminalRows) {
|
|
7988
8429
|
const key = this.staticReconcileKey(row);
|
|
7989
8430
|
const item = this.staticReconcileItems.get(key);
|
|
@@ -8404,8 +8845,9 @@ export class Manager {
|
|
|
8404
8845
|
throw e;
|
|
8405
8846
|
}
|
|
8406
8847
|
}
|
|
8407
|
-
/** Managed agents cross-referenced with live presence (the manager sees the roster).
|
|
8408
|
-
|
|
8848
|
+
/** Managed agents cross-referenced with live presence (the manager sees the roster).
|
|
8849
|
+
*
|
|
8850
|
+
* `ownerFilter`: restrict to agents whose spawn-time stored `userOwner` equals it (the ps/status
|
|
8409
8851
|
* owner-domain bound); undefined = unbounded. {@link NO_OWNER_MATCHES} matches nothing. */
|
|
8410
8852
|
list(ownerFilter) {
|
|
8411
8853
|
const roster = new Map(this.ep.getRoster().map((p) => [p.card.name, p]));
|
|
@@ -8427,8 +8869,33 @@ export class Manager {
|
|
|
8427
8869
|
: undefined;
|
|
8428
8870
|
// The connector-reported provider from presence (#785): the live roster card's `meta.provider`,
|
|
8429
8871
|
// when it is a non-empty string. Absent when the connector reported none — never fabricated.
|
|
8430
|
-
const
|
|
8872
|
+
const seen = roster.get(a.name);
|
|
8873
|
+
const providerMeta = seen?.card.meta?.provider;
|
|
8431
8874
|
const provider = typeof providerMeta === "string" && providerMeta ? providerMeta : undefined;
|
|
8875
|
+
// The harness-reported condition beside the mesh status (#618): a turn that died upstream reads
|
|
8876
|
+
// `waiting` with `rate_limit` here, not a bare `waiting`. Absent when the connector reported none.
|
|
8877
|
+
// Projected field by field like `provider`: a malformed peer row must not fail this reply's schema.
|
|
8878
|
+
const reported = roster.get(a.name)?.condition;
|
|
8879
|
+
const condition = reported && typeof reported.code === "string" && reported.code
|
|
8880
|
+
? {
|
|
8881
|
+
code: reported.code,
|
|
8882
|
+
...(typeof reported.source === "string" ? { source: reported.source } : {}),
|
|
8883
|
+
...(typeof reported.message === "string" ? { message: reported.message } : {}),
|
|
8884
|
+
...(typeof reported.since === "number" && Number.isFinite(reported.since) ? { since: reported.since } : {}),
|
|
8885
|
+
}
|
|
8886
|
+
: undefined;
|
|
8887
|
+
// The seat's last harness-reported work progress (#618). `ts` is only its heartbeat, so a turn
|
|
8888
|
+
// that stopped advancing reads as an old `activeAt` beside a live process. Absent when none was reported.
|
|
8889
|
+
const activeAtReported = roster.get(a.name)?.activeAt;
|
|
8890
|
+
const activeAt = typeof activeAtReported === "number" && Number.isFinite(activeAtReported) ? activeAtReported : undefined;
|
|
8891
|
+
// How long an offline verdict has stood (#1208): the seat's last heartbeat before it went
|
|
8892
|
+
// offline. Only beside a verdict, since a stale or unpopulated view dates nothing, only from the
|
|
8893
|
+
// seat's own record, since `seen` is matched by name and a same-named foreign peer or an older
|
|
8894
|
+
// incarnation must not date this seat, and only when the seat's unvalidated `ts` fits the row
|
|
8895
|
+
// schema, so one odd record cannot fail the reply.
|
|
8896
|
+
const offlineSince = view.state === "current" && this.isOwnPresence(a, seen) && seen.status === "offline" && Number.isSafeInteger(seen.ts) && seen.ts >= 0
|
|
8897
|
+
? seen.ts
|
|
8898
|
+
: undefined;
|
|
8432
8899
|
return {
|
|
8433
8900
|
name: a.name,
|
|
8434
8901
|
// The spawned agent's id (nkey, or the user-mode principal) — lets an operator tool (e.g.
|
|
@@ -8441,10 +8908,13 @@ export class Manager {
|
|
|
8441
8908
|
mode: a.handle.kind,
|
|
8442
8909
|
status: a.handle.status(),
|
|
8443
8910
|
uptimeMs: Date.now() - a.startedAt,
|
|
8444
|
-
mesh:
|
|
8911
|
+
mesh: seen?.status ?? "absent",
|
|
8912
|
+
...(condition ? { condition } : {}),
|
|
8913
|
+
...(activeAt !== undefined ? { activeAt } : {}),
|
|
8445
8914
|
// `current` is the only state in which `mesh` is a verdict; the other two are the
|
|
8446
8915
|
// observer's own condition and travel on the row (older CLIs ignore the field).
|
|
8447
8916
|
meshView: view.state,
|
|
8917
|
+
...(offlineSince !== undefined ? { offlineSince } : {}),
|
|
8448
8918
|
// The incarnation coordinate (SPEC 13.1) — with `id`, exactly what a v0.4 caller needs to
|
|
8449
8919
|
// build a targeted (`despawn`/`attach`) request against THIS incarnation.
|
|
8450
8920
|
lifecycleUid: a.lifecycleUid,
|
|
@@ -8460,6 +8930,9 @@ export class Manager {
|
|
|
8460
8930
|
spawner: a.spawner,
|
|
8461
8931
|
instanceId: this.managerInstanceId,
|
|
8462
8932
|
host: hostname(),
|
|
8933
|
+
// #1500: the session a `--resume` seat forked, with its title and transcript hash once the
|
|
8934
|
+
// seat has recorded them.
|
|
8935
|
+
...(a.launch.forkSource ? { resume: this.forkProvenance(a) ?? { source: a.launch.forkSource } } : {}),
|
|
8463
8936
|
...(health && health.state !== "ok" ? { authHealth: health.state, authReason: health.reason } : {}),
|
|
8464
8937
|
};
|
|
8465
8938
|
});
|