@cotal-ai/manager 0.48.2 → 0.50.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -2
- package/dist/commands.d.ts +1 -0
- package/dist/commands.d.ts.map +1 -1
- package/dist/commands.js +136 -41
- package/dist/commands.js.map +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js.map +1 -1
- package/dist/manager-service-contract.d.ts.map +1 -1
- package/dist/manager-service-contract.js +17 -2
- package/dist/manager-service-contract.js.map +1 -1
- package/dist/manager.d.ts +190 -26
- package/dist/manager.d.ts.map +1 -1
- package/dist/manager.js +772 -203
- package/dist/manager.js.map +1 -1
- package/dist/remote-authority.d.ts +22 -2
- package/dist/remote-authority.d.ts.map +1 -1
- package/dist/remote-authority.js +124 -2
- package/dist/remote-authority.js.map +1 -1
- package/dist/remote-register.d.ts.map +1 -1
- package/dist/remote-register.js +5 -1
- package/dist/remote-register.js.map +1 -1
- package/dist/resume.d.ts +5 -0
- package/dist/resume.d.ts.map +1 -1
- package/dist/resume.js +14 -2
- package/dist/resume.js.map +1 -1
- package/dist/run-hosting.d.ts +24 -3
- package/dist/run-hosting.d.ts.map +1 -1
- package/dist/run-hosting.js +97 -8
- package/dist/run-hosting.js.map +1 -1
- package/dist/runtime/custodial-pty.d.ts +10 -3
- package/dist/runtime/custodial-pty.d.ts.map +1 -1
- package/dist/runtime/custodial-pty.js +27 -5
- package/dist/runtime/custodial-pty.js.map +1 -1
- package/dist/runtime/index.d.ts +50 -0
- package/dist/runtime/index.d.ts.map +1 -1
- package/dist/runtime/index.js +34 -0
- package/dist/runtime/index.js.map +1 -1
- package/dist/runtime/pty.d.ts.map +1 -1
- package/dist/runtime/pty.js +28 -26
- package/dist/runtime/pty.js.map +1 -1
- package/dist/session/plane.d.ts +2 -1
- package/dist/session/plane.d.ts.map +1 -1
- package/dist/session/plane.js +3 -2
- package/dist/session/plane.js.map +1 -1
- package/dist/static-lifecycle.d.ts +1 -0
- package/dist/static-lifecycle.d.ts.map +1 -1
- package/dist/static-lifecycle.js +5 -0
- package/dist/static-lifecycle.js.map +1 -1
- package/package.json +11 -10
package/dist/manager.js
CHANGED
|
@@ -4,9 +4,9 @@ import { hostname } from "node:os";
|
|
|
4
4
|
import { credsAuthenticator } from "@nats-io/transport-node";
|
|
5
5
|
import { existsSync, lstatSync, readFileSync, rmSync } from "node:fs";
|
|
6
6
|
import { join, dirname, resolve } from "node:path";
|
|
7
|
-
import { CotalEndpoint, DEFAULT_SERVER, DEV_OWNER, MANAGER_LEASE_RENEW_MS, STANDING_RENEWABLE_TTL_SEC, agentFilePath, clearSpaceHistory, dialerFor, connectorServers, spawnEnvAllow, deprovisionAgent, firstFreeName, spawnNameError, idFromCreds, inspectCredHealth, loadAgentFile, listPersonaCatalog, personaCatalogDescription, personaCatalogReadable, loadCotalConfig, mintCreds, mintLifecycleUid, mkSecretDir, newIdentity, actionContext, parsePrincipalKey, parseShareSelection, principalKey, probeConnect, provisionAgent, provisionAgentDurables, registry, resolveAuthProvider, saveAgentFile, subjectMatches, AUTH_ENDPOINT, EP_CMD_RETIRE_LIFECYCLE, epRequestSubject, epCallerReplyFilter, parseEpSubject, controlServiceSubject, eventChannelPrincipal, } from "@cotal-ai/core";
|
|
8
|
-
import { agentAuthState, agentCredsDir, agentLifecycleSecretFilePaths, agentSecretFilePaths, agentSecretKeyForFile, authDir, connectorInstallHint, DEFAULT_CONNECTOR, defaultAgentType, DELIVERY_CREDS_KIND, extensionConnectors, findCotalRoot, getSpaceAuth, hasUserAuthState, loadExtensionsManifest, loadManagerInstanceIdentity, loadMeshes, manifestExtensionNames, materializeFromManifest, materializeSecretToFile, MEMBERSHIP_RW_CREDS_KIND, mergeLaunchOptions, remintDaemonCreds, resolveOnPath,
|
|
9
|
-
import { createRuntime, requireRuntimeAdopt, } from "./runtime/index.js";
|
|
7
|
+
import { CotalEndpoint, DEFAULT_SERVER, DEV_OWNER, MANAGER_LEASE_RENEW_MS, STANDING_RENEWABLE_TTL_SEC, divergentSecretStoreNotice, parseSecretStoreIdentity, sameSecretStoreIdentity, agentFilePath, clearSpaceHistory, dialerFor, connectorServers, spawnEnvAllow, deprovisionAgent, firstFreeName, spawnNameError, idFromCreds, inspectCredHealth, composeWirePersona, loadAgentFile, listPersonaCatalog, personaCatalogDescription, personaCatalogReadable, loadCotalConfig, mintCreds, mintGeneration, mintAcceptedToken, openIssuedStore, acceptedBucket, issuedBucket, mintLifecycleUid, managedRetirementOpId, mkSecretDir, newIdentity, actionContext, parsePrincipalKey, parseShareSelection, principalKey, probeConnect, provisionAgent, provisionAgentDurables, registry, resolveAuthProvider, saveAgentFile, subjectMatches, AUTH_ENDPOINT, EP_CMD_RETIRE_LIFECYCLE, epRequestSubject, epCallerReplyFilter, parseEpSubject, controlServiceSubject, eventChannelPrincipal, } from "@cotal-ai/core";
|
|
8
|
+
import { agentAuthState, agentCredsDir, agentLifecycleSecretFilePaths, agentSecretFilePaths, agentSecretKeyForFile, authDir, connectorInstallHint, DEFAULT_CONNECTOR, defaultAgentType, DELIVERY_CREDS_KIND, extensionConnectors, findCotalRoot, getSpaceAuth, hasUserAuthState, loadExtensionsManifest, loadManagerInstanceIdentity, loadMeshes, manifestExtensionNames, materializeFromManifest, materializeSecretToFile, MEMBERSHIP_RW_CREDS_KIND, mergeLaunchOptions, remintDaemonCreds, resolveOnPath, createManagerInstanceIdentity, spaceMaterialKey, SYSTEM_CREDS_FILES, userAuthStateDir, workspaceSecretStore, writeRenewalRecord } from "@cotal-ai/workspace";
|
|
9
|
+
import { createRuntime, isCustodialRuntime, requireRuntimeAdopt, requireRuntimeReap, RuntimeReapUnproven, } from "./runtime/index.js";
|
|
10
10
|
import { AttachEndpoint } from "./attach-endpoint.js";
|
|
11
11
|
import { makeManagerEndpointEvictionEvidence, makeManagerEndpointEvictor } from "./endpoint-evict.js";
|
|
12
12
|
import { makeManagerHolderLivenessProbe } from "./holder-liveness.js";
|
|
@@ -16,7 +16,7 @@ import { authorizeLaunch, authorizeNamedControl } from "./authorize.js";
|
|
|
16
16
|
import { controlShutdown } from "./control-shutdown.js";
|
|
17
17
|
import { RunHosting } from "./run-hosting.js";
|
|
18
18
|
import { controlSession } from "./control-session.js";
|
|
19
|
-
import { parseResumeCommitArgs, parseResumeControlArgs, parseResumeFinalizeArgs } from "./resume.js";
|
|
19
|
+
import { MAX_AGENTS, parseResumeCommitArgs, parseResumeControlArgs, parseResumeFinalizeArgs } from "./resume.js";
|
|
20
20
|
// Unit B (the static §13.1 lifecycle executor): the shared grammar/stores from core plus the
|
|
21
21
|
// manager-side adapter (transport + slot orchestration + the F1 terminal) — see static-lifecycle.ts.
|
|
22
22
|
import { jetstream, jetstreamManager } from "@nats-io/jetstream";
|
|
@@ -29,12 +29,9 @@ import { ManagerSessionPlane, openSessionLedgerKv } from "./session/index.js";
|
|
|
29
29
|
// P2 item 1 (1a-serve): the manager as an ordinary v0.4 `service` endpoint — the §13.1
|
|
30
30
|
// endpoint-serve credential subsystem (gate provisioning, registration barrier, mint fence) plus
|
|
31
31
|
// the register/authorize/serve seams, all driven over a scoped one-shot executor connection.
|
|
32
|
-
import { provisionEndpointGateOpen, endpointRegistrationBarrier, serveIssuanceGateKv, commitSiblingIssuance, markLedgerRowRevoked, epcredRowKey, epgateKey, parseEndpointGate, registerServiceInstance, deregisterServiceInstance, authorizeServeGrant, writeServiceStatus, SERVICE_READY, serveEndpoint, bindGoal, createGoal, transitionGoal, commitGoalResult, settleGoalUncertain, readGoalResult, readGoalStatus, readGoalSpec, recordGoalIndex, readGoalIndex, clearGoalIndex, listGoalIndex, mintCheckpoint, resumeCheckpoint, readCheckpointSettle, readCheckpointSpec, expireCheckpoint, GOAL_TERMINAL_STATES, goalRefOf, goalProgressTopic, epeSubject, submissionFingerprint, EpEnvelopeError, lifecycleBlocked, lifecycleBlockedFrom, renderLifecycleBlocked, } from "@cotal-ai/core";
|
|
32
|
+
import { provisionEndpointGateOpen, endpointRegistrationBarrier, serveIssuanceGateKv, commitSiblingIssuance, markLedgerRowRevoked, epcredRowKey, epgateKey, parseEndpointGate, readEndpointGateGeneration, registerServiceInstance, deregisterServiceInstance, authorizeServeGrant, writeServiceStatus, SERVICE_READY, serveEndpoint, bindGoal, createGoal, transitionGoal, commitGoalResult, settleGoalUncertain, readGoalResult, readGoalStatus, readGoalSpec, recordGoalIndex, readGoalIndex, clearGoalIndex, listGoalIndex, mintCheckpoint, resumeCheckpoint, readCheckpointSettle, readCheckpointSpec, expireCheckpoint, GOAL_TERMINAL_STATES, goalRefOf, goalProgressTopic, epeSubject, submissionFingerprint, EpEnvelopeError, lifecycleBlocked, lifecycleBlockedFrom, renderLifecycleBlocked, } from "@cotal-ai/core";
|
|
33
33
|
import { MANAGER_ENDPOINT, managerClusterArtifacts, managerCommandDefs, managerContractArtifactValues } from "./manager-service-contract.js";
|
|
34
34
|
import { staticLifecycleTransport, activateStaticLifecycle, runStaticTerminal, readStaticSlot, casStaticSlot, recordSlotCredential, appendStaticCredentialRow, planStaticSlotResume, observeStaticSlot, renderStaticSlotObservation, StaticSlotReadError, STATIC_SLOT_READ_FAILED_DETAIL, } from "./static-lifecycle.js";
|
|
35
|
-
/** Concurrency ceiling — the manager refuses to hold more than this many live + in-flight +
|
|
36
|
-
* cooling slots at once (P4a). Bounds a fork-bomb: spawn is a full agent process per call. */
|
|
37
|
-
const MAX_AGENTS = 50;
|
|
38
35
|
/** Minimum slot lifetime for rate-flooring (P4c). A slot freed (by despawn OR natural exit/reap)
|
|
39
36
|
* before living this long leaves a cooling stamp that still counts toward the ceiling until it
|
|
40
37
|
* expires — so churn (spawn↔despawn or spawn↔fast-exit) can't outrun the concurrency bound. */
|
|
@@ -73,6 +70,19 @@ const DELIVERY_ADMIN_RELOAD_TIMEOUT_MS = 15_000;
|
|
|
73
70
|
/** A hard preservation stop should settle quickly. The manager still waits and reports a partial
|
|
74
71
|
* cut rather than pretending a child is gone. Held in ManagerOptions so fake runtimes can shorten it. */
|
|
75
72
|
const PRESERVE_STOP_TIMEOUT_MS = 10_000;
|
|
73
|
+
/** How much of an `input` text is written to a seat's pty at a time. Comfortably under the 4096-byte
|
|
74
|
+
* kernel input buffer, so a slice is never re-split at a boundary the manager did not choose: the
|
|
75
|
+
* tail of an oversized write stays queued and swallows the submit key that follows it (issue #1649).
|
|
76
|
+
* Sized in CHARS against a byte buffer on purpose — a multi-byte character can only make a slice
|
|
77
|
+
* shorter in characters than in bytes, never longer, so the margin cannot be eaten from below. */
|
|
78
|
+
const INPUT_SLICE_CHARS = 2048;
|
|
79
|
+
/** Bracketed-paste delimiters (DECSET 2004). `input` wraps a seat's text in these so the return it
|
|
80
|
+
* writes afterwards is a KEYSTROKE rather than the newline trailing a pasted block: a TUI that
|
|
81
|
+
* classifies a fast burst as a paste consumes that trailing newline and the text sits unsubmitted
|
|
82
|
+
* in the composer until the next call's return (issue #1649). Framing, never content - these bytes
|
|
83
|
+
* are not counted into the receipt, so `bytes` still reports what the caller asked to deliver. */
|
|
84
|
+
export const PASTE_START = "\x1b[200~";
|
|
85
|
+
export const PASTE_END = "\x1b[201~";
|
|
76
86
|
/** Pick a `setInterval` period for {@link Manager.renewDaemonCreds} that guarantees at least one
|
|
77
87
|
* tick lands inside every renewal owner's `[renewAt, exp)` window.
|
|
78
88
|
*
|
|
@@ -106,7 +116,7 @@ const STATIC_RECONCILE_MAX_ATTEMPTS = STATIC_RECONCILE_RETRY_DELAYS_MS.length +
|
|
|
106
116
|
* operation (the rail's idempotence table needs exactly one op per retiring incarnation). 26 hex
|
|
107
117
|
* chars = in the lifecycle-token grammar `[a-z0-9]{26,32}`, collision-resistant. */
|
|
108
118
|
function retireOpId(lifecycleUid) {
|
|
109
|
-
return
|
|
119
|
+
return managedRetirementOpId(lifecycleUid);
|
|
110
120
|
}
|
|
111
121
|
/** Sentinel owner-filter value that matches NO agent's `userOwner` (owner tokens never contain a
|
|
112
122
|
* dash) — what {@link Manager.psOwnerFilter} returns for an unparseable caller so a malformed
|
|
@@ -252,6 +262,20 @@ function foreignEventChannels(channels, owner, actor) {
|
|
|
252
262
|
return p !== null && !(p.owner === owner && p.actor === actor);
|
|
253
263
|
});
|
|
254
264
|
}
|
|
265
|
+
function injectedManagerStoreIdentity(store) {
|
|
266
|
+
if (store.identity !== undefined)
|
|
267
|
+
return parseSecretStoreIdentity(store.identity);
|
|
268
|
+
const coordinate = process.env.COTAL_SECRET_STORE;
|
|
269
|
+
if (!coordinate)
|
|
270
|
+
throw new Error("ManagerOptions.secretStore must declare its identity or set COTAL_SECRET_STORE so the manager and delivery daemon can name the same authority (never a silent local-root fallback)");
|
|
271
|
+
return { kind: "injected", coordinate };
|
|
272
|
+
}
|
|
273
|
+
/** True only when the delivery-admin rail has no bound responder. A timeout is a hung rail, not absence. */
|
|
274
|
+
function isAbsentDeliveryAdmin(msg) {
|
|
275
|
+
if (/timeout/i.test(msg))
|
|
276
|
+
return false;
|
|
277
|
+
return /no responders|\b503\b/i.test(msg);
|
|
278
|
+
}
|
|
255
279
|
export class Manager {
|
|
256
280
|
space;
|
|
257
281
|
servers;
|
|
@@ -265,12 +289,15 @@ export class Manager {
|
|
|
265
289
|
/** The ONE secret store for every kind this manager touches (daemon-cred remint + agent kinds).
|
|
266
290
|
* See {@link ManagerOptions.secretStore}. */
|
|
267
291
|
secrets;
|
|
292
|
+
/** Named identity of {@link secrets}: the same coordinate the delivery daemon must reload from. */
|
|
293
|
+
secretStoreIdentity;
|
|
268
294
|
/** See {@link ManagerOptions.installedExtensions}. */
|
|
269
295
|
installedExtensions;
|
|
270
296
|
runtime;
|
|
271
297
|
/** Internal test seam. Production leaves this undefined and uses the scoped delivery-admin evictor. */
|
|
272
298
|
staticLifecycleEvict;
|
|
273
299
|
preserveStopTimeoutMs;
|
|
300
|
+
endpointServeExecutorExpiresInSeconds;
|
|
274
301
|
agents = new Map();
|
|
275
302
|
/** Names whose spawn is in flight (reserved synchronously before the provision await) — counted
|
|
276
303
|
* toward the ceiling so two concurrent same-name spawns can't both pass the gate (P4a). */
|
|
@@ -286,6 +313,12 @@ export class Manager {
|
|
|
286
313
|
* truth is the auth-side lifecycle head itself (an unretired head refuses issuance — the
|
|
287
314
|
* named residual this belt narrows, not replaces). */
|
|
288
315
|
retiring = new Map();
|
|
316
|
+
/** Exact predecessor incarnations whose full hosted retirement reached a terminal answer. Presence
|
|
317
|
+
* is advisory and can retain that old lifecycle briefly after its process exits. Resume may ignore
|
|
318
|
+
* only this exact (alias, principal, lifecycleUid) row when adopting a different lifecycle; every
|
|
319
|
+
* unknown, current, or differently identified live row still refuses. A later retirement of the
|
|
320
|
+
* same alias replaces its prior predecessor coordinate. */
|
|
321
|
+
confirmedRetiredPredecessors = new Map();
|
|
289
322
|
/** SINGLE-FLIGHT guard for {@link requestRetirement} (audit #1): one in-flight rail round-trip per
|
|
290
323
|
* (name, lifecycleUid). The detached `deprovision` call and every same-name-spawn nudge for THAT
|
|
291
324
|
* lifecycle JOIN the same promise instead of stacking independent requests that dual-enter the
|
|
@@ -457,6 +490,11 @@ export class Manager {
|
|
|
457
490
|
leaseRenewInFlight = false;
|
|
458
491
|
/** The class-2 renewal owner's half-TTL schedule (D5 slice 5); armed only on auth meshes. */
|
|
459
492
|
credRenewTimer;
|
|
493
|
+
/** #1634: does THIS manager own the delivery daemon's credential renewal? True only while the
|
|
494
|
+
* daemon reloads from this manager's store AND this manager holds the per-space renewal lease.
|
|
495
|
+
* Starts false: a manager that has not proved both must not remint. */
|
|
496
|
+
daemonRenewalOwner = false;
|
|
497
|
+
daemonRenewalLeaseTimer;
|
|
460
498
|
maintenanceState = "active";
|
|
461
499
|
lifecycleInFlight = 0;
|
|
462
500
|
lifecycleDrainWaiters = [];
|
|
@@ -502,9 +540,13 @@ export class Manager {
|
|
|
502
540
|
if (opts.remoteAuthority)
|
|
503
541
|
this.managerLifecycleUid = opts.remoteAuthority.lifecycleUid;
|
|
504
542
|
this.secrets = opts.secretStore ?? workspaceSecretStore(this.workspaceRoot);
|
|
543
|
+
this.secretStoreIdentity = opts.secretStore
|
|
544
|
+
? injectedManagerStoreIdentity(opts.secretStore)
|
|
545
|
+
: { kind: "fs", root: resolve(this.workspaceRoot) };
|
|
505
546
|
this.installedExtensions = opts.installedExtensions ?? false;
|
|
506
547
|
this.runtime = createRuntime(opts.runtime ?? "auto", `cotal-${this.space}`);
|
|
507
548
|
this.preserveStopTimeoutMs = opts.preserveStopTimeoutMs ?? PRESERVE_STOP_TIMEOUT_MS;
|
|
549
|
+
this.endpointServeExecutorExpiresInSeconds = opts.endpointServeExecutorExpiresInSeconds;
|
|
508
550
|
if (opts.resumeAttemptId && !/^[A-Za-z0-9_-]{1,128}$/.test(opts.resumeAttemptId))
|
|
509
551
|
throw new Error("resumeAttemptId must be a safe token (letters, digits, _, -; max 128)");
|
|
510
552
|
if (opts.resumeDurableCommitToken && !/^[a-f0-9]{64}$/.test(opts.resumeDurableCommitToken))
|
|
@@ -532,6 +574,10 @@ export class Manager {
|
|
|
532
574
|
get runtimeKind() {
|
|
533
575
|
return this.runtime.kind;
|
|
534
576
|
}
|
|
577
|
+
/** Whether a default process stop can drop manager-local custody without taking agents with it. */
|
|
578
|
+
get canSpareAgents() {
|
|
579
|
+
return this.runtime.kind !== "pty" || this.runtime.supportsRelease === true;
|
|
580
|
+
}
|
|
535
581
|
/** Reattach this manager's runtime to a durable handle. Refuses by name when adopt is absent. */
|
|
536
582
|
adoptRuntimeHandle(reference) {
|
|
537
583
|
return requireRuntimeAdopt(this.runtime, reference);
|
|
@@ -620,6 +666,12 @@ export class Manager {
|
|
|
620
666
|
// refused loud (no-fallbacks - a restart never silently becomes a fresh instance). A second
|
|
621
667
|
// manager in a DIFFERENT workspace root is a DIFFERENT logical id by construction (its own state
|
|
622
668
|
// dir) - two managers in ONE space are two workspace roots.
|
|
669
|
+
//
|
|
670
|
+
// First mint on a fresh root is exclusive create (#1263). Two processes that both observe no
|
|
671
|
+
// file must not keep different in-memory ids: of N concurrent starts exactly one creates the
|
|
672
|
+
// file (`link` / O_EXCL) and the others adopt that identity before taking a lease, or refuse
|
|
673
|
+
// with manager-instance-identity-create-lost. Atomic rename is not enough, because the loser
|
|
674
|
+
// of a replace would still serve under the id it minted.
|
|
623
675
|
{
|
|
624
676
|
if (this.remoteAuthority) {
|
|
625
677
|
this.managerInstanceId = this.remoteAuthority.instanceId;
|
|
@@ -632,9 +684,12 @@ export class Manager {
|
|
|
632
684
|
this.managerServeIdentity = persisted.serveIdentity;
|
|
633
685
|
}
|
|
634
686
|
else {
|
|
635
|
-
this.
|
|
636
|
-
|
|
637
|
-
|
|
687
|
+
const claimed = createManagerInstanceIdentity(this.workspaceRoot, this.space, {
|
|
688
|
+
instanceId: mintLifecycleUid(),
|
|
689
|
+
serveIdentity: newIdentity(),
|
|
690
|
+
});
|
|
691
|
+
this.managerInstanceId = claimed.instanceId;
|
|
692
|
+
this.managerServeIdentity = claimed.serveIdentity;
|
|
638
693
|
}
|
|
639
694
|
}
|
|
640
695
|
}
|
|
@@ -714,9 +769,10 @@ export class Manager {
|
|
|
714
769
|
this.leaseTimer.unref?.();
|
|
715
770
|
// Unit B (static §13.1): after this instance holds its lease, collect the durable static rows
|
|
716
771
|
// now, but do not let their exact-op terminals make the whole space unreachable. The service
|
|
717
|
-
// comes up below, then the sweep overlaps the remaining registration work.
|
|
718
|
-
//
|
|
719
|
-
// attach THAT alias until its terminal
|
|
772
|
+
// comes up below, then the sweep overlaps the remaining registration work. Two-window
|
|
773
|
+
// heal-then-register is slower than one executor, but the overlap is still the no-outage
|
|
774
|
+
// property: `reconcilingAliases` refuses spawn or attach of THAT alias until its terminal
|
|
775
|
+
// attempt returns.
|
|
720
776
|
const startupReconcile = this.auth && !this.userMode ? this.reconcileStaticLifecycles() : undefined;
|
|
721
777
|
if (startupReconcile)
|
|
722
778
|
void startupReconcile.catch((e) => console.error(`! ${STARTUP_RECONCILING}: ${e.message} - no per-alias retry could be planned; a later manager start re-reads unfinished durable terminals`));
|
|
@@ -731,6 +787,15 @@ export class Manager {
|
|
|
731
787
|
// every half-TTL: re-sign the daemon creds files for their EXISTING nkeys, request the explicit
|
|
732
788
|
// `reloadCreds` adoption on the delivery-admin rail, and persist the audit record doctor renders.
|
|
733
789
|
if (this.auth) {
|
|
790
|
+
// #1634: a space may be served by several managers, so ownership of the daemon's credential
|
|
791
|
+
// renewal is decided here rather than by being the only manager able to start.
|
|
792
|
+
await this.claimDaemonRenewalOwnership();
|
|
793
|
+
// ARMED BEFORE the first remint, not after. That first `renewDaemonCreds()` re-signs through
|
|
794
|
+
// the SecretStore, and a slow or contended store makes the one pass outlast the lease's
|
|
795
|
+
// MANAGER_LEASE_TTL_MS. Arming afterwards leaves the whole first remint unprotected: the lease
|
|
796
|
+
// expires mid-pass, a second manager's CAS create succeeds, and both remint into one store.
|
|
797
|
+
this.daemonRenewalLeaseTimer = setInterval(() => { void this.keepDaemonRenewalLease(); }, MANAGER_LEASE_RENEW_MS);
|
|
798
|
+
this.daemonRenewalLeaseTimer.unref?.();
|
|
734
799
|
await this.renewDaemonCreds();
|
|
735
800
|
this.credRenewTimer = setInterval(() => { void this.renewDaemonCreds(); }, credRenewIntervalMs(STANDING_RENEWABLE_TTL_SEC));
|
|
736
801
|
this.credRenewTimer.unref?.();
|
|
@@ -785,6 +850,81 @@ export class Manager {
|
|
|
785
850
|
// manager records each agent's read ACL at spawn (`commitAcl`, in provisionAgent) so the daemon can
|
|
786
851
|
// re-authorize it; that is the only Plane-3 state the manager touches, and it rides minting.
|
|
787
852
|
}
|
|
853
|
+
/** Proof that this manager remints into the store the delivery daemon reloads from.
|
|
854
|
+
* Fingerprint-only `reloadCreds` is safe only after this. Two named, different stores
|
|
855
|
+
* refuse the pass. A daemon that is not bound yet is not a named store: start and
|
|
856
|
+
* remint still proceed (tests, delayed delivery), writing this manager's store. The
|
|
857
|
+
* challenge runs again before every remint, so a later daemon on a foreign store is
|
|
858
|
+
* refused then rather than certified by an earlier absence. A request timeout is not
|
|
859
|
+
* absence: a hung rail would skip the proof, so it fails closed. */
|
|
860
|
+
async daemonStoreRelation() {
|
|
861
|
+
let reply;
|
|
862
|
+
try {
|
|
863
|
+
reply = await this.ep.requestDeliveryAdmin("reloadStoreIdentity", {}, 5_000);
|
|
864
|
+
}
|
|
865
|
+
catch (e) {
|
|
866
|
+
const msg = e.message;
|
|
867
|
+
if (isAbsentDeliveryAdmin(msg))
|
|
868
|
+
return "absent";
|
|
869
|
+
throw new Error(`could not challenge the delivery daemon's SecretStore: ${msg}`);
|
|
870
|
+
}
|
|
871
|
+
if (!reply.ok)
|
|
872
|
+
throw new Error(reply.error ?? "delivery daemon refused to name the SecretStore it reloads from");
|
|
873
|
+
let daemon;
|
|
874
|
+
try {
|
|
875
|
+
daemon = parseSecretStoreIdentity(reply.data);
|
|
876
|
+
}
|
|
877
|
+
catch (e) {
|
|
878
|
+
throw new Error(`delivery daemon named an unreadable SecretStore: ${e.message}`);
|
|
879
|
+
}
|
|
880
|
+
if (!sameSecretStoreIdentity(this.secretStoreIdentity, daemon)) {
|
|
881
|
+
console.error(`! ${divergentSecretStoreNotice(this.secretStoreIdentity, daemon)}`);
|
|
882
|
+
return "divergent";
|
|
883
|
+
}
|
|
884
|
+
return "shared";
|
|
885
|
+
}
|
|
886
|
+
/** Keep this manager's renewal lease alive between credential passes (#1634). The manager bucket
|
|
887
|
+
* expires keys at MANAGER_LEASE_TTL_MS, far below the credential pass's cadence, so the lease
|
|
888
|
+
* needs its own heartbeat like the per-instance liveness lease. A non-owner calls this too: the
|
|
889
|
+
* CAS is how a survivor picks up a dead owner's expired lease with no operator step. Never
|
|
890
|
+
* throws; a failed heartbeat leaves ownership to the next pass rather than ending the process. */
|
|
891
|
+
async keepDaemonRenewalLease() {
|
|
892
|
+
if (!this.auth)
|
|
893
|
+
return;
|
|
894
|
+
try {
|
|
895
|
+
const held = await this.ep.holdDaemonRenewalLease(this.managerInstanceId);
|
|
896
|
+
if (this.daemonRenewalOwner && !held)
|
|
897
|
+
console.error(`! daemon credential renewal for space "${this.space}" passed to another manager (this one lost the renewal lease) - it keeps serving its seats`);
|
|
898
|
+
this.daemonRenewalOwner = held;
|
|
899
|
+
}
|
|
900
|
+
catch (e) {
|
|
901
|
+
console.error(`! could not refresh the daemon renewal lease: ${e.message} - ownership is re-decided on the next pass`);
|
|
902
|
+
}
|
|
903
|
+
}
|
|
904
|
+
/** Decide whether THIS manager owns the daemon's credential renewal (#1634). Both conditions are
|
|
905
|
+
* required, because each answers a question the other cannot:
|
|
906
|
+
*
|
|
907
|
+
* 1. The daemon must reload from this manager's store (#1447). Reminting into a store the daemon
|
|
908
|
+
* never reads writes bytes nobody adopts, and fingerprint-only `reloadCreds` would then name a
|
|
909
|
+
* generation the daemon cannot see.
|
|
910
|
+
* 2. This manager must hold the per-space renewal lease. Store identity CANNOT pick one owner on
|
|
911
|
+
* its own: `sameSecretStoreIdentity` is pure equality with no holder and no tiebreak, so every
|
|
912
|
+
* manager sharing one store passes it, and two owners reminting on independent timers have no
|
|
913
|
+
* ordering between them - one's write lands between the other's re-sign and its adoption
|
|
914
|
+
* request, which is the #773 refusal the challenge exists to prevent.
|
|
915
|
+
*
|
|
916
|
+
* A manager failing either test serves the space normally and skips only the daemon remint. */
|
|
917
|
+
async claimDaemonRenewalOwnership() {
|
|
918
|
+
if ((await this.daemonStoreRelation()) === "divergent") {
|
|
919
|
+
// Not our daemon's store: drop any lease we hold rather than sit on it, so the manager that
|
|
920
|
+
// CAN remint is not blocked behind us.
|
|
921
|
+
if (this.daemonRenewalOwner)
|
|
922
|
+
await this.ep.releaseDaemonRenewalLease();
|
|
923
|
+
this.daemonRenewalOwner = false;
|
|
924
|
+
return;
|
|
925
|
+
}
|
|
926
|
+
await this.keepDaemonRenewalLease();
|
|
927
|
+
}
|
|
788
928
|
/** One class-2 renewal pass (D5 slice 5): re-sign `.cotal/delivery.creds` + `.cotal/membership-rw.creds`
|
|
789
929
|
* for their existing nkeys, then request the delivery daemon's EXPLICIT `reloadCreds` adoption on the
|
|
790
930
|
* delivery-admin rail and persist the audit record (`.cotal/renewal.json`) that `cotal doctor auth`
|
|
@@ -796,6 +936,10 @@ export class Manager {
|
|
|
796
936
|
if (!release)
|
|
797
937
|
return;
|
|
798
938
|
try {
|
|
939
|
+
// #1634: re-decide every pass, so a daemon that moved onto this manager's store becomes
|
|
940
|
+
// ownable, one that moved off retires this manager's ownership, and a crashed owner's expired
|
|
941
|
+
// lease is taken over by a survivor.
|
|
942
|
+
await this.claimDaemonRenewalOwnership();
|
|
799
943
|
// Re-sign through the manager's ONE store — the SAME store the delivery daemon reads
|
|
800
944
|
// (`runDelivery(args, store)`), so a hosted remint writes the store the daemon renews from,
|
|
801
945
|
// never a divergent one. Locally this is the workstation FS store (`.cotal/*.creds`).
|
|
@@ -803,9 +947,11 @@ export class Manager {
|
|
|
803
947
|
// overwriting from ANY signer form (full or stripped). A same-label alternate full bundle is
|
|
804
948
|
// self-bound, not broker-bound, so the manager-hosted path proves every re-sign before it could
|
|
805
949
|
// clobber the last-good with a broker-dead cred; a wrong-account signer's cred is refused here.
|
|
806
|
-
const results =
|
|
807
|
-
|
|
808
|
-
|
|
950
|
+
const results = this.daemonRenewalOwner
|
|
951
|
+
? await remintDaemonCreds(this.workspaceRoot, this.space, this.secrets, {
|
|
952
|
+
preflight: (creds) => this.probeStaticCredential(creds).then((r) => r.ok),
|
|
953
|
+
})
|
|
954
|
+
: [];
|
|
809
955
|
const resigned = results.filter((r) => r.ok);
|
|
810
956
|
let adoption;
|
|
811
957
|
if (resigned.length) {
|
|
@@ -836,7 +982,10 @@ export class Manager {
|
|
|
836
982
|
console.error(`! credential renewal: daemon adoption failed: ${adoption.error}`);
|
|
837
983
|
// `writeRenewalRecord` redacts the ephemeral fingerprint at the persistence boundary (covering
|
|
838
984
|
// the `doctor auth --fix` writer too), so the results pass straight through.
|
|
839
|
-
|
|
985
|
+
// A non-owner (#1634) re-signed nothing, so it writes no record: an empty one renders in
|
|
986
|
+
// `doctor auth` as a pass that never ran.
|
|
987
|
+
if (this.daemonRenewalOwner)
|
|
988
|
+
writeRenewalRecord(this.workspaceRoot, { ts: new Date().toISOString(), owner: "manager", results, adoption });
|
|
840
989
|
this.warnOnSystemCredExpiry();
|
|
841
990
|
// F5(b) (Unit B): the MANAGER is the renewal owner for its managed-static agent creds —
|
|
842
991
|
// supervisor-side PUSH remint for recorded LIVE slots (the child JWT is never proof of
|
|
@@ -1305,7 +1454,7 @@ export class Manager {
|
|
|
1305
1454
|
? (() => {
|
|
1306
1455
|
if (!files?.creds)
|
|
1307
1456
|
throw new Error(`managed agent ${a.name} is static-auth but its credential path was not recorded`);
|
|
1308
|
-
return { mode: "static", id: principal.actor, lifecycleUid: a.lifecycleUid, credential: { kind: "file", path: files.creds, sha256: this.fileDigestOrEmpty(files.creds) } };
|
|
1457
|
+
return { mode: "static", id: principal.actor, lifecycleUid: a.lifecycleUid, credential: { kind: "file", path: files.creds, sha256: this.fileDigestOrEmpty(files.creds) }, ...(a.issued ? { issued: a.issued } : {}) };
|
|
1309
1458
|
})()
|
|
1310
1459
|
: { mode: "open", id: principal.actor, lifecycleUid: a.lifecycleUid };
|
|
1311
1460
|
const dependencies = [a.launch.source.configPath];
|
|
@@ -1339,7 +1488,7 @@ export class Manager {
|
|
|
1339
1488
|
startedAt: new Date(a.startedAt).toISOString(),
|
|
1340
1489
|
};
|
|
1341
1490
|
}
|
|
1342
|
-
/** Tear down every managed agent's footprint on
|
|
1491
|
+
/** Tear down every managed agent's footprint on an explicit destructive {@link stop}. A manager exit is
|
|
1343
1492
|
* a mass agent-exit, and without this its agents' footprints (creds files + `dm_`/`dlv_` durables + ACL
|
|
1344
1493
|
* rows) would orphan exactly as the per-agent exit path prevents. Hard-stop each child (an exit has no
|
|
1345
1494
|
* time for the graceful grace window) and AWAIT its deprovision — bounded per agent (`withTimeout`) and
|
|
@@ -1372,6 +1521,29 @@ export class Manager {
|
|
|
1372
1521
|
if (failures.length)
|
|
1373
1522
|
throw new Error(`manager shutdown could not prove every seat exited: ${failures.join("; ")}`);
|
|
1374
1523
|
}
|
|
1524
|
+
/** Drop only this manager's local custody handles. Validate the complete snapshot before releasing
|
|
1525
|
+
* any handle so a non-detachable runtime cannot leave a partially detached manager behind. */
|
|
1526
|
+
releaseManagedAgents() {
|
|
1527
|
+
const managed = [...this.agents.values()];
|
|
1528
|
+
const blocked = managed.filter((a) => a.handle.kind === "pty" && typeof a.handle.release !== "function");
|
|
1529
|
+
if (blocked.length) {
|
|
1530
|
+
return [
|
|
1531
|
+
`manager shutdown cannot detach ${blocked.map((a) => `${a.name} (${a.handle.kind})`).join(", ")}: runtime handle does not support release`,
|
|
1532
|
+
];
|
|
1533
|
+
}
|
|
1534
|
+
const failures = [];
|
|
1535
|
+
for (const a of managed) {
|
|
1536
|
+
try {
|
|
1537
|
+
a.handle.release?.();
|
|
1538
|
+
a.suppressCleanup = true;
|
|
1539
|
+
this.agents.delete(a.name);
|
|
1540
|
+
}
|
|
1541
|
+
catch (e) {
|
|
1542
|
+
failures.push(`${a.name}: ${e.message}`);
|
|
1543
|
+
}
|
|
1544
|
+
}
|
|
1545
|
+
return failures;
|
|
1546
|
+
}
|
|
1375
1547
|
async stopRetainedAgentsOnExit() {
|
|
1376
1548
|
const managed = [...this.agents.values()];
|
|
1377
1549
|
for (const a of managed)
|
|
@@ -1396,7 +1568,7 @@ export class Manager {
|
|
|
1396
1568
|
if (failures.length)
|
|
1397
1569
|
throw new Error(`manager preservation shutdown incomplete: ${failures.join("; ")}`);
|
|
1398
1570
|
}
|
|
1399
|
-
async stop() {
|
|
1571
|
+
async stop(options = {}) {
|
|
1400
1572
|
this.staticReconcileStopping = true;
|
|
1401
1573
|
const starting = this.startTask;
|
|
1402
1574
|
for (const item of this.staticReconcileItems.values()) {
|
|
@@ -1411,16 +1583,25 @@ export class Manager {
|
|
|
1411
1583
|
clearInterval(this.leaseTimer);
|
|
1412
1584
|
if (this.credRenewTimer)
|
|
1413
1585
|
clearInterval(this.credRenewTimer);
|
|
1586
|
+
if (this.daemonRenewalLeaseTimer)
|
|
1587
|
+
clearInterval(this.daemonRenewalLeaseTimer);
|
|
1414
1588
|
if (this.sessionKeyRenewTimer)
|
|
1415
1589
|
clearInterval(this.sessionKeyRenewTimer);
|
|
1590
|
+
let releaseFailures = [];
|
|
1416
1591
|
if (this.maintenanceState === "active" && !this.resumeRequired) {
|
|
1417
|
-
|
|
1592
|
+
if (options.withAgents)
|
|
1593
|
+
await this.teardownManagedAgents();
|
|
1594
|
+
else
|
|
1595
|
+
releaseFailures = this.releaseManagedAgents();
|
|
1418
1596
|
}
|
|
1419
1597
|
else {
|
|
1420
1598
|
// A signal after a partial preservation must never fall back into destructive teardown.
|
|
1421
1599
|
await this.stopRetainedAgentsOnExit();
|
|
1422
1600
|
}
|
|
1423
1601
|
await this.ep.releaseManagerLease(this.managerInstanceId, this.leaseRevision);
|
|
1602
|
+
// #1634: hand the renewal lease back on a clean stop so a sibling picks it up on its next pass
|
|
1603
|
+
// rather than waiting out the bucket TTL. A non-owner holds no revision and this is a no-op.
|
|
1604
|
+
await this.ep.releaseDaemonRenewalLease();
|
|
1424
1605
|
// Capture BEFORE the serve loop is torn down: `stopServiceServe` clears the state, and what is
|
|
1425
1606
|
// being asked here is "did this process register a service instance", which only the state
|
|
1426
1607
|
// before teardown can answer. Deregistration runs AFTER the serve loop has drained, so no
|
|
@@ -1437,6 +1618,8 @@ export class Manager {
|
|
|
1437
1618
|
await this.stopSessionPlane();
|
|
1438
1619
|
await this.ep.stop();
|
|
1439
1620
|
await this.attach.stop();
|
|
1621
|
+
if (releaseFailures.length)
|
|
1622
|
+
throw new Error(`manager shutdown could not release every detachable seat: ${releaseFailures.join("; ")}`);
|
|
1440
1623
|
}
|
|
1441
1624
|
/**
|
|
1442
1625
|
* REMOVE THIS INSTANCE'S SERVICE REGISTRATION on a clean stop (§13.5 deregistration).
|
|
@@ -1470,7 +1653,9 @@ export class Manager {
|
|
|
1470
1653
|
},
|
|
1471
1654
|
});
|
|
1472
1655
|
try {
|
|
1473
|
-
const outcome = await (this.auth
|
|
1656
|
+
const outcome = await ((this.auth || this.remoteAuthority)
|
|
1657
|
+
? this.withEndpointServeExecutor(dereg)
|
|
1658
|
+
: this.withOpenServeConnection(dereg));
|
|
1474
1659
|
if (outcome.removed)
|
|
1475
1660
|
console.error(`✓ deregistered manager instance ${iid} from the ${MANAGER_ENDPOINT} service registry (spec revision ${outcome.specRevision})`);
|
|
1476
1661
|
else if (outcome.reason === "superseded")
|
|
@@ -1791,23 +1976,25 @@ export class Manager {
|
|
|
1791
1976
|
* only into operator instruments (§13.2: `any` is operator-policy-mintable; the agent/spawn
|
|
1792
1977
|
* rollups never carry them), so REACHING the handler is holding the admin tier, exactly as
|
|
1793
1978
|
* holding `ctl.<admin>` is today. User mesh: the caller's CURRENT ledger scope must carry
|
|
1794
|
-
* `admin
|
|
1795
|
-
*
|
|
1796
|
-
*
|
|
1979
|
+
* `admin`. A local-host manager reads its own ledger fresh. A remote-authority manager relays the
|
|
1980
|
+
* subject-parsed caller tuple to its host-owned authorization callback, with no participant-ledger
|
|
1981
|
+
* fallback. The same fresh authority drives {@link psOwnerFilter}, so a revoked scope demotes the
|
|
1982
|
+
* very next call even on a still-valid bearer. Local ledger read failures authorize nothing;
|
|
1983
|
+
* remote host-state faults throw and fail the operation closed. NAMED RESIDUAL (critic,
|
|
1984
|
+
* 1c.2b): the static `true` has no serve-time
|
|
1797
1985
|
* re-check — a LEAKED static admin instrument keeps its reach until the credential's bounded
|
|
1798
1986
|
* TTL (the one-shot 5-minute profile), the same static-revoke≠reconnect-death class ruled
|
|
1799
1987
|
* across this campaign; static revocation is the TTL, not a ledger. */
|
|
1800
1988
|
async epAdminReach(caller) {
|
|
1801
1989
|
if (!this.userMode)
|
|
1802
1990
|
return true;
|
|
1803
|
-
|
|
1804
|
-
|
|
1805
|
-
return false;
|
|
1991
|
+
if (this.remoteAuthority)
|
|
1992
|
+
return this.remoteAuthority.authorizeAdmin({ owner: caller.owner, actor: caller.actor, lifecycleUid: caller.uid });
|
|
1806
1993
|
try {
|
|
1807
1994
|
const scope = await resolveAuthProvider().actorScope({
|
|
1808
1995
|
dir: userAuthStateDir(this.workspaceRoot, this.space),
|
|
1809
|
-
owner:
|
|
1810
|
-
actor:
|
|
1996
|
+
owner: caller.owner,
|
|
1997
|
+
actor: caller.actor,
|
|
1811
1998
|
});
|
|
1812
1999
|
return scope?.includes("admin") === true;
|
|
1813
2000
|
}
|
|
@@ -1823,7 +2010,7 @@ export class Manager {
|
|
|
1823
2010
|
async epAnyModeAdmin(ctx) {
|
|
1824
2011
|
if (ctx.subject.target?.mode !== "any")
|
|
1825
2012
|
return false;
|
|
1826
|
-
if (!(await this.epAdminReach(
|
|
2013
|
+
if (!(await this.epAdminReach(ctx.subject.caller)))
|
|
1827
2014
|
throw new EpEnvelopeError("permission-denied", `an any-mode ${ctx.subject.command} is operator reach; the caller's current ledger grant does not carry "admin" (SPEC 13.2)`);
|
|
1828
2015
|
return true;
|
|
1829
2016
|
}
|
|
@@ -1901,7 +2088,7 @@ export class Manager {
|
|
|
1901
2088
|
// call instead of riding the bearer's remaining JWT-row lifetime. The resume family keeps its
|
|
1902
2089
|
// serveGated BYPASS (those ops must run while the maintenance fence holds) but not the gate.
|
|
1903
2090
|
const adminGated = async (ctx, fn) => {
|
|
1904
|
-
if (!(await this.epAdminReach(
|
|
2091
|
+
if (!(await this.epAdminReach(ctx.subject.caller)))
|
|
1905
2092
|
throw new EpEnvelopeError("permission-denied", `${ctx.subject.command} is operator reach; the caller's current ledger grant does not carry "admin" (SPEC 13.2)`);
|
|
1906
2093
|
return fn();
|
|
1907
2094
|
};
|
|
@@ -1914,10 +2101,10 @@ export class Manager {
|
|
|
1914
2101
|
};
|
|
1915
2102
|
return managerCommandDefs({
|
|
1916
2103
|
status: (ctx) => this.serveGated(ctx, () => this.managerStatusData()),
|
|
1917
|
-
ps: (ctx) => this.serveGated(ctx, async () => this.list(await this.psOwnerFilter(
|
|
2104
|
+
ps: (ctx) => this.serveGated(ctx, async () => this.list(await this.psOwnerFilter(ctx.subject.caller, false))),
|
|
1918
2105
|
inspect: (ctx) => this.serveGated(ctx, async () => {
|
|
1919
2106
|
const name = String(args(ctx).name ?? "").trim();
|
|
1920
|
-
const row = this.list(await this.psOwnerFilter(
|
|
2107
|
+
const row = this.list(await this.psOwnerFilter(ctx.subject.caller, false)).find((x) => x.name === name);
|
|
1921
2108
|
if (!row) {
|
|
1922
2109
|
// The live hit above stays entirely local. Only a miss widens into durable static state,
|
|
1923
2110
|
// where `not-found` is honest only after the slot read itself succeeds and returns absent.
|
|
@@ -1957,14 +2144,14 @@ export class Manager {
|
|
|
1957
2144
|
spawn: (ctx) => this.serveGated(ctx, () => this.serveSpawnGoal(ctx, (h) => this.opStart(args(ctx), callerOf(ctx), h))),
|
|
1958
2145
|
despawn: (ctx) => this.serveGated(ctx, async () => {
|
|
1959
2146
|
const a = targetAgent(ctx);
|
|
1960
|
-
const denied = await this.authorizeNamed(a, callerOf(ctx), await this.epAnyModeAdmin(ctx));
|
|
2147
|
+
const denied = await this.authorizeNamed(a, callerOf(ctx), await this.epAnyModeAdmin(ctx), ctx.subject.caller);
|
|
1961
2148
|
if (denied)
|
|
1962
2149
|
throw new EpEnvelopeError("permission-denied", denied);
|
|
1963
2150
|
return unwrap(this.despawnAuthorized(a, args(ctx).graceful !== false, true));
|
|
1964
2151
|
}),
|
|
1965
2152
|
attach: (ctx) => this.serveGated(ctx, async () => {
|
|
1966
2153
|
const a = targetAgent(ctx);
|
|
1967
|
-
const denied = await this.authorizeNamed(a, callerOf(ctx), await this.epAnyModeAdmin(ctx));
|
|
2154
|
+
const denied = await this.authorizeNamed(a, callerOf(ctx), await this.epAnyModeAdmin(ctx), ctx.subject.caller);
|
|
1968
2155
|
if (denied)
|
|
1969
2156
|
throw new EpEnvelopeError("permission-denied", denied);
|
|
1970
2157
|
return unwrap(await this.attachAuthorized(a, ctx.subject.caller));
|
|
@@ -1974,7 +2161,7 @@ export class Manager {
|
|
|
1974
2161
|
// place for one of them to quietly acquire a condition the other does not have.
|
|
1975
2162
|
input: (ctx) => this.serveGated(ctx, async () => {
|
|
1976
2163
|
const a = targetAgent(ctx);
|
|
1977
|
-
const denied = await this.authorizeNamed(a, callerOf(ctx), await this.epAnyModeAdmin(ctx));
|
|
2164
|
+
const denied = await this.authorizeNamed(a, callerOf(ctx), await this.epAnyModeAdmin(ctx), ctx.subject.caller);
|
|
1978
2165
|
if (denied)
|
|
1979
2166
|
throw new EpEnvelopeError("permission-denied", denied);
|
|
1980
2167
|
return this.inputAuthorized(a, args(ctx));
|
|
@@ -1984,7 +2171,7 @@ export class Manager {
|
|
|
1984
2171
|
// reason `input` is (a shared policy, not a shared body).
|
|
1985
2172
|
turn: (ctx) => this.serveGated(ctx, async () => {
|
|
1986
2173
|
const a = targetAgent(ctx);
|
|
1987
|
-
const denied = await this.authorizeNamed(a, callerOf(ctx), await this.epAnyModeAdmin(ctx));
|
|
2174
|
+
const denied = await this.authorizeNamed(a, callerOf(ctx), await this.epAnyModeAdmin(ctx), ctx.subject.caller);
|
|
1988
2175
|
if (denied)
|
|
1989
2176
|
throw new EpEnvelopeError("permission-denied", denied);
|
|
1990
2177
|
return this.serveTurnGoal(ctx, a);
|
|
@@ -2025,7 +2212,7 @@ export class Manager {
|
|
|
2025
2212
|
finalizeResume: (ctx) => adminGated(ctx, async () => unwrap(await this.opFinalizeResume(args(ctx)))),
|
|
2026
2213
|
// The workflow-run family (SPEC 14.3): the manager hosts the driver. Reach is the broker's
|
|
2027
2214
|
// (`run` capability / privileged instrument rows); the serve gate is the maintenance fence.
|
|
2028
|
-
runStart: (ctx) => this.serveGated(ctx, () => this.runHost().start(args(ctx))),
|
|
2215
|
+
runStart: (ctx) => this.serveGated(ctx, () => this.runHost().start(ctx, args(ctx))),
|
|
2029
2216
|
runResume: (ctx) => this.serveGated(ctx, () => this.runHost().resume(args(ctx))),
|
|
2030
2217
|
runAnswer: (ctx) => this.serveGated(ctx, () => this.runHost().answer(args(ctx), this.runAnswerer(ctx))),
|
|
2031
2218
|
runStatus: (ctx) => this.serveGated(ctx, () => this.runHost().status(args(ctx))),
|
|
@@ -2075,11 +2262,27 @@ export class Manager {
|
|
|
2075
2262
|
* instruments). With admin=true (any-mode) any named target is allowed (operator). Otherwise
|
|
2076
2263
|
* a named target is allowed if it's the caller's OWN child (`spawner == caller`) — and, on a
|
|
2077
2264
|
* user mesh, if it runs under the CALLER'S OWNER (owner-domain) or the caller's ledger row
|
|
2078
|
-
* holds `admin`, read fresh.
|
|
2079
|
-
*
|
|
2080
|
-
*
|
|
2081
|
-
*
|
|
2082
|
-
|
|
2265
|
+
* holds `admin`, read fresh. Local managers run the pure {@link authorizeNamedControl} policy
|
|
2266
|
+
* against their provider-backed ledger. Remote managers preserve own-child and owner-domain
|
|
2267
|
+
* decisions locally, then send only the residual cross-owner admin question through the registered
|
|
2268
|
+
* host callback with the full endpoint caller tuple. Error string when denied, `undefined` when
|
|
2269
|
+
* allowed; remote host-state faults throw rather than being collapsed into a denial. */
|
|
2270
|
+
async authorizeNamed(target, caller, admin, epCaller) {
|
|
2271
|
+
if (this.remoteAuthority && epCaller && !admin) {
|
|
2272
|
+
if (target.spawner === caller)
|
|
2273
|
+
return undefined;
|
|
2274
|
+
const principal = parsePrincipalKey(caller);
|
|
2275
|
+
if (principal && target.userOwner === principal.owner)
|
|
2276
|
+
return undefined;
|
|
2277
|
+
// Unlike the local pure-policy adapter below, host authority faults are not collapsed into an
|
|
2278
|
+
// ordinary denial. The registered remote manager must fail the operation closed and surface
|
|
2279
|
+
// unavailable/corrupt authoritative state rather than implying a healthy `authorized:false`.
|
|
2280
|
+
if (await this.epAdminReach(epCaller))
|
|
2281
|
+
return undefined;
|
|
2282
|
+
return (`not authorized: ${target.name} runs under another owner - your grant covers agents under your own owner; ` +
|
|
2283
|
+
`cross-owner stop/attach/input needs scope "admin" on your actor. Re-grant with "admin" ADDED to your current ` +
|
|
2284
|
+
`scope (the upsert replaces the list; see \`cotal actor list\`)`);
|
|
2285
|
+
}
|
|
2083
2286
|
return authorizeNamedControl({
|
|
2084
2287
|
target: { name: target.name, spawner: target.spawner, userOwner: target.userOwner },
|
|
2085
2288
|
caller,
|
|
@@ -2407,14 +2610,17 @@ export class Manager {
|
|
|
2407
2610
|
// static retirement the detached deprovision below drives) — the alias frees only when the
|
|
2408
2611
|
// gate+head terminal completes, exactly the user-mode discipline. The wire principal is the
|
|
2409
2612
|
// incarnation-unique nkey (F5-bind); owner is the dev owner.
|
|
2410
|
-
this.retiring.set(a.name, { opId: retireOpId(a.lifecycleUid), lifecycleUid: a.lifecycleUid, owner: DEV_OWNER, actor: a.id, agentId: a.id, secretPaths: a.secretPaths, startedAt: Date.now() });
|
|
2613
|
+
this.retiring.set(a.name, { opId: retireOpId(a.lifecycleUid), lifecycleUid: a.lifecycleUid, owner: DEV_OWNER, actor: a.id, agentId: a.id, secretPaths: a.secretPaths, runtime: a.handle?.reference, startedAt: Date.now() });
|
|
2411
2614
|
}
|
|
2412
2615
|
// Auth mode: tear down the departed agent's minted broker footprint + creds file (#159 B2). The
|
|
2413
2616
|
// process is already gone, so this must never block the slot free or throw into the caller — it runs
|
|
2414
2617
|
// detached, and a failure is logged loudly (never swallowed), not retried. The `agents` guard above
|
|
2415
2618
|
// makes this fire exactly once per agent across every free path (despawn / self-stop / reap / exit).
|
|
2619
|
+
// The departed agent's own custody reference rides the teardown: the exit that freed this slot
|
|
2620
|
+
// is usually the seat ending, but a stop that could not prove the process gone leaves one the
|
|
2621
|
+
// terminal must still address before it frees the alias.
|
|
2416
2622
|
if (!a.suppressCleanup && (this.maintenanceState === "active" || acceptedBeforeFence))
|
|
2417
|
-
this.trackDeprovision(a);
|
|
2623
|
+
this.trackDeprovision({ ...a, runtime: a.handle?.reference });
|
|
2418
2624
|
}
|
|
2419
2625
|
/** Tear down a departed agent's minted footprint (#159 B2, auth mode): its local-principal durables
|
|
2420
2626
|
* (`dm_local-<id>`, `dlv_local-<id>`), its read-ACL row, and its creds file — everything the spawn's
|
|
@@ -2429,7 +2635,7 @@ export class Manager {
|
|
|
2429
2635
|
* is the separate per-user-auth work, not this. Tearing down the durables + ACL row still shrinks the
|
|
2430
2636
|
* delivery surface a stale copy could use. */
|
|
2431
2637
|
async deprovision(a) {
|
|
2432
|
-
if (!this.auth)
|
|
2638
|
+
if (!this.auth && !this.remoteAuthority)
|
|
2433
2639
|
return; // open mesh mints no creds/durables — nothing to tear down
|
|
2434
2640
|
// SINGLE-FLIGHT per (name, lifecycleUid) (INT-2/C): join an in-flight teardown for this exact
|
|
2435
2641
|
// lifecycle rather than launching a second concurrent one whose delayed name-keyed revoke could
|
|
@@ -2447,6 +2653,20 @@ export class Manager {
|
|
|
2447
2653
|
}
|
|
2448
2654
|
/** The actual footprint teardown (wrapped by {@link deprovision}'s single-flight). */
|
|
2449
2655
|
async driveDeprovision(a) {
|
|
2656
|
+
if (this.remoteAuthority) {
|
|
2657
|
+
// A hosted composition owns grant revocation and resumable footprint release. It calls this
|
|
2658
|
+
// prerequisite before the terminal rail. The callback must preserve this UID and its pending
|
|
2659
|
+
// state until the host decides terminal retirement, never map deprovisioning to suspension.
|
|
2660
|
+
const target = parsePrincipalKey(a.id);
|
|
2661
|
+
if (!target)
|
|
2662
|
+
throw new Error(`hosted retirement cannot derive the managed principal ${a.id}`);
|
|
2663
|
+
await this.remoteAuthority.prepareAgentRetirement({
|
|
2664
|
+
target: { owner: target.owner, actor: target.actor, lifecycleUid: a.lifecycleUid },
|
|
2665
|
+
opId: retireOpId(a.lifecycleUid),
|
|
2666
|
+
});
|
|
2667
|
+
await this.requestRetirement(a);
|
|
2668
|
+
return;
|
|
2669
|
+
}
|
|
2450
2670
|
if (!this.auth)
|
|
2451
2671
|
return; // guaranteed by deprovision; re-checked for the deprovisionBroker narrowing
|
|
2452
2672
|
if (!this.userMode && !a.userOwner) {
|
|
@@ -2552,8 +2772,8 @@ export class Manager {
|
|
|
2552
2772
|
}
|
|
2553
2773
|
/** The rail round-trip for one retirement (wrapped by {@link requestRetirement}'s single-flight). */
|
|
2554
2774
|
async driveRetirement(a) {
|
|
2555
|
-
if (!this.auth)
|
|
2556
|
-
return;
|
|
2775
|
+
if (!this.auth && !this.remoteAuthority)
|
|
2776
|
+
return;
|
|
2557
2777
|
const held = this.retiring.get(a.name);
|
|
2558
2778
|
const target = parsePrincipalKey(a.id);
|
|
2559
2779
|
if (!target) {
|
|
@@ -2594,10 +2814,18 @@ export class Manager {
|
|
|
2594
2814
|
// mismatch in the owner half the moment a manager ran under a user-shaped identity. This is
|
|
2595
2815
|
// also the more honest attribution: the authority being exercised is "I am the registered
|
|
2596
2816
|
// serving instance", which is exactly what the gate records.
|
|
2597
|
-
const caller =
|
|
2598
|
-
|
|
2599
|
-
|
|
2600
|
-
|
|
2817
|
+
const caller = this.remoteAuthority
|
|
2818
|
+
? { owner: this.remoteAuthority.owner, actor: this.remoteAuthority.actors.serve, uid: this.managerLifecycleUid }
|
|
2819
|
+
: { owner: DEV_OWNER, actor: serveIdentity.id, uid: this.managerLifecycleUid };
|
|
2820
|
+
const requestIdentity = newIdentity();
|
|
2821
|
+
const retirementTarget = { owner: target.owner, actor: target.actor, lifecycleUid: a.lifecycleUid };
|
|
2822
|
+
const opId = retireOpId(a.lifecycleUid);
|
|
2823
|
+
const serveEpoch = this.serviceServe?.grant.epoch ?? 0;
|
|
2824
|
+
const creds = this.remoteAuthority
|
|
2825
|
+
? await this.remoteAuthority.mintRetirementRequester({ identity: requestIdentity, target: retirementTarget, opId, serveEpoch })
|
|
2826
|
+
: await mintCreds(this.auth, requestIdentity, "retirement-requester", {
|
|
2827
|
+
retirementRequester: { ...caller, target: retirementTarget },
|
|
2828
|
+
});
|
|
2601
2829
|
const nc = await this.dial({ authenticator: credsAuthenticator(new TextEncoder().encode(creds)), maxReconnectAttempts: 0 });
|
|
2602
2830
|
try {
|
|
2603
2831
|
// §13.2 nonce: >=128 bits of CSPRNG entropy, base64url (the `endpoint-invoke` idiom).
|
|
@@ -2625,30 +2853,12 @@ export class Manager {
|
|
|
2625
2853
|
// caller's own subject-derived principal. A superseded predecessor (same instanceId, OLD
|
|
2626
2854
|
// epoch after a restart) is still refused by the epoch comparison.
|
|
2627
2855
|
JSON.stringify({ id: requestId, op: "retireLifecycle", args: {
|
|
2628
|
-
opId
|
|
2629
|
-
serveEndpoint: MANAGER_ENDPOINT, serveInstanceId: this.managerInstanceId, serveEpoch
|
|
2856
|
+
opId,
|
|
2857
|
+
serveEndpoint: MANAGER_ENDPOINT, serveInstanceId: this.managerInstanceId, serveEpoch,
|
|
2630
2858
|
} }), 20_000);
|
|
2631
2859
|
const r = m;
|
|
2632
2860
|
if (r.ok) {
|
|
2633
|
-
|
|
2634
|
-
// lifecycle's - a late reply for a retired predecessor must never clear a successor's newer hold.
|
|
2635
|
-
const cur = this.retiring.get(a.name);
|
|
2636
|
-
if (cur && cur.lifecycleUid === a.lifecycleUid) {
|
|
2637
|
-
if (cur.standingAuthorityLive) {
|
|
2638
|
-
// INT-2: the auth-plane lifecycle retired, but the manager-side STANDING mint authority is
|
|
2639
|
-
// not yet revoked (a failed revoke). Freeing the name here would be a false terminal (a
|
|
2640
|
-
// copied token could still mint), so keep the hold with its revoke-failure copy; a retry
|
|
2641
|
-
// re-drives the full teardown (revoke included).
|
|
2642
|
-
console.error(`despawn ${a.name}: the auth-plane lifecycle retired, but the standing mint authority is not yet revoked; the name stays held. ${cur.lastError ?? ""}`);
|
|
2643
|
-
}
|
|
2644
|
-
else {
|
|
2645
|
-
this.retiring.delete(a.name);
|
|
2646
|
-
console.error(`despawn ${a.name}: the agent's retirement completed; the name is free for reuse`);
|
|
2647
|
-
}
|
|
2648
|
-
}
|
|
2649
|
-
else {
|
|
2650
|
-
console.error(`despawn ${a.name}: retirement confirmed for a prior lifecycle of "${a.name}"; the current hold is left intact`);
|
|
2651
|
-
}
|
|
2861
|
+
this.confirmRetirement(a);
|
|
2652
2862
|
}
|
|
2653
2863
|
else {
|
|
2654
2864
|
// The rail's refusal is already the operator copy (lease-loss/stale/foreign-op faces,
|
|
@@ -2669,6 +2879,30 @@ export class Manager {
|
|
|
2669
2879
|
console.error(`despawn ${a.name}: ${copy}`);
|
|
2670
2880
|
}
|
|
2671
2881
|
}
|
|
2882
|
+
/** Apply one terminal retirement answer. The predecessor coordinate is recorded only when the
|
|
2883
|
+
* answer clears this exact lifecycle's hold, never for an ABA-late answer or an incomplete hosted
|
|
2884
|
+
* teardown whose standing authority still lives. */
|
|
2885
|
+
confirmRetirement(a) {
|
|
2886
|
+
// CAS the hold clear (audit #1 ABA): free the alias ONLY if the current hold is still THIS
|
|
2887
|
+
// lifecycle's. A late reply for a retired predecessor must never clear a successor's newer hold.
|
|
2888
|
+
const cur = this.retiring.get(a.name);
|
|
2889
|
+
if (cur && cur.lifecycleUid === a.lifecycleUid) {
|
|
2890
|
+
if (cur.standingAuthorityLive) {
|
|
2891
|
+
// INT-2: the auth-plane lifecycle retired, but the manager-side STANDING mint authority is
|
|
2892
|
+
// not yet revoked. A copied token could still mint, so neither free the alias nor classify
|
|
2893
|
+
// its presence as a completed predecessor.
|
|
2894
|
+
console.error(`despawn ${a.name}: the auth-plane lifecycle retired, but the standing mint authority is not yet revoked; the name stays held. ${cur.lastError ?? ""}`);
|
|
2895
|
+
}
|
|
2896
|
+
else {
|
|
2897
|
+
this.confirmedRetiredPredecessors.set(a.name, { principal: a.id, lifecycleUid: a.lifecycleUid });
|
|
2898
|
+
this.retiring.delete(a.name);
|
|
2899
|
+
console.error(`despawn ${a.name}: the agent's retirement completed; the name is free for reuse`);
|
|
2900
|
+
}
|
|
2901
|
+
}
|
|
2902
|
+
else {
|
|
2903
|
+
console.error(`despawn ${a.name}: retirement confirmed for a prior lifecycle of "${a.name}"; the current hold is left intact`);
|
|
2904
|
+
}
|
|
2905
|
+
}
|
|
2672
2906
|
/** The teardown's ASYNC BROKER PHASE: mint the ephemeral target-pinned deprovisioner cred and
|
|
2673
2907
|
* delete the agent's broker footprint (dm_/dlv_ durables + read-ACL row). Split from
|
|
2674
2908
|
* {@link deprovision} because it runs LAST in the ordered teardown chain — after the creds/secret
|
|
@@ -2831,7 +3065,12 @@ export class Manager {
|
|
|
2831
3065
|
const joinedAfter = this.ep.getRoster()
|
|
2832
3066
|
.filter((p) => p.card.id === wanted && p.lifecycleUid === a.lifecycleUid)
|
|
2833
3067
|
.reduce((max, p) => Math.max(max, p.ts), 0) + 1;
|
|
2834
|
-
|
|
3068
|
+
// Same ordering as the spawn path: the replacement seat's reference is recorded on the
|
|
3069
|
+
// still-active slot before its processes exist, so a crash here never strands it.
|
|
3070
|
+
const custody = this.reserveCustody();
|
|
3071
|
+
if (custody)
|
|
3072
|
+
await this.recordSlotCustody(a, custody);
|
|
3073
|
+
const handle = await this.spawnCustodied(a.name, spec, a.launch.cwd, custody);
|
|
2835
3074
|
replacement = handle;
|
|
2836
3075
|
restart.sessionStatePath = spec.sessionStatePath ?? restart.sessionStatePath;
|
|
2837
3076
|
if (continueSession !== undefined)
|
|
@@ -2867,6 +3106,7 @@ export class Manager {
|
|
|
2867
3106
|
replacement = undefined;
|
|
2868
3107
|
restart.opts = opts;
|
|
2869
3108
|
restart.recovering = false;
|
|
3109
|
+
await this.recordSlotRuntime(a);
|
|
2870
3110
|
if (continueSession !== undefined)
|
|
2871
3111
|
console.error(`! ${a.name}: recovered Pi session ${continueSession} after crash (${restart.crashes.length}/${limit})`);
|
|
2872
3112
|
else
|
|
@@ -3177,14 +3417,13 @@ export class Manager {
|
|
|
3177
3417
|
async psOwnerFilter(caller, admin) {
|
|
3178
3418
|
if (!this.userMode || admin)
|
|
3179
3419
|
return undefined;
|
|
3180
|
-
|
|
3181
|
-
|
|
3182
|
-
return NO_OWNER_MATCHES;
|
|
3420
|
+
if (this.remoteAuthority)
|
|
3421
|
+
return (await this.epAdminReach(caller)) ? undefined : caller.owner;
|
|
3183
3422
|
try {
|
|
3184
3423
|
const scope = await resolveAuthProvider().actorScope({
|
|
3185
3424
|
dir: userAuthStateDir(this.workspaceRoot, this.space),
|
|
3186
|
-
owner:
|
|
3187
|
-
actor:
|
|
3425
|
+
owner: caller.owner,
|
|
3426
|
+
actor: caller.actor,
|
|
3188
3427
|
});
|
|
3189
3428
|
if (scope?.includes("admin"))
|
|
3190
3429
|
return undefined;
|
|
@@ -3192,7 +3431,7 @@ export class Manager {
|
|
|
3192
3431
|
catch {
|
|
3193
3432
|
/* unreadable ledger authorizes nothing extra: fall through to the own-owner bound */
|
|
3194
3433
|
}
|
|
3195
|
-
return
|
|
3434
|
+
return caller.owner;
|
|
3196
3435
|
}
|
|
3197
3436
|
/** Boot one resolved agent from a mesh-manifest launch spec, for `cotal spawn -f` onto a RUNNING
|
|
3198
3437
|
* manager. The request carries `{ runId, name }` — plus, for a deploy from another checkout or
|
|
@@ -3554,6 +3793,11 @@ export class Manager {
|
|
|
3554
3793
|
// AFTER provisioning (buildLaunch / runtime.spawn) — the orphan-rollback tears it down. Carries
|
|
3555
3794
|
// `userOwner` for a user-mode spawn so that rollback runs the revoke+shred branch, not just the
|
|
3556
3795
|
// static durable teardown (the freelance found this window leaking the managed grant + files).
|
|
3796
|
+
// The RESERVED custody reference rides the rollback object: a spawn that launched the seat and
|
|
3797
|
+
// then threw (the activation CAS refuses a slot that is not this spawn's intent) reaches the
|
|
3798
|
+
// `finally` with the handle only in a local, so without this the terminal has nothing to reap
|
|
3799
|
+
// and retires over a live process. The slot row holds the same reference; this is the copy the
|
|
3800
|
+
// in-process rollback can actually read.
|
|
3557
3801
|
let provisioned;
|
|
3558
3802
|
try {
|
|
3559
3803
|
// A stable nkey identity assigned at spawn: the public key is the agent's card.id (threaded via
|
|
@@ -3563,6 +3807,11 @@ export class Manager {
|
|
|
3563
3807
|
// broker resource (dm_/dlv_/chathist_ durables, ACL row, memberships) and the teardown
|
|
3564
3808
|
// credential carry it, so a same-name successor's footprint is name-disjoint by construction.
|
|
3565
3809
|
const lifecycleUid = mintLifecycleUid();
|
|
3810
|
+
// Reserve the seat's custody reference NOW, before any durable row and long before the
|
|
3811
|
+
// launch. Minting it inside `runtime.spawn` (as it used to be) put the processes on disk
|
|
3812
|
+
// before anything recorded how to address them, so a crash in that window left a live seat
|
|
3813
|
+
// outside every manager and the successor retired the slot over it.
|
|
3814
|
+
const custody = this.reserveCustody();
|
|
3566
3815
|
// ACCEPT SEAM (P2 item 2 spawn-as-action): the incarnation identity is minted and NOTHING has
|
|
3567
3816
|
// been provisioned yet — the action serve path binds the goal + replies the acceptance HERE. A
|
|
3568
3817
|
// throw (bind conflict / duplicate goalId) aborts the spawn before provisioning: the catch below
|
|
@@ -3625,6 +3874,7 @@ export class Manager {
|
|
|
3625
3874
|
// spawned session reads them (COTAL_CREDS path). Open mesh → no creds. Scope = the resolved
|
|
3626
3875
|
// subscribe/allowSubscribe (read) + allowPublish (post, default-deny).
|
|
3627
3876
|
let credsPath;
|
|
3877
|
+
let issued;
|
|
3628
3878
|
let userLaunch;
|
|
3629
3879
|
let userOwner;
|
|
3630
3880
|
if (this.userMode) {
|
|
@@ -3645,7 +3895,7 @@ export class Manager {
|
|
|
3645
3895
|
}
|
|
3646
3896
|
userLaunch = prep.launch;
|
|
3647
3897
|
userOwner = prep.owner;
|
|
3648
|
-
provisioned = { id: principalKey(prep.owner, name).key, name, lifecycleUid, userOwner: prep.owner, secretPaths: prep.files };
|
|
3898
|
+
provisioned = { id: principalKey(prep.owner, name).key, name, lifecycleUid, userOwner: prep.owner, secretPaths: prep.files, ...(custody ? { runtime: custody } : {}) };
|
|
3649
3899
|
}
|
|
3650
3900
|
else if (this.auth) {
|
|
3651
3901
|
// Unit B (§13.1): reserve + activate this incarnation's DURABLE identity BEFORE any
|
|
@@ -3653,18 +3903,25 @@ export class Manager {
|
|
|
3653
3903
|
// then the SHARED core activation saga (reserve uid -> gate frozen -> head CAS -> reopen
|
|
3654
3904
|
// LAST) over the key-pinned executor. The wire AUTHORITY principal is the incarnation-
|
|
3655
3905
|
// unique nkey (F5-bind); the alias is protected by the name-keyed slot + freeSlot hold.
|
|
3656
|
-
await this.withLifecycleExecutor({ owner: DEV_OWNER, actor: identity.id, lifecycleUid, alias: name }, (t) => activateStaticLifecycle(t, { owner: DEV_OWNER, alias: name, actor: identity.id, lifecycleUid, managerInstance: this.managerLifecycleUid, ownerInstanceId: this.managerInstanceId }));
|
|
3906
|
+
await this.withLifecycleExecutor({ owner: DEV_OWNER, actor: identity.id, lifecycleUid, alias: name }, (t) => activateStaticLifecycle(t, { owner: DEV_OWNER, alias: name, actor: identity.id, lifecycleUid, managerInstance: this.managerLifecycleUid, ownerInstanceId: this.managerInstanceId, runtime: custody }));
|
|
3657
3907
|
// From here the DURABLE registration exists: arm the rollback BEFORE minting, so a throw
|
|
3658
3908
|
// between activation and provisioning still drives the exact-op static terminal (the
|
|
3659
3909
|
// finally's deprovision tolerates absent files; the broker teardown is idempotent).
|
|
3660
|
-
provisioned = { id: identity.id, name, lifecycleUid };
|
|
3910
|
+
provisioned = { id: identity.id, name, lifecycleUid, ...(custody ? { runtime: custody } : {}) };
|
|
3661
3911
|
// Pre-create the agent's bind-only chat (+ DM + role TASK) durables and mint its scoped creds
|
|
3662
3912
|
// — the shared onboarding step (provisionAgent). It runs on a short-lived PROVISIONER connection
|
|
3663
3913
|
// (NOT the supervisor's long-lived endpoint), so the DM/DLV consumer-create surface exists only
|
|
3664
3914
|
// for the provisioning window, never as a standing grant on the always-on daemon (residual 2).
|
|
3665
3915
|
// F5(b): the credential is BOUNDED (`expiresAt`) — the manager push-renews it ahead of expiry.
|
|
3666
3916
|
const exp = Math.floor(Date.now() / 1000) + MANAGED_STATIC_TTL_SEC;
|
|
3667
|
-
|
|
3917
|
+
// The credential is an ISSUANCE (SPEC 13.15): its caller rails carry a fresh generation
|
|
3918
|
+
// and its ceiling is recorded as evidence bound to this incarnation's ledger family. The
|
|
3919
|
+
// §13.1 ledger append (ledger BEFORE materialization) is the release's finalizer, so the
|
|
3920
|
+
// evidence activates only once the row exists, and the row exists only after the evidence
|
|
3921
|
+
// was staged. The agent learns the generation by reading the accepted row under its token.
|
|
3922
|
+
issued = { generation: mintGeneration(), acceptedToken: mintAcceptedToken() };
|
|
3923
|
+
const issuedAgent = { name, id: identity.id, lifecycleUid };
|
|
3924
|
+
const creds = await this.withIssuer((i) => this.withProvisioner((prov) => provisionAgent(prov, this.auth, identity, {
|
|
3668
3925
|
subscribe,
|
|
3669
3926
|
allowSubscribe,
|
|
3670
3927
|
allowPublish,
|
|
@@ -3672,15 +3929,9 @@ export class Manager {
|
|
|
3672
3929
|
capabilities,
|
|
3673
3930
|
lifecycleUid,
|
|
3674
3931
|
expiresAt: exp,
|
|
3675
|
-
|
|
3676
|
-
|
|
3677
|
-
|
|
3678
|
-
// it — a credential is never materialized before its ledger row exists.
|
|
3679
|
-
const credentialId = rawDigest(creds).replace("sha256:", "sha256-");
|
|
3680
|
-
await this.withLifecycleExecutor({ owner: DEV_OWNER, actor: identity.id, lifecycleUid, alias: name }, async (t) => {
|
|
3681
|
-
await recordSlotCredential(t, DEV_OWNER, name, lifecycleUid, credentialId);
|
|
3682
|
-
await appendStaticCredentialRow(t, { lifecycleUid, credentialId, holderPrincipal: principalKey(DEV_OWNER, identity.id).key, exp });
|
|
3683
|
-
});
|
|
3932
|
+
issued,
|
|
3933
|
+
issuance: this.issuanceSeam(i, "issue", issuedAgent, exp),
|
|
3934
|
+
})));
|
|
3684
3935
|
// Store first (the source of truth), then materialize: `buildLaunch` hands the CHILD this
|
|
3685
3936
|
// file path, so the cred must exist as a file regardless of the store behind the seam. The
|
|
3686
3937
|
// manager's ONE store (injected for hosted, workstation FS locally).
|
|
@@ -3690,7 +3941,7 @@ export class Manager {
|
|
|
3690
3941
|
credsPath = agentLifecycleSecretFilePaths(this.workspaceRoot, this.space, name, lifecycleUid).creds;
|
|
3691
3942
|
await secrets.put(agentSecretKeyForFile(credsPath, this.space), creds);
|
|
3692
3943
|
await materializeSecretToFile(secrets, agentSecretKeyForFile(credsPath, this.space), credsPath);
|
|
3693
|
-
provisioned = { id: identity.id, name, lifecycleUid, secretPaths: { creds: credsPath } }; // footprint now exists — the finally rolls it back if the spawn throws
|
|
3944
|
+
provisioned = { id: identity.id, name, lifecycleUid, secretPaths: { creds: credsPath }, ...(custody ? { runtime: custody } : {}) }; // footprint now exists — the finally rolls it back if the spawn throws
|
|
3694
3945
|
}
|
|
3695
3946
|
// Personal MCP servers the operator opted to share with manager-spawned agents of this type
|
|
3696
3947
|
// (cotal config; default none → isolated, the memory-safe default this guards), narrowed by
|
|
@@ -3723,6 +3974,7 @@ export class Manager {
|
|
|
3723
3974
|
// chathist durables by this exact value (its creds pin the same names, so a mismatch fails
|
|
3724
3975
|
// at the broker, never silently).
|
|
3725
3976
|
lifecycleUid,
|
|
3977
|
+
acceptedToken: issued?.acceptedToken,
|
|
3726
3978
|
servers: this.servers,
|
|
3727
3979
|
configPath,
|
|
3728
3980
|
model,
|
|
@@ -3751,7 +4003,7 @@ export class Manager {
|
|
|
3751
4003
|
workspaceRoot: this.workspaceRoot,
|
|
3752
4004
|
};
|
|
3753
4005
|
const spec = connector.buildLaunch(launchOpts);
|
|
3754
|
-
const handle = this.
|
|
4006
|
+
const handle = await this.spawnCustodied(name, spec, cwd, custody);
|
|
3755
4007
|
hooks?.onLaunched?.(); // P2 item 2: the "launched" progress edge (process spawned, pre-presence)
|
|
3756
4008
|
const managed = {
|
|
3757
4009
|
name,
|
|
@@ -3763,6 +4015,7 @@ export class Manager {
|
|
|
3763
4015
|
// recorded truth teardown/preservation/health consume, never re-derived by name.
|
|
3764
4016
|
secretPaths: provisioned?.secretPaths,
|
|
3765
4017
|
...(userLaunch ? { userOwner } : { seed: identity.seed }),
|
|
4018
|
+
...(issued ? { issued } : {}),
|
|
3766
4019
|
spawner: spawner ?? this.ep.ref().id,
|
|
3767
4020
|
authorityParent: userLaunch && spawner && parsePrincipalKey(spawner) ? spawner : undefined,
|
|
3768
4021
|
startedAt: Date.now(),
|
|
@@ -3818,7 +4071,9 @@ export class Manager {
|
|
|
3818
4071
|
const slot = await readStaticSlot(t, DEV_OWNER, name);
|
|
3819
4072
|
if (slot === undefined || slot.row.lifecycleUid !== lifecycleUid || slot.row.phase !== "provisioning")
|
|
3820
4073
|
throw new Error(`the static slot for "${name}" is ${slot === undefined ? "absent" : `${slot.row.phase} at uid ${slot.row.lifecycleUid}`}, not this spawn's provisioning intent; refusing to take the slot`);
|
|
3821
|
-
|
|
4074
|
+
// The custody reference rides the SAME CAS as activation: a crash after this write leaves
|
|
4075
|
+
// an active slot a successor can reap by reference, never a live seat nobody addresses.
|
|
4076
|
+
await casStaticSlot(t, { ...slot.row, phase: "active", ...(handle.reference ? { runtime: handle.reference } : {}) }, slot.revision);
|
|
3822
4077
|
});
|
|
3823
4078
|
}
|
|
3824
4079
|
this.agents.set(name, managed);
|
|
@@ -3915,15 +4170,33 @@ export class Manager {
|
|
|
3915
4170
|
const seen = new Set();
|
|
3916
4171
|
const principals = new Set();
|
|
3917
4172
|
await this.ep.waitForPresenceSnapshot();
|
|
3918
|
-
const
|
|
3919
|
-
.filter((presence) => presence.status !== "offline")
|
|
3920
|
-
.map((presence) => presence.card.id));
|
|
4173
|
+
const liveRoster = this.ep.getRoster().filter((presence) => presence.status !== "offline");
|
|
3921
4174
|
if (this.agents.size + this.reserved.size + this.coolingCount() + inventory.agents.length > MAX_AGENTS)
|
|
3922
4175
|
return { ok: false, agents: [], error: `resume inventory would exceed manager capacity (${MAX_AGENTS})` };
|
|
3923
4176
|
for (const entry of inventory.agents) {
|
|
3924
4177
|
if (seen.has(entry.name))
|
|
3925
4178
|
return { ok: false, agents: [], error: `resume inventory contains duplicate agent name "${entry.name}"` };
|
|
3926
4179
|
seen.add(entry.name);
|
|
4180
|
+
// A manager-local retirement hold is stronger and more precise than presence. The stopped
|
|
4181
|
+
// predecessor can remain roster-live until its TTL/offline update lands, but that stale row
|
|
4182
|
+
// must not hide the durable teardown state the manager already owns. Refuse and re-drive the
|
|
4183
|
+
// exact held lifecycle before consulting generic principal liveness. A confirmed terminal has
|
|
4184
|
+
// already removed the hold, so fresh-lifecycle resume still reaches the fail-closed roster gate.
|
|
4185
|
+
const held = this.retiring.get(entry.name);
|
|
4186
|
+
if (held) {
|
|
4187
|
+
void this.deprovision({
|
|
4188
|
+
id: held.agentId,
|
|
4189
|
+
name: entry.name,
|
|
4190
|
+
lifecycleUid: held.lifecycleUid,
|
|
4191
|
+
userOwner: held.userOwner,
|
|
4192
|
+
secretPaths: held.secretPaths,
|
|
4193
|
+
}).catch(() => { });
|
|
4194
|
+
return {
|
|
4195
|
+
ok: false,
|
|
4196
|
+
agents: [],
|
|
4197
|
+
error: `retained agent "${entry.name}" is reserved pending retirement: its previous lifecycle ${held.lifecycleUid} still owns the alias${held.lastError ? ` (last attempt: ${held.lastError})` : ""}; retrying re-drives that exact teardown`,
|
|
4198
|
+
};
|
|
4199
|
+
}
|
|
3927
4200
|
let principal;
|
|
3928
4201
|
try {
|
|
3929
4202
|
principal = entry.identity.mode === "user"
|
|
@@ -3936,7 +4209,17 @@ export class Manager {
|
|
|
3936
4209
|
if (principals.has(principal))
|
|
3937
4210
|
return { ok: false, agents: [], error: `resume inventory contains duplicate principal "${principal}"` };
|
|
3938
4211
|
principals.add(principal);
|
|
3939
|
-
|
|
4212
|
+
const confirmedPredecessor = this.confirmedRetiredPredecessors.get(entry.name);
|
|
4213
|
+
const principalIsLive = liveRoster.some((presence) => {
|
|
4214
|
+
if (presence.card.id !== principal)
|
|
4215
|
+
return false;
|
|
4216
|
+
return confirmedPredecessor === undefined ||
|
|
4217
|
+
confirmedPredecessor.principal !== principal ||
|
|
4218
|
+
confirmedPredecessor.lifecycleUid !== presence.lifecycleUid ||
|
|
4219
|
+
presence.card.name !== entry.name ||
|
|
4220
|
+
entry.identity.lifecycleUid === confirmedPredecessor.lifecycleUid;
|
|
4221
|
+
});
|
|
4222
|
+
if (principalIsLive)
|
|
3940
4223
|
return { ok: false, agents: [], error: `retained principal "${principal}" is already live and this runtime cannot authoritatively adopt it` };
|
|
3941
4224
|
if (this.agents.has(entry.name) || this.reserved.has(entry.name))
|
|
3942
4225
|
return { ok: false, agents: [], error: `retained agent "${entry.name}" is already managed or reserved` };
|
|
@@ -4088,15 +4371,23 @@ export class Manager {
|
|
|
4088
4371
|
const sentinelCreds = await secrets.get(agentSecretKeyForFile(recordedSentinel, this.space));
|
|
4089
4372
|
if (actorToken === undefined || sentinelCreds === undefined)
|
|
4090
4373
|
throw new Error("the retained actor token / sentinel credential is not in the secret store");
|
|
4091
|
-
const adopted =
|
|
4092
|
-
|
|
4093
|
-
|
|
4094
|
-
|
|
4095
|
-
|
|
4096
|
-
|
|
4097
|
-
|
|
4098
|
-
|
|
4099
|
-
|
|
4374
|
+
const adopted = this.remoteAuthority
|
|
4375
|
+
? await this.remoteAuthority.validateRetainedAgent({
|
|
4376
|
+
owner: entry.identity.owner,
|
|
4377
|
+
actor: entry.identity.actor,
|
|
4378
|
+
lifecycleUid: entry.identity.lifecycleUid,
|
|
4379
|
+
actorToken,
|
|
4380
|
+
sentinelCreds,
|
|
4381
|
+
})
|
|
4382
|
+
: await provider.validateRetainedAgent({
|
|
4383
|
+
store: secrets,
|
|
4384
|
+
dir: userAuthStateDir(this.workspaceRoot, this.space),
|
|
4385
|
+
space: this.space,
|
|
4386
|
+
owner: entry.identity.owner,
|
|
4387
|
+
actor: entry.identity.actor,
|
|
4388
|
+
actorToken,
|
|
4389
|
+
sentinelCreds,
|
|
4390
|
+
});
|
|
4100
4391
|
if (adopted.owner !== entry.identity.owner || adopted.actor !== entry.identity.actor)
|
|
4101
4392
|
throw new Error(`auth provider returned a replacement principal; expected ${entry.identity.owner}.${entry.identity.actor}`);
|
|
4102
4393
|
// Bind the inventory's uid to the CURRENT authority row BEFORE any spawn: a corrupt or
|
|
@@ -4119,7 +4410,9 @@ export class Manager {
|
|
|
4119
4410
|
...process.execArgv,
|
|
4120
4411
|
process.argv[1],
|
|
4121
4412
|
provider.agentBearerCommand,
|
|
4122
|
-
|
|
4413
|
+
...(this.remoteAuthority
|
|
4414
|
+
? ["--exchange-url", this.remoteAuthority.agentBearerExchangeUrl]
|
|
4415
|
+
: ["--dir", userAuthStateDir(this.workspaceRoot, this.space)]),
|
|
4123
4416
|
"--space", this.space,
|
|
4124
4417
|
"--owner", entry.identity.owner,
|
|
4125
4418
|
"--actor", entry.identity.actor,
|
|
@@ -4251,6 +4544,7 @@ export class Manager {
|
|
|
4251
4544
|
// with no COTAL_LIFECYCLE_UID: static/user fail the connector auth gate and open self-mints a
|
|
4252
4545
|
// fresh uid that orphans the preserved durables and never matches the readiness fence.
|
|
4253
4546
|
lifecycleUid: entry.identity.lifecycleUid,
|
|
4547
|
+
acceptedToken: entry.identity.mode === "static" ? entry.identity.issued?.acceptedToken : undefined,
|
|
4254
4548
|
servers: this.servers,
|
|
4255
4549
|
configPath: entry.launch.source.configPath,
|
|
4256
4550
|
model: entry.launch.model,
|
|
@@ -4300,13 +4594,18 @@ export class Manager {
|
|
|
4300
4594
|
const readinessTimeoutMs = connector.readinessTimeoutMs ?? this.readinessTimeoutMs;
|
|
4301
4595
|
if (!Number.isSafeInteger(readinessTimeoutMs) || readinessTimeoutMs <= 0)
|
|
4302
4596
|
return { ok: false, error: `connector ${connector.name} declares invalid readinessTimeoutMs ${JSON.stringify(connector.readinessTimeoutMs)}; expected a positive safe integer` };
|
|
4303
|
-
const
|
|
4597
|
+
const resumeId = entry.identity.mode === "user" ? principalKey(entry.identity.owner, entry.identity.actor).key : entry.identity.id;
|
|
4598
|
+
const custody = this.reserveCustody();
|
|
4599
|
+
if (custody)
|
|
4600
|
+
await this.recordSlotCustody({ name: entry.name, id: resumeId, lifecycleUid: entry.identity.lifecycleUid }, custody);
|
|
4601
|
+
const handle = await this.spawnCustodied(entry.name, prepared.spec, entry.launch.cwd, custody);
|
|
4304
4602
|
const managed = {
|
|
4305
4603
|
name: entry.name,
|
|
4306
4604
|
role: entry.role,
|
|
4307
4605
|
agent: entry.launch.connector,
|
|
4308
|
-
id:
|
|
4606
|
+
id: resumeId,
|
|
4309
4607
|
seed: adoptedSeed,
|
|
4608
|
+
...(entry.identity.mode === "static" && entry.identity.issued ? { issued: entry.identity.issued } : {}),
|
|
4310
4609
|
// Recover the ORIGINAL incarnation uid the durables are keyed by (never a fresh mint on resume).
|
|
4311
4610
|
lifecycleUid: entry.identity.lifecycleUid,
|
|
4312
4611
|
// Adopt the INVENTORY's recorded family (possibly a pre-split name-keyed layout) — the
|
|
@@ -4345,6 +4644,7 @@ export class Manager {
|
|
|
4345
4644
|
this.agents.set(entry.name, managed);
|
|
4346
4645
|
if (this.resumeAttemptId)
|
|
4347
4646
|
this.resumedAgentNames.add(entry.name);
|
|
4647
|
+
await this.recordSlotRuntime(managed);
|
|
4348
4648
|
const readiness = await this.awaitReadiness(managed, readinessTimeoutMs);
|
|
4349
4649
|
if (!readiness.ok && !readiness.uncertain)
|
|
4350
4650
|
return { ok: false, error: readiness.detail };
|
|
@@ -4643,16 +4943,56 @@ export class Manager {
|
|
|
4643
4943
|
await nc.drain().catch(() => nc.close());
|
|
4644
4944
|
}
|
|
4645
4945
|
}
|
|
4646
|
-
/** Run one
|
|
4946
|
+
/** Run one SPEC 13.15 issuance operation over an ephemeral `issuer` connection: the evidence
|
|
4947
|
+
* stage/release of one mint, or the retirement walk of one lifecycle's issuances. The store
|
|
4948
|
+
* handle and the accepted-row KV live only for this window; the standing supervisor holds no
|
|
4949
|
+
* grant on either bucket. */
|
|
4950
|
+
async withIssuer(fn) {
|
|
4951
|
+
if (!this.auth)
|
|
4952
|
+
throw new Error("withIssuer: no space auth (an open mesh issues nothing)");
|
|
4953
|
+
const creds = await mintCreds(this.auth, newIdentity(), "issuer");
|
|
4954
|
+
const nc = await this.dial({ ...standaloneConnectOpts({ creds, /* not yet wired to a recorded transport */ tls: false }), maxReconnectAttempts: 0 });
|
|
4955
|
+
try {
|
|
4956
|
+
const kvm = new Kvm(nc);
|
|
4957
|
+
const store = openIssuedStore(await kvm.open(issuedBucket(this.space)), await jetstreamManager(nc), this.space);
|
|
4958
|
+
return await fn({ store, accepted: await kvm.open(acceptedBucket(this.space)) });
|
|
4959
|
+
}
|
|
4960
|
+
finally {
|
|
4961
|
+
await nc.drain().catch(() => nc.close());
|
|
4962
|
+
}
|
|
4963
|
+
}
|
|
4964
|
+
/** The issuance seam of ONE static credential mint (SPEC 13.15): the evidence is bound to the
|
|
4965
|
+
* incarnation's own `cred.<uid>` ledger family, so the family's revocation (the static
|
|
4966
|
+
* terminal) retires every issuance that named it. The finalizer is the §13.1 ledger append
|
|
4967
|
+
* (slot credentialId + `cred.<uid>.<credId>` row); release runs it, then activates. */
|
|
4968
|
+
issuanceSeam(i, mode, a, exp) {
|
|
4969
|
+
return {
|
|
4970
|
+
mode,
|
|
4971
|
+
store: i.store,
|
|
4972
|
+
accepted: i.accepted,
|
|
4973
|
+
sources: mode === "issue" ? [{ space: this.space, bucket: epAuthBucket(this.space), key: `cred.${a.lifecycleUid}` }] : [],
|
|
4974
|
+
finalize: async (credentialId) => {
|
|
4975
|
+
await this.withLifecycleExecutor({ owner: DEV_OWNER, actor: a.id, lifecycleUid: a.lifecycleUid, alias: a.name }, async (t) => {
|
|
4976
|
+
await recordSlotCredential(t, DEV_OWNER, a.name, a.lifecycleUid, credentialId);
|
|
4977
|
+
await appendStaticCredentialRow(t, { lifecycleUid: a.lifecycleUid, credentialId, holderPrincipal: principalKey(DEV_OWNER, a.id).key, exp });
|
|
4978
|
+
});
|
|
4979
|
+
},
|
|
4980
|
+
};
|
|
4981
|
+
}
|
|
4982
|
+
/** Run one §13.1 endpoint-instance maintenance operation over an ephemeral,
|
|
4647
4983
|
* key-pinned `endpoint-serve-executor` connection: the credential's grants name exactly the
|
|
4648
4984
|
* manager instance's `epgate`/`epcred` keys plus its registration's two records keys, so the
|
|
4649
|
-
* gate CAS,
|
|
4650
|
-
*
|
|
4985
|
+
* gate CAS, mint fence, spec/governance writes, and clean deregistration ride a one-shot scoped
|
|
4986
|
+
* authority. The remote boot goal-index sweep instead uses the authenticated host-owned scan.
|
|
4987
|
+
* This is NEVER the manager's standing seed/supervisor connection. */
|
|
4651
4988
|
async withEndpointServeExecutor(fn) {
|
|
4652
4989
|
const identity = this.remoteAuthority?.identities.executor ?? newIdentity();
|
|
4653
4990
|
const creds = this.remoteAuthority?.executorCreds ?? (this.auth
|
|
4654
4991
|
? await mintCreds(this.auth, identity, "endpoint-serve-executor", {
|
|
4655
4992
|
endpointServeExecutor: { endpoint: MANAGER_ENDPOINT, instanceId: this.managerInstanceId },
|
|
4993
|
+
...(this.endpointServeExecutorExpiresInSeconds !== undefined
|
|
4994
|
+
? { expiresInSeconds: this.endpointServeExecutorExpiresInSeconds }
|
|
4995
|
+
: {}),
|
|
4656
4996
|
})
|
|
4657
4997
|
: undefined);
|
|
4658
4998
|
if (!creds)
|
|
@@ -4671,8 +5011,8 @@ export class Manager {
|
|
|
4671
5011
|
* writes ride a bare one-shot connection (the broker enforces nothing on an open mesh; the
|
|
4672
5012
|
* ceremony still produces the real gate, epoch, and registration the serve rails run on). */
|
|
4673
5013
|
async withOpenServeConnection(fn) {
|
|
4674
|
-
if (this.auth)
|
|
4675
|
-
throw new Error("withOpenServeConnection: an
|
|
5014
|
+
if (this.auth || this.remoteAuthority)
|
|
5015
|
+
throw new Error("withOpenServeConnection: an authenticated mesh must use the scoped endpoint-serve executor");
|
|
4676
5016
|
const nc = await this.dial({ maxReconnectAttempts: 0 });
|
|
4677
5017
|
try {
|
|
4678
5018
|
const kvm = new Kvm(nc);
|
|
@@ -4867,7 +5207,23 @@ export class Manager {
|
|
|
4867
5207
|
// the takeover barrier revokes a deposed manager's ledger cred alongside its goal-writer. The
|
|
4868
5208
|
// per-session serving creds join the same family, each with its own fresh identity.
|
|
4869
5209
|
this.sessionLedgerIdentity = newIdentity();
|
|
4870
|
-
|
|
5210
|
+
// One registration operation for this boot. Fresh executors may replace a dead connection, but
|
|
5211
|
+
// they must resume THIS freeze rather than minting a new op that discards Phase-2 progress.
|
|
5212
|
+
const registrationOpId = mintLifecycleUid();
|
|
5213
|
+
const executorExpired = (e) => /closed connection/i.test(e?.message ?? String(e));
|
|
5214
|
+
const retryExpiredExecutor = async (label, fn) => {
|
|
5215
|
+
for (;;) {
|
|
5216
|
+
try {
|
|
5217
|
+
return await fn();
|
|
5218
|
+
}
|
|
5219
|
+
catch (e) {
|
|
5220
|
+
if (!executorExpired(e))
|
|
5221
|
+
throw e;
|
|
5222
|
+
console.error(`manager ${label} executor expired; retrying${label === "registration" ? ` operation ${registrationOpId}` : ""} with fresh scoped authority`);
|
|
5223
|
+
}
|
|
5224
|
+
}
|
|
5225
|
+
};
|
|
5226
|
+
const publishAndProvision = async ({ authKv, nc: execNc }) => {
|
|
4871
5227
|
// §13.7 contract-artifact publication (1c): every schema root + its closure manifest, plus
|
|
4872
5228
|
// the cluster document + ITS manifest, land in the EPC store BEFORE the registration that
|
|
4873
5229
|
// advertises their digests — so a caller can always fetch-verify-compile a registered
|
|
@@ -4886,14 +5242,9 @@ export class Manager {
|
|
|
4886
5242
|
// so the gate's principal binding is unchanged either way.
|
|
4887
5243
|
if ((await serveIssuanceGateKv(authKv, this.space, { endpoint: MANAGER_ENDPOINT, instanceId: iid }).observe()) === null)
|
|
4888
5244
|
await provisionEndpointGateOpen(authKv, { endpoint: MANAGER_ENDPOINT, instanceId: iid, principal: servePrincipal });
|
|
4889
|
-
|
|
4890
|
-
|
|
4891
|
-
|
|
4892
|
-
// holder-gone evidence BEFORE this incarnation freezes a new one. Auth only: the
|
|
4893
|
-
// CONNZ oracle rides delivery-admin, which an open mesh does not have.
|
|
4894
|
-
if (auth) {
|
|
4895
|
-
await this.healFrozenRegistrationGate(authKv, iid, auth, recordsKv);
|
|
4896
|
-
}
|
|
5245
|
+
};
|
|
5246
|
+
const completeRegistration = async ({ recordsKv, authKv, nc: execNc }) => {
|
|
5247
|
+
await publishAndProvision({ recordsKv, authKv, nc: execNc });
|
|
4897
5248
|
// P2 item 3 (slice 3a): on an AUTH mesh a RE-registration (restart of the persisted instanceId)
|
|
4898
5249
|
// must VERIFY-EVICT the superseded serve family BEFORE the epoch advances (§13.1 "old authority
|
|
4899
5250
|
// dies before new authority is visible"). Inject the SCOPED delivery-admin evictor; the OPEN
|
|
@@ -4902,12 +5253,18 @@ export class Manager {
|
|
|
4902
5253
|
// evictor THROWS naming the cure, so PHASE 2 fails closed with the delivery-daemon fix in the
|
|
4903
5254
|
// error text — a crash-restart never silently skips eviction (no-fallbacks).
|
|
4904
5255
|
const barrier = endpointRegistrationBarrier(authKv, this.space, {
|
|
4905
|
-
endpoint: MANAGER_ENDPOINT, instanceId: iid, opId:
|
|
5256
|
+
endpoint: MANAGER_ENDPOINT, instanceId: iid, opId: registrationOpId,
|
|
4906
5257
|
...(auth ? { evict: makeManagerEndpointEvictor({ space: this.space, servers: this.servers ?? DEFAULT_SERVER, auth, log: (line) => console.error(line) }) } : {}),
|
|
4907
5258
|
});
|
|
4908
5259
|
const spec = { endpoint: MANAGER_ENDPOINT, owner: DEV_OWNER, clusterDigests: [artifacts.closureDigest], protocol: { v: 1 } };
|
|
4909
5260
|
const { registrationRevision } = await registerServiceInstance(recordsKv, {
|
|
4910
5261
|
space: this.space, spec, instanceId: iid, registrant: { owner: DEV_OWNER }, authority, barrier, readClusterArtifact,
|
|
5262
|
+
// #1393: when a FOREIGN instance holds the endpoint governance slot, let core tell an
|
|
5263
|
+
// in-flight registration from one abandoned by a predecessor that died between its
|
|
5264
|
+
// slot-take and its spec publish. The read rides THIS executor's existing auth-bucket read
|
|
5265
|
+
// grant (no new grant, no new writer); core refuses on anything but a gate provably
|
|
5266
|
+
// reopened past the slot's stamp.
|
|
5267
|
+
observeHolderGeneration: (holderInstanceId) => readEndpointGateGeneration(authKv, { endpoint: MANAGER_ENDPOINT, instanceId: holderInstanceId }),
|
|
4911
5268
|
});
|
|
4912
5269
|
// processEpoch comes from the GATE (checklist 4: never derived from the uid string); the
|
|
4913
5270
|
// fence below is also the mint's §13.1 release CAS.
|
|
@@ -4960,7 +5317,32 @@ export class Manager {
|
|
|
4960
5317
|
const sessionLedgerCreds = auth ? await this.mintAndStageSessionLedger(authKv) : undefined;
|
|
4961
5318
|
return { grant, creds, goalWriterCreds, sessionLedgerCreds };
|
|
4962
5319
|
};
|
|
4963
|
-
|
|
5320
|
+
let registration;
|
|
5321
|
+
if (auth) {
|
|
5322
|
+
// Boot heal may consume an arbitrarily large prior family. End that executor window here and
|
|
5323
|
+
// mint fresh scoped authority for the new registration, rather than forcing heal + takeover
|
|
5324
|
+
// through one fixed-lifetime connection. A closed connection during heal retries the SAME
|
|
5325
|
+
// frozen predecessor op via reconcile's durable cursor; a closed connection during
|
|
5326
|
+
// registration retries THIS boot's operationId without heal, so Phase-2 progress stays bound.
|
|
5327
|
+
// #783/#871: a predecessor that died mid-barrier leaves this gate frozen under a
|
|
5328
|
+
// registration op. registerServiceInstance will then refuse SPEC 13.8 forever, even
|
|
5329
|
+
// when the freeze-holder is gone. Complete that SAME op (abort-reopen) on independent
|
|
5330
|
+
// holder-gone evidence BEFORE this incarnation freezes a new one. Auth only: the
|
|
5331
|
+
// CONNZ oracle rides delivery-admin, which an open mesh does not have.
|
|
5332
|
+
await retryExpiredExecutor("boot-heal", () => this.withEndpointServeExecutor(async ({ recordsKv, authKv, nc: execNc }) => {
|
|
5333
|
+
await publishAndProvision({ recordsKv, authKv, nc: execNc });
|
|
5334
|
+
if (auth) {
|
|
5335
|
+
await this.healFrozenRegistrationGate(authKv, iid, auth, recordsKv);
|
|
5336
|
+
}
|
|
5337
|
+
}));
|
|
5338
|
+
registration = await retryExpiredExecutor("registration", () => this.withEndpointServeExecutor(completeRegistration));
|
|
5339
|
+
}
|
|
5340
|
+
else {
|
|
5341
|
+
registration = await this.withOpenServeConnection(completeRegistration);
|
|
5342
|
+
}
|
|
5343
|
+
if (!registration)
|
|
5344
|
+
throw new Error("manager registration produced no serve authority");
|
|
5345
|
+
const { grant, creds, goalWriterCreds, sessionLedgerCreds } = registration;
|
|
4964
5346
|
this.goalWriterCreds = goalWriterCreds;
|
|
4965
5347
|
this.sessionLedgerCreds = sessionLedgerCreds;
|
|
4966
5348
|
// The serve connection presents the CURRENT credential on every (re)connect (the state object
|
|
@@ -5380,10 +5762,11 @@ export class Manager {
|
|
|
5380
5762
|
}
|
|
5381
5763
|
/** P2 item 2 must-5 Q-B — the boot reconcile: a fresh incarnation (a manager restart takes a NEW
|
|
5382
5764
|
* instanceId, so the in-memory acceptance map starts empty) inherits the endpoint's accepted-but-
|
|
5383
|
-
* unterminal goals from any predecessor.
|
|
5384
|
-
*
|
|
5385
|
-
*
|
|
5386
|
-
*
|
|
5765
|
+
* unterminal goals from any predecessor. The local signer enumerates with an ephemeral PROVISIONER.
|
|
5766
|
+
* A remote manager calls the authenticated host-owned scan, which keeps the sealed consumer and its
|
|
5767
|
+
* lifecycle authority on the host; the instance executor, goal-writer, and supervisor hold NO records
|
|
5768
|
+
* consumer grant. Settle each orphan so an accepted goal is NEVER dropped across a restart. Open mesh:
|
|
5769
|
+
* a bare connection (the broker enforces nothing). Runs ONCE at start, BEFORE spawn-as-action begins
|
|
5387
5770
|
* accepting (the `goalReconcileDone` gate), so it never races a live goal's acceptance. Never
|
|
5388
5771
|
* fatal — a reconcile failure is logged and the gate opens either way. */
|
|
5389
5772
|
async reconcileGoalIndex() {
|
|
@@ -5394,16 +5777,26 @@ export class Manager {
|
|
|
5394
5777
|
}
|
|
5395
5778
|
try {
|
|
5396
5779
|
let entries = [];
|
|
5397
|
-
|
|
5398
|
-
|
|
5399
|
-
|
|
5400
|
-
|
|
5401
|
-
|
|
5402
|
-
|
|
5403
|
-
|
|
5780
|
+
if (this.remoteAuthority) {
|
|
5781
|
+
entries = (await this.remoteAuthority.scanGoalIndex()).map((entry) => ({
|
|
5782
|
+
ref: { endpoint: entry.endpoint, caller: { owner: entry.owner, actor: entry.actor, uid: entry.uid }, goalId: entry.goalId },
|
|
5783
|
+
iid: entry.iid,
|
|
5784
|
+
...(entry.allocated !== undefined ? { allocated: entry.allocated } : {}),
|
|
5785
|
+
...(entry.note !== undefined ? { note: entry.note } : {}),
|
|
5786
|
+
}));
|
|
5404
5787
|
}
|
|
5405
|
-
|
|
5406
|
-
|
|
5788
|
+
else {
|
|
5789
|
+
const nc = this.auth
|
|
5790
|
+
? await this.dial({ ...standaloneConnectOpts({ creds: await mintCreds(this.auth, newIdentity(), "provisioner"), /* not yet wired to a recorded transport */ tls: false }), maxReconnectAttempts: 0 })
|
|
5791
|
+
: await this.dial({ maxReconnectAttempts: 0 });
|
|
5792
|
+
try {
|
|
5793
|
+
const kvm = new Kvm(nc);
|
|
5794
|
+
await ensureAuthorityStores(await jetstreamManager(nc), kvm, this.space);
|
|
5795
|
+
entries = await listGoalIndex(await kvm.open(recordsBucket(this.space)), MANAGER_ENDPOINT);
|
|
5796
|
+
}
|
|
5797
|
+
finally {
|
|
5798
|
+
await nc.drain().catch(() => nc.close());
|
|
5799
|
+
}
|
|
5407
5800
|
}
|
|
5408
5801
|
// Single-manager item 2: EVERY inherited entry belongs to a DEAD predecessor (only one manager
|
|
5409
5802
|
// at a time), so all are reconciled. The `iid` field is the hook item-3's multi-instance sweep
|
|
@@ -6241,6 +6634,107 @@ export class Manager {
|
|
|
6241
6634
|
// the goal already terminalized (the readiness outcome won the settle race) - nothing to cancel.
|
|
6242
6635
|
}
|
|
6243
6636
|
}
|
|
6637
|
+
/** Re-record the custody reference of a managed agent's CURRENT handle on its active slot: a
|
|
6638
|
+
* same-lifecycle restart or a resume binds a new custody under the old uid, and the successor's
|
|
6639
|
+
* reap must address the live one. Static auth only; a runtime without durable custody records
|
|
6640
|
+
* nothing (there is nothing to reap by reference). Loud on failure, never fatal to the bind: the
|
|
6641
|
+
* handle is already live and a stale reference is refused by identity at reap time. */
|
|
6642
|
+
async recordSlotRuntime(a) {
|
|
6643
|
+
if (a.handle.reference === undefined)
|
|
6644
|
+
return;
|
|
6645
|
+
await this.recordSlotCustody(a, a.handle.reference);
|
|
6646
|
+
}
|
|
6647
|
+
/** Record one custody reference on an agent's ACTIVE slot. Split from {@link recordSlotRuntime}
|
|
6648
|
+
* so a restart or a resume can record the reference it RESERVED before spawning, while no
|
|
6649
|
+
* handle exists yet. Static auth only; a runtime without durable custody reserves nothing. */
|
|
6650
|
+
async recordSlotCustody(a, runtime) {
|
|
6651
|
+
if (!this.auth || this.userMode)
|
|
6652
|
+
return;
|
|
6653
|
+
try {
|
|
6654
|
+
await this.withLifecycleExecutor({ owner: DEV_OWNER, actor: a.id, lifecycleUid: a.lifecycleUid, alias: a.name }, async (t) => {
|
|
6655
|
+
const slot = await readStaticSlot(t, DEV_OWNER, a.name);
|
|
6656
|
+
if (slot === undefined || slot.row.lifecycleUid !== a.lifecycleUid || slot.row.phase !== "active")
|
|
6657
|
+
return;
|
|
6658
|
+
if (slot.row.runtime?.kind === runtime.kind && slot.row.runtime.id === runtime.id)
|
|
6659
|
+
return;
|
|
6660
|
+
await casStaticSlot(t, { ...slot.row, runtime }, slot.revision);
|
|
6661
|
+
});
|
|
6662
|
+
}
|
|
6663
|
+
catch (e) {
|
|
6664
|
+
console.error(`! ${a.name}: could not record the custody reference ${runtime.kind}:${runtime.id} on its static slot: ${e.message} - a successor reaps this seat only by an up-to-date reference`);
|
|
6665
|
+
}
|
|
6666
|
+
}
|
|
6667
|
+
/** Mint the custody reference for a seat this manager is about to spawn, BEFORE the processes
|
|
6668
|
+
* exist. Absent for a runtime with no durable custody (there is nothing a successor could reap
|
|
6669
|
+
* by reference), which is the only case that still spawns unreserved. */
|
|
6670
|
+
reserveCustody() {
|
|
6671
|
+
return isCustodialRuntime(this.runtime) ? this.runtime.reserve() : undefined;
|
|
6672
|
+
}
|
|
6673
|
+
/** Spawn under a reserved reference and prove the runtime honoured it. A runtime that minted its
|
|
6674
|
+
* own id instead would leave every durable record addressing a seat that does not exist, so the
|
|
6675
|
+
* mismatch tears the new seat down and throws rather than returning an unaddressable handle. */
|
|
6676
|
+
async spawnCustodied(name, spec, cwd, reserved) {
|
|
6677
|
+
const handle = this.runtime.spawn(name, spec, cwd, reserved);
|
|
6678
|
+
if (reserved === undefined)
|
|
6679
|
+
return handle;
|
|
6680
|
+
const got = handle.reference;
|
|
6681
|
+
if (got?.kind === reserved.kind && got.id === reserved.id)
|
|
6682
|
+
return handle;
|
|
6683
|
+
// A live seat now exists under a reference no durable row names, and the rollback will reap the
|
|
6684
|
+
// RESERVED one, which addresses nothing. `handle.stop` cannot close this: it is fire-and-forget
|
|
6685
|
+
// over the seat socket and swallows its own errors, so it proves nothing about the process. Reap
|
|
6686
|
+
// the reference the runtime ACTUALLY used — that path verifies identity and proves the exit —
|
|
6687
|
+
// and carry what the proof said into the throw, so a mismatch never reports a disposal it did
|
|
6688
|
+
// not make.
|
|
6689
|
+
let disposal;
|
|
6690
|
+
if (got === undefined) {
|
|
6691
|
+
try {
|
|
6692
|
+
handle.stop({ graceful: false });
|
|
6693
|
+
}
|
|
6694
|
+
catch { /* reported below */ }
|
|
6695
|
+
disposal = "the handle carries no reference, so nothing could reap it by identity and only a best-effort stop was issued";
|
|
6696
|
+
}
|
|
6697
|
+
else {
|
|
6698
|
+
try {
|
|
6699
|
+
const evidence = await requireRuntimeReap(this.runtime, got);
|
|
6700
|
+
disposal = `the spawned seat was reaped (${evidence.detail})`;
|
|
6701
|
+
}
|
|
6702
|
+
catch (e) {
|
|
6703
|
+
// Two unrelated failures used to share one sentence here, and the difference is the part the
|
|
6704
|
+
// reader has to act on. A runtime that cannot reap at all left nothing behind to find. An
|
|
6705
|
+
// UNPROVEN reap may have left a live seat under a reference no durable row names, which is
|
|
6706
|
+
// the one case where somebody has to go looking, so it says so and names what to look for.
|
|
6707
|
+
disposal =
|
|
6708
|
+
e instanceof RuntimeReapUnproven
|
|
6709
|
+
? `the spawned seat was NOT proved gone and may still be running as ${e.reference.kind}:${e.reference.id} (${e.message})`
|
|
6710
|
+
: `the spawned seat could NOT be reaped: ${e.message}`;
|
|
6711
|
+
}
|
|
6712
|
+
}
|
|
6713
|
+
throw new Error(`runtime "${this.runtime.kind}" reserved custody ${reserved.kind}:${reserved.id} for "${name}" but spawned ` +
|
|
6714
|
+
`${got ? `${got.kind}:${got.id}` : "a handle with no reference"}; the recorded reference would address no seat, so ${disposal}`);
|
|
6715
|
+
}
|
|
6716
|
+
/** The terminal's process step: an orphan (no live handle in THIS process) is reaped by the custody
|
|
6717
|
+
* reference the slot recorded, before its footprint goes. A successor must never retire a
|
|
6718
|
+
* lifecycle and free its alias while the predecessor's seat process is still running outside
|
|
6719
|
+
* every manager. The runtime verifies the process identity against its own record and proves the
|
|
6720
|
+
* exit; a runtime that cannot reap refuses by name and the lifecycle stays terminalizing. A row
|
|
6721
|
+
* with no recorded reference has nothing this manager can address. */
|
|
6722
|
+
async reapOrphanSeat(a) {
|
|
6723
|
+
if (a.runtime === undefined) {
|
|
6724
|
+
// No reference reached this terminal. Under a custodying runtime that is either a row written
|
|
6725
|
+
// before the reservation existed, or a caller that dropped the reference on the way here.
|
|
6726
|
+
// Do not name the slot row: the row may well hold a reference this caller never read, and
|
|
6727
|
+
// a diagnostic that guesses which is which sends the reader the wrong way.
|
|
6728
|
+
if (isCustodialRuntime(this.runtime))
|
|
6729
|
+
console.error(`static retirement ${a.name}: no custody reference reached this terminal, though runtime "${this.runtime.kind}" custodies its seats; any seat it launched is not addressable from here`);
|
|
6730
|
+
return;
|
|
6731
|
+
}
|
|
6732
|
+
// An unproven reap throws from here rather than printing, so it propagates into
|
|
6733
|
+
// driveStaticRetirement's catch, which records the failure and HOLDS the name. That is the whole
|
|
6734
|
+
// point of refusing: the alias must not be freed while a seat nobody proved gone may still run.
|
|
6735
|
+
const evidence = await requireRuntimeReap(this.runtime, a.runtime);
|
|
6736
|
+
console.error(`static retirement ${a.name}: orphan seat process ${evidence.detail}`);
|
|
6737
|
+
}
|
|
6244
6738
|
/** The static F1 terminal for one departed incarnation (Unit B): delegates the gate/head CAS
|
|
6245
6739
|
* sequence to the shared core saga over the executor transport; the footprint teardown (creds
|
|
6246
6740
|
* file + broker durables/ACL) runs INSIDE the barrier as its cleanup step. On completion the
|
|
@@ -6263,6 +6757,13 @@ export class Manager {
|
|
|
6263
6757
|
log: (line) => console.error(`static retirement ${a.name}: ${line}`),
|
|
6264
6758
|
});
|
|
6265
6759
|
const cleanup = async () => {
|
|
6760
|
+
// SPEC 13.15: every issuance bound to this incarnation's ledger family is retired here,
|
|
6761
|
+
// inside the barrier and after the ledger rows were revoked (the source is frozen first,
|
|
6762
|
+
// then the index is walked). A retirement that lost is surfaced, never skipped: an active
|
|
6763
|
+
// attempt row on a retired lifecycle would authorize a run the credential no longer can.
|
|
6764
|
+
const retired = await this.withIssuer((i) => i.store.retireSource({ space: this.space, bucket: epAuthBucket(this.space), key: `cred.${a.lifecycleUid}` }));
|
|
6765
|
+
if (retired > 0)
|
|
6766
|
+
console.error(`static retirement ${a.name}: retired ${retired} issuance(s) bound to uid ${a.lifecycleUid}`);
|
|
6266
6767
|
const secrets = this.secrets;
|
|
6267
6768
|
const files = a.secretPaths ?? agentLifecycleSecretFilePaths(this.workspaceRoot, this.space, a.name, a.lifecycleUid);
|
|
6268
6769
|
if (files.creds) {
|
|
@@ -6271,19 +6772,24 @@ export class Manager {
|
|
|
6271
6772
|
}
|
|
6272
6773
|
await this.deprovisionBroker(a);
|
|
6273
6774
|
};
|
|
6775
|
+
// The process goes before its footprint: reap the orphan seat by reference, then tear down.
|
|
6776
|
+
const reapThenCleanup = async () => {
|
|
6777
|
+
await this.reapOrphanSeat(a);
|
|
6778
|
+
await cleanup();
|
|
6779
|
+
};
|
|
6274
6780
|
try {
|
|
6275
6781
|
await this.withLifecycleExecutor({ owner: DEV_OWNER, actor: a.id, lifecycleUid: a.lifecycleUid, alias: a.name }, async (t) => {
|
|
6276
6782
|
const slot = await readStaticSlot(t, DEV_OWNER, a.name);
|
|
6277
6783
|
if (slot === undefined || slot.row.lifecycleUid !== a.lifecycleUid) {
|
|
6278
6784
|
// No durable registration for THIS incarnation: a pre-Unit-B spawn (or a slot already
|
|
6279
6785
|
// replaced by a successor — then this stale teardown must not touch the registry at all).
|
|
6280
|
-
await
|
|
6786
|
+
await reapThenCleanup();
|
|
6281
6787
|
return;
|
|
6282
6788
|
}
|
|
6283
6789
|
await runStaticTerminal(t, {
|
|
6284
6790
|
owner: DEV_OWNER, alias: a.name, actor: a.id, lifecycleUid: a.lifecycleUid, opId,
|
|
6285
6791
|
managerInstance: this.managerInstanceId, managerProcessUid: this.managerLifecycleUid,
|
|
6286
|
-
}, { cleanup, evict, log: (line) => console.error(`static retirement ${a.name}: ${line}`) });
|
|
6792
|
+
}, { cleanup: reapThenCleanup, evict, log: (line) => console.error(`static retirement ${a.name}: ${line}`) });
|
|
6287
6793
|
});
|
|
6288
6794
|
this.retiredPrincipals.add(principalKey(DEV_OWNER, a.id).key);
|
|
6289
6795
|
const cur = this.retiring.get(a.name);
|
|
@@ -6327,19 +6833,22 @@ export class Manager {
|
|
|
6327
6833
|
// The SAME permission scope the spawn minted (recorded on the managed row): allowSubscribe/
|
|
6328
6834
|
// allowPublish/role/capabilities are the JWT-shaping inputs; `subscribe` (the active read
|
|
6329
6835
|
// set) shapes durable membership only and is not a mint input.
|
|
6330
|
-
|
|
6836
|
+
if (!a.issued)
|
|
6837
|
+
throw new Error(`renewManagedStaticCred: ${a.name} carries no issuance; a static credential minted before SPEC 13.15 is not renewed under an unbound generation - respawn the agent`);
|
|
6838
|
+
// A RENEWAL keeps the generation (SPEC 13.15): the same JWT-shaping inputs produce the same
|
|
6839
|
+
// ceiling, which `confirm` verifies against the recorded evidence before the ledger append. A
|
|
6840
|
+
// changed ceiling is refused there: that is a fresh issuance on a new connection, never a renewal.
|
|
6841
|
+
const issued = a.issued;
|
|
6842
|
+
const creds = await this.withIssuer((i) => mintCreds(this.auth, { id: a.id, seed: a.seed }, "agent", {
|
|
6331
6843
|
allowSubscribe: a.launch.allowSubscribe,
|
|
6332
6844
|
allowPublish: a.launch.allowPublish,
|
|
6333
6845
|
role: a.role,
|
|
6334
6846
|
capabilities: a.launch.capabilities,
|
|
6335
6847
|
lifecycleUid: a.lifecycleUid,
|
|
6336
6848
|
expiresAt: exp,
|
|
6337
|
-
|
|
6338
|
-
|
|
6339
|
-
|
|
6340
|
-
await recordSlotCredential(t, DEV_OWNER, a.name, a.lifecycleUid, credentialId);
|
|
6341
|
-
await appendStaticCredentialRow(t, { lifecycleUid: a.lifecycleUid, credentialId, holderPrincipal: principalKey(DEV_OWNER, a.id).key, exp });
|
|
6342
|
-
});
|
|
6849
|
+
issued,
|
|
6850
|
+
issuance: this.issuanceSeam(i, "renew", a, exp),
|
|
6851
|
+
}));
|
|
6343
6852
|
const secrets = this.secrets;
|
|
6344
6853
|
const credsPath = a.secretPaths.creds;
|
|
6345
6854
|
await secrets.put(agentSecretKeyForFile(credsPath, this.space), creds);
|
|
@@ -6505,7 +7014,7 @@ export class Manager {
|
|
|
6505
7014
|
this.reconcilingAliases.add(row.alias);
|
|
6506
7015
|
console.error(`static reconcile terminal alias=${row.alias} phase=${row.phase} uid=${row.lifecycleUid} attempt=${item.attempts}/${item.maxAttempts}`);
|
|
6507
7016
|
try {
|
|
6508
|
-
await this.driveStaticRetirement({ id: row.actor, name: row.alias, lifecycleUid: row.lifecycleUid }, true);
|
|
7017
|
+
await this.driveStaticRetirement({ id: row.actor, name: row.alias, lifecycleUid: row.lifecycleUid, runtime: row.runtime }, true);
|
|
6509
7018
|
if (item.attempts === 1) {
|
|
6510
7019
|
this.staticReconcileItems.delete(key);
|
|
6511
7020
|
}
|
|
@@ -6799,12 +7308,14 @@ export class Manager {
|
|
|
6799
7308
|
/** Persist a peer-defined persona as config. After this, `start name` auto-discovers
|
|
6800
7309
|
* .cotal/agents/<name>.md and the connector applies its persona/model at spawn.
|
|
6801
7310
|
*
|
|
6802
|
-
* CONTENT vs POLICY (P6):
|
|
6803
|
-
*
|
|
6804
|
-
*
|
|
6805
|
-
*
|
|
6806
|
-
*
|
|
6807
|
-
*
|
|
7311
|
+
* CONTENT vs POLICY (P6): `capabilities` and `owner` still have no slot, so a peer cannot
|
|
7312
|
+
* self-grant spawn or claim ownership. Channel grants, role, and agent are content: they may
|
|
7313
|
+
* arrive as explicit arguments or as a leading frontmatter block inside `persona`. A prompt
|
|
7314
|
+
* that starts with `---` is parsed and merged (explicit args win) so the written file has one
|
|
7315
|
+
* frontmatter block; a malformed block is refused by name rather than wrapped. A fresh name is
|
|
7316
|
+
* created with owner = caller. Redefining an EXISTING file is allowed on the privileged tier
|
|
7317
|
+
* only if `file.owner == caller`, else admin is required. Fail-closed: an ownerless file
|
|
7318
|
+
* (legacy / operator-written) is admin-only. */
|
|
6808
7319
|
opDefinePersona(args, caller, admin) {
|
|
6809
7320
|
const name = String(args.name ?? "").trim();
|
|
6810
7321
|
if (!name)
|
|
@@ -6815,55 +7326,42 @@ export class Manager {
|
|
|
6815
7326
|
const persona = String(args.persona ?? "").trim();
|
|
6816
7327
|
if (!persona)
|
|
6817
7328
|
return { ok: false, error: "persona required" };
|
|
6818
|
-
const model = args.model ? String(args.model) : undefined;
|
|
7329
|
+
const model = args.model !== undefined && args.model !== null && String(args.model) !== "" ? String(args.model) : undefined;
|
|
6819
7330
|
const path = agentFilePath(this.workspaceRoot, name);
|
|
6820
|
-
let
|
|
7331
|
+
let existing;
|
|
6821
7332
|
if (existsSync(path)) {
|
|
6822
|
-
// Redefine: load, authorize by ownership, then overwrite ONLY content; preserve all policy.
|
|
6823
7333
|
try {
|
|
6824
|
-
|
|
7334
|
+
existing = loadAgentFile(path);
|
|
6825
7335
|
}
|
|
6826
7336
|
catch (e) {
|
|
6827
7337
|
return { ok: false, error: e.message };
|
|
6828
7338
|
}
|
|
6829
|
-
if (!admin &&
|
|
6830
|
-
const owner =
|
|
7339
|
+
if (!admin && existing.owner !== caller) {
|
|
7340
|
+
const owner = existing.owner ? `owned by ${existing.owner}` : "operator-owned (legacy file - no agent owner)";
|
|
6831
7341
|
return { ok: false, error: `not authorized to redefine ${name}: ${owner}; only its owner or an operator can` };
|
|
6832
7342
|
}
|
|
6833
|
-
// PATCH content: overwrite model only when provided, so a persona-only redefine can't wipe an existing model.
|
|
6834
|
-
if (model !== undefined)
|
|
6835
|
-
def.model = model;
|
|
6836
|
-
def.persona = persona;
|
|
6837
|
-
// A redefine cannot change scope, so a file that still has no channels keeps its marker. But
|
|
6838
|
-
// if an operator has since given it a real read set, the marker is a stale claim about a state
|
|
6839
|
-
// that no longer holds, and a marker that outlives its condition is worse than none: it tells
|
|
6840
|
-
// a census the scope was never chosen when someone chose it.
|
|
6841
|
-
//
|
|
6842
|
-
// The condition is the READ SET specifically, not any channel field. `allowSubscribe` alone is
|
|
6843
|
-
// a ceiling on what the agent may read, not the set it reads, so a persona given only that
|
|
6844
|
-
// still reads nothing and the marker still describes it correctly.
|
|
6845
|
-
if (def.meta?.scope_source === "wire-default" && def.subscribe?.length) {
|
|
6846
|
-
const { scope_source: _dropped, ...rest } = def.meta;
|
|
6847
|
-
def.meta = Object.keys(rest).length ? rest : undefined;
|
|
6848
|
-
}
|
|
6849
7343
|
}
|
|
6850
|
-
|
|
6851
|
-
|
|
6852
|
-
|
|
6853
|
-
|
|
6854
|
-
|
|
6855
|
-
|
|
6856
|
-
|
|
6857
|
-
|
|
6858
|
-
|
|
6859
|
-
|
|
6860
|
-
|
|
6861
|
-
|
|
6862
|
-
|
|
6863
|
-
|
|
6864
|
-
|
|
6865
|
-
|
|
6866
|
-
|
|
7344
|
+
const strList = (v) => {
|
|
7345
|
+
if (v === undefined || v === null)
|
|
7346
|
+
return undefined;
|
|
7347
|
+
return (Array.isArray(v) ? v : [v]).map((x) => String(x));
|
|
7348
|
+
};
|
|
7349
|
+
let def;
|
|
7350
|
+
try {
|
|
7351
|
+
def = composeWirePersona({
|
|
7352
|
+
name,
|
|
7353
|
+
owner: caller,
|
|
7354
|
+
prompt: persona,
|
|
7355
|
+
model,
|
|
7356
|
+
role: args.role !== undefined && args.role !== null && String(args.role) !== "" ? String(args.role) : undefined,
|
|
7357
|
+
agent: args.agent !== undefined && args.agent !== null && String(args.agent) !== "" ? String(args.agent) : undefined,
|
|
7358
|
+
subscribe: strList(args.subscribe),
|
|
7359
|
+
allowSubscribe: strList(args.allowSubscribe),
|
|
7360
|
+
allowPublish: strList(args.allowPublish),
|
|
7361
|
+
}, existing);
|
|
7362
|
+
}
|
|
7363
|
+
catch (e) {
|
|
7364
|
+
return { ok: false, error: e.message };
|
|
6867
7365
|
}
|
|
6868
7366
|
try {
|
|
6869
7367
|
saveAgentFile(path, def);
|
|
@@ -6968,7 +7466,7 @@ export class Manager {
|
|
|
6968
7466
|
*
|
|
6969
7467
|
* `enter` defaults to true: a harness command typed but not submitted has not been delivered.
|
|
6970
7468
|
* Nothing is echoed back; the caller reads the resulting turns from the event plane. */
|
|
6971
|
-
inputAuthorized(a, args) {
|
|
7469
|
+
async inputAuthorized(a, args) {
|
|
6972
7470
|
if (this.agents.get(a.name) !== a)
|
|
6973
7471
|
throw new EpEnvelopeError("failed-precondition", `agent "${a.name}" was replaced during authorization - retry`);
|
|
6974
7472
|
if (a.handle.status() !== "running")
|
|
@@ -6979,9 +7477,69 @@ export class Manager {
|
|
|
6979
7477
|
// The contract validated `text` (non-empty, <= 64KiB) and `enter` (boolean) before this ran, so
|
|
6980
7478
|
// the only decision left is the carriage return. `!== false` and not `?? true`: an ABSENT enter
|
|
6981
7479
|
// and an explicit `true` must behave identically, and only `false` may suppress the return.
|
|
6982
|
-
|
|
6983
|
-
|
|
6984
|
-
|
|
7480
|
+
//
|
|
7481
|
+
// THE TEXT IS DELIVERED AS A BRACKETED PASTE AND THE RETURN FOLLOWS IT AS ITS OWN WRITE, and
|
|
7482
|
+
// that is the whole of issue #1649. A TUI harness reads keystrokes, so a return fused onto the
|
|
7483
|
+
// tail of the text (`${text}\r`) is the text's last character rather than the submit key: the
|
|
7484
|
+
// line waits in the composer and the NEXT call's return submits the PREVIOUS call's text, so
|
|
7485
|
+
// every delivery runs one call behind while the op still reports the full byte count.
|
|
7486
|
+
//
|
|
7487
|
+
// Separating the return is necessary and NOT sufficient, which is the part that cost a second
|
|
7488
|
+
// round of measurement. A TUI classifies a fast burst of input as a PASTE and eats the newline
|
|
7489
|
+
// that trails it (Jcode spells this `paste_guard::consume_paste_trailing_enter`), so a text big
|
|
7490
|
+
// enough to need more than one write is still one call behind even when the return is written
|
|
7491
|
+
// alone. Measured on a real seat, delivering in 2048-char slices with a yield before the return:
|
|
7492
|
+
// 120 and 700 bytes submitted on their own call, 2500 and 3100 bytes did not - each appeared
|
|
7493
|
+
// only when the NEXT call's return arrived. That is the reported symptom exactly.
|
|
7494
|
+
//
|
|
7495
|
+
// Bracketed paste states the boundary instead of guessing at it. `ESC[200~ text ESC[201~`
|
|
7496
|
+
// delimits the pasted block, so the return after it is unambiguously a keystroke and the guard
|
|
7497
|
+
// has no trailing newline to consume. This is a CLASSIFIER, not a race, so a sleep would be the
|
|
7498
|
+
// wrong instrument: measured on a real seat, a 250ms pause before the return still lost the
|
|
7499
|
+
// submission while 1000ms carried it, and any margin chosen from those numbers is a guess a
|
|
7500
|
+
// loaded host breaks. Bracketed delivery submitted 120, 2600 and 5912 bytes (the issue's own
|
|
7501
|
+
// size) each on its own call. The terminal advertises support by enabling DECSET 2004, which
|
|
7502
|
+
// Jcode's TUI does.
|
|
7503
|
+
//
|
|
7504
|
+
// The bytes the SEAT receives as text are unchanged, and the receipt still counts the text plus
|
|
7505
|
+
// the return: the paste markers are framing this op adds, never content the caller asked to
|
|
7506
|
+
// deliver, so counting them would make `bytes` disagree with what the caller sent.
|
|
7507
|
+
const text = String(args.text);
|
|
7508
|
+
const submit = args.enter !== false;
|
|
7509
|
+
const intendedBytes = Buffer.byteLength(text, "utf8") + (submit ? 1 : 0);
|
|
7510
|
+
const parts = [{ data: PASTE_START, counted: false }];
|
|
7511
|
+
for (let i = 0; i < text.length; i += INPUT_SLICE_CHARS)
|
|
7512
|
+
parts.push({ data: text.slice(i, i + INPUT_SLICE_CHARS), counted: true });
|
|
7513
|
+
parts.push({ data: PASTE_END, counted: false });
|
|
7514
|
+
// `enter:false` still presses nothing: the text is delivered into the composer and left there.
|
|
7515
|
+
if (submit)
|
|
7516
|
+
parts.push({ data: "\r", counted: true });
|
|
7517
|
+
let bytes = 0;
|
|
7518
|
+
for (const [i, part] of parts.entries()) {
|
|
7519
|
+
// Yield BEFORE each write but the first, so the child drains the previous slice rather than
|
|
7520
|
+
// the pty buffer merging them. `setImmediate` runs after I/O, which a microtask would not.
|
|
7521
|
+
if (i > 0)
|
|
7522
|
+
await new Promise((resolve) => setImmediate(resolve));
|
|
7523
|
+
let accepted;
|
|
7524
|
+
try {
|
|
7525
|
+
accepted = await write(part.data);
|
|
7526
|
+
}
|
|
7527
|
+
catch (error) {
|
|
7528
|
+
throw new EpEnvelopeError("unavailable", `input for seat "${a.name}" failed: ${error.message}`);
|
|
7529
|
+
}
|
|
7530
|
+
if (!Number.isSafeInteger(accepted)) {
|
|
7531
|
+
throw new EpEnvelopeError("unavailable", `input for seat "${a.name}" failed: runtime accepted ${String(accepted)} of ${intendedBytes} bytes`);
|
|
7532
|
+
}
|
|
7533
|
+
// A short write on ANY part is a failed delivery, including a marker: a paste block that was
|
|
7534
|
+
// opened but not closed would leave the seat's composer in paste mode.
|
|
7535
|
+
if (accepted !== Buffer.byteLength(part.data, "utf8"))
|
|
7536
|
+
throw new EpEnvelopeError("unavailable", `input for seat "${a.name}" failed: runtime accepted ${accepted} of ${Buffer.byteLength(part.data, "utf8")} bytes of one write`);
|
|
7537
|
+
if (part.counted)
|
|
7538
|
+
bytes += accepted;
|
|
7539
|
+
}
|
|
7540
|
+
if (bytes !== intendedBytes)
|
|
7541
|
+
throw new EpEnvelopeError("unavailable", `input for seat "${a.name}" failed: runtime accepted ${String(bytes)} of ${intendedBytes} bytes`);
|
|
7542
|
+
return { name: a.name, bytes };
|
|
6985
7543
|
}
|
|
6986
7544
|
/** The post-authorization attach effect (P2 item 6): mint the holder-bound §13.6 offer, redeem it
|
|
6987
7545
|
* through the ONE session plane (one-use CAS + presenter-equality), and stand up the PTY bridge —
|
|
@@ -7075,6 +7633,14 @@ export class Manager {
|
|
|
7075
7633
|
* owner-domain bound); undefined = unbounded. {@link NO_OWNER_MATCHES} matches nothing. */
|
|
7076
7634
|
list(ownerFilter) {
|
|
7077
7635
|
const roster = new Map(this.ep.getRoster().map((p) => [p.card.name, p]));
|
|
7636
|
+
// The roster is only evidence while THIS observer's presence watch is fresh. A stale view
|
|
7637
|
+
// (whole-bucket silence past TTL) or an unpopulated one (snapshot not yet replayed) cannot
|
|
7638
|
+
// support "offline" or "absent" for anyone: a live deployment on 2026-09-09 rendered every live seat as
|
|
7639
|
+
// one of those for hours after the presence stream was recreated under a still-open watch.
|
|
7640
|
+
// Carry the view state on the row so the renderer can say "unknown" instead of a verdict.
|
|
7641
|
+
// An endpoint that reports no view (test doubles built on `getRoster` alone) is read as
|
|
7642
|
+
// `current`: that is exactly what every row meant before the field existed.
|
|
7643
|
+
const view = typeof this.ep.presenceView === "function" ? this.ep.presenceView() : { state: "current" };
|
|
7078
7644
|
return [...this.agents.values()].filter((a) => ownerFilter === undefined || a.userOwner === ownerFilter).map((a) => {
|
|
7079
7645
|
// USER MODE: a detached agent's bearer-refresh death is silent everywhere except here — its
|
|
7080
7646
|
// bearer command writes each attempt's outcome to the health file, and `ps` renders it
|
|
@@ -7096,6 +7662,9 @@ export class Manager {
|
|
|
7096
7662
|
status: a.handle.status(),
|
|
7097
7663
|
uptimeMs: Date.now() - a.startedAt,
|
|
7098
7664
|
mesh: roster.get(a.name)?.status ?? "absent",
|
|
7665
|
+
// `current` is the only state in which `mesh` is a verdict; the other two are the
|
|
7666
|
+
// observer's own condition and travel on the row (older CLIs ignore the field).
|
|
7667
|
+
meshView: view.state,
|
|
7099
7668
|
// The incarnation coordinate (SPEC 13.1) — with `id`, exactly what a v0.4 caller needs to
|
|
7100
7669
|
// build a targeted (`despawn`/`attach`) request against THIS incarnation.
|
|
7101
7670
|
lifecycleUid: a.lifecycleUid,
|