@cotal-ai/manager 0.61.0 → 0.63.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands.js +5 -5
- package/dist/commands.js.map +1 -1
- package/dist/endpoint-evict.d.ts +31 -9
- package/dist/endpoint-evict.d.ts.map +1 -1
- package/dist/endpoint-evict.js +125 -56
- package/dist/endpoint-evict.js.map +1 -1
- package/dist/holder-liveness.d.ts +5 -2
- package/dist/holder-liveness.d.ts.map +1 -1
- package/dist/holder-liveness.js +35 -25
- package/dist/holder-liveness.js.map +1 -1
- package/dist/manager.d.ts +81 -14
- package/dist/manager.d.ts.map +1 -1
- package/dist/manager.js +439 -75
- package/dist/manager.js.map +1 -1
- package/dist/remote-register.d.ts.map +1 -1
- package/dist/remote-register.js +9 -1
- package/dist/remote-register.js.map +1 -1
- package/dist/roster.d.ts +2 -1
- package/dist/roster.d.ts.map +1 -1
- package/dist/roster.js +12 -2
- package/dist/roster.js.map +1 -1
- package/dist/run-hosting.d.ts +16 -3
- package/dist/run-hosting.d.ts.map +1 -1
- package/dist/run-hosting.js +39 -11
- package/dist/run-hosting.js.map +1 -1
- package/dist/static-lifecycle.d.ts +3 -0
- package/dist/static-lifecycle.d.ts.map +1 -1
- package/dist/static-lifecycle.js +8 -5
- package/dist/static-lifecycle.js.map +1 -1
- package/package.json +12 -12
package/dist/manager.js
CHANGED
|
@@ -4,11 +4,11 @@ import { hostname } from "node:os";
|
|
|
4
4
|
import { credsAuthenticator } from "@nats-io/transport-node";
|
|
5
5
|
import { existsSync, lstatSync, readFileSync, realpathSync, rmSync, statSync } from "node:fs";
|
|
6
6
|
import { isAbsolute, join, dirname, resolve } from "node:path";
|
|
7
|
-
import { CotalEndpoint, accountFromCreds, credsClaims, DEFAULT_SERVER, DEV_OWNER, MANAGER_LEASE_RENEW_MS, STANDING_RENEWABLE_TTL_SEC, assertLifecycleToken, discardLaunchArtifacts, divergentSecretStoreNotice, parseDaemonStoreAnswer, parseSecretStoreIdentity, sameSecretStoreIdentity, agentFilePath, clearSpaceHistory, dialerFor, connectorServers, spawnEnvAllow, deprovisionAgent, DeprovisionError, isConcreteChannel, deprovisionTargetPrincipal, firstFreeName, spawnNameError, idFromCreds, inspectCredHealth, credsRenewalDelayMs, composeWirePersona, loadAgentFile, listPersonaCatalog, personaCatalogDescription, personaCatalogReadable, loadCotalConfig, mintCreds, mintGeneration, mintAcceptedToken, openIssuedStore, acceptedBucket, issuedBucket, mintLifecycleUid, managedRetirementOpId, mkSecretDir, newIdentity, actionContext, parsePrincipalKey, parseShareSelection, principalKey, probeConnect, provisionAgent, provisionAgentDurables, registry, resolveAuthProvider, saveAgentFile, subjectMatches, AUTH_ENDPOINT, EP_CMD_RETIRE_LIFECYCLE, controlServiceSubject, eventChannelPrincipal, resolveService, invokeCommand, } from "@cotal-ai/core";
|
|
7
|
+
import { CotalEndpoint, MANAGED_HANDOFF_KIND, accountFromCreds, credsClaims, DEFAULT_SERVER, DEV_OWNER, MANAGER_LEASE_RENEW_MS, STANDING_RENEWABLE_TTL_SEC, assertLifecycleToken, discardLaunchArtifacts, divergentSecretStoreNotice, parseDaemonStoreAnswer, parseSecretStoreIdentity, sameSecretStoreIdentity, agentFilePath, clearSpaceHistory, dialerFor, connectorServers, spawnEnvAllow, deprovisionAgent, DeprovisionError, isConcreteChannel, deprovisionTargetPrincipal, firstFreeName, spawnNameError, idFromCreds, inspectCredHealth, credsRenewalDelayMs, composeWirePersona, loadAgentFile, listPersonaCatalog, personaCatalogDescription, personaCatalogReadable, loadCotalConfig, modelPolicyRefusal, mintCreds, mintGeneration, mintAcceptedToken, openIssuedStore, acceptedBucket, issuedBucket, mintLifecycleUid, managedRetirementOpId, mkSecretDir, newIdentity, actionContext, parsePrincipalKey, parseShareSelection, principalKey, probeConnect, provisionAgent, provisionAgentDurables, registry, resolveAuthProvider, saveAgentFile, subjectMatches, AUTH_ENDPOINT, EP_CMD_RETIRE_LIFECYCLE, controlServiceSubject, eventChannelPrincipal, resolveService, invokeCommand, } from "@cotal-ai/core";
|
|
8
8
|
import { agentAuthState, agentCredsDir, agentLifecycleSecretFilePaths, agentSecretFilePaths, agentSecretKeyForFile, authDir, connectorInstallHint, DEFAULT_CONNECTOR, defaultAgentType, DELIVERY_CREDS_KIND, extensionConnectors, findCotalRoot, getSpaceAuth, hasUserAuthState, loadExtensionsManifest, loadManagerInstanceIdentity, loadMeshes, localTrustOfSpace, manifestExtensionNames, materializeFromManifest, materializeSecretToFile, MEMBERSHIP_RW_CREDS_KIND, mergeLaunchOptions, remintDaemonCreds, resolveOnPath, createManagerInstanceIdentity, spaceMaterialKey, SYSTEM_CREDS_FILES, userAuthStateDir, workspaceSecretStore, writeRenewalRecord } from "@cotal-ai/workspace";
|
|
9
9
|
import { createRuntime, isCustodialRuntime, requireRuntimeAdopt, requireRuntimeReap, RuntimeReapUnproven, } from "./runtime/index.js";
|
|
10
10
|
import { AttachEndpoint } from "./attach-endpoint.js";
|
|
11
|
-
import {
|
|
11
|
+
import { DELIVERY_ADMIN_BOOT_WAIT_MS, isUnansweredDeliveryAdmin, makeManagerEndpointEvictionEvidence, makeManagerEndpointHolderEvictor, untilDeliveryAdminAnswers } from "./endpoint-evict.js";
|
|
12
12
|
import { makeManagerHolderLivenessProbe } from "./holder-liveness.js";
|
|
13
13
|
import { GateReconcileRefused, reconcileEndpointGate } from "./reconcile-gate.js";
|
|
14
14
|
import { launchSpecForRun, materializePersona, launchAgentToStartOpts, parseLaunchSpec, persistLaunchSpec, readContinuityAssignment, writeContinuityAssignment } from "./launch.js";
|
|
@@ -70,6 +70,9 @@ const DELIVERY_ADMIN_RELOAD_TIMEOUT_MS = 15_000;
|
|
|
70
70
|
/** A hard preservation stop should settle quickly. The manager still waits and reports a partial
|
|
71
71
|
* cut rather than pretending a child is gone. Held in ManagerOptions so fake runtimes can shorten it. */
|
|
72
72
|
const PRESERVE_STOP_TIMEOUT_MS = 10_000;
|
|
73
|
+
/** How long a resume waits for a reaped orphan's principal to leave presence. Presence rows age
|
|
74
|
+
* out after their 6s TTL, so three TTLs covers a killed seat that never published its leave. */
|
|
75
|
+
const ORPHAN_PRESENCE_CLEAR_MS = 20_000;
|
|
73
76
|
/** How much of an `input` text is written to a seat's pty at a time. Comfortably under the 4096-byte
|
|
74
77
|
* kernel input buffer, so a slice is never re-split at a boundary the manager did not choose: the
|
|
75
78
|
* tail of an oversized write stays queued and swallows the submit key that follows it (issue #1649).
|
|
@@ -122,6 +125,17 @@ function retireOpId(lifecycleUid) {
|
|
|
122
125
|
* dash) — what {@link Manager.psOwnerFilter} returns for an unparseable caller so a malformed
|
|
123
126
|
* principal fail-closes to an empty `ps` instead of an unbounded one. */
|
|
124
127
|
const NO_OWNER_MATCHES = "-no-owner-";
|
|
128
|
+
/** A caught value's text for a refusal. A host-supplied store or callback may reject with any value,
|
|
129
|
+
* including one whose `message` is a Symbol or whose `message` getter or `toString` throws, and the
|
|
130
|
+
* refusal must still be returned, so the coercion to text runs inside the guard. */
|
|
131
|
+
function rejectionText(e) {
|
|
132
|
+
try {
|
|
133
|
+
return String(e?.message ?? e);
|
|
134
|
+
}
|
|
135
|
+
catch {
|
|
136
|
+
return "an unreadable rejection";
|
|
137
|
+
}
|
|
138
|
+
}
|
|
125
139
|
/** Run the agent's bearer argv once, pre-launch — the end-to-end auth preflight (state dir, daemon,
|
|
126
140
|
* ledger row, secret). Its stderr is the provider command's operator-exact sentence; surface it
|
|
127
141
|
* verbatim as the spawn refusal. */
|
|
@@ -237,6 +251,14 @@ function isAbsentDeliveryAdmin(msg) {
|
|
|
237
251
|
return false;
|
|
238
252
|
return /no responders|\b503\b/i.test(msg);
|
|
239
253
|
}
|
|
254
|
+
/** The store challenge went UNANSWERED while a delivery daemon exists: the rail timed out, or it had
|
|
255
|
+
* no responder while the lease row names a holder. Both are what a daemon that is starting or
|
|
256
|
+
* re-acquiring its lease looks like, so manager boot waits them out (#871); a renewal pass still
|
|
257
|
+
* refuses on the first one. A reply that does not decode was answered and is never this. */
|
|
258
|
+
class DeliveryAdminUnanswered extends Error {
|
|
259
|
+
}
|
|
260
|
+
/** How long one store challenge waits for the delivery daemon's answer. */
|
|
261
|
+
const STORE_CHALLENGE_TIMEOUT_MS = 5_000;
|
|
240
262
|
/**
|
|
241
263
|
* The agent supervisor: a long-lived mesh node that owns agent process lifecycle.
|
|
242
264
|
* It serves control requests on the "manager" service and spawns/kills agents
|
|
@@ -275,6 +297,14 @@ export class Manager {
|
|
|
275
297
|
* the launch settles, so no other spawn can take it, but the ceiling counts that seat through
|
|
276
298
|
* `agents` and, if it frees young, its cooling stamp, never through the reservation (#906). */
|
|
277
299
|
reservedLive = new Set();
|
|
300
|
+
/** Every numbered name {@link uniqueName} has issued. A suffix carries nothing but arrival order,
|
|
301
|
+
* so reissuing a freed one would give a later principal the name an earlier one posted under, and
|
|
302
|
+
* a reader of that name could not tell the two apart (#382). Never pruned in-process: one string
|
|
303
|
+
* per numbered spawn. */
|
|
304
|
+
issuedNumberedNames = new Set();
|
|
305
|
+
/** Lifecycle UIDs already handed to `Runtime.spawnDelegated`: a lifecycle is handed off at most
|
|
306
|
+
* once, so a lost create acknowledgement can never become a second create. */
|
|
307
|
+
delegatedLaunched = new Set();
|
|
278
308
|
/** Expiry stamps (`startedAt + MIN_LIFETIME`) for slots that freed while still young — a
|
|
279
309
|
* count-only, lazily-pruned recycle floor (P4c). Pruned + summed into the ceiling gate. */
|
|
280
310
|
cooling = [];
|
|
@@ -437,6 +467,10 @@ export class Manager {
|
|
|
437
467
|
/** The complete boot task. stop() joins it after fencing static reconciliation, so a registration
|
|
438
468
|
* already past an earlier shutdown check cannot finish after stop() returns. */
|
|
439
469
|
startTask;
|
|
470
|
+
/** The one stop this manager runs, with the agent policy it was asked for. A stop takes seats out
|
|
471
|
+
* of `agents` before they have exited, so a second teardown would find none and report success
|
|
472
|
+
* while the first still waits on a live seat. */
|
|
473
|
+
stopTask;
|
|
440
474
|
/** Held as an instance field so broker-owning smokes can compress the schedule without changing
|
|
441
475
|
* production semantics. It is never a fence expiry: the durable non-retired row stays authoritative. */
|
|
442
476
|
staticReconcileRetryDelaysMs = STATIC_RECONCILE_RETRY_DELAYS_MS;
|
|
@@ -474,7 +508,8 @@ export class Manager {
|
|
|
474
508
|
* for release would otherwise move the row out from under a same-cached-revision CAS delete. */
|
|
475
509
|
leaseRenewTask;
|
|
476
510
|
/** Set at the top of `stop()`. `renewLease` returns at once once this is set, and its `gone` arm
|
|
477
|
-
* does not re-acquire: a manager that is stopping never puts its own key back.
|
|
511
|
+
* does not re-acquire: a manager that is stopping never puts its own key back. It also closes the
|
|
512
|
+
* lifecycle fence ({@link beginLifecycle}), so no spawn can launch a seat the stop never sees. */
|
|
478
513
|
leaseStopping = false;
|
|
479
514
|
/** The class-2 renewal owner's half-TTL schedule (D5 slice 5); armed only on auth meshes. */
|
|
480
515
|
credRenewTimer;
|
|
@@ -487,6 +522,10 @@ export class Manager {
|
|
|
487
522
|
* daemon reloads from this manager's store AND this manager holds the per-space renewal lease.
|
|
488
523
|
* Starts false: a manager that has not proved both must not remint. */
|
|
489
524
|
daemonRenewalOwner = false;
|
|
525
|
+
/** #2561: may the lease heartbeat contend at all? Set by every claim, false while the daemon names
|
|
526
|
+
* another store or the challenge refused. Without it the heartbeat re-takes the lease the claim
|
|
527
|
+
* just released and holds it against every manager and `doctor auth --fix` that can remint. */
|
|
528
|
+
daemonRenewalEligible = false;
|
|
490
529
|
daemonRenewalLeaseTimer;
|
|
491
530
|
maintenanceState = "active";
|
|
492
531
|
lifecycleInFlight = 0;
|
|
@@ -585,9 +624,12 @@ export class Manager {
|
|
|
585
624
|
get runtimeKind() {
|
|
586
625
|
return this.runtime.kind;
|
|
587
626
|
}
|
|
588
|
-
/**
|
|
589
|
-
|
|
590
|
-
|
|
627
|
+
/** What a default process stop does with this manager's seats. `release` drops manager-local
|
|
628
|
+
* custody and leaves every seat running. `stop` means this runtime keeps its PTYs inside the
|
|
629
|
+
* manager process, so those seats close with it: the stop stops and deprovisions them, and still
|
|
630
|
+
* releases every seat that can outlive it. */
|
|
631
|
+
get spareSeats() {
|
|
632
|
+
return this.runtime.kind !== "pty" || this.runtime.supportsRelease === true ? "release" : "stop";
|
|
591
633
|
}
|
|
592
634
|
/** Reattach this manager's runtime to a durable handle. Refuses by name when adopt is absent. */
|
|
593
635
|
adoptRuntimeHandle(reference) {
|
|
@@ -826,8 +868,8 @@ export class Manager {
|
|
|
826
868
|
// second workspace root) has a distinct id ⇒ a distinct key ⇒ it coexists; the create THROWS only
|
|
827
869
|
// when the SAME instance id is already live (a same-root double-start, or a restart racing the
|
|
828
870
|
// crashed predecessor's not-yet-expired key), and we REFUSE loud. A crashed holder's key auto-expires
|
|
829
|
-
// (bucket TTL). Losing the key LATER
|
|
830
|
-
// retrying ({@link renewLease}).
|
|
871
|
+
// (bucket TTL). Losing the key LATER ends this process only when another process holds it; otherwise
|
|
872
|
+
// the renew loop puts it back or keeps retrying ({@link renewLease}).
|
|
831
873
|
this.leaseInfo = { holder: this.ep.ref().id, instanceId: this.managerInstanceId, runtime: this.runtime.kind, root: resolve(this.workspaceRoot), pid: process.pid };
|
|
832
874
|
try {
|
|
833
875
|
this.leaseRevision = await this.ep.acquireManagerLease(this.leaseInfo);
|
|
@@ -890,8 +932,10 @@ export class Manager {
|
|
|
890
932
|
// `reloadCreds` adoption on the delivery-admin rail, and persist the audit record doctor renders.
|
|
891
933
|
if (this.auth) {
|
|
892
934
|
// #1634: a space may be served by several managers, so ownership of the daemon's credential
|
|
893
|
-
// renewal is decided here rather than by being the only manager able to start.
|
|
894
|
-
|
|
935
|
+
// renewal is decided here rather than by being the only manager able to start. #871: a daemon
|
|
936
|
+
// that is still binding or re-acquiring its lease leaves this challenge unanswered for a few
|
|
937
|
+
// seconds, so boot waits that out instead of exiting over a daemon about to answer.
|
|
938
|
+
await this.claimDaemonRenewalOwnership(DELIVERY_ADMIN_BOOT_WAIT_MS);
|
|
895
939
|
// ARMED BEFORE the first remint, not after. That first `renewDaemonCreds()` re-signs through
|
|
896
940
|
// the SecretStore, and a slow or contended store makes the one pass outlast the lease's
|
|
897
941
|
// MANAGER_LEASE_TTL_MS. Arming afterwards leaves the whole first remint unprotected: the lease
|
|
@@ -1025,16 +1069,17 @@ export class Manager {
|
|
|
1025
1069
|
* daemon"), and only a row that is absent or holderless settles "absent", the state start()
|
|
1026
1070
|
* treats as not-yet-bound. An unreadable row (a failed read, or a key that exists but does not
|
|
1027
1071
|
* parse) is a genuine unknown and refuses fail-closed, like a hung rail. */
|
|
1028
|
-
async daemonStoreRelation() {
|
|
1072
|
+
async daemonStoreRelation(timeoutMs) {
|
|
1029
1073
|
let reply;
|
|
1030
1074
|
try {
|
|
1031
|
-
reply = await this.ep.requestDeliveryAdmin("reloadStoreIdentity", {},
|
|
1075
|
+
reply = await this.ep.requestDeliveryAdmin("reloadStoreIdentity", {}, timeoutMs);
|
|
1032
1076
|
}
|
|
1033
1077
|
catch (e) {
|
|
1034
1078
|
const msg = e.message;
|
|
1035
1079
|
if (isAbsentDeliveryAdmin(msg))
|
|
1036
|
-
return this.absentByLeaseRow();
|
|
1037
|
-
|
|
1080
|
+
return this.absentByLeaseRow(isUnansweredDeliveryAdmin(e));
|
|
1081
|
+
const failure = `could not challenge the delivery daemon's SecretStore: ${msg}`;
|
|
1082
|
+
throw isUnansweredDeliveryAdmin(e) ? new DeliveryAdminUnanswered(failure) : new Error(failure);
|
|
1038
1083
|
}
|
|
1039
1084
|
if (!reply.ok)
|
|
1040
1085
|
throw new Error(reply.error ?? "delivery daemon refused to name the SecretStore it reloads from");
|
|
@@ -1089,7 +1134,7 @@ export class Manager {
|
|
|
1089
1134
|
*
|
|
1090
1135
|
* `deliveryLeaseHolder` swallows its read failures into `undefined`, which would fold case 2
|
|
1091
1136
|
* into case 3 — so this reads the entry directly and lets the read's own failure refuse. */
|
|
1092
|
-
async absentByLeaseRow() {
|
|
1137
|
+
async absentByLeaseRow(unanswered) {
|
|
1093
1138
|
let entry;
|
|
1094
1139
|
try {
|
|
1095
1140
|
entry = await this.ep.readDeliveryLeaseEntry(Manager.DELIVERY_SHARD);
|
|
@@ -1099,9 +1144,11 @@ export class Manager {
|
|
|
1099
1144
|
`(${e.message}) - absence is undetermined, nothing reminted`);
|
|
1100
1145
|
}
|
|
1101
1146
|
const holder = entry?.info.holder;
|
|
1102
|
-
if (holder !== undefined && holder !== "")
|
|
1103
|
-
|
|
1104
|
-
`(${holder}) - absence cannot be read off the rail alone, and nothing reminted
|
|
1147
|
+
if (holder !== undefined && holder !== "") {
|
|
1148
|
+
const refusal = `the delivery-admin rail reported no responder while the delivery lease names a holder ` +
|
|
1149
|
+
`(${holder}) - absence cannot be read off the rail alone, and nothing reminted`;
|
|
1150
|
+
throw unanswered ? new DeliveryAdminUnanswered(refusal) : new Error(refusal);
|
|
1151
|
+
}
|
|
1105
1152
|
return "absent";
|
|
1106
1153
|
}
|
|
1107
1154
|
/** Keep this manager's renewal lease alive between credential passes (#1634). The manager bucket
|
|
@@ -1110,7 +1157,7 @@ export class Manager {
|
|
|
1110
1157
|
* CAS is how a survivor picks up a dead owner's expired lease with no operator step. Never
|
|
1111
1158
|
* throws; a failed heartbeat leaves ownership to the next pass rather than ending the process. */
|
|
1112
1159
|
async keepDaemonRenewalLease() {
|
|
1113
|
-
if (!this.auth)
|
|
1160
|
+
if (!this.auth || !this.daemonRenewalEligible)
|
|
1114
1161
|
return;
|
|
1115
1162
|
try {
|
|
1116
1163
|
const held = await this.ep.holdDaemonRenewalLease(this.managerInstanceId);
|
|
@@ -1135,7 +1182,7 @@ export class Manager {
|
|
|
1135
1182
|
* request, which is the #773 refusal the challenge exists to prevent.
|
|
1136
1183
|
*
|
|
1137
1184
|
* A manager failing either test serves the space normally and skips only the daemon remint. */
|
|
1138
|
-
async claimDaemonRenewalOwnership() {
|
|
1185
|
+
async claimDaemonRenewalOwnership(waitMs = 0) {
|
|
1139
1186
|
// EVERY refusal outcome drops a held lease, not only "divergent". The refusal paths THROW
|
|
1140
1187
|
// (#1694: a non-holder answer, an unbindable answerer, an unreadable or holder-naming row on a
|
|
1141
1188
|
// no-responder rail), and a thrown refusal caught by renewDaemonCreds would otherwise leave a
|
|
@@ -1144,14 +1191,18 @@ export class Manager {
|
|
|
1144
1191
|
// Deciding here, inside the claim, is what keeps the release on the refusal path itself.
|
|
1145
1192
|
let relation;
|
|
1146
1193
|
try {
|
|
1147
|
-
|
|
1194
|
+
// Only the read-only challenge is retried, so a wait that ends mid-attempt abandons a read,
|
|
1195
|
+
// never the lease work below.
|
|
1196
|
+
relation = await untilDeliveryAdminAnswers(waitMs, (requestMs) => this.daemonStoreRelation(requestMs(STORE_CHALLENGE_TIMEOUT_MS)), (reason, delayMs) => console.error(`! delivery-admin rail did not answer the store challenge (${reason}); retrying in ${delayMs / 1000}s`), (e) => e instanceof DeliveryAdminUnanswered);
|
|
1148
1197
|
}
|
|
1149
1198
|
catch (e) {
|
|
1199
|
+
this.daemonRenewalEligible = false;
|
|
1150
1200
|
if (this.daemonRenewalOwner)
|
|
1151
1201
|
await this.ep.releaseDaemonRenewalLease().catch(() => { });
|
|
1152
1202
|
this.daemonRenewalOwner = false;
|
|
1153
1203
|
throw e;
|
|
1154
1204
|
}
|
|
1205
|
+
this.daemonRenewalEligible = relation !== "divergent";
|
|
1155
1206
|
if (relation === "divergent") {
|
|
1156
1207
|
// Not our daemon's store: drop any lease we hold rather than sit on it, so the manager that
|
|
1157
1208
|
// CAN remint is not blocked behind us.
|
|
@@ -1522,9 +1573,9 @@ export class Manager {
|
|
|
1522
1573
|
}
|
|
1523
1574
|
}
|
|
1524
1575
|
/** Admit one lifecycle/control operation while active. The synchronous increment is the fence:
|
|
1525
|
-
* preserveState
|
|
1576
|
+
* preserveState and stop close it before their first await, so work is either counted or rejected. */
|
|
1526
1577
|
beginLifecycle(resumeOperation = false) {
|
|
1527
|
-
if (this.maintenanceState !== "active" || (this.resumeRequired && !resumeOperation))
|
|
1578
|
+
if (this.leaseStopping || this.maintenanceState !== "active" || (this.resumeRequired && !resumeOperation))
|
|
1528
1579
|
return undefined;
|
|
1529
1580
|
this.lifecycleInFlight++;
|
|
1530
1581
|
let released = false;
|
|
@@ -1558,6 +1609,8 @@ export class Manager {
|
|
|
1558
1609
|
await new Promise((resolve) => this.lifecycleDrainWaiters.push(resolve));
|
|
1559
1610
|
}
|
|
1560
1611
|
maintenanceError() {
|
|
1612
|
+
if (this.leaseStopping)
|
|
1613
|
+
return "manager is stopping; new lifecycle/control work is fenced";
|
|
1561
1614
|
if (this.resumeRequired)
|
|
1562
1615
|
return `manager is waiting for resume attempt ${this.resumeAttemptId}; ordinary lifecycle/control work is fenced`;
|
|
1563
1616
|
return `manager is in ${this.maintenanceState} mode; new lifecycle/control work is fenced`;
|
|
@@ -1622,6 +1675,11 @@ export class Manager {
|
|
|
1622
1675
|
error,
|
|
1623
1676
|
});
|
|
1624
1677
|
}
|
|
1678
|
+
// A later manager does not resume a delegated seat, and its close runs only inside its
|
|
1679
|
+
// retirement, so a cut that holds one is refused before any child stops.
|
|
1680
|
+
for (const a of this.agents.values())
|
|
1681
|
+
if (a.handedOff)
|
|
1682
|
+
failures.push({ name: a.name, id: a.id, error: "a delegated seat is not resumed by a later manager; stop it before preserving" });
|
|
1625
1683
|
const unverifiedStops = this.unverifiedStops.filter((stopped) => {
|
|
1626
1684
|
try {
|
|
1627
1685
|
if (!stopped.authoritative && stopped.handle.status() === "exited")
|
|
@@ -1895,15 +1953,15 @@ export class Manager {
|
|
|
1895
1953
|
...(backfillFloor !== undefined ? { backfillFloor } : {}),
|
|
1896
1954
|
};
|
|
1897
1955
|
}
|
|
1898
|
-
/** Tear down every managed agent's footprint on an explicit destructive {@link stop}
|
|
1956
|
+
/** Tear down every managed agent's footprint on an explicit destructive {@link stop}, or only the
|
|
1957
|
+
* in-process seats a default stop cannot detach. A manager exit is
|
|
1899
1958
|
* a mass agent-exit, and without this its agents' footprints (creds files + `dm_`/`dlv_` durables + ACL
|
|
1900
1959
|
* rows) would orphan exactly as the per-agent exit path prevents. Hard-stop each child (an exit has no
|
|
1901
1960
|
* time for the graceful grace window) and AWAIT its deprovision — bounded per agent (`withTimeout`) and
|
|
1902
1961
|
* best-effort (`allSettled` + a loud log), so one slow/failed teardown can neither hang nor abort exit.
|
|
1903
1962
|
* The creds file is dropped even if the broker teardown fails (see {@link deprovision}). Deliberately
|
|
1904
1963
|
* touches NEITHER the lease NOR the endpoints — the caller owns those. */
|
|
1905
|
-
async teardownManagedAgents() {
|
|
1906
|
-
const managed = [...this.agents.values()];
|
|
1964
|
+
async teardownManagedAgents(managed = [...this.agents.values()]) {
|
|
1907
1965
|
const failures = [];
|
|
1908
1966
|
for (const a of managed) {
|
|
1909
1967
|
// Free the slot + hard-stop each; `stopHandle` is best-effort (never throws — see it), so one bad
|
|
@@ -1930,20 +1988,15 @@ export class Manager {
|
|
|
1930
1988
|
}
|
|
1931
1989
|
}));
|
|
1932
1990
|
// Deprovision EVERY snapshot entry regardless of whether its stop failed (allSettled + a loud log).
|
|
1933
|
-
await Promise.allSettled(managed.filter((a) => !a.suppressCleanup).map((a) => this.deprovision(a).catch((e) => console.error(`deprovision ${a.name} (${a.id}) on shutdown: ${e.message}`))));
|
|
1991
|
+
await Promise.allSettled(managed.filter((a) => !a.suppressCleanup).map((a) => this.deprovision({ ...a, ...(a.handedOff ? { delegatedHandle: a.handle } : {}) }).catch((e) => console.error(`deprovision ${a.name} (${a.id}) on shutdown: ${e.message}`))));
|
|
1934
1992
|
if (failures.length)
|
|
1935
1993
|
throw new Error(`manager shutdown could not prove every seat exited: ${failures.join("; ")}`);
|
|
1936
1994
|
}
|
|
1937
|
-
/** Drop
|
|
1938
|
-
*
|
|
1995
|
+
/** Drop this manager's local custody of every seat that can outlive it. A pty seat whose handle
|
|
1996
|
+
* cannot be released lives in this process and closes with it, so it is left for {@link stop} to
|
|
1997
|
+
* stop instead. */
|
|
1939
1998
|
releaseManagedAgents() {
|
|
1940
|
-
const managed = [...this.agents.values()];
|
|
1941
|
-
const blocked = managed.filter((a) => a.handle.kind === "pty" && typeof a.handle.release !== "function");
|
|
1942
|
-
if (blocked.length) {
|
|
1943
|
-
return [
|
|
1944
|
-
`manager shutdown cannot detach ${blocked.map((a) => `${a.name} (${a.handle.kind})`).join(", ")}: runtime handle does not support release`,
|
|
1945
|
-
];
|
|
1946
|
-
}
|
|
1999
|
+
const managed = [...this.agents.values()].filter((a) => !(a.handle.kind === "pty" && typeof a.handle.release !== "function"));
|
|
1947
2000
|
const failures = [];
|
|
1948
2001
|
for (const a of managed) {
|
|
1949
2002
|
try {
|
|
@@ -1964,7 +2017,11 @@ export class Manager {
|
|
|
1964
2017
|
const failures = [];
|
|
1965
2018
|
await Promise.all(managed.map(async (a) => {
|
|
1966
2019
|
try {
|
|
1967
|
-
a
|
|
2020
|
+
// No cut holds a delegated seat (runPreparation refuses one), so stopping it is its retirement.
|
|
2021
|
+
if (a.handedOff)
|
|
2022
|
+
await this.deprovision({ ...a, delegatedHandle: a.handle });
|
|
2023
|
+
else
|
|
2024
|
+
a.handle.stop({ graceful: false });
|
|
1968
2025
|
}
|
|
1969
2026
|
catch (e) {
|
|
1970
2027
|
failures.push(`${a.name}: stop failed: ${e.message}`);
|
|
@@ -1981,7 +2038,16 @@ export class Manager {
|
|
|
1981
2038
|
if (failures.length)
|
|
1982
2039
|
throw new Error(`manager preservation shutdown incomplete: ${failures.join("; ")}`);
|
|
1983
2040
|
}
|
|
1984
|
-
|
|
2041
|
+
/** Every call joins the first stop and settles with it. A call that asks for a different
|
|
2042
|
+
* `withAgents` is refused, since the running stop has already applied its own policy. */
|
|
2043
|
+
stop(options = {}) {
|
|
2044
|
+
const withAgents = options.withAgents === true;
|
|
2045
|
+
this.stopTask ??= { withAgents, done: this.stopOnce(withAgents) };
|
|
2046
|
+
if (this.stopTask.withAgents === withAgents)
|
|
2047
|
+
return this.stopTask.done;
|
|
2048
|
+
return Promise.reject(new Error(`manager is already stopping ${this.stopTask.withAgents ? "with" : "without"} its agents`));
|
|
2049
|
+
}
|
|
2050
|
+
async stopOnce(withAgents) {
|
|
1985
2051
|
this.leaseStopping = true;
|
|
1986
2052
|
this.staticReconcileStopping = true;
|
|
1987
2053
|
const starting = this.startTask;
|
|
@@ -2007,14 +2073,22 @@ export class Manager {
|
|
|
2007
2073
|
// still move the key's revision after we clear the timer, and releasing against a cached
|
|
2008
2074
|
// revision it has since moved is a CAS the broker correctly refuses, silently, forever.
|
|
2009
2075
|
await this.leaseRenewTask?.catch(() => { });
|
|
2076
|
+
// A spawn admitted before the fence closed may still be launching. Its seat must be in the
|
|
2077
|
+
// snapshot below, or it would outlive a stop that reported success.
|
|
2078
|
+
await this.awaitLifecycleDrain();
|
|
2010
2079
|
let releaseFailures = [];
|
|
2011
2080
|
let seatFailure;
|
|
2012
2081
|
try {
|
|
2013
2082
|
if (this.maintenanceState === "active" && !this.resumeRequired) {
|
|
2014
|
-
if (
|
|
2083
|
+
if (withAgents)
|
|
2015
2084
|
await this.teardownManagedAgents();
|
|
2016
|
-
else
|
|
2085
|
+
else {
|
|
2086
|
+
// An in-process seat cannot be detached from the process that is exiting, so the default
|
|
2087
|
+
// stop stops and deprovisions it as `withAgents` does instead of orphaning its footprint.
|
|
2088
|
+
const inProcess = [...this.agents.values()].filter((a) => a.handle.kind === "pty" && typeof a.handle.release !== "function");
|
|
2017
2089
|
releaseFailures = this.releaseManagedAgents();
|
|
2090
|
+
await this.teardownManagedAgents(inProcess);
|
|
2091
|
+
}
|
|
2018
2092
|
}
|
|
2019
2093
|
else {
|
|
2020
2094
|
// A signal after a partial preservation must never fall back into destructive teardown.
|
|
@@ -2162,18 +2236,18 @@ export class Manager {
|
|
|
2162
2236
|
catch { /* best effort */ }
|
|
2163
2237
|
}
|
|
2164
2238
|
}
|
|
2165
|
-
/** Keep this instance's liveness key fresh, and
|
|
2239
|
+
/** Keep this instance's liveness key fresh, and end the process only when another process holds it.
|
|
2166
2240
|
*
|
|
2167
2241
|
* A renew that throws is a question, not a verdict: the request may have timed out with the write
|
|
2168
2242
|
* landed and only its acknowledgement lost, the key may have expired during a stall, or a same-id
|
|
2169
2243
|
* process may hold it. The verdict comes from RE-READING the key ({@link reconcileLease}), and each
|
|
2170
|
-
* answer has one response
|
|
2244
|
+
* answer has one response:
|
|
2171
2245
|
* - `held`: adopt the broker's revision and carry on.
|
|
2172
2246
|
* - `gone`: the key expired or was released while this process was still here, so put it back. The
|
|
2173
2247
|
* create is atomic, so a same-id process that got there first shows up as `taken` next tick.
|
|
2174
|
-
* - `taken`: a different process holds this instance's key
|
|
2175
|
-
*
|
|
2176
|
-
*
|
|
2248
|
+
* - `taken`: a different process holds this instance's key, so it took this instance over while
|
|
2249
|
+
* this one stalled. This process is the superseded one and stops serving by exiting (SPEC 13.1).
|
|
2250
|
+
* An open mesh has no barrier that would evict its serve connection, so nothing else stops it.
|
|
2177
2251
|
* - `unknown`: the broker could not be asked. Keep serving and ask again next tick, for as long as it
|
|
2178
2252
|
* takes. A manager that cannot reach its broker gains nothing by ending itself, and the seats it
|
|
2179
2253
|
* holds lose everything (a pty child dies with its parent).
|
|
@@ -2240,8 +2314,12 @@ export class Manager {
|
|
|
2240
2314
|
return;
|
|
2241
2315
|
}
|
|
2242
2316
|
case "taken":
|
|
2243
|
-
|
|
2244
|
-
|
|
2317
|
+
// No `stop()` on the way out: its deregistration deletes the instance's current
|
|
2318
|
+
// registration, which is the other process's now. That process took over on the premise
|
|
2319
|
+
// that this one was gone, so this exit leaves what a crash would, and seats a custodian or
|
|
2320
|
+
// multiplexer owns outlive it the same way.
|
|
2321
|
+
console.error(`! manager instance ${this.managerInstanceId} finds its liveness lease key held by ${verdict.by}, not by this process (renew: ${why}); that process serves this instance now, so this one stops serving and exits (space "${this.space}")`);
|
|
2322
|
+
process.exit(1);
|
|
2245
2323
|
case "unknown":
|
|
2246
2324
|
this.noteLease("unknown", `could not renew its liveness lease (${why}) or re-read it (${verdict.why}); serving, retrying until the broker answers`);
|
|
2247
2325
|
return;
|
|
@@ -2654,6 +2732,13 @@ export class Manager {
|
|
|
2654
2732
|
// A retired row affirms that no agent exists now, so it keeps the old `not-found`
|
|
2655
2733
|
// result. The changed refusal is reserved for durable NONTERMINAL state that contradicts
|
|
2656
2734
|
// the live-map miss and needs operator attention.
|
|
2735
|
+
//
|
|
2736
|
+
// A sibling's nonterminal row is not a contradiction: the name belongs to another
|
|
2737
|
+
// manager instance in this space. In a multi-manager space the class queue hands this
|
|
2738
|
+
// read to either instance, so a plain `not-found` here read the same as a name that
|
|
2739
|
+
// exists nowhere and a caller had no way to tell the two apart (#443).
|
|
2740
|
+
if (observed?.ownerInstanceId !== undefined)
|
|
2741
|
+
throw new EpEnvelopeError("failed-precondition", `the durable slot for "${name}" belongs to manager instance ${observed.ownerInstanceId}, not ${this.managerInstanceId} which answered; only that instance can act on it ${renderStaticSlotObservation(observed)}`, [observed]);
|
|
2657
2742
|
if (observed !== undefined && observed.slotPhase !== "retired")
|
|
2658
2743
|
throw new EpEnvelopeError("failed-precondition", `no live agent "${name}"; durable state contradicts absence ${renderStaticSlotObservation(observed)}`, [observed]);
|
|
2659
2744
|
}
|
|
@@ -2781,12 +2866,12 @@ export class Manager {
|
|
|
2781
2866
|
// The workflow-run family (SPEC 14.3): the manager hosts the driver. Reach is the broker's
|
|
2782
2867
|
// (`run` capability / privileged instrument rows); the serve gate is the maintenance fence.
|
|
2783
2868
|
runStart: (ctx) => this.serveGated(ctx, () => this.runHost().start(ctx, args(ctx))),
|
|
2784
|
-
runResume: (ctx) => this.serveGated(ctx, () => this.runHost().resume(args(ctx))),
|
|
2869
|
+
runResume: (ctx) => this.serveGated(ctx, () => this.runHost().resume(args(ctx), ctx)),
|
|
2785
2870
|
runAnswer: (ctx) => this.serveGated(ctx, () => {
|
|
2786
2871
|
const input = args(ctx);
|
|
2787
2872
|
if (input.amend === true)
|
|
2788
|
-
return this.runHost().amend(input, this.runAnswerer(ctx), (accepted) => this.authorizeRunAmend(ctx, accepted));
|
|
2789
|
-
return this.runHost().answer(input, this.runAnswerer(ctx), (open) => this.authorizeRunAnswer(ctx, input, open));
|
|
2873
|
+
return this.runHost().amend(input, this.runAnswerer(ctx), (accepted) => this.authorizeRunAmend(ctx, accepted), ctx);
|
|
2874
|
+
return this.runHost().answer(input, this.runAnswerer(ctx), (open) => this.authorizeRunAnswer(ctx, input, open), ctx);
|
|
2790
2875
|
}),
|
|
2791
2876
|
runStatus: (ctx) => this.serveGated(ctx, () => this.runHost().status(args(ctx))),
|
|
2792
2877
|
runPs: (ctx) => this.serveGated(ctx, () => this.runHost().list(args(ctx))),
|
|
@@ -2958,7 +3043,13 @@ export class Manager {
|
|
|
2958
3043
|
try {
|
|
2959
3044
|
if (graceful && process.platform === "win32" && a.control)
|
|
2960
3045
|
controlShutdown(a.control);
|
|
2961
|
-
|
|
3046
|
+
// A delegated seat's close runs only inside its retirement, after the host revoked its grant,
|
|
3047
|
+
// so stopping it starts that retirement. Every exit wait on a stop path then settles on that
|
|
3048
|
+
// close; the free path's own teardown joins the same flight.
|
|
3049
|
+
if (a.handedOff)
|
|
3050
|
+
this.trackDeprovision({ ...a, delegatedHandle: a.handle });
|
|
3051
|
+
else
|
|
3052
|
+
a.handle.stop({ graceful });
|
|
2962
3053
|
}
|
|
2963
3054
|
catch (e) {
|
|
2964
3055
|
console.error(`stop ${a.name} (${a.id}): ${e.message}`);
|
|
@@ -3018,6 +3109,19 @@ export class Manager {
|
|
|
3018
3109
|
* dead auth plane. Every failure is returned as the refusal sentence, with the grant + files
|
|
3019
3110
|
* rolled back. */
|
|
3020
3111
|
async provisionUserAgent(name, opts) {
|
|
3112
|
+
if (opts.delegatedIntent) {
|
|
3113
|
+
// startAgentActive refused a delegated launch on a manager without this callback.
|
|
3114
|
+
const execute = this.remoteAuthority.executeDelegatedUserIntent;
|
|
3115
|
+
const { intentId, owner } = opts.delegatedIntent;
|
|
3116
|
+
// The hosted arm with the intent's execution in place of the holder's own enrollment: the
|
|
3117
|
+
// spawning owner is the intent's, so its existing owner check refuses any other.
|
|
3118
|
+
return this.enrollUserAgent(name, { ...opts, specOwner: owner }, async ({ target }) => {
|
|
3119
|
+
const result = await execute({ intentId, operation: "launch", target: { actor: target.actor, tokenHash: target.tokenHash } });
|
|
3120
|
+
if (result.operation !== "launch" || result.intentId !== intentId)
|
|
3121
|
+
throw new Error(`the host answered delegated intent ${intentId} with another execution`);
|
|
3122
|
+
return result.material;
|
|
3123
|
+
});
|
|
3124
|
+
}
|
|
3021
3125
|
if (this.remoteAuthority?.enrollManagedAgent)
|
|
3022
3126
|
return this.enrollUserAgent(name, opts, this.remoteAuthority.enrollManagedAgent);
|
|
3023
3127
|
const spawnerPr = opts.spawner ? parsePrincipalKey(opts.spawner) : null;
|
|
@@ -3176,6 +3280,8 @@ export class Manager {
|
|
|
3176
3280
|
// Re-key the family onto the HOST's uid (SPEC 13.1 name-disjointness on the FS): the child's
|
|
3177
3281
|
// credential paths must embed the uid its broker footprint carries, never the provisional one.
|
|
3178
3282
|
files = agentLifecycleSecretFilePaths(this.workspaceRoot, this.space, name, material.lifecycleUid);
|
|
3283
|
+
// From here the host holds a grant at that uid.
|
|
3284
|
+
opts.onEnrolled?.({ owner: material.owner, lifecycleUid: material.lifecycleUid, files });
|
|
3179
3285
|
if (files.actorToken !== staged.actorToken) {
|
|
3180
3286
|
await secrets.put(agentSecretKeyForFile(files.actorToken, this.space), actorToken);
|
|
3181
3287
|
await materializeSecretToFile(secrets, agentSecretKeyForFile(files.actorToken, this.space), files.actorToken);
|
|
@@ -3204,21 +3310,70 @@ export class Manager {
|
|
|
3204
3310
|
lifecycleUid: material.lifecycleUid,
|
|
3205
3311
|
files,
|
|
3206
3312
|
launch: { owner: material.owner, actor: name, sentinelCredsPath: files.sentinelCreds, bearerCmd },
|
|
3313
|
+
enrolled: { material, actorToken },
|
|
3207
3314
|
};
|
|
3208
3315
|
}
|
|
3209
3316
|
catch (e) {
|
|
3210
3317
|
// Shred every local secret this attempt materialized. The HOST's rows are the host's to
|
|
3211
3318
|
// reconcile: a participant holds no writer, so it must not pretend to roll back a grant or a
|
|
3212
3319
|
// durable it never authored. A retry with the same requestId re-enters the same enrollment.
|
|
3320
|
+
// A failed removal joins the refusal instead of throwing, so every removal is attempted and
|
|
3321
|
+
// the refusal still reaches the caller.
|
|
3322
|
+
const unshredded = [];
|
|
3323
|
+
const shred = async (what, remove) => {
|
|
3324
|
+
try {
|
|
3325
|
+
await remove();
|
|
3326
|
+
}
|
|
3327
|
+
catch (err) {
|
|
3328
|
+
unshredded.push(`${what}: ${rejectionText(err)}`);
|
|
3329
|
+
}
|
|
3330
|
+
};
|
|
3213
3331
|
for (const family of files.actorToken === staged.actorToken ? [staged] : [staged, files]) {
|
|
3214
|
-
|
|
3215
|
-
|
|
3216
|
-
|
|
3217
|
-
|
|
3218
|
-
|
|
3219
|
-
}
|
|
3220
|
-
return { error: `agent auth preflight failed for "${name}": ${e
|
|
3221
|
-
}
|
|
3332
|
+
for (const path of [family.actorToken, family.sentinelCreds])
|
|
3333
|
+
await shred(`secret ${path}`, () => secrets.delete(agentSecretKeyForFile(path, this.space)));
|
|
3334
|
+
for (const path of [family.actorToken, family.sentinelCreds, family.health])
|
|
3335
|
+
await shred(`file ${path}`, () => rmSync(path, { force: true }));
|
|
3336
|
+
}
|
|
3337
|
+
const leftover = unshredded.length ? `; cleanup failed: ${unshredded.join("; ")}` : "";
|
|
3338
|
+
return { error: `agent auth preflight failed for "${name}": ${rejectionText(e)}${leftover}` };
|
|
3339
|
+
}
|
|
3340
|
+
}
|
|
3341
|
+
/** This manager's registry record for its space, which pins the broker and IdP a delegated seat's
|
|
3342
|
+
* handoff carries. It must pin the exchange the host enrolls against, or the child would register
|
|
3343
|
+
* a mesh whose bearers come from somewhere else. */
|
|
3344
|
+
delegatedMeshRecord() {
|
|
3345
|
+
const record = loadMeshes().find((m) => m.space === this.space);
|
|
3346
|
+
if (!record?.userAuth)
|
|
3347
|
+
throw new Error(`a delegated seat needs this manager's user-mode registry record for space "${this.space}", and there is none`);
|
|
3348
|
+
if (record.userAuth.endpoints?.url !== this.remoteAuthority?.agentBearerExchangeUrl)
|
|
3349
|
+
throw new Error(`the registry record for space "${this.space}" pins a different exchange than the host's agentBearerExchangeUrl`);
|
|
3350
|
+
return record;
|
|
3351
|
+
}
|
|
3352
|
+
/** The one handoff a delegated seat receives, built from values this manager holds: the
|
|
3353
|
+
* enrollment result, the token it generated, and its own registry record. No path crosses. */
|
|
3354
|
+
composeManagedHandoff(enrolled, actorToken) {
|
|
3355
|
+
const record = this.delegatedMeshRecord();
|
|
3356
|
+
if (enrolled.agentBearerExchangeUrl !== record.userAuth.endpoints?.url)
|
|
3357
|
+
throw new Error(`the host enrolled "${enrolled.actor}" against a different exchange than this manager's registry record pins`);
|
|
3358
|
+
const { url, issuer, audience } = record.userAuth.idp;
|
|
3359
|
+
return {
|
|
3360
|
+
kind: MANAGED_HANDOFF_KIND,
|
|
3361
|
+
space: this.space,
|
|
3362
|
+
owner: enrolled.owner,
|
|
3363
|
+
actor: enrolled.actor,
|
|
3364
|
+
lifecycleUid: enrolled.lifecycleUid,
|
|
3365
|
+
server: record.server,
|
|
3366
|
+
tlsRequired: record.tlsRequired === true,
|
|
3367
|
+
authProvider: record.userAuth.provider,
|
|
3368
|
+
idp: { url, issuer, audience },
|
|
3369
|
+
exchangeUrl: enrolled.agentBearerExchangeUrl,
|
|
3370
|
+
sentinelCreds: enrolled.sentinelCreds,
|
|
3371
|
+
actorToken,
|
|
3372
|
+
subscribe: enrolled.subscribe,
|
|
3373
|
+
allowSubscribe: enrolled.allowSubscribe,
|
|
3374
|
+
allowPublish: enrolled.allowPublish,
|
|
3375
|
+
...(record.policy ? { policy: record.policy } : {}),
|
|
3376
|
+
};
|
|
3222
3377
|
}
|
|
3223
3378
|
/**
|
|
3224
3379
|
* WHY A SEAT LEFT — one line, at the one place every free path passes through.
|
|
@@ -3325,7 +3480,7 @@ export class Manager {
|
|
|
3325
3480
|
if (this.userMode) {
|
|
3326
3481
|
const p = parsePrincipalKey(a.id);
|
|
3327
3482
|
if (p)
|
|
3328
|
-
this.retiring.set(a.name, { opId: retireOpId(a.lifecycleUid), lifecycleUid: a.lifecycleUid, owner: p.owner, actor: p.actor, agentId: a.id, userOwner: a.userOwner, secretPaths: a.secretPaths, launch: { allowSubscribe: a.launch.allowSubscribe }, startedAt: Date.now() });
|
|
3483
|
+
this.retiring.set(a.name, { opId: retireOpId(a.lifecycleUid), lifecycleUid: a.lifecycleUid, owner: p.owner, actor: p.actor, agentId: a.id, userOwner: a.userOwner, delegated: a.delegated, secretPaths: a.secretPaths, launch: { allowSubscribe: a.launch.allowSubscribe }, ...(a.handedOff ? { delegatedHandle: a.handle } : {}), startedAt: Date.now() });
|
|
3329
3484
|
}
|
|
3330
3485
|
else if (this.auth) {
|
|
3331
3486
|
// Unit B: a STATIC lifecycle now also holds its name pending its own terminal (the F1
|
|
@@ -3342,7 +3497,7 @@ export class Manager {
|
|
|
3342
3497
|
// is usually the seat ending, but a stop that could not prove the process gone leaves one the
|
|
3343
3498
|
// terminal must still address before it frees the alias.
|
|
3344
3499
|
if (!a.suppressCleanup && (this.maintenanceState === "active" || acceptedBeforeFence))
|
|
3345
|
-
this.trackDeprovision({ ...a, runtime: a.handle?.reference });
|
|
3500
|
+
this.trackDeprovision({ ...a, runtime: a.handle?.reference, ...(a.handedOff ? { delegatedHandle: a.handle } : {}) });
|
|
3346
3501
|
}
|
|
3347
3502
|
/** Tear down a departed agent's minted footprint (#159 B2, auth mode): its local-principal durables
|
|
3348
3503
|
* (`dm_local-<id>`, `dlv_local-<id>`), its read-ACL row, and its creds file — everything the spawn's
|
|
@@ -3376,6 +3531,14 @@ export class Manager {
|
|
|
3376
3531
|
/** The actual footprint teardown (wrapped by {@link deprovision}'s single-flight). */
|
|
3377
3532
|
async driveDeprovision(a) {
|
|
3378
3533
|
if (this.remoteAuthority) {
|
|
3534
|
+
if (a.delegated) {
|
|
3535
|
+
// The agent is its user's: only that user's retirement intent retires it, so the name stays
|
|
3536
|
+
// held until retireDelegatedAgent confirms the host retired this UID.
|
|
3537
|
+
const held = this.retiring.get(a.name);
|
|
3538
|
+
if (held?.lifecycleUid === a.lifecycleUid)
|
|
3539
|
+
held.lastError = `"${a.name}" was launched under a delegated user intent and retires only through its user's retirement intent`;
|
|
3540
|
+
return;
|
|
3541
|
+
}
|
|
3379
3542
|
// A hosted composition owns grant revocation and resumable footprint release. It calls this
|
|
3380
3543
|
// prerequisite before the terminal rail. The callback must preserve this UID and its pending
|
|
3381
3544
|
// state until the host decides terminal retirement, never map deprovisioning to suspension.
|
|
@@ -3386,6 +3549,13 @@ export class Manager {
|
|
|
3386
3549
|
target: { owner: target.owner, actor: target.actor, lifecycleUid: a.lifecycleUid },
|
|
3387
3550
|
opId: retireOpId(a.lifecycleUid),
|
|
3388
3551
|
});
|
|
3552
|
+
// A delegated seat's fenced close runs only now, after the grant is revoked so its child can
|
|
3553
|
+
// obtain no fresh bearer, and before the barrier, which must never run ahead of a create that
|
|
3554
|
+
// could still land. A close that cannot prove that yet throws here and keeps the alias held.
|
|
3555
|
+
if (a.delegatedHandle) {
|
|
3556
|
+
a.delegatedHandle.stop({ graceful: true });
|
|
3557
|
+
await this.awaitHandleExit(a.delegatedHandle);
|
|
3558
|
+
}
|
|
3389
3559
|
await this.requestRetirement(a);
|
|
3390
3560
|
return;
|
|
3391
3561
|
}
|
|
@@ -4031,10 +4201,16 @@ export class Manager {
|
|
|
4031
4201
|
* still refused downstream exactly as before. Offline rows do NOT occupy — a properly retired
|
|
4032
4202
|
* name stays reusable. A row this manager itself confirmed retired (its alias, principal, and
|
|
4033
4203
|
* lifecycleUid all matching a {@link confirmedRetiredPredecessors} entry) does not occupy
|
|
4034
|
-
* either, even while its advisory presence record ages out.
|
|
4204
|
+
* either, even while its advisory presence record ages out. A numbered name is never reissued
|
|
4205
|
+
* ({@link issuedNumberedNames}), even after its holder retired. The requested base is exempt
|
|
4206
|
+
* from that history: it is the persona's own name, which stays reusable even when it once came
|
|
4207
|
+
* out of another persona's numbering. */
|
|
4035
4208
|
uniqueName(base) {
|
|
4036
4209
|
const live = this.liveRosterNames();
|
|
4037
|
-
|
|
4210
|
+
const name = firstFreeName(base, (n) => this.nameInUse(n, live) || (n !== base && this.issuedNumberedNames.has(n)));
|
|
4211
|
+
if (name !== base)
|
|
4212
|
+
this.issuedNumberedNames.add(name);
|
|
4213
|
+
return name;
|
|
4038
4214
|
}
|
|
4039
4215
|
/** Spawn a teammate by persona ref (`name` loads `.cotal/agents/<name>.md`; the peer presents
|
|
4040
4216
|
* under that file's own `name:`), as if a peer asked via the control plane. Used to pre-spawn the
|
|
@@ -4434,6 +4610,14 @@ export class Manager {
|
|
|
4434
4610
|
}
|
|
4435
4611
|
}
|
|
4436
4612
|
async startAgentActive(opts, spawner, hooks) {
|
|
4613
|
+
if (opts.delegatedIntent) {
|
|
4614
|
+
if (!this.remoteAuthority?.executeDelegatedUserIntent)
|
|
4615
|
+
return { ok: false, error: `"${opts.name}" names a delegated user intent, and this manager has no host execution for one` };
|
|
4616
|
+
if (opts.supervise)
|
|
4617
|
+
return { ok: false, error: `"${opts.name}" names a delegated user intent, and a delegated agent is never restarted (SPEC 13.16)` };
|
|
4618
|
+
// A delegated launch's spawner is the user's principal, never the composition root's call.
|
|
4619
|
+
spawner = opts.delegatedIntent.parent;
|
|
4620
|
+
}
|
|
4437
4621
|
// The spawn argument is a persona REF — a filename in `.cotal/agents` (the unique spawn KEY), or
|
|
4438
4622
|
// a path via `--config`. It is NOT the mesh identity: the identity comes from inside the file
|
|
4439
4623
|
// (`name:`), so a persona can be filed descriptively (review-critic.md) yet present under a
|
|
@@ -4537,6 +4721,28 @@ export class Manager {
|
|
|
4537
4721
|
if (this.userMode)
|
|
4538
4722
|
return { ok: false, error: "supervise is a restart policy this host cannot enforce: a user-mode seat has no static slot to keep the incarnation owned across a process death" };
|
|
4539
4723
|
}
|
|
4724
|
+
// A delegated seat starts outside this host from values alone, so a choice only this host can
|
|
4725
|
+
// honour is refused here, before enrollment, and named rather than dropped.
|
|
4726
|
+
if (this.runtime.spawnDelegated) {
|
|
4727
|
+
const delegatedBy = `runtime "${this.runtime.kind}" starts seats outside this host`;
|
|
4728
|
+
if (!this.remoteAuthority?.enrollManagedAgent)
|
|
4729
|
+
return { ok: false, error: `${delegatedBy}, which needs a host that enrolls managed agents (remoteAuthority.enrollManagedAgent); a delegated seat has no local grant path` };
|
|
4730
|
+
try {
|
|
4731
|
+
this.delegatedMeshRecord();
|
|
4732
|
+
}
|
|
4733
|
+
catch (e) {
|
|
4734
|
+
return { ok: false, error: e.message };
|
|
4735
|
+
}
|
|
4736
|
+
if (opts.resume)
|
|
4737
|
+
return { ok: false, error: `${delegatedBy}, so it cannot resume a session held on this host (resume)` };
|
|
4738
|
+
if (opts.resolved?.continuity === "exact")
|
|
4739
|
+
return { ok: false, error: `${delegatedBy}, so it cannot reopen a session held on this host (continuity: exact)` };
|
|
4740
|
+
if (typeof opts.cwd === "string" && opts.cwd !== "")
|
|
4741
|
+
return { ok: false, error: `${delegatedBy}, so it cannot run in a directory on this host's filesystem (cwd); the seat runs in its runtime resource's own directory` };
|
|
4742
|
+
const shared = Object.keys(connectorServers(loadCotalConfig(this.workspaceRoot), agent, parseShareSelection(opts.shareTools)));
|
|
4743
|
+
if (shared.length)
|
|
4744
|
+
return { ok: false, error: `${delegatedBy}, so it cannot share MCP servers that run on this host (${shared.join(", ")}); pass --share-tools none` };
|
|
4745
|
+
}
|
|
4540
4746
|
// A placed spawn's cwd is checked on this host BEFORE anything is minted or reserved: an absolute
|
|
4541
4747
|
// path this manager cannot realpath, or one that is not a directory, is refused here with the
|
|
4542
4748
|
// asked path, the reason and this host, so a misplaced spawn in a multi-manager space is
|
|
@@ -4642,6 +4848,27 @@ export class Manager {
|
|
|
4642
4848
|
`An operator adds \`capabilities: [spawn]\` to ${configPath}; a peer-defined persona ` +
|
|
4643
4849
|
`(cotal_persona) cannot declare capabilities by design — ask an operator to grant it.`,
|
|
4644
4850
|
};
|
|
4851
|
+
// #581: the operator's per-role model allowlist, judged on the EFFECTIVE role and model resolved
|
|
4852
|
+
// above, before any reserve or mint. This is where the launch is performed, so no caller can
|
|
4853
|
+
// skip it, and a definition pinned to a stale model (or to none) is refused here instead of
|
|
4854
|
+
// serving on it. Read per spawn, so a policy edit reaches the next spawn without a restart.
|
|
4855
|
+
let policyRefusal;
|
|
4856
|
+
try {
|
|
4857
|
+
policyRefusal = modelPolicyRefusal(loadCotalConfig(this.workspaceRoot), {
|
|
4858
|
+
persona: opts.resolved ? `manifest agent "${ref}"` : `persona "${ref}" (${configPath})`,
|
|
4859
|
+
role, model, variant,
|
|
4860
|
+
// A manifest launch carries its own declared model/variant in these options
|
|
4861
|
+
// (launchAgentToStartOpts), so there they are the definition's field, never a flag.
|
|
4862
|
+
modelFlag: !opts.resolved && opts.model !== undefined,
|
|
4863
|
+
variantFlag: !opts.resolved && opts.variant !== undefined,
|
|
4864
|
+
launchOptions,
|
|
4865
|
+
});
|
|
4866
|
+
}
|
|
4867
|
+
catch (e) {
|
|
4868
|
+
return { ok: false, error: e.message };
|
|
4869
|
+
}
|
|
4870
|
+
if (policyRefusal)
|
|
4871
|
+
return { ok: false, error: policyRefusal };
|
|
4645
4872
|
// The alias-reuse gate (#29 piece 3): a name whose previous agent is still retiring REFUSES
|
|
4646
4873
|
// legibly (never a silent suffix), and the refusal re-drives the FULL durable teardown so
|
|
4647
4874
|
// "retry the spawn" is also the nudge. It routes through `deprovision` (not `requestRetirement`
|
|
@@ -4652,7 +4879,7 @@ export class Manager {
|
|
|
4652
4879
|
return { ok: false, error: `the name "${identityName}" is still reconciling at manager startup; its prior lifecycle terminal owns this alias until it completes. Retry shortly.` };
|
|
4653
4880
|
const held = this.retiring.get(identityName);
|
|
4654
4881
|
if (held !== undefined) {
|
|
4655
|
-
void this.deprovision({ id: held.agentId, name: identityName, lifecycleUid: held.lifecycleUid, userOwner: held.userOwner, secretPaths: held.secretPaths, launch: held.launch }).catch(() => { });
|
|
4882
|
+
void this.deprovision({ id: held.agentId, name: identityName, lifecycleUid: held.lifecycleUid, userOwner: held.userOwner, delegated: held.delegated, secretPaths: held.secretPaths, delegatedHandle: held.delegatedHandle, launch: held.launch }).catch(() => { });
|
|
4656
4883
|
const err = lifecycleBlocked("failed-precondition", `the name "${identityName}" is reserved pending retirement: its previous agent's despawn started that lifecycle's teardown (footprint + standing-authority revoke + auth-side retirement), and the name frees only when all of it completes${held.lastError !== undefined ? ` (last attempt: ${held.lastError})` : ""}. NEXT: wait a moment and retry this spawn (retrying re-drives the whole teardown), or pick another name.`, { blockedOp: "retirement", headState: "retiring", opId: held.opId, remedy: "retry" });
|
|
4657
4884
|
return { ok: false, error: renderLifecycleBlocked(err.message, err), details: err.details };
|
|
4658
4885
|
}
|
|
@@ -4675,6 +4902,10 @@ export class Manager {
|
|
|
4675
4902
|
return { ok: false, error: e.message };
|
|
4676
4903
|
}
|
|
4677
4904
|
}
|
|
4905
|
+
// The child takes launch options as `--opt k=v`, so only string values can cross to it.
|
|
4906
|
+
const unportable = this.runtime.spawnDelegated ? Object.keys(launchOptions ?? {}).filter((k) => typeof launchOptions[k] !== "string") : [];
|
|
4907
|
+
if (unportable.length)
|
|
4908
|
+
return { ok: false, error: `runtime "${this.runtime.kind}" starts seats outside this host, so it cannot pass non-string launch options (${unportable.join(", ")})` };
|
|
4678
4909
|
// #4 A4 (panel): the roster the allocation consults must reflect the initial presence snapshot,
|
|
4679
4910
|
// or a spawn immediately after manager boot races an already-live unmanaged peer and re-opens the
|
|
4680
4911
|
// very collision black-hole this closes. Await the snapshot (bounded internally, fail-safe on an
|
|
@@ -4833,6 +5064,7 @@ export class Manager {
|
|
|
4833
5064
|
let issued;
|
|
4834
5065
|
let userLaunch;
|
|
4835
5066
|
let userOwner;
|
|
5067
|
+
let enrolled;
|
|
4836
5068
|
if (this.userMode) {
|
|
4837
5069
|
const prep = await this.provisionUserAgent(name, {
|
|
4838
5070
|
spawner,
|
|
@@ -4844,6 +5076,15 @@ export class Manager {
|
|
|
4844
5076
|
capabilities,
|
|
4845
5077
|
label: ref,
|
|
4846
5078
|
lifecycleUid,
|
|
5079
|
+
delegatedIntent: opts.delegatedIntent,
|
|
5080
|
+
// Hold the name at the UID the host enrolled for the intent's user before any later local
|
|
5081
|
+
// step runs, so no later failure, its refusal included, releases a lifecycle that only that
|
|
5082
|
+
// user's retirement intent retires.
|
|
5083
|
+
onEnrolled: opts.delegatedIntent
|
|
5084
|
+
? (enrolled) => {
|
|
5085
|
+
provisioned = { id: principalKey(enrolled.owner, name).key, name, lifecycleUid: enrolled.lifecycleUid, userOwner: enrolled.owner, delegated: true, secretPaths: enrolled.files };
|
|
5086
|
+
}
|
|
5087
|
+
: undefined,
|
|
4847
5088
|
});
|
|
4848
5089
|
if ("error" in prep) {
|
|
4849
5090
|
this.reserved.delete(name);
|
|
@@ -4857,7 +5098,8 @@ export class Manager {
|
|
|
4857
5098
|
lifecycleUid = prep.lifecycleUid;
|
|
4858
5099
|
userLaunch = prep.launch;
|
|
4859
5100
|
userOwner = prep.owner;
|
|
4860
|
-
|
|
5101
|
+
enrolled = prep.enrolled;
|
|
5102
|
+
provisioned = { id: principalKey(prep.owner, name).key, name, lifecycleUid, userOwner: prep.owner, ...(opts.delegatedIntent ? { delegated: true } : {}), secretPaths: prep.files, ...(custody ? { runtime: custody } : {}) };
|
|
4861
5103
|
}
|
|
4862
5104
|
else if (this.auth) {
|
|
4863
5105
|
// Unit B (§13.1): reserve + activate this incarnation's DURABLE identity BEFORE any
|
|
@@ -4969,9 +5211,22 @@ export class Manager {
|
|
|
4969
5211
|
// (possibly per-agent) launch cwd below. The cwd itself rides runtime.spawn, not the launch.
|
|
4970
5212
|
workspaceRoot: this.workspaceRoot,
|
|
4971
5213
|
};
|
|
4972
|
-
|
|
4973
|
-
|
|
4974
|
-
|
|
5214
|
+
// A delegated seat gets no local launch at all: no launch-material file, control token or
|
|
5215
|
+
// manager path is produced for it, only the handoff its runtime writes into the child.
|
|
5216
|
+
let spec;
|
|
5217
|
+
let handle;
|
|
5218
|
+
if (this.runtime.spawnDelegated) {
|
|
5219
|
+
const handoff = this.composeManagedHandoff(enrolled.material, enrolled.actorToken);
|
|
5220
|
+
if (this.delegatedLaunched.has(lifecycleUid))
|
|
5221
|
+
throw new Error(`lifecycle ${lifecycleUid} was already handed to runtime "${this.runtime.kind}"; a lifecycle is handed off at most once`);
|
|
5222
|
+
this.delegatedLaunched.add(lifecycleUid);
|
|
5223
|
+
handle = this.runtime.spawnDelegated({ agent, persona: readFileSync(configPath, "utf8"), role, model, variant, prompt, launchOptions: launchOptions, events }, handoff);
|
|
5224
|
+
}
|
|
5225
|
+
else {
|
|
5226
|
+
spec = connector.buildLaunch(launchOpts);
|
|
5227
|
+
spec.env = { ...spec.env, COTAL_MANAGER_INSTANCE: this.managerInstanceId };
|
|
5228
|
+
handle = await this.spawnCustodied(name, spec, cwd, custody);
|
|
5229
|
+
}
|
|
4975
5230
|
hooks?.onLaunched?.(); // P2 item 2: the "launched" progress edge (process spawned, pre-presence)
|
|
4976
5231
|
const managed = {
|
|
4977
5232
|
name,
|
|
@@ -4983,12 +5238,14 @@ export class Manager {
|
|
|
4983
5238
|
// recorded truth teardown/preservation/health consume, never re-derived by name.
|
|
4984
5239
|
secretPaths: provisioned?.secretPaths,
|
|
4985
5240
|
...(userLaunch ? { userOwner } : { seed: identity.seed }),
|
|
5241
|
+
...(opts.delegatedIntent ? { delegated: true } : {}),
|
|
4986
5242
|
...(issued ? { issued } : {}),
|
|
4987
5243
|
spawner: spawner ?? this.ep.ref().id,
|
|
4988
5244
|
authorityParent: userLaunch && spawner && parsePrincipalKey(spawner) ? spawner : undefined,
|
|
4989
5245
|
startedAt: Date.now(),
|
|
4990
5246
|
handle,
|
|
4991
|
-
control: spec
|
|
5247
|
+
control: spec?.control,
|
|
5248
|
+
...(spec ? {} : { handedOff: true }),
|
|
4992
5249
|
launch: {
|
|
4993
5250
|
source: opts.resolved
|
|
4994
5251
|
? {
|
|
@@ -5014,14 +5271,15 @@ export class Manager {
|
|
|
5014
5271
|
events,
|
|
5015
5272
|
shareTools: opts.shareTools,
|
|
5016
5273
|
forkSource: opts.resume,
|
|
5017
|
-
...(opts.resume !== undefined && spec
|
|
5274
|
+
...(opts.resume !== undefined && spec?.resumeRecordPath ? { resumeRecordPath: spec.resumeRecordPath } : {}),
|
|
5018
5275
|
// Opaque values may contain secrets. Preserve only their keys and require the referenced
|
|
5019
5276
|
// persona/manifest to resolve the values again; imperative overrides have no safe payload.
|
|
5020
5277
|
unresolvedLaunchOptionKeys: opts.launchOptions && Object.keys(opts.launchOptions).length
|
|
5021
5278
|
? Object.keys(opts.launchOptions).sort()
|
|
5022
5279
|
: undefined,
|
|
5023
5280
|
},
|
|
5024
|
-
|
|
5281
|
+
// A delegated agent is never restarted (SPEC 13.16), so its slot arms no recovery.
|
|
5282
|
+
...(spec && (connector.supportsSessionContinuation || opts.supervise !== undefined) && !opts.delegatedIntent
|
|
5025
5283
|
? {
|
|
5026
5284
|
restart: {
|
|
5027
5285
|
opts: launchOpts,
|
|
@@ -5139,6 +5397,11 @@ export class Manager {
|
|
|
5139
5397
|
// orphan down (detached, fail-loud) so a failed spawn leaves no creds/durables behind (#159 B).
|
|
5140
5398
|
if (provisioned) {
|
|
5141
5399
|
const orphan = provisioned;
|
|
5400
|
+
// The host enrolled this UID for the intent's user, so only that user's retirement intent
|
|
5401
|
+
// retires it, and its name stays held for retireDelegatedAgent.
|
|
5402
|
+
const p = orphan.delegated ? parsePrincipalKey(orphan.id) : null;
|
|
5403
|
+
if (p)
|
|
5404
|
+
this.retiring.set(orphan.name, { opId: retireOpId(orphan.lifecycleUid), lifecycleUid: orphan.lifecycleUid, owner: p.owner, actor: p.actor, agentId: orphan.id, userOwner: orphan.userOwner, delegated: true, secretPaths: orphan.secretPaths, startedAt: Date.now() });
|
|
5142
5405
|
this.trackDeprovision(orphan, "(orphaned spawn)");
|
|
5143
5406
|
}
|
|
5144
5407
|
}
|
|
@@ -5159,6 +5422,7 @@ export class Manager {
|
|
|
5159
5422
|
return { ok: false, agents: [], error: `resume inventory belongs to space "${inventory.space}", not "${this.space}"` };
|
|
5160
5423
|
const seen = new Set();
|
|
5161
5424
|
const principals = new Set();
|
|
5425
|
+
const orphans = [];
|
|
5162
5426
|
await this.ep.waitForPresenceSnapshot();
|
|
5163
5427
|
const liveRoster = this.ep.getRoster().filter((presence) => presence.status !== "offline");
|
|
5164
5428
|
const occupancy = this.occupancy();
|
|
@@ -5180,7 +5444,9 @@ export class Manager {
|
|
|
5180
5444
|
name: entry.name,
|
|
5181
5445
|
lifecycleUid: held.lifecycleUid,
|
|
5182
5446
|
userOwner: held.userOwner,
|
|
5447
|
+
delegated: held.delegated,
|
|
5183
5448
|
secretPaths: held.secretPaths,
|
|
5449
|
+
delegatedHandle: held.delegatedHandle,
|
|
5184
5450
|
}).catch(() => { });
|
|
5185
5451
|
return {
|
|
5186
5452
|
ok: false,
|
|
@@ -5210,8 +5476,12 @@ export class Manager {
|
|
|
5210
5476
|
presence.card.name !== entry.name ||
|
|
5211
5477
|
entry.identity.lifecycleUid === confirmedPredecessor.lifecycleUid;
|
|
5212
5478
|
});
|
|
5213
|
-
if (principalIsLive)
|
|
5214
|
-
|
|
5479
|
+
if (principalIsLive) {
|
|
5480
|
+
const orphan = await this.orphanedResumeSeat(entry, principal, liveRoster);
|
|
5481
|
+
if (!orphan)
|
|
5482
|
+
return { ok: false, agents: [], error: `retained principal "${principal}" is already live and this runtime cannot authoritatively adopt it` };
|
|
5483
|
+
orphans.push({ principal, reference: orphan });
|
|
5484
|
+
}
|
|
5215
5485
|
if (this.agents.has(entry.name) || this.reserved.has(entry.name))
|
|
5216
5486
|
return { ok: false, agents: [], error: `retained agent "${entry.name}" is already managed or reserved` };
|
|
5217
5487
|
}
|
|
@@ -5231,6 +5501,13 @@ export class Manager {
|
|
|
5231
5501
|
agents: preflight,
|
|
5232
5502
|
error: `${preflightFailures.length} retained agent${preflightFailures.length === 1 ? "" : "s"} failed preflight`,
|
|
5233
5503
|
};
|
|
5504
|
+
// Only now, with the whole inventory through preflight, is an orphan reaped: a refused batch
|
|
5505
|
+
// leaves every seat as it found it.
|
|
5506
|
+
for (const { principal, reference } of orphans) {
|
|
5507
|
+
const refusal = await this.reapOrphanedResumeSeat(principal, reference);
|
|
5508
|
+
if (refusal)
|
|
5509
|
+
return { ok: false, agents: [], error: refusal };
|
|
5510
|
+
}
|
|
5234
5511
|
const agents = [];
|
|
5235
5512
|
for (let i = 0; i < inventory.agents.length; i++) {
|
|
5236
5513
|
const entry = inventory.agents[i];
|
|
@@ -5430,6 +5707,46 @@ export class Manager {
|
|
|
5430
5707
|
throw new Error(`retained user principal ${entry.identity.owner}.${entry.identity.actor} could not be reused: ${e.message}`);
|
|
5431
5708
|
}
|
|
5432
5709
|
}
|
|
5710
|
+
/** The seat reference behind a live retained principal, when that seat is provably one a lost
|
|
5711
|
+
* manager of this logical instance launched for this exact incarnation. The resume binds the
|
|
5712
|
+
* launched handle's reference to the incarnation's active slot before it waits for readiness, and
|
|
5713
|
+
* the coordinator journals `resume-active` only after that wait, so a coordinator crash between the
|
|
5714
|
+
* two leaves `resume-intent` behind a live seat. Undefined for everything else, which stays refused:
|
|
5715
|
+
* another incarnation's presence, a slot another instance owns, a runtime that cannot reap the
|
|
5716
|
+
* reference, and user or open meshes, which keep no slot. */
|
|
5717
|
+
async orphanedResumeSeat(entry, principal, liveRoster) {
|
|
5718
|
+
if (!this.auth || this.userMode || entry.identity.mode !== "static")
|
|
5719
|
+
return undefined;
|
|
5720
|
+
if (typeof this.runtime.reap !== "function")
|
|
5721
|
+
return undefined;
|
|
5722
|
+
if (liveRoster.some((p) => p.card.id === principal && p.lifecycleUid !== entry.identity.lifecycleUid))
|
|
5723
|
+
return undefined;
|
|
5724
|
+
const slot = await this.readStaticReconcileSlot(entry.name);
|
|
5725
|
+
if (slot?.phase !== "active" || slot.actor !== entry.identity.id || slot.lifecycleUid !== entry.identity.lifecycleUid)
|
|
5726
|
+
return undefined;
|
|
5727
|
+
if (slot.ownerInstanceId !== undefined && slot.ownerInstanceId !== this.managerInstanceId)
|
|
5728
|
+
return undefined;
|
|
5729
|
+
return slot.runtime?.kind === this.runtime.kind ? slot.runtime : undefined;
|
|
5730
|
+
}
|
|
5731
|
+
/** Reap a seat {@link orphanedResumeSeat} found, then wait for its principal to leave presence so
|
|
5732
|
+
* the relaunch's readiness cannot match the dead seat's row. Returns the refusal when the reap
|
|
5733
|
+
* proves nothing or the principal stays live, which means another process holds it. */
|
|
5734
|
+
async reapOrphanedResumeSeat(principal, reference) {
|
|
5735
|
+
try {
|
|
5736
|
+
const evidence = await requireRuntimeReap(this.runtime, reference);
|
|
5737
|
+
console.error(`resume: reaped orphaned seat ${reference.kind}:${reference.id} of retained principal "${principal}" (${evidence.detail})`);
|
|
5738
|
+
}
|
|
5739
|
+
catch (e) {
|
|
5740
|
+
return `retained principal "${principal}" is already live and its recorded seat ${reference.kind}:${reference.id} could not be reaped: ${e.message}`;
|
|
5741
|
+
}
|
|
5742
|
+
const deadline = Date.now() + ORPHAN_PRESENCE_CLEAR_MS;
|
|
5743
|
+
while (this.ep.getRoster().some((p) => p.card.id === principal && p.status !== "offline")) {
|
|
5744
|
+
if (Date.now() >= deadline)
|
|
5745
|
+
return `retained principal "${principal}" is still live after its recorded seat ${reference.kind}:${reference.id} was reaped, so another process holds it and this runtime cannot authoritatively adopt it`;
|
|
5746
|
+
await new Promise((resolveWait) => setTimeout(resolveWait, 250));
|
|
5747
|
+
}
|
|
5748
|
+
return undefined;
|
|
5749
|
+
}
|
|
5433
5750
|
/** Validate/relaunch one retained inventory entry. Called only through resumePreserved so all
|
|
5434
5751
|
* records pass the same preflight before the first child is exposed. */
|
|
5435
5752
|
async resumePreservedAgent(entry, preflightOnly = false, batchReserved = false, prepared) {
|
|
@@ -5986,6 +6303,53 @@ export class Manager {
|
|
|
5986
6303
|
void this.cancelAgentGoal(a.name, graceful ? "graceful" : "terminate"); // M4: cancel a live spawn goal
|
|
5987
6304
|
return { ok: true, data: { name: a.name, stopped: true, graceful } };
|
|
5988
6305
|
}
|
|
6306
|
+
/** Retire one agent this manager launched under a delegated user intent, through the host
|
|
6307
|
+
* (SPEC 13.16). It sends the user's retirement intent for the UID of the running slot or its
|
|
6308
|
+
* retirement hold, and only on `retired: true` for that exact target stops the slot and frees the
|
|
6309
|
+
* name; every other answer leaves the slot running or the name held. It never calls the holder's
|
|
6310
|
+
* own prepare or requester. */
|
|
6311
|
+
async retireDelegatedAgent(name, intentId) {
|
|
6312
|
+
const execute = this.remoteAuthority?.executeDelegatedUserIntent;
|
|
6313
|
+
if (!execute)
|
|
6314
|
+
return { ok: false, error: `this manager has no host execution for a delegated user intent; "${name}" was not retired` };
|
|
6315
|
+
const release = this.beginLifecycle();
|
|
6316
|
+
if (!release)
|
|
6317
|
+
return { ok: false, error: this.maintenanceError() };
|
|
6318
|
+
try {
|
|
6319
|
+
const live = this.agents.get(name);
|
|
6320
|
+
const held = live ? undefined : this.retiring.get(name);
|
|
6321
|
+
const principal = live?.delegated ? parsePrincipalKey(live.id) : held?.delegated ? held : null;
|
|
6322
|
+
const lifecycleUid = live?.lifecycleUid ?? held?.lifecycleUid;
|
|
6323
|
+
if (!principal || !lifecycleUid)
|
|
6324
|
+
return { ok: false, error: `no agent named "${name}" was launched on this manager under a delegated user intent` };
|
|
6325
|
+
const target = { owner: principal.owner, actor: principal.actor, lifecycleUid };
|
|
6326
|
+
try {
|
|
6327
|
+
const result = await execute({ intentId, operation: "retire", target });
|
|
6328
|
+
if (result.operation !== "retire" || result.intentId !== intentId || result.opId !== retireOpId(target.lifecycleUid) ||
|
|
6329
|
+
result.target.owner !== target.owner || result.target.actor !== target.actor || result.target.lifecycleUid !== target.lifecycleUid)
|
|
6330
|
+
throw new Error(`the host answered delegated intent ${intentId} with another execution`);
|
|
6331
|
+
if (result.retired !== true)
|
|
6332
|
+
return { ok: false, error: `the host did not confirm the delegated retirement of "${name}" at ${target.lifecycleUid}; "${name}" is unchanged` };
|
|
6333
|
+
}
|
|
6334
|
+
catch (e) {
|
|
6335
|
+
const error = `the delegated retirement under intent ${intentId} did not complete (${rejectionText(e)})`;
|
|
6336
|
+
const hold = this.retiring.get(name);
|
|
6337
|
+
if (hold?.lifecycleUid === target.lifecycleUid)
|
|
6338
|
+
hold.lastError = error;
|
|
6339
|
+
return { ok: false, error: `${error}; "${name}" is unchanged` };
|
|
6340
|
+
}
|
|
6341
|
+
// The host's retirement closed this UID's provider handle before its barrier, so the stop only
|
|
6342
|
+
// reaps the slot, and its hold clears with the confirmation.
|
|
6343
|
+
const slot = this.agents.get(name);
|
|
6344
|
+
if (slot?.lifecycleUid === target.lifecycleUid)
|
|
6345
|
+
this.despawnAuthorized(slot, true, false, `delegated retirement ${intentId}`);
|
|
6346
|
+
this.confirmRetirement({ id: principalKey(target.owner, target.actor).key, name, lifecycleUid: target.lifecycleUid });
|
|
6347
|
+
return { ok: true, data: { name, lifecycleUid: target.lifecycleUid, retired: true } };
|
|
6348
|
+
}
|
|
6349
|
+
finally {
|
|
6350
|
+
release();
|
|
6351
|
+
}
|
|
6352
|
+
}
|
|
5989
6353
|
/** Resolve a v0.4 TARGET triple (owner, actor, lifecycleUid — broker-validated subject/body
|
|
5990
6354
|
* agreement, currency re-checked by the serve boundary's resolver) to the live managed agent it
|
|
5991
6355
|
* names. Static agents key `(DEV_OWNER, nkey)`; user-mode agents store the principal dot-form
|
|
@@ -6167,7 +6531,7 @@ export class Manager {
|
|
|
6167
6531
|
try {
|
|
6168
6532
|
report = await reconcileEndpointGate({
|
|
6169
6533
|
kv: authKv, space: this.space, endpoint: MANAGER_ENDPOINT, instanceId,
|
|
6170
|
-
probeHolder: makeManagerHolderLivenessProbe({ space: this.space, servers, auth, log }),
|
|
6534
|
+
probeHolder: makeManagerHolderLivenessProbe({ space: this.space, servers, auth, log, unreachableWaitMs: DELIVERY_ADMIN_BOOT_WAIT_MS }),
|
|
6171
6535
|
evictHolders: makeManagerEndpointHolderEvictor({ space: this.space, servers, auth, log }),
|
|
6172
6536
|
log,
|
|
6173
6537
|
recordsKv,
|
|
@@ -6442,7 +6806,7 @@ export class Manager {
|
|
|
6442
6806
|
// error text — a crash-restart never silently skips eviction (no-fallbacks).
|
|
6443
6807
|
const barrier = endpointRegistrationBarrier(authKv, this.space, {
|
|
6444
6808
|
endpoint: MANAGER_ENDPOINT, instanceId: iid, opId: registrationOpId,
|
|
6445
|
-
...(auth ? { evict:
|
|
6809
|
+
...(auth ? { evict: makeManagerEndpointHolderEvictor({ space: this.space, servers: this.servers ?? DEFAULT_SERVER, auth, log: (line) => console.error(line), unreachableWaitMs: DELIVERY_ADMIN_BOOT_WAIT_MS }) } : {}),
|
|
6446
6810
|
});
|
|
6447
6811
|
const spec = { endpoint: MANAGER_ENDPOINT, owner: DEV_OWNER, clusterDigests: [artifacts.closureDigest], protocol: { v: 1 } };
|
|
6448
6812
|
const { registrationRevision } = await registerServiceInstance(recordsKv, {
|