@cotal-ai/manager 0.68.0 → 0.70.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands.js +35 -62
- package/dist/commands.js.map +1 -1
- package/dist/endpoint-evict.d.ts +10 -1
- package/dist/endpoint-evict.d.ts.map +1 -1
- package/dist/endpoint-evict.js +31 -51
- package/dist/endpoint-evict.js.map +1 -1
- package/dist/holder-liveness.d.ts.map +1 -1
- package/dist/holder-liveness.js +11 -48
- package/dist/holder-liveness.js.map +1 -1
- package/dist/manager-service-contract.d.ts +7 -7
- package/dist/manager-service-contract.d.ts.map +1 -1
- package/dist/manager-service-contract.js +16 -22
- package/dist/manager-service-contract.js.map +1 -1
- package/dist/manager.d.ts +35 -23
- package/dist/manager.d.ts.map +1 -1
- package/dist/manager.js +193 -152
- package/dist/manager.js.map +1 -1
- package/dist/remote-authority.d.ts +15 -2
- package/dist/remote-authority.d.ts.map +1 -1
- package/dist/remote-authority.js +72 -53
- package/dist/remote-authority.js.map +1 -1
- package/dist/remote-register.d.ts +1 -1
- package/dist/remote-register.d.ts.map +1 -1
- package/dist/remote-register.js +1 -9
- package/dist/remote-register.js.map +1 -1
- package/dist/resume.d.ts.map +1 -1
- package/dist/resume.js +24 -7
- package/dist/resume.js.map +1 -1
- package/dist/roster.d.ts.map +1 -1
- package/dist/roster.js +3 -9
- package/dist/roster.js.map +1 -1
- package/dist/run-hosting.js +1 -1
- package/dist/run-hosting.js.map +1 -1
- package/dist/static-lifecycle.d.ts +9 -5
- package/dist/static-lifecycle.d.ts.map +1 -1
- package/dist/static-lifecycle.js +14 -9
- package/dist/static-lifecycle.js.map +1 -1
- package/package.json +12 -12
package/dist/manager.js
CHANGED
|
@@ -4,8 +4,9 @@ import { hostname } from "node:os";
|
|
|
4
4
|
import { credsAuthenticator } from "@nats-io/transport-node";
|
|
5
5
|
import { existsSync, lstatSync, mkdirSync, readFileSync, realpathSync, rmSync, statSync } from "node:fs";
|
|
6
6
|
import { isAbsolute, join, dirname, resolve } from "node:path";
|
|
7
|
-
import
|
|
8
|
-
import {
|
|
7
|
+
import Headless from "@xterm/headless";
|
|
8
|
+
import { CotalEndpoint, MANAGED_HANDOFF_KIND, accountFromCreds, credsClaims, DEFAULT_SERVER, DEV_OWNER, MANAGER_LEASE_RENEW_MS, STANDING_RENEWABLE_TTL_SEC, assertLifecycleToken, discardLaunchArtifacts, divergentSecretStoreNotice, parseDaemonStoreAnswer, parseSecretStoreIdentity, sameSecretStoreIdentity, agentFilePath, clearSpaceHistory, dialerFor, connectorServers, spawnEnvAllow, deprovisionAgent, DeprovisionError, isConcreteChannel, deprovisionTargetPrincipal, firstFreeName, spawnNameError, idFromCreds, inspectCredHealth, credsRenewalDelayMs, composeWirePersona, loadAgentFile, listPersonaCatalog, personaCatalogDescription, personaCatalogReadable, loadCotalConfig, modelPolicyRefusal, mintCreds, mintGeneration, mintAcceptedToken, openIssuedStore, acceptedBucket, issuedBucket, mintLifecycleUid, managedRetirementOpId, mkSecretDir, newIdentity, actionContext, parsePrincipalKey, principalKey, probeConnect, provisionAgent, provisionAgentDurables, registry, resolveAuthProvider, resolveReadAcl, bearerCommandFailure, saveAgentFile, subjectMatches, AUTH_ENDPOINT, EP_CMD_RETIRE_LIFECYCLE, controlServiceSubject, eventChannelPrincipal, resolveService, invokeCommand, } from "@cotal-ai/core";
|
|
9
|
+
import { agentAuthState, agentCredsDir, agentLifecycleSecretFilePaths, agentSecretFilePaths, agentSecretKeyForFile, authDir, claimManagerSiblingIdentities, connectorInstallHint, DELIVERY_CREDS_KIND, extensionConnectors, findCotalRoot, getSpaceAuth, hasUserAuthState, loadExtensionsManifest, loadManagerInstanceIdentity, loadMeshes, localTrustOfSpace, manifestExtensionNames, materializeFromManifest, materializeSecretToFile, MEMBERSHIP_RW_CREDS_KIND, mergeLaunchOptions, remintDaemonCreds, resolveAgentType, resolveOnPath, createManagerInstanceIdentity, spaceMaterialKey, SYSTEM_CREDS_FILES, userAuthStateDir, workspaceSecretStore, writeRenewalRecord } from "@cotal-ai/workspace";
|
|
9
10
|
import { createRuntime, isCustodialRuntime, requireRuntimeAdopt, requireRuntimeReap, RuntimeReapUnproven, } from "./runtime/index.js";
|
|
10
11
|
import { AttachEndpoint } from "./attach-endpoint.js";
|
|
11
12
|
import { DELIVERY_ADMIN_BOOT_WAIT_MS, isUnansweredDeliveryAdmin, makeManagerEndpointEvictionEvidence, makeManagerEndpointHolderEvictor, untilDeliveryAdminAnswers } from "./endpoint-evict.js";
|
|
@@ -32,7 +33,7 @@ import { ManagerSessionPlane, openSessionLedgerKv } from "./session/index.js";
|
|
|
32
33
|
// the register/authorize/serve seams, all driven over a scoped one-shot executor connection.
|
|
33
34
|
import { provisionEndpointGateOpen, endpointRegistrationBarrier, serveIssuanceGateKv, commitSiblingIssuance, markLedgerRowRevoked, epcredRowKey, epgateKey, parseEndpointGate, readEndpointGateGeneration, registerServingInstance, deregisterServiceInstance, serveEndpoint, bindGoal, createGoal, transitionGoal, commitGoalResult, settleGoalUncertain, readGoalResult, readGoalStatus, readGoalSpec, recordGoalIndex, readGoalIndex, clearGoalIndex, listGoalIndex, mintCheckpoint, resumeCheckpoint, readCheckpointSettle, readCheckpointSpec, expireCheckpoint, GOAL_TERMINAL_STATES, goalRefOf, goalProgressTopic, epeSubject, submissionFingerprint, EpEnvelopeError, lifecycleBlocked, lifecycleBlockedFrom, renderLifecycleBlocked, } from "@cotal-ai/core";
|
|
34
35
|
import { MANAGER_ENDPOINT, managerClusterArtifacts, managerCommandDefs, managerContractArtifactValues } from "./manager-service-contract.js";
|
|
35
|
-
import { staticLifecycleTransport, activateStaticLifecycle, runStaticTerminal, readStaticSlot, casStaticSlot, recordSlotCredential, appendStaticCredentialRow, planStaticSlotResume, observeStaticSlot, listStaticSlotObservations, renderStaticSlotObservation, StaticSlotReadError, STATIC_SLOT_READ_FAILED_DETAIL, } from "./static-lifecycle.js";
|
|
36
|
+
import { staticLifecycleTransport, activateStaticLifecycle, runStaticTerminal, readStaticSlot, casStaticSlot, recordSlotCredential, appendStaticCredentialRow, planStaticSlotResume, ownedBySibling, observeStaticSlot, listStaticSlotObservations, renderStaticSlotObservation, StaticSlotReadError, STATIC_SLOT_READ_FAILED_DETAIL, } from "./static-lifecycle.js";
|
|
36
37
|
/** Minimum slot lifetime for rate-flooring (P4c). A slot freed (by despawn OR natural exit/reap)
|
|
37
38
|
* before living this long leaves a cooling stamp that still counts toward the ceiling until it
|
|
38
39
|
* expires — so churn (spawn↔despawn or spawn↔fast-exit) can't outrun the concurrency bound. */
|
|
@@ -293,7 +294,8 @@ export class Manager {
|
|
|
293
294
|
/** The ONE secret store for every kind this manager touches (daemon-cred remint + agent kinds).
|
|
294
295
|
* See {@link ManagerOptions.secretStore}. */
|
|
295
296
|
secrets;
|
|
296
|
-
/** Named identity of {@link secrets}: the same coordinate the delivery daemon must reload from.
|
|
297
|
+
/** Named identity of {@link secrets}: the same coordinate the delivery daemon must reload from.
|
|
298
|
+
* Set at construction for an injected store; the workspace store is named on first challenge. */
|
|
297
299
|
secretStoreIdentity;
|
|
298
300
|
/** See {@link ManagerOptions.installedExtensions}. */
|
|
299
301
|
installedExtensions;
|
|
@@ -379,7 +381,7 @@ export class Manager {
|
|
|
379
381
|
* standing connection DISJOINT from the serve credential (Q2), scoped to exactly this endpoint's
|
|
380
382
|
* goal bind/terminal facts + goal-record writes ({@link goalWriterGrants}). Auth mode mints the
|
|
381
383
|
* `goal-writer` cred; an open mesh uses a bare connection (no credential system to mint from).
|
|
382
|
-
* `gate` (auth mode) is the own-issuance-gate READER for the
|
|
384
|
+
* `gate` (auth mode) is the own-issuance-gate READER for the own-gate currency belt — the
|
|
383
385
|
* manager reads its OWN `epgate.<e>.<iid>` epoch over this connection before a terminal commit
|
|
384
386
|
* and skips a superseded commit (the fast-fail belt paired with the (b) barrier-revoke fence). */
|
|
385
387
|
goalWriter;
|
|
@@ -387,7 +389,7 @@ export class Manager {
|
|
|
387
389
|
* per-run credential and connection. Absent under a remote authority, which mints no driver
|
|
388
390
|
* credentials, so the `run-*` family refuses there rather than connecting on a weaker identity. */
|
|
389
391
|
runHosting;
|
|
390
|
-
/**
|
|
392
|
+
/** The STABLE goal-writer identity (auth mode) — minted once at
|
|
391
393
|
* registration alongside the serve identity; a renewal re-mints the SAME nkey with a fresh
|
|
392
394
|
* bounded exp and re-stages its distinct credId into the §13.1 revocation family. The current
|
|
393
395
|
* goal-writer credential is minted INSIDE {@link registerManagerService}'s run block (fence
|
|
@@ -418,10 +420,10 @@ export class Manager {
|
|
|
418
420
|
sessionLedgerCreds;
|
|
419
421
|
/** P2 item 2: the acceptance replied for each in-flight goalId this incarnation accepted, so an
|
|
420
422
|
* idempotent same-goalId retry serves the IDENTICAL acceptance (same allocated name/triple) without
|
|
421
|
-
* a second spawn. Durable cross-incarnation reconstruction rides the
|
|
423
|
+
* a second spawn. Durable cross-incarnation reconstruction rides the goal index; here the
|
|
422
424
|
* live map covers same-incarnation retries, with the committed result fact as the fallback. */
|
|
423
425
|
goalAcceptances = new Map();
|
|
424
|
-
/**
|
|
426
|
+
/** The boot reconcile of the durable goal index runs ONCE at start (a
|
|
425
427
|
* fresh incarnation inherits the endpoint's accepted-but-unterminal goals from any predecessor).
|
|
426
428
|
* Spawn-as-action REFUSES to accept until it completes, so the sweep never races a live goal's
|
|
427
429
|
* acceptance (settling one mid-flight would steal its real terminal). */
|
|
@@ -604,9 +606,8 @@ export class Manager {
|
|
|
604
606
|
if (opts.remoteAuthority)
|
|
605
607
|
this.managerLifecycleUid = opts.remoteAuthority.lifecycleUid;
|
|
606
608
|
this.secrets = opts.secretStore ?? workspaceSecretStore(this.workspaceRoot);
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
: { kind: "fs", root: resolve(this.workspaceRoot) };
|
|
609
|
+
if (opts.secretStore)
|
|
610
|
+
this.secretStoreIdentity = injectedManagerStoreIdentity(opts.secretStore);
|
|
610
611
|
this.installedExtensions = opts.installedExtensions ?? false;
|
|
611
612
|
this.runtime = createRuntime(opts.runtime ?? "auto", `cotal-${this.space}`);
|
|
612
613
|
if (opts.pooled && isCustodialRuntime(this.runtime))
|
|
@@ -801,7 +802,7 @@ export class Manager {
|
|
|
801
802
|
// First mint on a fresh root is exclusive create (#1263). Two processes that both observe no
|
|
802
803
|
// file must not keep different in-memory ids: of N concurrent starts exactly one creates the
|
|
803
804
|
// file (`link` / O_EXCL) and the others adopt that identity before taking a lease, or refuse
|
|
804
|
-
// with
|
|
805
|
+
// with identity-record-create-lost. Atomic rename is not enough, because the loser
|
|
805
806
|
// of a replace would still serve under the id it minted.
|
|
806
807
|
{
|
|
807
808
|
if (this.remoteAuthority) {
|
|
@@ -1018,7 +1019,7 @@ export class Manager {
|
|
|
1018
1019
|
// rides the serve grant's epoch + the family-staged session-ledger cred), on its own standing
|
|
1019
1020
|
// connection disjoint from both the serve and goal-writer creds.
|
|
1020
1021
|
await this.startSessionPlane();
|
|
1021
|
-
//
|
|
1022
|
+
// Reconcile any accepted-but-unterminal goals inherited from a predecessor
|
|
1022
1023
|
// BEFORE spawn-as-action begins accepting (the goalReconcileDone gate) — a fresh incarnation
|
|
1023
1024
|
// never drops a goal a dead predecessor accepted. Never fatal; the gate opens either way.
|
|
1024
1025
|
await this.reconcileGoalIndex();
|
|
@@ -1140,6 +1141,9 @@ export class Manager {
|
|
|
1140
1141
|
`this space's delivery lease (${holder}), so its SecretStore is not the store the daemon ` +
|
|
1141
1142
|
`reloads from - nothing reminted`);
|
|
1142
1143
|
const daemon = answer.identity;
|
|
1144
|
+
// The workspace store records its id the first time it is named, so it is named here, where
|
|
1145
|
+
// only an auth manager reaches, rather than in a constructor every open-mode manager runs.
|
|
1146
|
+
this.secretStoreIdentity ??= parseSecretStoreIdentity(this.secrets.identity);
|
|
1143
1147
|
if (!sameSecretStoreIdentity(this.secretStoreIdentity, daemon)) {
|
|
1144
1148
|
console.error(`! ${divergentSecretStoreNotice(this.secretStoreIdentity, daemon)}`);
|
|
1145
1149
|
return "divergent";
|
|
@@ -1407,7 +1411,7 @@ export class Manager {
|
|
|
1407
1411
|
console.error(`! endpoint-serve renewal: ${e.message} - the manager's service endpoint dies loud at this cred's expiry unless it is re-registered`);
|
|
1408
1412
|
}
|
|
1409
1413
|
}
|
|
1410
|
-
//
|
|
1414
|
+
// The manager is also the goal-writer's renewal owner — re-mint the SAME
|
|
1411
1415
|
// goal-writer nkey with a fresh bounded exp AND re-stage its new credId into the §13.1 family,
|
|
1412
1416
|
// through the scoped executor (never the standing seed). Without this the standing goal-writer
|
|
1413
1417
|
// connection dies at its TTL and spawn-as-action stops accepting until a restart. The
|
|
@@ -1687,7 +1691,7 @@ export class Manager {
|
|
|
1687
1691
|
// actually saw.
|
|
1688
1692
|
const backfillFloor = await this.chatFrontierForPreservation();
|
|
1689
1693
|
const inventory = this.preservationInventory ?? {
|
|
1690
|
-
version: "cotal-manager-resume/
|
|
1694
|
+
version: "cotal-manager-resume/v2",
|
|
1691
1695
|
space: this.space,
|
|
1692
1696
|
createdAt: new Date().toISOString(),
|
|
1693
1697
|
agents: [...this.agents.values()].map((a) => this.resumeEntry(a, backfillFloor)),
|
|
@@ -2150,17 +2154,17 @@ export class Manager {
|
|
|
2150
2154
|
await step("release renewal lease", () => this.ep.releaseDaemonRenewalLease());
|
|
2151
2155
|
}
|
|
2152
2156
|
// Capture BEFORE the serve loop is torn down: `stopServiceServe` clears the state, and what is
|
|
2153
|
-
// being asked here is "did this process
|
|
2157
|
+
// being asked here is "which registration did this process write", which only the state
|
|
2154
2158
|
// before teardown can answer. Deregistration runs AFTER the serve loop has drained, so no
|
|
2155
2159
|
// in-flight command can write a status back onto the record it just removed.
|
|
2156
|
-
const
|
|
2160
|
+
const registrationRevision = this.serviceServe?.grant.registrationRevision;
|
|
2157
2161
|
await step("stop service serve", () => this.stopServiceServe());
|
|
2158
2162
|
// The drives AFTER the serve loop (no new `run-start` can land) and BEFORE deregistration: a
|
|
2159
2163
|
// released run's status write is the last thing this incarnation says about it.
|
|
2160
2164
|
await step("stop run hosting", () => this.runHosting?.stop());
|
|
2161
2165
|
this.runHosting = undefined;
|
|
2162
|
-
if (
|
|
2163
|
-
await step("deregister service", () => this.deregisterServiceOnStop());
|
|
2166
|
+
if (registrationRevision !== undefined && !seatFailure)
|
|
2167
|
+
await step("deregister service", () => this.deregisterServiceOnStop(registrationRevision));
|
|
2164
2168
|
await step("stop goal writer", () => this.stopGoalWriter());
|
|
2165
2169
|
await step("stop session plane", () => this.stopSessionPlane());
|
|
2166
2170
|
await step("stop endpoint", () => this.ep.stop());
|
|
@@ -2206,8 +2210,8 @@ export class Manager {
|
|
|
2206
2210
|
* deadline — on every `cotal ps`, `stop` and `attach` in that space, for good. A manager that is
|
|
2207
2211
|
* shutting down is the one participant that KNOWS it is going away, so it says so.
|
|
2208
2212
|
*
|
|
2209
|
-
* The delete is
|
|
2210
|
-
* persists the SAME instanceId and has already re-registered loses nothing to a slow stop here.
|
|
2213
|
+
* The delete is pinned to the `registrationRevision` this process registered at, so a successor
|
|
2214
|
+
* that persists the SAME instanceId and has already re-registered loses nothing to a slow stop here.
|
|
2211
2215
|
*
|
|
2212
2216
|
* Best-effort and LOUD, matching every other teardown step: a broker that is already gone must not
|
|
2213
2217
|
* turn a stop into a failure, but a registration that survives a stop is the exact defect this
|
|
@@ -2215,11 +2219,12 @@ export class Manager {
|
|
|
2215
2219
|
* rather than swallowed. It is not fatal because a crash leaves the same state and the operator
|
|
2216
2220
|
* verb handles both.
|
|
2217
2221
|
*/
|
|
2218
|
-
async deregisterServiceOnStop() {
|
|
2222
|
+
async deregisterServiceOnStop(registrationRevision) {
|
|
2219
2223
|
const iid = this.managerInstanceId;
|
|
2220
2224
|
const dereg = ({ recordsKv, authKv }) => deregisterServiceInstance(recordsKv, {
|
|
2221
2225
|
endpoint: MANAGER_ENDPOINT,
|
|
2222
2226
|
instanceId: iid,
|
|
2227
|
+
registrationRevision,
|
|
2223
2228
|
observeGeneration: async () => {
|
|
2224
2229
|
const key = epgateKey(MANAGER_ENDPOINT, iid);
|
|
2225
2230
|
const entry = await authKv.get(key);
|
|
@@ -2235,7 +2240,7 @@ export class Manager {
|
|
|
2235
2240
|
if (outcome.removed)
|
|
2236
2241
|
console.error(`✓ deregistered manager instance ${iid} from the ${MANAGER_ENDPOINT} service registry (spec revision ${outcome.specRevision})`);
|
|
2237
2242
|
else if (outcome.reason === "superseded")
|
|
2238
|
-
console.error(`! manager instance ${iid} was not deregistered:
|
|
2243
|
+
console.error(`! manager instance ${iid} was not deregistered: another incarnation has registered it since this process did - leaving it alone`);
|
|
2239
2244
|
else if (outcome.reason === "registration-in-flight")
|
|
2240
2245
|
console.error(`! manager instance ${iid} was not deregistered: a registration is still in flight (governance slot held at the live gate generation) - leaving it alone`);
|
|
2241
2246
|
// `absent` is silent: there was nothing registered to remove, which is not news at shutdown.
|
|
@@ -3361,16 +3366,22 @@ export class Manager {
|
|
|
3361
3366
|
// A failed removal joins the refusal instead of throwing, so every removal is attempted and
|
|
3362
3367
|
// the refusal still reaches the caller.
|
|
3363
3368
|
const unshredded = [];
|
|
3364
|
-
for (const family of files.actorToken === staged.actorToken ? [staged] : [staged, files])
|
|
3365
|
-
|
|
3366
|
-
await attemptCleanup(unshredded, `secret ${path}`, () => secrets.delete(agentSecretKeyForFile(path, this.space)));
|
|
3367
|
-
for (const path of [family.actorToken, family.sentinelCreds, family.health])
|
|
3368
|
-
await attemptCleanup(unshredded, `file ${path}`, () => rmSync(path, { force: true }));
|
|
3369
|
-
}
|
|
3369
|
+
for (const family of files.actorToken === staged.actorToken ? [staged] : [staged, files])
|
|
3370
|
+
await this.shredUserSecrets(unshredded, family);
|
|
3370
3371
|
const leftover = unshredded.length ? `; cleanup failed: ${unshredded.join("; ")}` : "";
|
|
3371
3372
|
return { error: `agent auth preflight failed for "${name}": ${rejectionText(e)}${leftover}` };
|
|
3372
3373
|
}
|
|
3373
3374
|
}
|
|
3375
|
+
/** Shred a user agent's secret family from the store and from disk. Every removal is attempted,
|
|
3376
|
+
* and each failure joins `unshredded`, so one fault cannot leave the rest of the family behind. */
|
|
3377
|
+
async shredUserSecrets(unshredded, family) {
|
|
3378
|
+
for (const path of [family.actorToken, family.sentinelCreds])
|
|
3379
|
+
if (path)
|
|
3380
|
+
await attemptCleanup(unshredded, `secret ${path}`, () => this.secrets.delete(agentSecretKeyForFile(path, this.space)));
|
|
3381
|
+
for (const path of [family.actorToken, family.sentinelCreds, family.health])
|
|
3382
|
+
if (path)
|
|
3383
|
+
await attemptCleanup(unshredded, `file ${path}`, () => rmSync(path, { force: true }));
|
|
3384
|
+
}
|
|
3374
3385
|
/** This manager's registry record for its space, which pins the broker and IdP a delegated seat's
|
|
3375
3386
|
* handoff carries. It must pin the exchange the host enrolls against, or the child would register
|
|
3376
3387
|
* a mesh whose bearers come from somewhere else. */
|
|
@@ -3631,12 +3642,7 @@ export class Manager {
|
|
|
3631
3642
|
// the agent's standing mint authority, so delete it (next exchange refused, next connect
|
|
3632
3643
|
// denied) and shred the secret/sentinel/health files. A copied actor token dies here; a
|
|
3633
3644
|
// still-LIVE connection ends at its bearer-bound JWT expiry (≤ the agent TTL).
|
|
3634
|
-
|
|
3635
|
-
if (path)
|
|
3636
|
-
await attemptCleanup(unshredded, `secret ${path}`, () => secrets.delete(agentSecretKeyForFile(path, this.space)));
|
|
3637
|
-
for (const path of [files.actorToken, files.sentinelCreds, files.health])
|
|
3638
|
-
if (path)
|
|
3639
|
-
await attemptCleanup(unshredded, `file ${path}`, () => rmSync(path, { force: true }));
|
|
3645
|
+
await this.shredUserSecrets(unshredded, files);
|
|
3640
3646
|
// The ledger row IS the agent's STANDING mint authority (a different store from the auth-plane
|
|
3641
3647
|
// cred ledger the rail retirement covers): while it lives, a copied actor token can still mint a
|
|
3642
3648
|
// fresh connect credential. So a FAILED revoke must NOT be swallowed into a clean terminal (INT-2):
|
|
@@ -4196,7 +4202,7 @@ export class Manager {
|
|
|
4196
4202
|
restart.armed = false;
|
|
4197
4203
|
let tail = "";
|
|
4198
4204
|
try {
|
|
4199
|
-
tail = this.tail(
|
|
4205
|
+
tail = await this.tail((replacement ?? a.handle).attach());
|
|
4200
4206
|
}
|
|
4201
4207
|
catch { /* runtime has no readable tail */ }
|
|
4202
4208
|
console.error(`! ${a.name}: ${supervised ? "supervised restart" : "Pi session recovery"} failed: ${error.message}${tail ? ` - last output: ${tail}` : ""} - retiring the managed seat`);
|
|
@@ -4275,7 +4281,7 @@ export class Manager {
|
|
|
4275
4281
|
nameInUse(name, live = this.liveRosterNames()) {
|
|
4276
4282
|
return this.agents.has(name) || this.reserved.has(name) || this.retiring.has(name) || live.has(name);
|
|
4277
4283
|
}
|
|
4278
|
-
/** First free name in the series `base`, `
|
|
4284
|
+
/** First free name in the series `base`, `base_2`, `base_3`, … — checked against live slots,
|
|
4279
4285
|
* in-flight (reserved) slots, names held pending retirement, AND the live mesh roster. The
|
|
4280
4286
|
* roster check covers occupants this manager does not manage (a foreground `cotal spawn`, a
|
|
4281
4287
|
* connector session, another manager's agent): allocating their name would mint a sibling the
|
|
@@ -4356,11 +4362,12 @@ export class Manager {
|
|
|
4356
4362
|
throw new Error(`${flag}: expected an array of strings`);
|
|
4357
4363
|
return v;
|
|
4358
4364
|
};
|
|
4359
|
-
let subscribe, allowSubscribe, allowPublish;
|
|
4365
|
+
let subscribe, allowSubscribe, allowPublish, shareTools;
|
|
4360
4366
|
try {
|
|
4361
4367
|
subscribe = strList(args.subscribe, "subscribe");
|
|
4362
4368
|
allowSubscribe = strList(args.allowSubscribe, "allowSubscribe");
|
|
4363
4369
|
allowPublish = strList(args.allowPublish, "allowPublish");
|
|
4370
|
+
shareTools = strList(args.shareTools, "shareTools");
|
|
4364
4371
|
}
|
|
4365
4372
|
catch (e) {
|
|
4366
4373
|
return Promise.resolve({ ok: false, error: e.message });
|
|
@@ -4410,7 +4417,7 @@ export class Manager {
|
|
|
4410
4417
|
subscribe,
|
|
4411
4418
|
allowSubscribe,
|
|
4412
4419
|
allowPublish,
|
|
4413
|
-
shareTools
|
|
4420
|
+
shareTools,
|
|
4414
4421
|
...(supervise !== undefined ? { supervise } : {}),
|
|
4415
4422
|
route,
|
|
4416
4423
|
}, caller, hooks);
|
|
@@ -4722,6 +4729,9 @@ export class Manager {
|
|
|
4722
4729
|
}
|
|
4723
4730
|
}
|
|
4724
4731
|
async startAgentActive(opts, spawner, hooks) {
|
|
4732
|
+
// The caller keeps its array, so the selection is copied before the first await: a later write
|
|
4733
|
+
// to it cannot change what this launch shares or what preservation retains.
|
|
4734
|
+
const shareTools = opts.shareTools && [...opts.shareTools];
|
|
4725
4735
|
if (opts.delegatedIntent) {
|
|
4726
4736
|
if (!this.remoteAuthority?.executeDelegatedUserIntent)
|
|
4727
4737
|
return { ok: false, error: `"${opts.name}" names a delegated user intent, and this manager has no host execution for one` };
|
|
@@ -4778,10 +4788,7 @@ export class Manager {
|
|
|
4778
4788
|
return { ok: false, error: e.message };
|
|
4779
4789
|
}
|
|
4780
4790
|
}
|
|
4781
|
-
|
|
4782
|
-
// this manager's COTAL_DEFAULT_AGENT > DEFAULT_CONNECTOR. Both environment values are defaults,
|
|
4783
|
-
// never overrides: neither may beat a deliberate per-persona pin.
|
|
4784
|
-
const agent = opts.agent ?? def?.agent ?? opts.defaultAgent ?? defaultAgentType(DEFAULT_CONNECTOR);
|
|
4791
|
+
const agent = resolveAgentType({ flag: opts.agent, pin: def?.agent, callerDefault: opts.defaultAgent });
|
|
4785
4792
|
// Materialize the requested connector up front — the ONE async step in the spawn path (a lazy
|
|
4786
4793
|
// `cotal ext` manifest import on the published binary). It runs BEFORE the capacity/reserve span
|
|
4787
4794
|
// below so that span stays fully SYNCHRONOUS and atomic. On the manifest binary this also honors a
|
|
@@ -4862,7 +4869,7 @@ export class Manager {
|
|
|
4862
4869
|
return { ok: false, error: `${delegatedBy}, so it cannot reopen a session held on this host (continuity: exact)` };
|
|
4863
4870
|
if (typeof opts.cwd === "string" && opts.cwd !== "")
|
|
4864
4871
|
return { ok: false, error: `${delegatedBy}, so it cannot run in a directory on this host's filesystem (cwd); the seat runs in its runtime resource's own directory` };
|
|
4865
|
-
const shared = Object.keys(connectorServers(loadCotalConfig(this.workspaceRoot), agent,
|
|
4872
|
+
const shared = Object.keys(connectorServers(loadCotalConfig(this.workspaceRoot), agent, shareTools));
|
|
4866
4873
|
if (shared.length)
|
|
4867
4874
|
return { ok: false, error: `${delegatedBy}, so it cannot share MCP servers that run on this host (${shared.join(", ")}); pass --share-tools none` };
|
|
4868
4875
|
}
|
|
@@ -4883,7 +4890,7 @@ export class Manager {
|
|
|
4883
4890
|
// Resolve the launch profile: IDENTITY (free-form `name:`) + role + read/post ACL + capabilities
|
|
4884
4891
|
// + model/variant. Either from a fully-resolved manifest launch object (`opts.resolved`, whose `config`
|
|
4885
4892
|
// is a materialized transient persona — the file is NOT the access authority), or from the
|
|
4886
|
-
// persona file. The number rides the IDENTITY (socrates →
|
|
4893
|
+
// persona file. The number rides the IDENTITY (socrates → socrates_2), not the file ref — a
|
|
4887
4894
|
// redelivered identical spawn yields a fresh numbered agent (MAX_AGENTS bounds the blast radius).
|
|
4888
4895
|
let identityName;
|
|
4889
4896
|
let role;
|
|
@@ -4898,13 +4905,13 @@ export class Manager {
|
|
|
4898
4905
|
if (opts.resolved) {
|
|
4899
4906
|
// A manifest launch is the access + identity authority: imperative overrides arriving
|
|
4900
4907
|
// alongside `resolved` are a caller contract error, not something to merge (no fallbacks).
|
|
4901
|
-
if (opts.subscribe || opts.allowSubscribe || opts.allowPublish || opts.prompt ||
|
|
4908
|
+
if (opts.subscribe || opts.allowSubscribe || opts.allowPublish || opts.prompt || shareTools || opts.identity)
|
|
4902
4909
|
return { ok: false, error: "a manifest launch (resolved) rejects imperative overrides (identity/subscribe/allow*/prompt/shareTools)" };
|
|
4903
4910
|
const r = opts.resolved;
|
|
4904
4911
|
identityName = r.name;
|
|
4905
4912
|
role = opts.role ?? r.role;
|
|
4906
4913
|
subscribe = r.subscribe;
|
|
4907
|
-
allowSubscribe = r.allowSubscribe
|
|
4914
|
+
allowSubscribe = r.allowSubscribe;
|
|
4908
4915
|
allowPublish = r.allowPublish;
|
|
4909
4916
|
capabilities = r.capabilities;
|
|
4910
4917
|
model = opts.model ?? r.model;
|
|
@@ -4926,9 +4933,7 @@ export class Manager {
|
|
|
4926
4933
|
// paths of the merged grammar can't diverge. One source feeds BOTH the minted creds and the
|
|
4927
4934
|
// connector env below.
|
|
4928
4935
|
subscribe = opts.subscribe ?? def.subscribe;
|
|
4929
|
-
|
|
4930
|
-
// boundary); runtime durable joins are re-authorized against the committed ACL by the daemon.
|
|
4931
|
-
allowSubscribe = opts.allowSubscribe ?? def.allowSubscribe ?? subscribe ?? [];
|
|
4936
|
+
allowSubscribe = opts.allowSubscribe ?? def.allowSubscribe ?? [];
|
|
4932
4937
|
allowPublish = opts.allowPublish ?? def.allowPublish;
|
|
4933
4938
|
capabilities = def.capabilities;
|
|
4934
4939
|
// #651: fold the persona's model into the launch record, mirroring the variant line below
|
|
@@ -4939,6 +4944,14 @@ export class Manager {
|
|
|
4939
4944
|
variant = opts.variant ?? def.variant;
|
|
4940
4945
|
launchOptions = mergeLaunchOptions(def.launchOptions, opts.launchOptions);
|
|
4941
4946
|
}
|
|
4947
|
+
// Resolved once for both branches, because this one list is granted, minted, recorded on the
|
|
4948
|
+
// launch and forwarded to the session, and each of those must read the list the provisioner does.
|
|
4949
|
+
try {
|
|
4950
|
+
allowSubscribe = resolveReadAcl(subscribe ?? [], allowSubscribe);
|
|
4951
|
+
}
|
|
4952
|
+
catch (e) {
|
|
4953
|
+
return { ok: false, error: opts.resolved ? `launch agent: ${e.message}` : `persona ${configPath}: ${e.message}` };
|
|
4954
|
+
}
|
|
4942
4955
|
// #651: an empty or whitespace-only model string is not a pin. Coerce it to undefined here, at
|
|
4943
4956
|
// the single point every path (persona, manifest, imperative) has resolved `model`, so it
|
|
4944
4957
|
// serializes ABSENT rather than present-but-empty (`"model": ""`), which a key-presence consumer
|
|
@@ -5026,6 +5039,30 @@ export class Manager {
|
|
|
5026
5039
|
const unportable = this.runtime.spawnDelegated ? Object.keys(launchOptions ?? {}).filter((k) => typeof launchOptions[k] !== "string") : [];
|
|
5027
5040
|
if (unportable.length)
|
|
5028
5041
|
return { ok: false, error: `runtime "${this.runtime.kind}" starts seats outside this host, so it cannot pass non-string launch options (${unportable.join(", ")})` };
|
|
5042
|
+
// The AG-UI event plane is on unless the launch explicitly opted out. Refused HERE, before
|
|
5043
|
+
// anything is minted: a connector that cannot
|
|
5044
|
+
// emit must fail before provisioning rather than after, exactly as an unsupported `resume` does.
|
|
5045
|
+
// The GRANT itself cannot be derived yet. It is keyed on the agent's PRINCIPAL, and in user mode
|
|
5046
|
+
// the principal's owner is resolved further down, so deriving it from anything in scope here
|
|
5047
|
+
// would mean guessing at the identity the child will actually connect as. It is added at the
|
|
5048
|
+
// accept seam below, where the allocated triple exists.
|
|
5049
|
+
if (this.eventsRequired && opts.events === false)
|
|
5050
|
+
return { ok: false, error: `space "${this.space}" requires the event plane by registration policy; --no-events (events: false on the start op) is not allowed` };
|
|
5051
|
+
const events = this.eventsRequired || opts.events !== false;
|
|
5052
|
+
if (events && !connector.eventChannel)
|
|
5053
|
+
return { ok: false, error: this.eventsRequired
|
|
5054
|
+
? `space "${this.space}" requires the event plane by registration policy, but connector "${connector.name}" does not publish one`
|
|
5055
|
+
: `connector "${connector.name}" does not publish an AG-UI event plane; pass --no-events (events: false on the start op) to launch it without one` };
|
|
5056
|
+
// F2 (Unit B): a STATIC managed spawn REFUSES endpoint capabilities, fail-closed IN CODE (not
|
|
5057
|
+
// a doc note): the static terminal has no obligation-drain/frontier steps yet, so an accepted-
|
|
5058
|
+
// but-uncompleted endpoint obligation could execute AFTER its uid is declared retired. The
|
|
5059
|
+
// refusal sits at spawn-accept, before any provisioning, over the same records a persona or
|
|
5060
|
+
// manifest self-claim would ride in on — capabilities cannot slip past it into the grant path.
|
|
5061
|
+
if (this.auth && !this.userMode) {
|
|
5062
|
+
const claims = [opts, (opts.resolved ?? {})];
|
|
5063
|
+
if (claims.some((c) => c.endpointCapabilities !== undefined))
|
|
5064
|
+
return { ok: false, error: "a static managed spawn refuses endpointCapabilities (Unit B F2): the static lifecycle terminal carries no obligation-drain/frontier steps, so endpoint-rail grants are not containable in static mode" };
|
|
5065
|
+
}
|
|
5029
5066
|
// #4 A4 (panel): the roster the allocation consults must reflect the initial presence snapshot,
|
|
5030
5067
|
// or a spawn immediately after manager boot races an already-live unmanaged peer and re-opens the
|
|
5031
5068
|
// very collision black-hole this closes. Await the snapshot (bounded internally, fail-safe on an
|
|
@@ -5035,10 +5072,10 @@ export class Manager {
|
|
|
5035
5072
|
await this.ep.waitForPresenceSnapshot();
|
|
5036
5073
|
// M6 (P2 item 2 spawn-as-action): a HARD-PINNED name — an imperative `--name`/identity override
|
|
5037
5074
|
// or a manifest-declared name (opts.resolved) — that collides with a LIVE/provisioning/reserved
|
|
5038
|
-
// incarnation REFUSES loud at accept, BEFORE any reserve/mint/bind (pin 1), never a silent
|
|
5075
|
+
// incarnation REFUSES loud at accept, BEFORE any reserve/mint/bind (pin 1), never a silent `_2`
|
|
5039
5076
|
// suffix (so an address-by-triple caller's pinned name can't be re-pointed). A PERSONA-DERIVED
|
|
5040
5077
|
// base name (no pin) keeps uniqueName's collision numbering, so multi-peer `spawn reviewer` twice
|
|
5041
|
-
// still yields reviewer +
|
|
5078
|
+
// still yields reviewer + reviewer_2. The retiring-hold refuse (~2472) is orthogonal and already fired.
|
|
5042
5079
|
const hardPinned = opts.identity !== undefined || opts.resolved !== undefined;
|
|
5043
5080
|
let name;
|
|
5044
5081
|
if (hardPinned) {
|
|
@@ -5054,42 +5091,6 @@ export class Manager {
|
|
|
5054
5091
|
else {
|
|
5055
5092
|
name = this.uniqueName(identityName);
|
|
5056
5093
|
}
|
|
5057
|
-
this.reserved.add(name);
|
|
5058
|
-
// The AG-UI event plane is on unless the launch explicitly opted out. Refused HERE, before
|
|
5059
|
-
// anything is minted: a connector that cannot
|
|
5060
|
-
// emit must fail before provisioning rather than after, exactly as an unsupported `resume` does.
|
|
5061
|
-
// The GRANT itself cannot be derived yet. It is keyed on the agent's PRINCIPAL, and in user mode
|
|
5062
|
-
// the principal's owner is resolved further down, so deriving it from anything in scope here
|
|
5063
|
-
// would mean guessing at the identity the child will actually connect as. It is added at the
|
|
5064
|
-
// accept seam below, where the allocated triple exists.
|
|
5065
|
-
if (this.eventsRequired && opts.events === false) {
|
|
5066
|
-
this.reserved.delete(name);
|
|
5067
|
-
return { ok: false, error: `space "${this.space}" requires the event plane by registration policy; --no-events (events: false on the start op) is not allowed` };
|
|
5068
|
-
}
|
|
5069
|
-
const events = this.eventsRequired || opts.events !== false;
|
|
5070
|
-
if (events && !connector.eventChannel) {
|
|
5071
|
-
// Release the just-reserved name on this fail-fast path. A leaked reserve is silent: it costs
|
|
5072
|
-
// the next spawn of this persona its un-suffixed name and nothing reports why.
|
|
5073
|
-
this.reserved.delete(name);
|
|
5074
|
-
return { ok: false, error: this.eventsRequired
|
|
5075
|
-
? `space "${this.space}" requires the event plane by registration policy, but connector "${connector.name}" does not publish one`
|
|
5076
|
-
: `connector "${connector.name}" does not publish an AG-UI event plane; pass --no-events (events: false on the start op) to launch it without one` };
|
|
5077
|
-
}
|
|
5078
|
-
// F2 (Unit B): a STATIC managed spawn REFUSES endpoint capabilities, fail-closed IN CODE (not
|
|
5079
|
-
// a doc note): the static terminal has no obligation-drain/frontier steps yet, so an accepted-
|
|
5080
|
-
// but-uncompleted endpoint obligation could execute AFTER its uid is declared retired. The
|
|
5081
|
-
// refusal sits at spawn-accept, before any provisioning, over the same records a persona or
|
|
5082
|
-
// manifest self-claim would ride in on — capabilities cannot slip past it into the grant path.
|
|
5083
|
-
if (this.auth && !this.userMode) {
|
|
5084
|
-
const claims = [opts, (opts.resolved ?? {})];
|
|
5085
|
-
if (claims.some((c) => c.endpointCapabilities !== undefined)) {
|
|
5086
|
-
this.reserved.delete(name);
|
|
5087
|
-
return { ok: false, error: "a static managed spawn refuses endpointCapabilities (Unit B F2): the static lifecycle terminal carries no obligation-drain/frontier steps, so endpoint-rail grants are not containable in static mode" };
|
|
5088
|
-
}
|
|
5089
|
-
}
|
|
5090
|
-
// Set once the agent's creds + durables are minted; cleared the moment a live slot takes ownership
|
|
5091
|
-
// (`agents.set`, after which freeSlot deprovisions on exit). If it survives to `finally`, the spawn
|
|
5092
|
-
// threw AFTER minting (buildLaunch / runtime.spawn) — tear the orphan down so no footprint leaks (#159 B).
|
|
5093
5094
|
// Set once the agent's footprint (durables + creds, or the user-mode grant + secret files)
|
|
5094
5095
|
// exists; cleared when a live slot takes ownership. If it survives to `finally`, the spawn threw
|
|
5095
5096
|
// AFTER provisioning (buildLaunch / runtime.spawn) — the orphan-rollback tears it down. Carries
|
|
@@ -5103,6 +5104,9 @@ export class Manager {
|
|
|
5103
5104
|
let provisioned;
|
|
5104
5105
|
// A carried resume's seat-private home, removed by the finally unless a live seat took it.
|
|
5105
5106
|
let seatHome;
|
|
5107
|
+
// Taken right before the try so its finally is the only release; a refusal belongs above the
|
|
5108
|
+
// allocation, where there is no name to give back.
|
|
5109
|
+
this.reserved.add(name);
|
|
5106
5110
|
try {
|
|
5107
5111
|
// A stable nkey identity assigned at spawn: the public key is the agent's card.id (threaded via
|
|
5108
5112
|
// COTAL_ID); the seed is retained to mint matching creds later.
|
|
@@ -5125,7 +5129,8 @@ export class Manager {
|
|
|
5125
5129
|
// been provisioned yet — the action serve path binds the goal + replies the acceptance HERE. A
|
|
5126
5130
|
// throw (bind conflict / duplicate goalId) aborts the spawn before provisioning: the catch below
|
|
5127
5131
|
// returns the failure and the finally releases the reserve, so a refused accept leaves zero
|
|
5128
|
-
// footprint (pin 1).
|
|
5132
|
+
// footprint (pin 1). The hosted enrollment arm is the exception, below. Blocking callers
|
|
5133
|
+
// (roster boot) pass no hooks and this is a no-op.
|
|
5129
5134
|
// The ALLOCATED agent's addressing triple (the acceptance floor names what was actually
|
|
5130
5135
|
// allocated, never the requested-but-unallocated name). Static/open key on DEV_OWNER + the
|
|
5131
5136
|
// freshly-minted nkey; user mode keys on the derived owner (opts.owner, else a u_-owner spawner)
|
|
@@ -5178,7 +5183,11 @@ export class Manager {
|
|
|
5178
5183
|
if (events)
|
|
5179
5184
|
allowPublish = [...(allowPublish ?? []), connector.eventChannel({ owner: agentTriple.owner, actor: agentTriple.actor })];
|
|
5180
5185
|
await hooks?.onReadinessWindow?.(readinessTimeoutMs);
|
|
5181
|
-
|
|
5186
|
+
// The hosted enrollment arm accepts only once the host has answered: the floor names the uid
|
|
5187
|
+
// the agent runs at, and before the host picks it there is no such uid to name.
|
|
5188
|
+
const hostedEnrollment = this.userMode && (opts.delegatedIntent !== undefined || this.remoteAuthority?.enrollManagedAgent !== undefined);
|
|
5189
|
+
if (!hostedEnrollment)
|
|
5190
|
+
await hooks?.onAccepted?.({ name, identity, lifecycleUid, agentTriple });
|
|
5182
5191
|
// In auth mode, mint the agent's creds from the space signing key and write them where the
|
|
5183
5192
|
// spawned session reads them (COTAL_CREDS path). Open mesh → no creds. Scope = the resolved
|
|
5184
5193
|
// subscribe/allowSubscribe (read) + allowPublish (post, default-deny).
|
|
@@ -5208,10 +5217,8 @@ export class Manager {
|
|
|
5208
5217
|
}
|
|
5209
5218
|
: undefined,
|
|
5210
5219
|
});
|
|
5211
|
-
if ("error" in prep)
|
|
5212
|
-
this.reserved.delete(name);
|
|
5220
|
+
if ("error" in prep)
|
|
5213
5221
|
return { ok: false, error: prep.error };
|
|
5214
|
-
}
|
|
5215
5222
|
// THE HOST'S UID WINS (#1972). On the local arm this is the value passed in, unchanged. On
|
|
5216
5223
|
// the hosted enrollment arm the host selected it, and from here every lifecycle-keyed thing
|
|
5217
5224
|
// this spawn records — the slot, the launch, the managed row, the teardown credential — must
|
|
@@ -5222,6 +5229,20 @@ export class Manager {
|
|
|
5222
5229
|
userOwner = prep.owner;
|
|
5223
5230
|
enrolled = prep.enrolled;
|
|
5224
5231
|
provisioned = { id: principalKey(prep.owner, name).key, name, lifecycleUid, userOwner: prep.owner, ...(opts.delegatedIntent ? { delegated: true } : {}), secretPaths: prep.files, ...(custody ? { runtime: custody } : {}) };
|
|
5232
|
+
if (hostedEnrollment) {
|
|
5233
|
+
try {
|
|
5234
|
+
await hooks?.onAccepted?.({ name, identity, lifecycleUid, agentTriple: { owner: prep.owner, actor: name, uid: lifecycleUid } });
|
|
5235
|
+
}
|
|
5236
|
+
catch (e) {
|
|
5237
|
+
// A refused accept rolls the enrollment back: the host's grant through `provisioned`, and
|
|
5238
|
+
// this participant's secret family here, because the hosted retirement leaves it on disk.
|
|
5239
|
+
const unshredded = [];
|
|
5240
|
+
await this.shredUserSecrets(unshredded, prep.files);
|
|
5241
|
+
if (unshredded.length)
|
|
5242
|
+
throw new Error(`${rejectionText(e)}; cleanup failed: ${unshredded.join("; ")}`, { cause: e });
|
|
5243
|
+
throw e;
|
|
5244
|
+
}
|
|
5245
|
+
}
|
|
5225
5246
|
}
|
|
5226
5247
|
else if (this.auth) {
|
|
5227
5248
|
// Unit B (§13.1): reserve + activate this incarnation's DURABLE identity BEFORE any
|
|
@@ -5273,7 +5294,7 @@ export class Manager {
|
|
|
5273
5294
|
// (cotal config; default none → isolated, the memory-safe default this guards), narrowed by
|
|
5274
5295
|
// an optional --share-tools selection (absent → all declared, the pre-merge behavior).
|
|
5275
5296
|
const cotalConfig = loadCotalConfig(this.workspaceRoot);
|
|
5276
|
-
const mcpServers = connectorServers(cotalConfig, agent,
|
|
5297
|
+
const mcpServers = connectorServers(cotalConfig, agent, shareTools);
|
|
5277
5298
|
// The operator's spawn-env policy travels the same route: absent means no extras (the OS
|
|
5278
5299
|
// allow-list + operator knobs + connector-declared inputs), present means those names too.
|
|
5279
5300
|
// A connector never reads the config itself.
|
|
@@ -5400,7 +5421,7 @@ export class Manager {
|
|
|
5400
5421
|
allowPublish,
|
|
5401
5422
|
capabilities,
|
|
5402
5423
|
events,
|
|
5403
|
-
shareTools
|
|
5424
|
+
shareTools,
|
|
5404
5425
|
forkSource: opts.resume,
|
|
5405
5426
|
...(opts.resume !== undefined && spec?.resumeRecordPath ? { resumeRecordPath: spec.resumeRecordPath } : {}),
|
|
5406
5427
|
...(carried ? { carried: { transcriptSha256: carried.sha256, host: carried.sourceHost, transferredAt: carried.stagedAt } } : {}),
|
|
@@ -5573,19 +5594,22 @@ export class Manager {
|
|
|
5573
5594
|
const batchReservations = [];
|
|
5574
5595
|
const prepared = new Map();
|
|
5575
5596
|
try {
|
|
5576
|
-
if (inventory.version !== "cotal-manager-resume/
|
|
5597
|
+
if (inventory.version !== "cotal-manager-resume/v2")
|
|
5577
5598
|
return { ok: false, agents: [], error: `unsupported manager resume inventory version ${String(inventory.version)}` };
|
|
5578
5599
|
if (inventory.space !== this.space)
|
|
5579
5600
|
return { ok: false, agents: [], error: `resume inventory belongs to space "${inventory.space}", not "${this.space}"` };
|
|
5601
|
+
// The caller keeps its inventory, so each selection is copied before the first await: a later
|
|
5602
|
+
// write to it cannot change what a resumed seat shares or what the next preservation retains.
|
|
5603
|
+
const entries = inventory.agents.map((entry) => ({ ...entry, launch: { ...entry.launch, shareTools: entry.launch.shareTools && [...entry.launch.shareTools] } }));
|
|
5580
5604
|
const seen = new Set();
|
|
5581
5605
|
const principals = new Set();
|
|
5582
5606
|
const orphans = [];
|
|
5583
5607
|
await this.ep.waitForPresenceSnapshot();
|
|
5584
5608
|
const liveRoster = this.ep.getRoster().filter((presence) => presence.status !== "offline");
|
|
5585
5609
|
const occupancy = this.occupancy();
|
|
5586
|
-
if (occupancy.used +
|
|
5587
|
-
return { ok: false, agents: [], error: `resume inventory of ${
|
|
5588
|
-
for (const entry of
|
|
5610
|
+
if (occupancy.used + entries.length > MAX_AGENTS)
|
|
5611
|
+
return { ok: false, agents: [], error: `resume inventory of ${entries.length} would exceed manager capacity (${this.occupancyText(occupancy)})` };
|
|
5612
|
+
for (const entry of entries) {
|
|
5589
5613
|
if (seen.has(entry.name))
|
|
5590
5614
|
return { ok: false, agents: [], error: `resume inventory contains duplicate agent name "${entry.name}"` };
|
|
5591
5615
|
seen.add(entry.name);
|
|
@@ -5642,12 +5666,12 @@ export class Manager {
|
|
|
5642
5666
|
if (this.agents.has(entry.name) || this.reserved.has(entry.name))
|
|
5643
5667
|
return { ok: false, agents: [], error: `retained agent "${entry.name}" is already managed or reserved` };
|
|
5644
5668
|
}
|
|
5645
|
-
for (const entry of
|
|
5669
|
+
for (const entry of entries) {
|
|
5646
5670
|
this.reserved.add(entry.name);
|
|
5647
5671
|
batchReservations.push(entry.name);
|
|
5648
5672
|
}
|
|
5649
5673
|
const preflight = [];
|
|
5650
|
-
for (const entry of
|
|
5674
|
+
for (const entry of entries) {
|
|
5651
5675
|
const reply = await this.resumePreservedAgent(entry, true, true, prepared);
|
|
5652
5676
|
preflight.push({ name: entry.name, reply });
|
|
5653
5677
|
}
|
|
@@ -5666,12 +5690,12 @@ export class Manager {
|
|
|
5666
5690
|
return { ok: false, agents: [], error: refusal };
|
|
5667
5691
|
}
|
|
5668
5692
|
const agents = [];
|
|
5669
|
-
for (let i = 0; i <
|
|
5670
|
-
const entry =
|
|
5693
|
+
for (let i = 0; i < entries.length; i++) {
|
|
5694
|
+
const entry = entries[i];
|
|
5671
5695
|
const reply = await this.resumePreservedAgent(entry, false, true, prepared);
|
|
5672
5696
|
agents.push({ name: entry.name, reply });
|
|
5673
5697
|
if (!reply.ok) {
|
|
5674
|
-
for (const skipped of
|
|
5698
|
+
for (const skipped of entries.slice(i + 1))
|
|
5675
5699
|
agents.push({ name: skipped.name, reply: { ok: false, error: `not launched because ${entry.name} failed` } });
|
|
5676
5700
|
return { ok: false, agents, error: reply.error };
|
|
5677
5701
|
}
|
|
@@ -5881,7 +5905,7 @@ export class Manager {
|
|
|
5881
5905
|
const slot = await this.readStaticReconcileSlot(entry.name);
|
|
5882
5906
|
if (slot?.phase !== "active" || slot.actor !== entry.identity.id || slot.lifecycleUid !== entry.identity.lifecycleUid)
|
|
5883
5907
|
return undefined;
|
|
5884
|
-
if (slot
|
|
5908
|
+
if (ownedBySibling(slot, this.managerInstanceId))
|
|
5885
5909
|
return undefined;
|
|
5886
5910
|
return slot.runtime?.kind === this.runtime.kind ? slot.runtime : undefined;
|
|
5887
5911
|
}
|
|
@@ -6013,7 +6037,7 @@ export class Manager {
|
|
|
6013
6037
|
}
|
|
6014
6038
|
try {
|
|
6015
6039
|
const resumeConfig = loadCotalConfig(this.workspaceRoot);
|
|
6016
|
-
const mcpServers = connectorServers(resumeConfig, entry.launch.connector,
|
|
6040
|
+
const mcpServers = connectorServers(resumeConfig, entry.launch.connector, entry.launch.shareTools);
|
|
6017
6041
|
const envAllow = spawnEnvAllow(resumeConfig);
|
|
6018
6042
|
const launchOpts = {
|
|
6019
6043
|
space: this.space,
|
|
@@ -6141,19 +6165,7 @@ export class Manager {
|
|
|
6141
6165
|
await this.recordSlotRuntime(managed);
|
|
6142
6166
|
}
|
|
6143
6167
|
catch (error) {
|
|
6144
|
-
|
|
6145
|
-
this.stopHandle(managed, false);
|
|
6146
|
-
try {
|
|
6147
|
-
await this.awaitHandleExit(managed.handle);
|
|
6148
|
-
}
|
|
6149
|
-
catch (exit) {
|
|
6150
|
-
// A resumed seat keeps its retained credentials, so freeing it runs no deprovision and
|
|
6151
|
-
// nothing would reap it: it stays managed until it exits or the manager stops it.
|
|
6152
|
-
this.watchExit(managed);
|
|
6153
|
-
return { ok: false, error: `${detail}; it stays managed because its stop is unproven: ${exit.message}` };
|
|
6154
|
-
}
|
|
6155
|
-
this.freeSlot(managed, true, "resume-custody-unrecorded", true);
|
|
6156
|
-
return { ok: false, error: detail };
|
|
6168
|
+
return await this.stopFailedResume(managed, "resume-custody-unrecorded", `${managed.name} resumed, but ${error.message}`);
|
|
6157
6169
|
}
|
|
6158
6170
|
const readiness = await this.awaitReadiness(managed, readinessTimeoutMs);
|
|
6159
6171
|
if (!readiness.ok && !readiness.uncertain)
|
|
@@ -6180,9 +6192,7 @@ export class Manager {
|
|
|
6180
6192
|
}
|
|
6181
6193
|
}
|
|
6182
6194
|
catch (error) {
|
|
6183
|
-
this.
|
|
6184
|
-
this.freeSlot(managed, true, "resume-session-rebind-failed", true);
|
|
6185
|
-
return { ok: false, error: `${managed.name} resumed, but its exact host session could not be rebound: ${error.message}` };
|
|
6195
|
+
return await this.stopFailedResume(managed, "resume-session-rebind-failed", `${managed.name} resumed, but its exact host session could not be rebound: ${error.message}`);
|
|
6186
6196
|
}
|
|
6187
6197
|
}
|
|
6188
6198
|
if (!this.resumeAttemptId)
|
|
@@ -6205,6 +6215,21 @@ export class Manager {
|
|
|
6205
6215
|
}
|
|
6206
6216
|
}
|
|
6207
6217
|
}
|
|
6218
|
+
/** A resumed seat keeps its retained credentials, so freeing it runs no deprovision and nothing
|
|
6219
|
+
* would reap it: it is freed only once its exit is proved, and otherwise stays managed until it
|
|
6220
|
+
* exits or the manager stops it. */
|
|
6221
|
+
async stopFailedResume(managed, cause, detail) {
|
|
6222
|
+
this.stopHandle(managed, false);
|
|
6223
|
+
try {
|
|
6224
|
+
await this.awaitHandleExit(managed.handle);
|
|
6225
|
+
}
|
|
6226
|
+
catch (exit) {
|
|
6227
|
+
this.watchExit(managed);
|
|
6228
|
+
return { ok: false, error: `${detail}; it stays managed because its stop is unproven: ${exit.message}` };
|
|
6229
|
+
}
|
|
6230
|
+
this.freeSlot(managed, true, cause, true);
|
|
6231
|
+
return { ok: false, error: detail };
|
|
6232
|
+
}
|
|
6208
6233
|
probeStaticCredential(creds) {
|
|
6209
6234
|
return probeConnect(this.servers ?? DEFAULT_SERVER, { creds, timeoutMs: 5_000 });
|
|
6210
6235
|
}
|
|
@@ -6328,7 +6353,7 @@ export class Manager {
|
|
|
6328
6353
|
// waitForExit may close the attach stream before this snapshot
|
|
6329
6354
|
let tail = "";
|
|
6330
6355
|
try {
|
|
6331
|
-
tail = this.tail(
|
|
6356
|
+
tail = await this.tail(s);
|
|
6332
6357
|
}
|
|
6333
6358
|
catch { }
|
|
6334
6359
|
if (opts.reapOnExit !== false)
|
|
@@ -6364,19 +6389,34 @@ export class Manager {
|
|
|
6364
6389
|
onPresence();
|
|
6365
6390
|
});
|
|
6366
6391
|
}
|
|
6367
|
-
/** Last non-empty line of
|
|
6368
|
-
*
|
|
6369
|
-
*
|
|
6370
|
-
|
|
6371
|
-
|
|
6372
|
-
|
|
6373
|
-
|
|
6374
|
-
|
|
6375
|
-
|
|
6376
|
-
|
|
6377
|
-
|
|
6378
|
-
|
|
6379
|
-
|
|
6392
|
+
/** Last non-empty line of the session's rendered backlog as a single trimmed snippet (≤160 chars)
|
|
6393
|
+
* — a readable one-line cause for an early-exit diagnostic. The backlog repaints cells a TUI
|
|
6394
|
+
* skipped with cursor moves, so it is rendered and the line is read off the terminal; stripping
|
|
6395
|
+
* its escapes would glue the words those moves separate. */
|
|
6396
|
+
async tail(session) {
|
|
6397
|
+
const backlog = await session.backlog();
|
|
6398
|
+
// Sized after the snapshot: a custodial seat's snapshot carries the geometry it was taken at.
|
|
6399
|
+
// The headless build exposes `buffer` only as proposed API.
|
|
6400
|
+
const term = new Headless.Terminal({ cols: session.cols, rows: session.rows, allowProposedApi: true });
|
|
6401
|
+
await new Promise((done) => term.write(backlog, done));
|
|
6402
|
+
const screen = term.buffer.active;
|
|
6403
|
+
let text = "";
|
|
6404
|
+
for (let y = screen.length - 1; y >= 0; y--) {
|
|
6405
|
+
const row = screen.getLine(y);
|
|
6406
|
+
// A line wider than the screen continues across wrapped rows, so it is read whole.
|
|
6407
|
+
text = row.translateToString(text === "") + text;
|
|
6408
|
+
if (row.isWrapped)
|
|
6409
|
+
continue;
|
|
6410
|
+
// Blanks a program painted survive the right trim, so a line of only blanks is skipped.
|
|
6411
|
+
if (text.trim())
|
|
6412
|
+
break;
|
|
6413
|
+
text = "";
|
|
6414
|
+
}
|
|
6415
|
+
term.dispose();
|
|
6416
|
+
text = text.trim();
|
|
6417
|
+
// Cut on code points: half a surrogate pair is not I-JSON, and a launch terminal carrying it is refused.
|
|
6418
|
+
const chars = [...text];
|
|
6419
|
+
return chars.length > 160 ? `…${chars.slice(-160).join("")}` : text;
|
|
6380
6420
|
}
|
|
6381
6421
|
/** Subscribe to a managed agent's process-exit so a self-driven exit frees its slot and reaps
|
|
6382
6422
|
* its children (P4b/P4c). Only pty streams exit (via the attach session's `onExit`); external runtimes'
|
|
@@ -6927,7 +6967,7 @@ export class Manager {
|
|
|
6927
6967
|
const servePrincipal = principalKey(DEV_OWNER, serveIdentity.id).key;
|
|
6928
6968
|
// An open mesh mints no credentials, so it has no family for a fresh pair to grow.
|
|
6929
6969
|
const siblings = auth ? claimManagerSiblingIdentities(this.workspaceRoot, this.space) : { goalWriter: newIdentity(), sessionLedger: newIdentity() };
|
|
6930
|
-
//
|
|
6970
|
+
// The STABLE goal-writer identity — a SIBLING credential in the same §13.1 family
|
|
6931
6971
|
// (not the gate's bound serving principal), so the run block can family-stage it.
|
|
6932
6972
|
this.goalWriterIdentity = siblings.goalWriter;
|
|
6933
6973
|
// P2 item 6: the STABLE session-LEDGER identity — another SIBLING in the SAME §13.1 family, so
|
|
@@ -6998,7 +7038,7 @@ export class Manager {
|
|
|
6998
7038
|
// Open mesh: NO mint - the §13.1 fence is issuance-only and nothing is ever issued, so the
|
|
6999
7039
|
// gate keeps an empty `epcred` family; the serve connection below stays bare.
|
|
7000
7040
|
const creds = auth ? await mintCreds(auth, serveIdentity, "endpoint-serve", { serveIssuance: fence, endpointServe: grant }) : undefined;
|
|
7001
|
-
//
|
|
7041
|
+
// Mint + family-stage the goal-writer credential HERE, over this executor's
|
|
7002
7042
|
// authKv (the fence is live), so its credId lands in `epcred.<e>.<iid>` and the takeover
|
|
7003
7043
|
// barrier revokes it. Open mesh: no mint (no credential system; the goal-writer conn is bare).
|
|
7004
7044
|
const goalWriterCreds = auth ? await this.mintAndStageGoalWriter(authKv) : undefined;
|
|
@@ -7053,7 +7093,7 @@ export class Manager {
|
|
|
7053
7093
|
if (state.creds !== undefined)
|
|
7054
7094
|
this.scheduleServeRenewal(state.creds);
|
|
7055
7095
|
}
|
|
7056
|
-
/**
|
|
7096
|
+
/** Mint the standing `goal-writer` credential and STAGE it into this
|
|
7057
7097
|
* instance's §13.1 revocation family (`epcred.<e>.<iid>`), over the passed executor's `authKv`
|
|
7058
7098
|
* (the scoped `endpoint-serve-executor`, which holds the epcred write grant). The GRANT profile
|
|
7059
7099
|
* stays goal-writer-only (Q2 — disjoint from the serve credential); only the FAMILY membership
|
|
@@ -7491,7 +7531,7 @@ export class Manager {
|
|
|
7491
7531
|
}
|
|
7492
7532
|
}
|
|
7493
7533
|
}
|
|
7494
|
-
/**
|
|
7534
|
+
/** The boot reconcile: a fresh incarnation (a manager restart takes a NEW
|
|
7495
7535
|
* instanceId, so the in-memory acceptance map starts empty) inherits the endpoint's accepted-but-
|
|
7496
7536
|
* unterminal goals from any predecessor. The local signer enumerates with an ephemeral PROVISIONER.
|
|
7497
7537
|
* A remote manager calls the authenticated host-owned scan, which keeps the sealed consumer and its
|
|
@@ -7652,19 +7692,20 @@ export class Manager {
|
|
|
7652
7692
|
if (observed === null)
|
|
7653
7693
|
throw new EpEnvelopeError("expired", `the manager's issuance gate for ${MANAGER_ENDPOINT}/${this.managerInstanceId} is gone; a retired incarnation never commits a goal terminal (SPEC 13.1/13.6)`);
|
|
7654
7694
|
if (observed.processEpoch !== epoch)
|
|
7655
|
-
throw new EpEnvelopeError("expired", `the manager's issuance gate epoch is ${observed.processEpoch} but this goal was accepted under epoch ${epoch}; a superseded incarnation never commits a goal terminal (
|
|
7695
|
+
throw new EpEnvelopeError("expired", `the manager's issuance gate epoch is ${observed.processEpoch} but this goal was accepted under epoch ${epoch}; a superseded incarnation never commits a goal terminal (SPEC 13.6)`);
|
|
7656
7696
|
}
|
|
7657
7697
|
/** Serve `spawn`/`launch` as an ACTION (P2 item 2). Authz already ran in {@link serveGated}. The
|
|
7658
7698
|
* accept path runs INLINE on the handler ({@link startAgent} with hooks): the goal binds + the
|
|
7659
|
-
* acceptance replies the moment the identity is minted, BEFORE any provision (pin 1)
|
|
7660
|
-
*
|
|
7699
|
+
* acceptance replies the moment the identity is minted, BEFORE any provision (pin 1), or on the
|
|
7700
|
+
* hosted enrollment arm once the host has enrolled the agent; progress and the terminal are driven
|
|
7701
|
+
* OFF-handler, so the ~30s readiness wait no longer blocks the reply.
|
|
7661
7702
|
* Returns the acceptance floor payload {name, owner, actor, uid, goalId, fingerprint, executor}
|
|
7662
7703
|
* (the ALLOCATED identity). goalId = the request id (env.id, Q3). */
|
|
7663
7704
|
async serveSpawnGoal(ctx, run) {
|
|
7664
7705
|
const gw = this.goalWriter;
|
|
7665
7706
|
if (!gw)
|
|
7666
7707
|
throw new EpEnvelopeError("unavailable", "the manager goal-writer connection is not standing; spawn-as-action cannot accept (SPEC 13.6)");
|
|
7667
|
-
//
|
|
7708
|
+
// Refuse to accept until the boot reconcile of inherited goals completes, so a fresh
|
|
7668
7709
|
// acceptance never races the sweep (settling a live goal mid-flight would steal its real terminal).
|
|
7669
7710
|
if (!this.goalReconcileDone)
|
|
7670
7711
|
throw new EpEnvelopeError("unavailable", "the manager is still reconciling accepted goals at boot; retry shortly (SPEC 13.6)");
|
|
@@ -7709,7 +7750,7 @@ export class Manager {
|
|
|
7709
7750
|
// message as the caller-visible outcome.
|
|
7710
7751
|
let ownsGoal = false;
|
|
7711
7752
|
// The terminal commits OFF-handler on the goal-writer connection (manager-only authority; a
|
|
7712
|
-
// caller cannot publish it). TWO COMPOSED FENCES (defense in depth):
|
|
7753
|
+
// caller cannot publish it). TWO COMPOSED FENCES (defense in depth): (a) reads THIS
|
|
7713
7754
|
// incarnation's OWN gate epoch and REFUSES a superseded commit (the currency belt), and (b)
|
|
7714
7755
|
// barrier-revoke evicts this connection on takeover. The terminal lands on the ONE subject
|
|
7715
7756
|
// SPEC §13.2 (reserved subjects) reserves; first-terminal-fact-wins is global, so a committed outcome is visible
|
|
@@ -7727,7 +7768,7 @@ export class Manager {
|
|
|
7727
7768
|
return;
|
|
7728
7769
|
terminalEntered = true; // entered, not succeeded — see the catch below
|
|
7729
7770
|
try {
|
|
7730
|
-
await this.assertGoalWriterEpochCurrent(epoch); //
|
|
7771
|
+
await this.assertGoalWriterEpochCurrent(epoch); // a superseded corpse never commits
|
|
7731
7772
|
let fact;
|
|
7732
7773
|
if (o.kind === "succeeded") {
|
|
7733
7774
|
this.emitGoalProgress(ref, epoch, { phase: "presence" });
|
|
@@ -7743,7 +7784,7 @@ export class Manager {
|
|
|
7743
7784
|
({ fact } = await settleGoalUncertain(gw.ctx, { ref, now: Date.now(), committer: { instanceId: this.managerInstanceId, epoch }, ...(typeof why === "string" && why.length > 0 ? { reason: why } : {}) }));
|
|
7744
7785
|
}
|
|
7745
7786
|
this.emitGoalProgress(ref, epoch, { phase: "terminal", state: fact.state, ...(fact.data !== undefined ? { data: fact.data } : {}) });
|
|
7746
|
-
await clearGoalIndex(gw.ctx, ref); //
|
|
7787
|
+
await clearGoalIndex(gw.ctx, ref); // terminal reached - the successor never reconciles it
|
|
7747
7788
|
if (acceptance)
|
|
7748
7789
|
this.agentGoals.delete(acceptance.name); // goal terminal - no cancel path left
|
|
7749
7790
|
}
|
|
@@ -7760,7 +7801,7 @@ export class Manager {
|
|
|
7760
7801
|
};
|
|
7761
7802
|
const bg = run({
|
|
7762
7803
|
onAccepted: async ({ name, agentTriple }) => {
|
|
7763
|
-
//
|
|
7804
|
+
// Record the goal in the reconcile index BEFORE the bind (index-CAS-before-bind),
|
|
7764
7805
|
// so a successor incarnation finds + settles this goal if we crash before its terminal. A
|
|
7765
7806
|
// crash between this write and the bind leaves an index entry whose goal status is absent —
|
|
7766
7807
|
// the sweep clears it as a no-goal; a crash before it leaves no entry (never durable). A
|
|
@@ -8353,11 +8394,11 @@ export class Manager {
|
|
|
8353
8394
|
this.agentGoals.delete(name);
|
|
8354
8395
|
const epoch = this.serviceServe?.grant.epoch ?? 0;
|
|
8355
8396
|
try {
|
|
8356
|
-
await this.assertGoalWriterEpochCurrent(epoch); //
|
|
8397
|
+
await this.assertGoalWriterEpochCurrent(epoch); // a superseded corpse never commits a cancel terminal either
|
|
8357
8398
|
await transitionGoal(gw.ctx, ref, "cancelling", { fields: { cancelMode: mode } });
|
|
8358
8399
|
const r = await commitGoalResult(gw.ctx, { ref, now: Date.now(), cause: "cancel", data: { cancelledBy: "despawn" }, committer: { instanceId: this.managerInstanceId, epoch } });
|
|
8359
8400
|
this.emitGoalProgress(ref, epoch, { phase: "terminal", state: r.fact.state, ...(r.fact.data !== undefined ? { data: r.fact.data } : {}) });
|
|
8360
|
-
await clearGoalIndex(gw.ctx, ref); //
|
|
8401
|
+
await clearGoalIndex(gw.ctx, ref); // terminal reached - the successor never reconciles it
|
|
8361
8402
|
}
|
|
8362
8403
|
catch {
|
|
8363
8404
|
// the goal already terminalized (the readiness outcome won the settle race) - nothing to cancel.
|
|
@@ -8705,7 +8746,7 @@ export class Manager {
|
|
|
8705
8746
|
console.error(`✓ static reconcile recovered alias=${item.alias} phase=retired uid=${item.lifecycleUid} attempts=${item.attempts}/${item.maxAttempts}`);
|
|
8706
8747
|
return;
|
|
8707
8748
|
}
|
|
8708
|
-
if (row
|
|
8749
|
+
if (ownedBySibling(row, this.managerInstanceId)) {
|
|
8709
8750
|
item.disposition = "refused";
|
|
8710
8751
|
item.remedy = "the owning manager must reconcile or an operator must perform an explicit CAS takeover";
|
|
8711
8752
|
item.lastError = `the durable slot belongs to manager instance ${row.ownerInstanceId}, not ${this.managerInstanceId}`;
|
|
@@ -8898,7 +8939,7 @@ export class Manager {
|
|
|
8898
8939
|
// hazard, now cross-instance). A legacy row (pre-3b-2, no owner recorded) predates multi-manager,
|
|
8899
8940
|
// so this manager is its legitimate single-manager-past successor and reconciles it. An orphaned
|
|
8900
8941
|
// sibling row is reclaimed only by an explicit operator CAS takeover (ruling 1), never here.
|
|
8901
|
-
if (row
|
|
8942
|
+
if (ownedBySibling(row, this.managerInstanceId)) {
|
|
8902
8943
|
preservedForeign++;
|
|
8903
8944
|
continue;
|
|
8904
8945
|
}
|