@cotal-ai/manager 0.66.1 → 0.68.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands.js +9 -0
- package/dist/commands.js.map +1 -1
- package/dist/manager-service-contract.d.ts +8 -2
- package/dist/manager-service-contract.d.ts.map +1 -1
- package/dist/manager-service-contract.js +46 -6
- package/dist/manager-service-contract.js.map +1 -1
- package/dist/manager.d.ts +39 -7
- package/dist/manager.d.ts.map +1 -1
- package/dist/manager.js +292 -95
- package/dist/manager.js.map +1 -1
- package/dist/remote-authority.d.ts +1 -1
- package/dist/remote-authority.d.ts.map +1 -1
- package/dist/remote-authority.js +7 -7
- package/dist/remote-authority.js.map +1 -1
- package/dist/run-hosting.d.ts +6 -2
- package/dist/run-hosting.d.ts.map +1 -1
- package/dist/run-hosting.js +23 -9
- package/dist/run-hosting.js.map +1 -1
- package/dist/static-lifecycle.js +12 -12
- package/dist/static-lifecycle.js.map +1 -1
- package/dist/transcript-receive.d.ts +63 -0
- package/dist/transcript-receive.d.ts.map +1 -0
- package/dist/transcript-receive.js +178 -0
- package/dist/transcript-receive.js.map +1 -0
- package/package.json +12 -12
package/dist/manager.js
CHANGED
|
@@ -2,10 +2,10 @@ import { execFile } from "node:child_process";
|
|
|
2
2
|
import { createHash, randomUUID, randomBytes } from "node:crypto";
|
|
3
3
|
import { hostname } from "node:os";
|
|
4
4
|
import { credsAuthenticator } from "@nats-io/transport-node";
|
|
5
|
-
import { existsSync, lstatSync, readFileSync, realpathSync, rmSync, statSync } from "node:fs";
|
|
5
|
+
import { existsSync, lstatSync, mkdirSync, readFileSync, realpathSync, rmSync, statSync } from "node:fs";
|
|
6
6
|
import { isAbsolute, join, dirname, resolve } from "node:path";
|
|
7
|
-
import { CotalEndpoint, MANAGED_HANDOFF_KIND, accountFromCreds, credsClaims, DEFAULT_SERVER, DEV_OWNER, MANAGER_LEASE_RENEW_MS, STANDING_RENEWABLE_TTL_SEC, assertLifecycleToken, discardLaunchArtifacts, divergentSecretStoreNotice, parseDaemonStoreAnswer, parseSecretStoreIdentity, sameSecretStoreIdentity, agentFilePath, clearSpaceHistory, dialerFor, connectorServers, spawnEnvAllow, deprovisionAgent, DeprovisionError, isConcreteChannel, deprovisionTargetPrincipal, firstFreeName, spawnNameError, idFromCreds, inspectCredHealth, credsRenewalDelayMs, composeWirePersona, loadAgentFile, listPersonaCatalog, personaCatalogDescription, personaCatalogReadable, loadCotalConfig, modelPolicyRefusal, mintCreds, mintGeneration, mintAcceptedToken, openIssuedStore, acceptedBucket, issuedBucket, mintLifecycleUid, managedRetirementOpId, mkSecretDir, newIdentity, actionContext, parsePrincipalKey, parseShareSelection, principalKey, probeConnect, provisionAgent, provisionAgentDurables, registry, resolveAuthProvider, saveAgentFile, subjectMatches, AUTH_ENDPOINT, EP_CMD_RETIRE_LIFECYCLE, controlServiceSubject, eventChannelPrincipal, resolveService, invokeCommand, } from "@cotal-ai/core";
|
|
8
|
-
import { agentAuthState, agentCredsDir, agentLifecycleSecretFilePaths, agentSecretFilePaths, agentSecretKeyForFile, authDir, connectorInstallHint, DEFAULT_CONNECTOR, defaultAgentType, DELIVERY_CREDS_KIND, extensionConnectors, findCotalRoot, getSpaceAuth, hasUserAuthState, loadExtensionsManifest, loadManagerInstanceIdentity, loadMeshes, localTrustOfSpace, manifestExtensionNames, materializeFromManifest, materializeSecretToFile, MEMBERSHIP_RW_CREDS_KIND, mergeLaunchOptions, remintDaemonCreds, resolveOnPath, createManagerInstanceIdentity, spaceMaterialKey, SYSTEM_CREDS_FILES, userAuthStateDir, workspaceSecretStore, writeRenewalRecord } from "@cotal-ai/workspace";
|
|
7
|
+
import { CotalEndpoint, MANAGED_HANDOFF_KIND, accountFromCreds, credsClaims, DEFAULT_SERVER, DEV_OWNER, MANAGER_LEASE_RENEW_MS, STANDING_RENEWABLE_TTL_SEC, assertLifecycleToken, discardLaunchArtifacts, divergentSecretStoreNotice, parseDaemonStoreAnswer, parseSecretStoreIdentity, sameSecretStoreIdentity, agentFilePath, clearSpaceHistory, dialerFor, connectorServers, spawnEnvAllow, deprovisionAgent, DeprovisionError, isConcreteChannel, deprovisionTargetPrincipal, firstFreeName, spawnNameError, idFromCreds, inspectCredHealth, credsRenewalDelayMs, composeWirePersona, loadAgentFile, listPersonaCatalog, personaCatalogDescription, personaCatalogReadable, loadCotalConfig, modelPolicyRefusal, mintCreds, mintGeneration, mintAcceptedToken, openIssuedStore, acceptedBucket, issuedBucket, mintLifecycleUid, managedRetirementOpId, mkSecretDir, newIdentity, actionContext, parsePrincipalKey, parseShareSelection, principalKey, probeConnect, provisionAgent, provisionAgentDurables, registry, resolveAuthProvider, bearerCommandFailure, saveAgentFile, subjectMatches, AUTH_ENDPOINT, EP_CMD_RETIRE_LIFECYCLE, controlServiceSubject, eventChannelPrincipal, resolveService, invokeCommand, } from "@cotal-ai/core";
|
|
8
|
+
import { agentAuthState, agentCredsDir, agentLifecycleSecretFilePaths, agentSecretFilePaths, agentSecretKeyForFile, authDir, claimManagerSiblingIdentities, connectorInstallHint, DEFAULT_CONNECTOR, defaultAgentType, DELIVERY_CREDS_KIND, extensionConnectors, findCotalRoot, getSpaceAuth, hasUserAuthState, loadExtensionsManifest, loadManagerInstanceIdentity, loadMeshes, localTrustOfSpace, manifestExtensionNames, materializeFromManifest, materializeSecretToFile, MEMBERSHIP_RW_CREDS_KIND, mergeLaunchOptions, remintDaemonCreds, resolveOnPath, createManagerInstanceIdentity, spaceMaterialKey, SYSTEM_CREDS_FILES, userAuthStateDir, workspaceSecretStore, writeRenewalRecord } from "@cotal-ai/workspace";
|
|
9
9
|
import { createRuntime, isCustodialRuntime, requireRuntimeAdopt, requireRuntimeReap, RuntimeReapUnproven, } from "./runtime/index.js";
|
|
10
10
|
import { AttachEndpoint } from "./attach-endpoint.js";
|
|
11
11
|
import { DELIVERY_ADMIN_BOOT_WAIT_MS, isUnansweredDeliveryAdmin, makeManagerEndpointEvictionEvidence, makeManagerEndpointHolderEvictor, untilDeliveryAdminAnswers } from "./endpoint-evict.js";
|
|
@@ -17,11 +17,12 @@ import { controlShutdown } from "./control-shutdown.js";
|
|
|
17
17
|
import { RunHosting } from "./run-hosting.js";
|
|
18
18
|
import { controlSession } from "./control-session.js";
|
|
19
19
|
import { MAX_AGENTS, parseResumeCommitArgs, parseResumeControlArgs, parseResumeFinalizeArgs } from "./resume.js";
|
|
20
|
+
import { TranscriptReceiver } from "./transcript-receive.js";
|
|
20
21
|
// Unit B (the static §13.1 lifecycle executor): the shared grammar/stores from core plus the
|
|
21
22
|
// manager-side adapter (transport + slot orchestration + the F1 terminal) — see static-lifecycle.ts.
|
|
22
23
|
import { jetstream, jetstreamManager } from "@nats-io/jetstream";
|
|
23
24
|
import { Kvm } from "@nats-io/kv";
|
|
24
|
-
import { recordsBucket, epAuthBucket, ensureAuthorityStores, ensureContractStore, createEndpointStreams, contractStoreContext, publishContractArtifact, contractArtifactCanonicalBytes, standaloneConnectOpts, STATIC_SLOT_PREFIX, rawDigest, STANDING_RENEWABLE_TTL_SEC as MANAGED_STATIC_TTL_SEC, newArtifactSigner, RotatingSigner, generationAnchor, sessionsBucket, SESSION_GRANT_MAX_TTL_MS, } from "@cotal-ai/core";
|
|
25
|
+
import { recordsBucket, epAuthBucket, ensureAuthorityStores, ensureContractStore, createEndpointStreams, contractStoreContext, publishContractArtifact, contractArtifactCanonicalBytes, standaloneConnectOpts, STATIC_SLOT_PREFIX, rawDigest, STANDING_RENEWABLE_TTL_SEC as MANAGED_STATIC_TTL_SEC, newArtifactSigner, RotatingSigner, generationAnchor, sessionsBucket, SESSION_GRANT_MAX_TTL_MS, REMOTE_MANAGER_IDENTITY_NAMES, } from "@cotal-ai/core";
|
|
25
26
|
// P2 item 6: the manager's ONE §13.6 session plane — offer mint + one-use redeem + PTY-bridge
|
|
26
27
|
// standup for `attach`, over a dedicated standing session-LEDGER connection (the byte rails ride
|
|
27
28
|
// per-session credentials on their own short-lived connections).
|
|
@@ -136,14 +137,25 @@ function rejectionText(e) {
|
|
|
136
137
|
return "an unreadable rejection";
|
|
137
138
|
}
|
|
138
139
|
}
|
|
140
|
+
/** Run one cleanup step, recording its failure in `failed` as `<what>: <reason>`. A step that threw
|
|
141
|
+
* would otherwise skip every step after it, and a swallowed one would leave nothing to report. */
|
|
142
|
+
async function attemptCleanup(failed, what, step) {
|
|
143
|
+
try {
|
|
144
|
+
await step();
|
|
145
|
+
}
|
|
146
|
+
catch (err) {
|
|
147
|
+
failed.push(`${what}: ${rejectionText(err)}`);
|
|
148
|
+
}
|
|
149
|
+
}
|
|
139
150
|
/** Run the agent's bearer argv once, pre-launch — the end-to-end auth preflight (state dir, daemon,
|
|
140
151
|
* ledger row, secret). Its stderr is the provider command's operator-exact sentence; surface it
|
|
141
152
|
* verbatim as the spawn refusal. */
|
|
142
153
|
function execBearerPreflight(argv) {
|
|
154
|
+
const timeout = 30_000;
|
|
143
155
|
return new Promise((res, rej) => {
|
|
144
|
-
execFile(argv[0], argv.slice(1), { timeout
|
|
156
|
+
execFile(argv[0], argv.slice(1), { timeout, maxBuffer: 64 * 1024 }, (err, _stdout, stderr) => {
|
|
145
157
|
if (err)
|
|
146
|
-
return rej(
|
|
158
|
+
return rej(bearerCommandFailure(err, stderr, timeout));
|
|
147
159
|
res();
|
|
148
160
|
});
|
|
149
161
|
});
|
|
@@ -172,6 +184,8 @@ const FREE_SLOT_CAUSE_TEXT = {
|
|
|
172
184
|
"supervise-recovery-failed": "this manager retired it after a supervised restart failed",
|
|
173
185
|
"session-bind-failed": "this manager stopped it: its host session could not be bound at launch",
|
|
174
186
|
"resume-session-rebind-failed": "this manager stopped it: its host session could not be rebound on resume",
|
|
187
|
+
"carried-fork-refused": "this manager stopped it: it recorded no fork of the carried transcript",
|
|
188
|
+
"resume-custody-unrecorded": "this manager stopped it: its custody reference could not be recorded on resume",
|
|
175
189
|
};
|
|
176
190
|
/** The stop family's sentences — the same job {@link FREE_SLOT_CAUSE_TEXT} does for the string
|
|
177
191
|
* causes, beside it so both stay the single place a cause becomes a sentence. A requested stop's
|
|
@@ -467,6 +481,8 @@ export class Manager {
|
|
|
467
481
|
/** The complete boot task. stop() joins it after fencing static reconciliation, so a registration
|
|
468
482
|
* already past an earlier shutdown check cannot finish after stop() returns. */
|
|
469
483
|
startTask;
|
|
484
|
+
/** This instance's side of a carried resume transcript (#1499); absent where it has no transfer reader. */
|
|
485
|
+
transcripts;
|
|
470
486
|
/** The one stop this manager runs, with the agent policy it was asked for. A stop takes seats out
|
|
471
487
|
* of `agents` before they have exited, so a second teardown would find none and report success
|
|
472
488
|
* while the first still waits on a live seat. */
|
|
@@ -877,11 +893,11 @@ export class Manager {
|
|
|
877
893
|
catch (e) {
|
|
878
894
|
// Our OWN instance id already holds a live key ⇒ refuse. Anything else (e.g. a KV/JS error) is a
|
|
879
895
|
// real failure to surface, not a silent "held" — keep the cause so it isn't misread as a conflict.
|
|
880
|
-
const held = await this.ep.
|
|
896
|
+
const held = (await this.ep.readOwnManagerLease(this.managerInstanceId).catch(() => undefined))?.info;
|
|
881
897
|
await this.ep.stop();
|
|
882
898
|
await this.attach.stop();
|
|
883
899
|
throw new Error(held
|
|
884
|
-
? `manager instance ${this.managerInstanceId} already serves space "${this.space}" from this workspace root (${held.runtime}, pid ${held.pid}, root ${held.root}) - stop it first before restarting the same instance`
|
|
900
|
+
? `manager instance ${this.managerInstanceId} already serves space "${this.space}" from ${held.root === this.leaseInfo.root ? "this" : "another"} workspace root (${held.runtime}, pid ${held.pid}, root ${held.root}) - stop it first before restarting the same instance`
|
|
885
901
|
: `could not acquire the manager lease for space "${this.space}": ${e.message}`);
|
|
886
902
|
}
|
|
887
903
|
this.leaseTimer = setInterval(() => { void this.renewLease(); }, MANAGER_LEASE_RENEW_MS);
|
|
@@ -1042,6 +1058,18 @@ export class Manager {
|
|
|
1042
1058
|
});
|
|
1043
1059
|
await this.runHosting.reconcile();
|
|
1044
1060
|
}
|
|
1061
|
+
// A carried resume (#1499) needs this instance's transfer reader: minted from the space's signing
|
|
1062
|
+
// seed, issued by the host to a remote manager, or none on an open mesh.
|
|
1063
|
+
if (!this.remoteAuthority || this.remoteAuthority.mintTransferReader) {
|
|
1064
|
+
this.transcripts = new TranscriptReceiver({
|
|
1065
|
+
space: this.space,
|
|
1066
|
+
instanceId: this.managerInstanceId,
|
|
1067
|
+
workspaceRoot: this.workspaceRoot,
|
|
1068
|
+
withReader: (fn) => this.withTransferReader(fn),
|
|
1069
|
+
log: (line) => console.error(line),
|
|
1070
|
+
});
|
|
1071
|
+
await this.transcripts.start();
|
|
1072
|
+
}
|
|
1045
1073
|
// Plane-3 (durable backstop) is NOT the manager's job — the manager only manages agent lifecycle.
|
|
1046
1074
|
// The server-side delivery daemon hosts the fan-out writer + trusted reader, owns the durable
|
|
1047
1075
|
// membership registry, and serves the runtime durable join/leave/list ops (on `ctl.delivery`). The
|
|
@@ -1483,10 +1511,9 @@ export class Manager {
|
|
|
1483
1511
|
let candidate;
|
|
1484
1512
|
try {
|
|
1485
1513
|
candidate = await remote.renewStandingBundle(processEpoch);
|
|
1486
|
-
|
|
1487
|
-
if (Object.keys(candidate).sort().join(",") !== [...names].sort().join(","))
|
|
1514
|
+
if (Object.keys(candidate).sort().join(",") !== [...REMOTE_MANAGER_IDENTITY_NAMES].sort().join(","))
|
|
1488
1515
|
throw new Error("host returned an incomplete or widened standing credential family");
|
|
1489
|
-
for (const name of
|
|
1516
|
+
for (const name of REMOTE_MANAGER_IDENTITY_NAMES) {
|
|
1490
1517
|
if (typeof candidate[name] !== "string" || inspectCredHealth(candidate[name]).state === "unreadable")
|
|
1491
1518
|
throw new Error(`host returned an unreadable ${name} credential`);
|
|
1492
1519
|
if (!remote.accountPublicKey || accountFromCreds(candidate[name]) !== remote.accountPublicKey)
|
|
@@ -1676,10 +1703,14 @@ export class Manager {
|
|
|
1676
1703
|
});
|
|
1677
1704
|
}
|
|
1678
1705
|
// A later manager does not resume a delegated seat, and its close runs only inside its
|
|
1679
|
-
// retirement, so a cut that holds one is refused before any child stops.
|
|
1680
|
-
|
|
1706
|
+
// retirement, so a cut that holds one is refused before any child stops. A carried resume seat's
|
|
1707
|
+
// private home is removed when its slot is freed, so a later manager would find nothing to fork.
|
|
1708
|
+
for (const a of this.agents.values()) {
|
|
1681
1709
|
if (a.handedOff)
|
|
1682
1710
|
failures.push({ name: a.name, id: a.id, error: "a delegated seat is not resumed by a later manager; stop it before preserving" });
|
|
1711
|
+
if (a.seatHome)
|
|
1712
|
+
failures.push({ name: a.name, id: a.id, error: "a carried resume seat's private home is not kept for a later manager; stop it before preserving" });
|
|
1713
|
+
}
|
|
1683
1714
|
const unverifiedStops = this.unverifiedStops.filter((stopped) => {
|
|
1684
1715
|
try {
|
|
1685
1716
|
if (!stopped.authoritative && stopped.handle.status() === "exited")
|
|
@@ -2049,6 +2080,7 @@ export class Manager {
|
|
|
2049
2080
|
}
|
|
2050
2081
|
async stopOnce(withAgents) {
|
|
2051
2082
|
this.leaseStopping = true;
|
|
2083
|
+
this.transcripts?.stop();
|
|
2052
2084
|
this.staticReconcileStopping = true;
|
|
2053
2085
|
const starting = this.startTask;
|
|
2054
2086
|
for (const item of this.staticReconcileItems.values()) {
|
|
@@ -2554,8 +2586,13 @@ export class Manager {
|
|
|
2554
2586
|
async epAdminReach(caller) {
|
|
2555
2587
|
if (!this.userMode)
|
|
2556
2588
|
return true;
|
|
2557
|
-
if (this.remoteAuthority)
|
|
2558
|
-
|
|
2589
|
+
if (this.remoteAuthority) {
|
|
2590
|
+
// A hosted run's own caller is derived from its id and holds no ledger row: it acts for the
|
|
2591
|
+
// caller its run was admitted for, so that caller's current grant decides, and a revoked
|
|
2592
|
+
// login demotes the run's next spawn.
|
|
2593
|
+
const asked = this.runHosting?.admittedCaller(caller) ?? caller;
|
|
2594
|
+
return this.remoteAuthority.authorizeAdmin({ owner: asked.owner, actor: asked.actor, lifecycleUid: asked.uid });
|
|
2595
|
+
}
|
|
2559
2596
|
try {
|
|
2560
2597
|
const scope = await resolveAuthProvider().actorScope({
|
|
2561
2598
|
dir: userAuthStateDir(this.workspaceRoot, this.space),
|
|
@@ -2878,6 +2915,11 @@ export class Manager {
|
|
|
2878
2915
|
preparePreservation: (ctx) => adminGated(ctx, async () => unwrap(await this.opPreservationCtl("preparePreservation", args(ctx)))),
|
|
2879
2916
|
commitPreservation: (ctx) => adminGated(ctx, async () => unwrap(await this.opPreservationCtl("commitPreservation", args(ctx)))),
|
|
2880
2917
|
abortPreservation: (ctx) => adminGated(ctx, async () => unwrap(await this.opPreservationCtl("abortPreservation", args(ctx)))),
|
|
2918
|
+
transcriptReceive: (ctx) => this.serveGated(ctx, () => adminGated(ctx, () => {
|
|
2919
|
+
if (!this.transcripts)
|
|
2920
|
+
throw new EpEnvelopeError("failed-precondition", "transcript-receive: this manager does not receive carried transcripts (its remote authority issues no transfer reader)");
|
|
2921
|
+
return this.transcripts.receive(args(ctx));
|
|
2922
|
+
})),
|
|
2881
2923
|
});
|
|
2882
2924
|
}
|
|
2883
2925
|
async commitResumeActivation(attemptId) {
|
|
@@ -3208,15 +3250,14 @@ export class Manager {
|
|
|
3208
3250
|
// Roll back everything this attempt materialized — a refused spawn must leave no standing
|
|
3209
3251
|
// secret, no ledger row, no durable footprint — and AWAIT the broker teardown: the caller
|
|
3210
3252
|
// may respawn the moment it reads the refusal, and a detached teardown would race (and
|
|
3211
|
-
// delete) that fresh spawn's just-provisioned durables.
|
|
3212
|
-
|
|
3213
|
-
|
|
3214
|
-
|
|
3215
|
-
|
|
3216
|
-
|
|
3217
|
-
|
|
3218
|
-
|
|
3219
|
-
return { error: `agent auth preflight failed for "${name}": ${e.message}` };
|
|
3253
|
+
// delete) that fresh spawn's just-provisioned durables. deprovision shreds the secret family
|
|
3254
|
+
// and revokes the grant, but only logs a failed revoke, so the revoke is attempted here first.
|
|
3255
|
+
// Each failure joins the refusal, so the caller learns what this spawn left behind.
|
|
3256
|
+
const failed = [];
|
|
3257
|
+
await attemptCleanup(failed, "revoke agent grant", () => provider.revokeAgent({ dir, owner, actor: name }));
|
|
3258
|
+
await attemptCleanup(failed, "deprovision", () => this.deprovision({ id: principalKey(owner, name).key, name, lifecycleUid: opts.lifecycleUid, userOwner: owner, secretPaths: files }));
|
|
3259
|
+
const leftover = failed.length ? `; cleanup failed: ${failed.join("; ")}` : "";
|
|
3260
|
+
return { error: `agent auth preflight failed for "${name}": ${rejectionText(e)}${leftover}` };
|
|
3220
3261
|
}
|
|
3221
3262
|
}
|
|
3222
3263
|
/**
|
|
@@ -3285,7 +3326,7 @@ export class Manager {
|
|
|
3285
3326
|
if (files.actorToken !== staged.actorToken) {
|
|
3286
3327
|
await secrets.put(agentSecretKeyForFile(files.actorToken, this.space), actorToken);
|
|
3287
3328
|
await materializeSecretToFile(secrets, agentSecretKeyForFile(files.actorToken, this.space), files.actorToken);
|
|
3288
|
-
await secrets.delete(agentSecretKeyForFile(staged.actorToken, this.space))
|
|
3329
|
+
await secrets.delete(agentSecretKeyForFile(staged.actorToken, this.space));
|
|
3289
3330
|
rmSync(staged.actorToken, { force: true });
|
|
3290
3331
|
}
|
|
3291
3332
|
await secrets.put(agentSecretKeyForFile(files.sentinelCreds, this.space), material.sentinelCreds);
|
|
@@ -3320,19 +3361,11 @@ export class Manager {
|
|
|
3320
3361
|
// A failed removal joins the refusal instead of throwing, so every removal is attempted and
|
|
3321
3362
|
// the refusal still reaches the caller.
|
|
3322
3363
|
const unshredded = [];
|
|
3323
|
-
const shred = async (what, remove) => {
|
|
3324
|
-
try {
|
|
3325
|
-
await remove();
|
|
3326
|
-
}
|
|
3327
|
-
catch (err) {
|
|
3328
|
-
unshredded.push(`${what}: ${rejectionText(err)}`);
|
|
3329
|
-
}
|
|
3330
|
-
};
|
|
3331
3364
|
for (const family of files.actorToken === staged.actorToken ? [staged] : [staged, files]) {
|
|
3332
3365
|
for (const path of [family.actorToken, family.sentinelCreds])
|
|
3333
|
-
await
|
|
3366
|
+
await attemptCleanup(unshredded, `secret ${path}`, () => secrets.delete(agentSecretKeyForFile(path, this.space)));
|
|
3334
3367
|
for (const path of [family.actorToken, family.sentinelCreds, family.health])
|
|
3335
|
-
await
|
|
3368
|
+
await attemptCleanup(unshredded, `file ${path}`, () => rmSync(path, { force: true }));
|
|
3336
3369
|
}
|
|
3337
3370
|
const leftover = unshredded.length ? `; cleanup failed: ${unshredded.join("; ")}` : "";
|
|
3338
3371
|
return { error: `agent auth preflight failed for "${name}": ${rejectionText(e)}${leftover}` };
|
|
@@ -3459,6 +3492,8 @@ export class Manager {
|
|
|
3459
3492
|
this.recordContinuityAtStop(a, "stop");
|
|
3460
3493
|
if (a.restart?.sessionStatePath)
|
|
3461
3494
|
rmSync(a.restart.sessionStatePath, { force: true });
|
|
3495
|
+
if (a.seatHome)
|
|
3496
|
+
rmSync(a.seatHome, { recursive: true, force: true });
|
|
3462
3497
|
// P2 item 6 (pin 4): end any live §13.6 attach session bound to THIS incarnation with the honest
|
|
3463
3498
|
// `target-despawn` reason. Fires once per agent on every free path (despawn / self-stop / reap /
|
|
3464
3499
|
// exit) via the `agents` guard above; a no-op when no plane or no live session for the target.
|
|
@@ -3583,22 +3618,25 @@ export class Manager {
|
|
|
3583
3618
|
// unowned same-name file is the exact successor-clobber this ownership discipline removes.
|
|
3584
3619
|
const secrets = this.secrets;
|
|
3585
3620
|
const files = a.secretPaths ?? agentLifecycleSecretFilePaths(this.workspaceRoot, this.space, a.name, a.lifecycleUid);
|
|
3621
|
+
// Every removal is attempted, and a failed one is thrown only after the broker teardown below,
|
|
3622
|
+
// so a file fault cannot strand the agent's durables and ACL row on the broker.
|
|
3623
|
+
const unshredded = [];
|
|
3586
3624
|
if (files.creds) {
|
|
3587
|
-
|
|
3588
|
-
|
|
3625
|
+
const creds = files.creds;
|
|
3626
|
+
await attemptCleanup(unshredded, `secret ${creds}`, () => secrets.delete(agentSecretKeyForFile(creds, this.space)));
|
|
3627
|
+
await attemptCleanup(unshredded, `file ${creds}`, () => rmSync(creds, { force: true }));
|
|
3589
3628
|
}
|
|
3590
3629
|
if (a.userOwner) {
|
|
3591
3630
|
// USER MODE: this teardown IS revocation, not just footprint reduction — the ledger row is
|
|
3592
3631
|
// the agent's standing mint authority, so delete it (next exchange refused, next connect
|
|
3593
3632
|
// denied) and shred the secret/sentinel/health files. A copied actor token dies here; a
|
|
3594
3633
|
// still-LIVE connection ends at its bearer-bound JWT expiry (≤ the agent TTL).
|
|
3595
|
-
|
|
3596
|
-
|
|
3597
|
-
|
|
3598
|
-
|
|
3599
|
-
|
|
3600
|
-
|
|
3601
|
-
rmSync(f, { force: true });
|
|
3634
|
+
for (const path of [files.actorToken, files.sentinelCreds])
|
|
3635
|
+
if (path)
|
|
3636
|
+
await attemptCleanup(unshredded, `secret ${path}`, () => secrets.delete(agentSecretKeyForFile(path, this.space)));
|
|
3637
|
+
for (const path of [files.actorToken, files.sentinelCreds, files.health])
|
|
3638
|
+
if (path)
|
|
3639
|
+
await attemptCleanup(unshredded, `file ${path}`, () => rmSync(path, { force: true }));
|
|
3602
3640
|
// The ledger row IS the agent's STANDING mint authority (a different store from the auth-plane
|
|
3603
3641
|
// cred ledger the rail retirement covers): while it lives, a copied actor token can still mint a
|
|
3604
3642
|
// fresh connect credential. So a FAILED revoke must NOT be swallowed into a clean terminal (INT-2):
|
|
@@ -3626,6 +3664,7 @@ export class Manager {
|
|
|
3626
3664
|
console.error(`revoke agent grant ${a.name}: ${e.message}`);
|
|
3627
3665
|
}
|
|
3628
3666
|
}
|
|
3667
|
+
const shredFailure = unshredded.length ? `could not shred "${a.name}": ${unshredded.join("; ")}` : undefined;
|
|
3629
3668
|
try {
|
|
3630
3669
|
const res = await this.deprovisionBroker(a);
|
|
3631
3670
|
const h = this.retiring.get(a.name);
|
|
@@ -3639,8 +3678,12 @@ export class Manager {
|
|
|
3639
3678
|
if (e instanceof DeprovisionError)
|
|
3640
3679
|
h.lastResources = e.accounting;
|
|
3641
3680
|
}
|
|
3681
|
+
if (shredFailure)
|
|
3682
|
+
throw new Error(`${rejectionText(e)}; ${shredFailure}`, { cause: e });
|
|
3642
3683
|
throw e;
|
|
3643
3684
|
}
|
|
3685
|
+
if (shredFailure)
|
|
3686
|
+
throw new Error(shredFailure);
|
|
3644
3687
|
// #29 piece 3: after the footprint teardown, ask the AUTH plane to RETIRE the lifecycle over
|
|
3645
3688
|
// the auth endpoint rail. The rail re-checks the SERVE-ISSUANCE GATE at serve time (not the
|
|
3646
3689
|
// space-manager lease - that check was replaced in 02794b2f) and refuses unless the registration
|
|
@@ -3929,8 +3972,15 @@ export class Manager {
|
|
|
3929
3972
|
* record cannot be read or does not name this seat's source, it is absent: `ps` then shows the
|
|
3930
3973
|
* source id alone rather than failing the listing for every other seat. */
|
|
3931
3974
|
forkProvenance(a) {
|
|
3932
|
-
|
|
3975
|
+
// A carried seat's provenance is set only by refuseCarriedFork, once its record names the claim.
|
|
3976
|
+
if (a.launch.resumed || a.launch.carried)
|
|
3933
3977
|
return a.launch.resumed;
|
|
3978
|
+
a.launch.resumed = this.readForkRecord(a);
|
|
3979
|
+
return a.launch.resumed;
|
|
3980
|
+
}
|
|
3981
|
+
readForkRecord(a) {
|
|
3982
|
+
if (!a.launch.resumeRecordPath || !a.launch.forkSource)
|
|
3983
|
+
return undefined;
|
|
3934
3984
|
let record;
|
|
3935
3985
|
try {
|
|
3936
3986
|
record = JSON.parse(readFileSync(a.launch.resumeRecordPath, "utf8"));
|
|
@@ -3942,8 +3992,43 @@ export class Manager {
|
|
|
3942
3992
|
if (r.source !== a.launch.forkSource || typeof r.transcriptSha256 !== "string" || !/^[a-f0-9]{64}$/.test(r.transcriptSha256) ||
|
|
3943
3993
|
!(r.title === undefined || r.title === null || (typeof r.title === "string" && r.title.length <= 1024)))
|
|
3944
3994
|
return undefined;
|
|
3945
|
-
|
|
3946
|
-
|
|
3995
|
+
return { source: r.source, ...(typeof r.title === "string" && r.title ? { title: r.title } : {}), transcriptSha256: r.transcriptSha256 };
|
|
3996
|
+
}
|
|
3997
|
+
/** Design section 9: a carried seat must fork the bytes the operator carried. Wait for its fork
|
|
3998
|
+
* record and hold it to the claim; on a match it becomes the seat's provenance, otherwise the
|
|
3999
|
+
* refusal is returned. A seat that writes no record cannot be shown to have forked the claim. */
|
|
4000
|
+
async refuseCarriedFork(a, timeoutMs) {
|
|
4001
|
+
const claim = a.launch.carried;
|
|
4002
|
+
const deadline = Date.now() + timeoutMs;
|
|
4003
|
+
let record;
|
|
4004
|
+
while (!(record = this.readForkRecord(a))) {
|
|
4005
|
+
if (a.terminalizing || a.handle.status() === "exited")
|
|
4006
|
+
return `${a.name} stopped before it wrote a fork record`;
|
|
4007
|
+
if (Date.now() >= deadline)
|
|
4008
|
+
return `${a.name} wrote no fork record within ${timeoutMs} ms, so it cannot be shown to have forked the carried transcript`;
|
|
4009
|
+
await new Promise((resolve) => setTimeout(resolve, 100));
|
|
4010
|
+
}
|
|
4011
|
+
if (record.transcriptSha256 !== claim.transcriptSha256)
|
|
4012
|
+
return `${a.name} forked a transcript with sha256:${record.transcriptSha256}, not the carried sha256:${claim.transcriptSha256}`;
|
|
4013
|
+
a.launch.resumed = { ...record, host: claim.host, transferredAt: claim.transferredAt };
|
|
4014
|
+
return undefined;
|
|
4015
|
+
}
|
|
4016
|
+
/** A carried seat whose launch was uncertain is held to the claim once it joins, as a started one
|
|
4017
|
+
* is; the wait ends when the seat stops or exits, which its exit watcher reaps. */
|
|
4018
|
+
async refuseLateCarriedFork(a, timeoutMs) {
|
|
4019
|
+
for (;;) {
|
|
4020
|
+
const late = await this.awaitReadiness(a, timeoutMs, { reapOnExit: false });
|
|
4021
|
+
if (late.ok)
|
|
4022
|
+
break;
|
|
4023
|
+
if (!late.uncertain || a.terminalizing)
|
|
4024
|
+
return;
|
|
4025
|
+
}
|
|
4026
|
+
const refused = await this.refuseCarriedFork(a, timeoutMs);
|
|
4027
|
+
if (refused === undefined || a.terminalizing || a.handle.status() === "exited")
|
|
4028
|
+
return;
|
|
4029
|
+
console.error(`! ${refused}; stopping it`);
|
|
4030
|
+
this.stopHandle(a, false);
|
|
4031
|
+
this.freeSlot(a, true, "carried-fork-refused");
|
|
3947
4032
|
}
|
|
3948
4033
|
readManagedSession(a) {
|
|
3949
4034
|
return this.readManagedSessionState(a).sessionId;
|
|
@@ -4096,10 +4181,10 @@ export class Manager {
|
|
|
4096
4181
|
}
|
|
4097
4182
|
a.handle = handle;
|
|
4098
4183
|
a.control = spec.control;
|
|
4184
|
+
await this.recordSlotRuntime(a);
|
|
4099
4185
|
replacement = undefined;
|
|
4100
4186
|
restart.opts = opts;
|
|
4101
4187
|
restart.recovering = false;
|
|
4102
|
-
await this.recordSlotRuntime(a);
|
|
4103
4188
|
if (continueSession !== undefined)
|
|
4104
4189
|
console.error(`! ${a.name}: recovered Pi session ${continueSession} after crash (${restart.crashes.length}/${limit})`);
|
|
4105
4190
|
else
|
|
@@ -4316,6 +4401,8 @@ export class Manager {
|
|
|
4316
4401
|
variant: args.variant ? String(args.variant) : undefined,
|
|
4317
4402
|
launchOptions: args.launchOptions,
|
|
4318
4403
|
resume: args.resume ? String(args.resume) : undefined,
|
|
4404
|
+
resumeClaim: args.resumeClaim !== undefined ? String(args.resumeClaim) : undefined,
|
|
4405
|
+
resumeAgent: args.resumeAgent !== undefined ? String(args.resumeAgent) : undefined,
|
|
4319
4406
|
events,
|
|
4320
4407
|
eventsNotice,
|
|
4321
4408
|
cwd: args.cwd ? String(args.cwd) : undefined,
|
|
@@ -4445,6 +4532,31 @@ export class Manager {
|
|
|
4445
4532
|
return reason;
|
|
4446
4533
|
return `${reason}. This manager cannot launch that harness. The space may have other managers; pin one with --on <instance> (the whole id, as ps prints it)`;
|
|
4447
4534
|
}
|
|
4535
|
+
/** The harness binaries a managed launch runs, as boot resolved them, or the refusal naming what is
|
|
4536
|
+
* missing. Spawn and resume both take it, so a seat launches from the same files whichever path
|
|
4537
|
+
* starts it. A connector registered after boot has no inventory row and is checked on PATH now. */
|
|
4538
|
+
launchHarness(connector) {
|
|
4539
|
+
const row = this.connectorStatuses.find((status) => status.agent === connector.name);
|
|
4540
|
+
if (row?.state === "unavailable")
|
|
4541
|
+
return { refusal: row.reason ?? `${connector.name} harness is unavailable` };
|
|
4542
|
+
if (row)
|
|
4543
|
+
return { binaries: row.binaries };
|
|
4544
|
+
const missing = (connector.requires ?? []).filter((bin) => !resolveOnPath(bin));
|
|
4545
|
+
if (missing.length)
|
|
4546
|
+
return { refusal: `${connector.name} harness needs ${missing.join(", ")} on PATH - not found` };
|
|
4547
|
+
return {};
|
|
4548
|
+
}
|
|
4549
|
+
/** The refusal for a launch choice the connector cannot honour. Spawn and resume both ask before
|
|
4550
|
+
* any reserve or mint, so a rule changed here holds on both paths. */
|
|
4551
|
+
capabilityRefusal(connector, launch) {
|
|
4552
|
+
if (launch.variant && !connector.supportsModelVariant)
|
|
4553
|
+
return `${connector.name} connector does not support model variants (variant)`;
|
|
4554
|
+
if (launch.prompt !== undefined && !connector.supportsPrompt)
|
|
4555
|
+
return `${connector.name} connector does not support an initial prompt (prompt)`;
|
|
4556
|
+
if (launch.exact && (!connector.supportsSessionReopen || !connector.supportsSessionContinuation))
|
|
4557
|
+
return `${connector.name} connector does not support exact session continuity (continuity: exact)`;
|
|
4558
|
+
return undefined;
|
|
4559
|
+
}
|
|
4448
4560
|
/** Return connector-provided model catalogs for selector UIs. Optional by connector: a host with no
|
|
4449
4561
|
* local model-list API reports `supported:false` rather than blocking the manager. A connector that
|
|
4450
4562
|
* fails to import shows an `error:` row (from manifest enumeration) and never blocks the others. */
|
|
@@ -4695,22 +4807,33 @@ export class Manager {
|
|
|
4695
4807
|
// Harness preflight before reserving a slot or minting — a missing `claude`/`opencode` binary
|
|
4696
4808
|
// fails here with a clear name, not obscurely at process spawn. No fallback. All synchronous, so
|
|
4697
4809
|
// the reserve gate stays atomic. (The connector itself was resolved up top, before the capacity gate.)
|
|
4698
|
-
const
|
|
4699
|
-
|
|
4700
|
-
|
|
4701
|
-
return { ok: false, error: this.harnessUnavailableError(bootStatus.reason ?? `${agent} harness is unavailable`, route) };
|
|
4702
|
-
// A connector registered after boot has no inventory row. Keep the existing pre-mint backstop
|
|
4703
|
-
// for that dynamic library-composition case; ordinary installed connectors were checked at boot.
|
|
4704
|
-
if (!bootStatus) {
|
|
4705
|
-
const missing = (connector.requires ?? []).filter((bin) => !resolveOnPath(bin));
|
|
4706
|
-
if (missing.length)
|
|
4707
|
-
return { ok: false, error: this.harnessUnavailableError(`${agent} harness needs ${missing.join(", ")} on PATH - not found`, route) };
|
|
4708
|
-
}
|
|
4810
|
+
const harness = this.launchHarness(connector);
|
|
4811
|
+
if ("refusal" in harness)
|
|
4812
|
+
return { ok: false, error: this.harnessUnavailableError(harness.refusal, opts.route ?? "inst") };
|
|
4709
4813
|
// Resume is a connector capability: reject an unsupported resume HERE, before the reserve/mint, so
|
|
4710
4814
|
// it can never provision creds + durables and then throw at buildLaunch (mint-then-orphan). Same
|
|
4711
4815
|
// reject-before-side-effects window as the harness preflight above; buildLaunch stays the backstop.
|
|
4712
4816
|
if (opts.resume && !connector.supportsResume)
|
|
4713
4817
|
return { ok: false, error: `${agent} connector does not support resuming an existing session (resume)` };
|
|
4818
|
+
// A carried resume (#1499) is checked here too, before anything is minted. The claim is only
|
|
4819
|
+
// looked at: the connector may still refuse the launch, and a refused launch leaves it unspent.
|
|
4820
|
+
let carried;
|
|
4821
|
+
if (opts.resumeClaim !== undefined) {
|
|
4822
|
+
if (opts.resume === undefined)
|
|
4823
|
+
return { ok: false, error: "resumeClaim: names carried bytes of a resume; the launch carries no resume" };
|
|
4824
|
+
if (!connector.resumeTranscript)
|
|
4825
|
+
return { ok: false, error: `${agent} connector does not take a carried transcript (resumeClaim)` };
|
|
4826
|
+
if (opts.resumeAgent !== agent)
|
|
4827
|
+
return { ok: false, error: `resumeClaim: the session was carried for agent "${opts.resumeAgent ?? "<none>"}", and this launch resolves agent "${agent}"` };
|
|
4828
|
+
if (!this.transcripts)
|
|
4829
|
+
return { ok: false, error: "resumeClaim: this manager does not receive carried transcripts" };
|
|
4830
|
+
try {
|
|
4831
|
+
carried = this.transcripts.peek(opts.resumeClaim, opts.resume);
|
|
4832
|
+
}
|
|
4833
|
+
catch (e) {
|
|
4834
|
+
return { ok: false, error: e.message };
|
|
4835
|
+
}
|
|
4836
|
+
}
|
|
4714
4837
|
// A restart policy this host cannot honour is refused at accept, never accepted and ignored.
|
|
4715
4838
|
// External runtimes (tmux/cmux/orca/herdr) attach to a process they do not own and stream no
|
|
4716
4839
|
// exit, so a name cannot be respawned in place. User-mode seats have no static slot that
|
|
@@ -4883,18 +5006,15 @@ export class Manager {
|
|
|
4883
5006
|
const err = lifecycleBlocked("failed-precondition", `the name "${identityName}" is reserved pending retirement: its previous agent's despawn started that lifecycle's teardown (footprint + standing-authority revoke + auth-side retirement), and the name frees only when all of it completes${held.lastError !== undefined ? ` (last attempt: ${held.lastError})` : ""}. NEXT: wait a moment and retry this spawn (retrying re-drives the whole teardown), or pick another name.`, { blockedOp: "retirement", opId: held.opId, remedy: "retry" });
|
|
4884
5007
|
return { ok: false, error: renderLifecycleBlocked(err.message, err), details: err.details };
|
|
4885
5008
|
}
|
|
4886
|
-
if (variant && !connector.supportsModelVariant)
|
|
4887
|
-
return { ok: false, error: `${agent} connector does not support model variants (variant)` };
|
|
4888
|
-
if (prompt !== undefined && !connector.supportsPrompt)
|
|
4889
|
-
return { ok: false, error: `${agent} connector does not support an initial prompt (prompt)` };
|
|
4890
5009
|
// A manifest `continuity: exact` agent reopens the session this manager last bound to its
|
|
4891
5010
|
// declared name. Refused before any reserve or mint when the connector cannot reopen an exact
|
|
4892
5011
|
// session or the recorded assignment does not match this declaration.
|
|
4893
5012
|
const exact = opts.resolved?.continuity === "exact";
|
|
5013
|
+
const unsupported = this.capabilityRefusal(connector, { variant, prompt, exact });
|
|
5014
|
+
if (unsupported)
|
|
5015
|
+
return { ok: false, error: unsupported };
|
|
4894
5016
|
let reopenSession;
|
|
4895
5017
|
if (exact) {
|
|
4896
|
-
if (!connector.supportsSessionReopen || !connector.supportsSessionContinuation)
|
|
4897
|
-
return { ok: false, error: `${agent} connector does not support exact session continuity (continuity: exact)` };
|
|
4898
5018
|
try {
|
|
4899
5019
|
reopenSession = readContinuityAssignment(this.workspaceRoot, { space: this.space, name: identityName, connector: agent, cwd: resolvedCwd ?? this.workspaceRoot });
|
|
4900
5020
|
}
|
|
@@ -4981,6 +5101,8 @@ export class Manager {
|
|
|
4981
5101
|
// and retires over a live process. The slot row holds the same reference; this is the copy the
|
|
4982
5102
|
// in-process rollback can actually read.
|
|
4983
5103
|
let provisioned;
|
|
5104
|
+
// A carried resume's seat-private home, removed by the finally unless a live seat took it.
|
|
5105
|
+
let seatHome;
|
|
4984
5106
|
try {
|
|
4985
5107
|
// A stable nkey identity assigned at spawn: the public key is the agent's card.id (threaded via
|
|
4986
5108
|
// COTAL_ID); the seed is retained to mint matching creds later.
|
|
@@ -5206,10 +5328,11 @@ export class Manager {
|
|
|
5206
5328
|
eventsRequired: this.eventsRequired,
|
|
5207
5329
|
mcpServers,
|
|
5208
5330
|
envAllow,
|
|
5209
|
-
resolvedBinaries:
|
|
5331
|
+
resolvedBinaries: harness.binaries,
|
|
5210
5332
|
// So a connector that keeps per-agent local state can root it at the workspace, not the
|
|
5211
|
-
// (possibly per-agent) launch cwd
|
|
5333
|
+
// (possibly per-agent) launch cwd.
|
|
5212
5334
|
workspaceRoot: this.workspaceRoot,
|
|
5335
|
+
cwd,
|
|
5213
5336
|
};
|
|
5214
5337
|
// A delegated seat gets no local launch at all: no launch-material file, control token or
|
|
5215
5338
|
// manager path is produced for it, only the handoff its runtime writes into the child.
|
|
@@ -5223,7 +5346,14 @@ export class Manager {
|
|
|
5223
5346
|
handle = this.runtime.spawnDelegated({ agent, persona: readFileSync(configPath, "utf8"), role, model, variant, prompt, launchOptions: launchOptions, events }, handoff);
|
|
5224
5347
|
}
|
|
5225
5348
|
else {
|
|
5349
|
+
if (carried) {
|
|
5350
|
+
seatHome = join(this.workspaceRoot, ".cotal", "seat-homes", lifecycleUid);
|
|
5351
|
+
mkdirSync(seatHome, { recursive: true, mode: 0o700 });
|
|
5352
|
+
launchOpts.carried = { home: seatHome, transcript: carried.path, cwd };
|
|
5353
|
+
}
|
|
5226
5354
|
spec = connector.buildLaunch(launchOpts);
|
|
5355
|
+
if (carried)
|
|
5356
|
+
this.transcripts.consume(opts.resumeClaim, opts.resume);
|
|
5227
5357
|
spec.env = { ...spec.env, COTAL_MANAGER_INSTANCE: this.managerInstanceId };
|
|
5228
5358
|
handle = await this.spawnCustodied(name, spec, cwd, custody);
|
|
5229
5359
|
}
|
|
@@ -5232,6 +5362,7 @@ export class Manager {
|
|
|
5232
5362
|
name,
|
|
5233
5363
|
role,
|
|
5234
5364
|
agent,
|
|
5365
|
+
...(seatHome ? { seatHome } : {}),
|
|
5235
5366
|
id: userLaunch ? principalKey(userLaunch.owner, name).key : identity.id,
|
|
5236
5367
|
lifecycleUid,
|
|
5237
5368
|
// The lifecycle-keyed family this spawn just materialized (absent on an open mesh) — the
|
|
@@ -5272,6 +5403,7 @@ export class Manager {
|
|
|
5272
5403
|
shareTools: opts.shareTools,
|
|
5273
5404
|
forkSource: opts.resume,
|
|
5274
5405
|
...(opts.resume !== undefined && spec?.resumeRecordPath ? { resumeRecordPath: spec.resumeRecordPath } : {}),
|
|
5406
|
+
...(carried ? { carried: { transcriptSha256: carried.sha256, host: carried.sourceHost, transferredAt: carried.stagedAt } } : {}),
|
|
5275
5407
|
// Opaque values may contain secrets. Preserve only their keys and require the referenced
|
|
5276
5408
|
// persona/manifest to resolve the values again; imperative overrides have no safe payload.
|
|
5277
5409
|
unresolvedLaunchOptionKeys: opts.launchOptions && Object.keys(opts.launchOptions).length
|
|
@@ -5311,6 +5443,7 @@ export class Manager {
|
|
|
5311
5443
|
// The live slot now owns teardown — freeSlot deprovisions this identity on exit — so the
|
|
5312
5444
|
// orphan-rollback in `finally` no longer applies to it.
|
|
5313
5445
|
provisioned = undefined;
|
|
5446
|
+
seatHome = undefined;
|
|
5314
5447
|
// #159 B1: reply on a REAL outcome, not a timer. Wait for the agent to actually join the mesh
|
|
5315
5448
|
// (presence) → started, the child to exit → failed (with its last output; already reaped), or
|
|
5316
5449
|
// neither in time → uncertain. `✓ started` therefore means "it joined", never just "a process
|
|
@@ -5336,9 +5469,24 @@ export class Manager {
|
|
|
5336
5469
|
// death — including one that follows an `uncertain` verdict, which deliberately does NOT deprovision).
|
|
5337
5470
|
if (!readiness.ok) {
|
|
5338
5471
|
this.watchExit(managed);
|
|
5472
|
+
if (managed.launch.carried)
|
|
5473
|
+
void this.refuseLateCarriedFork(managed, readinessTimeoutMs);
|
|
5339
5474
|
await hooks?.onOutcome?.({ kind: "uncertain", data: { reason: readiness.detail } });
|
|
5340
5475
|
return { ok: false, error: readiness.detail };
|
|
5341
5476
|
}
|
|
5477
|
+
if (managed.launch.carried) {
|
|
5478
|
+
const refused = await this.refuseCarriedFork(managed, readinessTimeoutMs);
|
|
5479
|
+
if (refused !== undefined && managed.terminalizing) {
|
|
5480
|
+
hooks?.onTerminalDeferred?.();
|
|
5481
|
+
return { ok: false, error: refused };
|
|
5482
|
+
}
|
|
5483
|
+
if (refused !== undefined) {
|
|
5484
|
+
this.stopHandle(managed, false);
|
|
5485
|
+
this.freeSlot(managed, true, "carried-fork-refused");
|
|
5486
|
+
await hooks?.onOutcome?.({ kind: "failed", data: { error: refused } });
|
|
5487
|
+
return { ok: false, error: refused };
|
|
5488
|
+
}
|
|
5489
|
+
}
|
|
5342
5490
|
if (managed.restart) {
|
|
5343
5491
|
if (connector.supportsSessionContinuation) {
|
|
5344
5492
|
try {
|
|
@@ -5385,14 +5533,23 @@ export class Manager {
|
|
|
5385
5533
|
// Failure after reserve (provision / launch threw): the slot was never live, so no cold-start
|
|
5386
5534
|
// was paid — the reserved rollback (finally) is enough, no cooling stamp.
|
|
5387
5535
|
// A lifecycle-blocked envelope already named the barrier; keep its details on the ControlReply
|
|
5388
|
-
// so follow/CLI/cotal_spawn do not collapse it to a generic string (#873).
|
|
5389
|
-
|
|
5390
|
-
|
|
5391
|
-
|
|
5536
|
+
// so follow/CLI/cotal_spawn do not collapse it to a generic string (#873). An extension can throw
|
|
5537
|
+
// any value, and an envelope's `message` is only typed as a string, so the refusal text always
|
|
5538
|
+
// comes from rejectionText; telling an envelope apart is guarded because a thrown Proxy can
|
|
5539
|
+
// throw from `instanceof` or from its own fields.
|
|
5540
|
+
const error = rejectionText(e);
|
|
5541
|
+
try {
|
|
5542
|
+
if (e instanceof EpEnvelopeError)
|
|
5543
|
+
return { ok: false, error: renderLifecycleBlocked(error, e), ...(e.details ? { details: e.details } : {}) };
|
|
5544
|
+
}
|
|
5545
|
+
catch { /* not a readable envelope */ }
|
|
5546
|
+
return { ok: false, error };
|
|
5392
5547
|
}
|
|
5393
5548
|
finally {
|
|
5394
5549
|
this.reserved.delete(name);
|
|
5395
5550
|
this.reservedLive.delete(name);
|
|
5551
|
+
if (seatHome)
|
|
5552
|
+
rmSync(seatHome, { recursive: true, force: true });
|
|
5396
5553
|
// Minted but never handed to a live slot (buildLaunch / runtime.spawn threw after mint) → tear the
|
|
5397
5554
|
// orphan down (detached, fail-loud) so a failed spawn leaves no creds/durables behind (#159 B).
|
|
5398
5555
|
if (provisioned) {
|
|
@@ -5799,13 +5956,9 @@ export class Manager {
|
|
|
5799
5956
|
catch (e) {
|
|
5800
5957
|
return { ok: false, error: e.message };
|
|
5801
5958
|
}
|
|
5802
|
-
const
|
|
5803
|
-
if (
|
|
5804
|
-
return { ok: false, error:
|
|
5805
|
-
if (entry.launch.variant && !connector.supportsModelVariant)
|
|
5806
|
-
return { ok: false, error: `${connector.name} connector does not support model variants (variant)` };
|
|
5807
|
-
if (entry.launch.prompt !== undefined && !connector.supportsPrompt)
|
|
5808
|
-
return { ok: false, error: `${connector.name} connector does not support an initial prompt (prompt)` };
|
|
5959
|
+
const harness = this.launchHarness(connector);
|
|
5960
|
+
if ("refusal" in harness)
|
|
5961
|
+
return { ok: false, error: harness.refusal };
|
|
5809
5962
|
let retainedSession;
|
|
5810
5963
|
try {
|
|
5811
5964
|
retainedSession = this.retainedSessionId(entry, connector);
|
|
@@ -5837,10 +5990,6 @@ export class Manager {
|
|
|
5837
5990
|
return { ok: false, error: `retained manifest agent ${launchSource.requested} is missing or its hash changed; refusing same-principal resume` };
|
|
5838
5991
|
launchOptions = spec.launchOptions;
|
|
5839
5992
|
exact = spec.continuity === "exact";
|
|
5840
|
-
// An exact seat reopens its retained session or fails, like its manifest launch, so the
|
|
5841
|
-
// connector must honor reopenSession before the batch starts any child.
|
|
5842
|
-
if (exact && (!connector.supportsSessionReopen || !connector.supportsSessionContinuation))
|
|
5843
|
-
return { ok: false, error: `${connector.name} connector does not support exact session continuity (continuity: exact)` };
|
|
5844
5993
|
}
|
|
5845
5994
|
else {
|
|
5846
5995
|
try {
|
|
@@ -5850,6 +5999,11 @@ export class Manager {
|
|
|
5850
5999
|
return { ok: false, error: e.message };
|
|
5851
6000
|
}
|
|
5852
6001
|
}
|
|
6002
|
+
// Asked once `exact` is known: an exact seat reopens its retained session or fails, like its
|
|
6003
|
+
// manifest launch, so the connector must honor reopenSession before the batch starts any child.
|
|
6004
|
+
const unsupported = this.capabilityRefusal(connector, { variant: entry.launch.variant, prompt: entry.launch.prompt, exact });
|
|
6005
|
+
if (unsupported)
|
|
6006
|
+
return { ok: false, error: unsupported };
|
|
5853
6007
|
let authority;
|
|
5854
6008
|
try {
|
|
5855
6009
|
authority = await this.validateRetainedAuthority(entry);
|
|
@@ -5891,7 +6045,9 @@ export class Manager {
|
|
|
5891
6045
|
events: entry.launch.events,
|
|
5892
6046
|
mcpServers,
|
|
5893
6047
|
envAllow,
|
|
6048
|
+
resolvedBinaries: harness.binaries,
|
|
5894
6049
|
workspaceRoot: this.workspaceRoot,
|
|
6050
|
+
cwd: entry.launch.cwd,
|
|
5895
6051
|
};
|
|
5896
6052
|
const spec = connector.buildLaunch(launchOpts);
|
|
5897
6053
|
spec.env = { ...spec.env, COTAL_MANAGER_INSTANCE: this.managerInstanceId };
|
|
@@ -5981,7 +6137,24 @@ export class Manager {
|
|
|
5981
6137
|
this.reservedLive.add(entry.name);
|
|
5982
6138
|
if (this.resumeAttemptId)
|
|
5983
6139
|
this.resumedAgentNames.add(entry.name);
|
|
5984
|
-
|
|
6140
|
+
try {
|
|
6141
|
+
await this.recordSlotRuntime(managed);
|
|
6142
|
+
}
|
|
6143
|
+
catch (error) {
|
|
6144
|
+
const detail = `${managed.name} resumed, but ${error.message}`;
|
|
6145
|
+
this.stopHandle(managed, false);
|
|
6146
|
+
try {
|
|
6147
|
+
await this.awaitHandleExit(managed.handle);
|
|
6148
|
+
}
|
|
6149
|
+
catch (exit) {
|
|
6150
|
+
// A resumed seat keeps its retained credentials, so freeing it runs no deprovision and
|
|
6151
|
+
// nothing would reap it: it stays managed until it exits or the manager stops it.
|
|
6152
|
+
this.watchExit(managed);
|
|
6153
|
+
return { ok: false, error: `${detail}; it stays managed because its stop is unproven: ${exit.message}` };
|
|
6154
|
+
}
|
|
6155
|
+
this.freeSlot(managed, true, "resume-custody-unrecorded", true);
|
|
6156
|
+
return { ok: false, error: detail };
|
|
6157
|
+
}
|
|
5985
6158
|
const readiness = await this.awaitReadiness(managed, readinessTimeoutMs);
|
|
5986
6159
|
if (!readiness.ok && !readiness.uncertain)
|
|
5987
6160
|
return { ok: false, error: readiness.detail };
|
|
@@ -6752,13 +6925,15 @@ export class Manager {
|
|
|
6752
6925
|
// with a fresh bounded exp; a restart re-provisions the same (idempotent) gate + re-registers.
|
|
6753
6926
|
const serveIdentity = this.managerServeIdentity;
|
|
6754
6927
|
const servePrincipal = principalKey(DEV_OWNER, serveIdentity.id).key;
|
|
6928
|
+
// An open mesh mints no credentials, so it has no family for a fresh pair to grow.
|
|
6929
|
+
const siblings = auth ? claimManagerSiblingIdentities(this.workspaceRoot, this.space) : { goalWriter: newIdentity(), sessionLedger: newIdentity() };
|
|
6755
6930
|
// must-5 (b): the STABLE goal-writer identity — a SIBLING credential in the same §13.1 family
|
|
6756
|
-
// (not the gate's bound serving principal),
|
|
6757
|
-
this.goalWriterIdentity =
|
|
6931
|
+
// (not the gate's bound serving principal), so the run block can family-stage it.
|
|
6932
|
+
this.goalWriterIdentity = siblings.goalWriter;
|
|
6758
6933
|
// P2 item 6: the STABLE session-LEDGER identity — another SIBLING in the SAME §13.1 family, so
|
|
6759
6934
|
// the takeover barrier revokes a deposed manager's ledger cred alongside its goal-writer. The
|
|
6760
6935
|
// per-session serving creds join the same family, each with its own fresh identity.
|
|
6761
|
-
this.sessionLedgerIdentity =
|
|
6936
|
+
this.sessionLedgerIdentity = siblings.sessionLedger;
|
|
6762
6937
|
// One registration operation for this boot. Fresh executors may replace a dead connection, but
|
|
6763
6938
|
// they must resume THIS freeze rather than minting a new op that discards Phase-2 progress.
|
|
6764
6939
|
const registrationOpId = mintLifecycleUid();
|
|
@@ -7468,9 +7643,7 @@ export class Manager {
|
|
|
7468
7643
|
* 3. OPEN MESH. No credential family exists, so the revoke/evict loop is vacuous and this belt
|
|
7469
7644
|
* is COOPERATIVE only: a non-conformant process simply does not run it.
|
|
7470
7645
|
* The named follow-up that would close 2 and 3 is the gate-linearized commit (routing the
|
|
7471
|
-
* terminal through the issuance gate's own CAS), deliberately deferred as substrate territory.
|
|
7472
|
-
* An earlier revision of this comment claimed the residual was "closed by item-3 slice 3.0,
|
|
7473
|
-
* never a permanent residual". That asserted a closure that does not exist. */
|
|
7646
|
+
* terminal through the issuance gate's own CAS), deliberately deferred as substrate territory. */
|
|
7474
7647
|
async assertGoalWriterEpochCurrent(epoch) {
|
|
7475
7648
|
const gate = this.goalWriter?.gate;
|
|
7476
7649
|
if (!gate)
|
|
@@ -8193,8 +8366,7 @@ export class Manager {
|
|
|
8193
8366
|
/** Re-record the custody reference of a managed agent's CURRENT handle on its active slot: a
|
|
8194
8367
|
* same-lifecycle restart or a resume binds a new custody under the old uid, and the successor's
|
|
8195
8368
|
* reap must address the live one. Static auth only; a runtime without durable custody records
|
|
8196
|
-
* nothing (there is nothing to reap by reference).
|
|
8197
|
-
* handle is already live and a stale reference is refused by identity at reap time. */
|
|
8369
|
+
* nothing (there is nothing to reap by reference). */
|
|
8198
8370
|
async recordSlotRuntime(a) {
|
|
8199
8371
|
if (a.handle.reference === undefined)
|
|
8200
8372
|
return;
|
|
@@ -8202,7 +8374,9 @@ export class Manager {
|
|
|
8202
8374
|
}
|
|
8203
8375
|
/** Record one custody reference on an agent's ACTIVE slot. Split from {@link recordSlotRuntime}
|
|
8204
8376
|
* so a restart or a resume can record the reference it RESERVED before spawning, while no
|
|
8205
|
-
* handle exists yet. Static auth only; a runtime without durable custody reserves nothing.
|
|
8377
|
+
* handle exists yet. Static auth only; a runtime without durable custody reserves nothing.
|
|
8378
|
+
* Throws when the slot does not take the reference: a slot left on the previous one reaps that
|
|
8379
|
+
* seat, already gone, so a successor would retire the lifecycle over the new one. */
|
|
8206
8380
|
async recordSlotCustody(a, runtime) {
|
|
8207
8381
|
if (!this.auth || this.userMode)
|
|
8208
8382
|
return;
|
|
@@ -8210,14 +8384,14 @@ export class Manager {
|
|
|
8210
8384
|
await this.withLifecycleExecutor({ owner: DEV_OWNER, actor: a.id, lifecycleUid: a.lifecycleUid, alias: a.name }, async (t) => {
|
|
8211
8385
|
const slot = await readStaticSlot(t, DEV_OWNER, a.name);
|
|
8212
8386
|
if (slot === undefined || slot.row.lifecycleUid !== a.lifecycleUid || slot.row.phase !== "active")
|
|
8213
|
-
|
|
8387
|
+
throw new Error(`the slot is ${slot === undefined ? "absent" : `${slot.row.phase} at uid ${slot.row.lifecycleUid}`}`);
|
|
8214
8388
|
if (slot.row.runtime?.kind === runtime.kind && slot.row.runtime.id === runtime.id)
|
|
8215
8389
|
return;
|
|
8216
8390
|
await casStaticSlot(t, { ...slot.row, runtime }, slot.revision);
|
|
8217
8391
|
});
|
|
8218
8392
|
}
|
|
8219
8393
|
catch (e) {
|
|
8220
|
-
|
|
8394
|
+
throw new Error(`could not record the custody reference ${runtime.kind}:${runtime.id} on the static slot of "${a.name}": ${e.message}`);
|
|
8221
8395
|
}
|
|
8222
8396
|
}
|
|
8223
8397
|
/** Mint the custody reference for a seat this manager is about to spawn, BEFORE the processes
|
|
@@ -8905,6 +9079,28 @@ export class Manager {
|
|
|
8905
9079
|
await prov.stop();
|
|
8906
9080
|
}
|
|
8907
9081
|
}
|
|
9082
|
+
/** A connection holding this instance's transfer reader grants (docs/design/resume-transfer.md
|
|
9083
|
+
* section 6) for this one receive or sweep: minted from the space's signing seed, or issued by the
|
|
9084
|
+
* host to a remote manager. An open mesh enforces no grants, so a bare one-shot connection is that
|
|
9085
|
+
* reader there. */
|
|
9086
|
+
async withTransferReader(fn) {
|
|
9087
|
+
const issue = this.remoteAuthority?.mintTransferReader;
|
|
9088
|
+
if (this.remoteAuthority && !issue)
|
|
9089
|
+
throw new Error("withTransferReader: this manager's remote authority issues no transfer reader");
|
|
9090
|
+
const identity = newIdentity();
|
|
9091
|
+
const creds = issue
|
|
9092
|
+
? await issue(identity)
|
|
9093
|
+
: this.auth && await mintCreds(this.auth, identity, "transfer-reader", { transferReader: { instanceId: this.managerInstanceId } });
|
|
9094
|
+
const nc = await this.dial(creds
|
|
9095
|
+
? { ...standaloneConnectOpts({ creds, /* not yet wired to a recorded transport */ tls: false }), maxReconnectAttempts: 0 }
|
|
9096
|
+
: { maxReconnectAttempts: 0 });
|
|
9097
|
+
try {
|
|
9098
|
+
return await fn(nc);
|
|
9099
|
+
}
|
|
9100
|
+
finally {
|
|
9101
|
+
await nc.drain().catch(() => nc.close());
|
|
9102
|
+
}
|
|
9103
|
+
}
|
|
8908
9104
|
/** Purge the space's retained message backlog (chat, optionally DMs). Privileged — the manager mints a
|
|
8909
9105
|
* short-lived "purger" cred (same destructive grant as `cotal history clear`, isolated off the
|
|
8910
9106
|
* supervisor); regular agents are denied STREAM.PURGE under auth. Cleanup only: leaves live agents and
|
|
@@ -9200,7 +9396,8 @@ export class Manager {
|
|
|
9200
9396
|
}
|
|
9201
9397
|
// establishAttach releases the claim it was handed on every exit; nothing to unwind here.
|
|
9202
9398
|
const { grant } = await this.sessionPlane.establishAttach(caller, { name: a.name, lifecycleUid: a.lifecycleUid }, session, slot);
|
|
9203
|
-
|
|
9399
|
+
const carried = a.launch.resumed?.host;
|
|
9400
|
+
return { ok: true, data: { grant, ...(carried ? { resumedFrom: { host: carried, source: a.launch.resumed.source } } : {}) } };
|
|
9204
9401
|
}
|
|
9205
9402
|
/** P2 item 6: the console's mesh §13.6 session establisher (backing `POST /session/<name>` on the
|
|
9206
9403
|
* loopback face). Drives THE ONE plane — same establishAttach as the ep `attach` command — with
|