@cotal-ai/manager 0.85.0 → 0.87.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands.js +25 -11
- package/dist/commands.js.map +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/dist/manager-service-contract.d.ts +7 -0
- package/dist/manager-service-contract.d.ts.map +1 -1
- package/dist/manager-service-contract.js +13 -35
- package/dist/manager-service-contract.js.map +1 -1
- package/dist/manager.d.ts +60 -12
- package/dist/manager.d.ts.map +1 -1
- package/dist/manager.js +376 -45
- package/dist/manager.js.map +1 -1
- package/dist/resume.d.ts +110 -0
- package/dist/resume.d.ts.map +1 -1
- package/dist/resume.js +57 -1
- package/dist/resume.js.map +1 -1
- package/dist/static-lifecycle.d.ts +2 -2
- package/dist/static-lifecycle.js +2 -2
- package/package.json +12 -12
package/dist/manager.js
CHANGED
|
@@ -4,8 +4,8 @@ import { credsAuthenticator } from "@nats-io/transport-node";
|
|
|
4
4
|
import { existsSync, lstatSync, mkdirSync, readFileSync, realpathSync, rmSync, statSync } from "node:fs";
|
|
5
5
|
import { isAbsolute, join, dirname, resolve } from "node:path";
|
|
6
6
|
import Headless from "@xterm/headless";
|
|
7
|
-
import { CotalEndpoint, canonicalJson, MANAGED_HANDOFF_KIND, accountFromCreds, credsClaims, DEFAULT_SERVER, DEV_OWNER, MANAGER_LEASE_RENEW_MS, STANDING_RENEWABLE_TTL_SEC, assertLifecycleToken, discardLaunchArtifacts, divergentSecretStoreNotice, injectedSecretStoreIdentity, parseDaemonStoreAnswer, parseSecretStoreIdentity, sameSecretStoreIdentity, agentFilePath, clearSpaceHistory, dialerFor, connectorServers, spawnEnvAllow, deprovisionAgent, DeprovisionError, isConcreteChannel, deprovisionTargetPrincipal, firstFreeName, spawnNameError, idFromCreds, inspectCredHealth, credsRenewalDelayMs, composeWirePersona, loadAgentFile, listPersonaCatalog, personaCatalogDescription, personaCatalogReadable, loadCotalConfig, modelPolicyRefusal, mintCreds, mintGeneration, mintAcceptedToken, openIssuedStore, acceptedBucket, issuedBucket, mintLifecycleUid, managedRetirementOpId, mkSecretDir, newIdentity, actionContext, parsePrincipalKey, principalKey, probeConnect, provisionAgent, provisionAgentDurables, registry, resolveAuthProvider, resolveReadAcl, runAgentBearer, saveAgentFile, subjectMatches, AUTH_ENDPOINT, EP_CMD_RETIRE_LIFECYCLE, controlServiceSubject, eventChannelPrincipal, resolveService, invokeCommand, } from "@cotal-ai/core";
|
|
8
|
-
import { agentAuthState, agentCredsDir, agentLifecycleSecretFilePaths, agentSecretFilePaths, agentSecretKeyForFile, authDir, claimIdentityRecord, claimManagerSiblingIdentities, connectorInstallHint, DELIVERY_CREDS_KIND, extensionConnectors, findCotalRoot, getSpaceAuth, hasUserAuthState, loadExtensionsManifest, loadManagerInstanceIdentity, loadMeshes, localTrustOfSpace, manifestExtensionNames, materializeFromManifest, materializeSecretToFile, MEMBERSHIP_RW_CREDS_KIND, mergeLaunchOptions, remintDaemonCreds, resolveAgentType, resolveOnPath, createManagerInstanceIdentity, spaceKey, spaceMaterialKey, SYSTEM_CREDS_FILES, userAuthStateDir, workspaceSecretStore, writeRenewalRecord } from "@cotal-ai/workspace";
|
|
7
|
+
import { CotalEndpoint, canonicalJson, MANAGED_HANDOFF_KIND, accountFromCreds, credsClaims, DEFAULT_SERVER, DEV_OWNER, MANAGER_LEASE_RENEW_MS, MANAGER_LEASE_TTL_MS, awaitLeaseExpiry, leaseRemainingMs, STANDING_RENEWABLE_TTL_SEC, assertLifecycleToken, discardLaunchArtifacts, divergentSecretStoreNotice, injectedSecretStoreIdentity, parseDaemonStoreAnswer, parseSecretStoreIdentity, sameSecretStoreIdentity, agentFilePath, clearSpaceHistory, dialerFor, connectorServers, spawnEnvAllow, deprovisionAgent, DeprovisionError, isConcreteChannel, deprovisionTargetPrincipal, firstFreeName, spawnNameError, idFromCreds, identityFromCreds, inspectCredHealth, credsRenewalDelayMs, composeWirePersona, loadAgentFile, listPersonaCatalog, personaCatalogDescription, personaCatalogReadable, loadCotalConfig, modelPolicyRefusal, mintCreds, mintGeneration, mintAcceptedToken, openIssuedStore, acceptedBucket, issuedBucket, mintLifecycleUid, managedRetirementOpId, mkSecretDir, newIdentity, actionContext, parsePrincipalKey, principalKey, probeConnect, provisionAgent, provisionAgentDurables, registry, resolveAuthProvider, resolveReadAcl, runAgentBearer, saveAgentFile, subjectMatches, AUTH_ENDPOINT, EP_CMD_RETIRE_LIFECYCLE, controlServiceSubject, eventChannelPrincipal, resolveService, invokeCommand, digestIdToken, } from "@cotal-ai/core";
|
|
8
|
+
import { agentAuthState, agentCredsDir, agentLifecycleSecretFilePaths, agentSecretFilePaths, agentSecretKeyForFile, authDir, claimIdentityRecord, claimManagerSiblingIdentities, connectorInstallHint, DELIVERY_CREDS_KIND, extensionConnectors, findCotalRoot, getSpaceAuth, hasUserAuthState, loadExtensionsManifest, loadManagerInstanceIdentity, loadMeshes, localTrustOfSpace, manifestExtensionNames, materializeFromManifest, materializeSecretToFile, MEMBERSHIP_RW_CREDS_KIND, mergeLaunchOptions, probeLiveness, remintDaemonCreds, resolveAgentType, resolveOnPath, createManagerInstanceIdentity, spaceKey, spaceMaterialKey, SYSTEM_CREDS_FILES, userAuthStateDir, workspaceSecretStore, writeRenewalRecord } from "@cotal-ai/workspace";
|
|
9
9
|
import { createRuntime, exitProof, isCustodialRuntime, pollProvedExit, requireRuntimeAdopt, requireRuntimeReap, RuntimeReapUnproven, watchProvedExit, } from "./runtime/index.js";
|
|
10
10
|
import { AttachEndpoint } from "./attach-endpoint.js";
|
|
11
11
|
import { DELIVERY_ADMIN_BOOT_WAIT_MS, isUnansweredDeliveryAdmin, makeManagerEndpointEvictionEvidence, makeManagerEndpointHolderEvictor, untilDeliveryAdminAnswers, withScopedEndpoint } from "./endpoint-evict.js";
|
|
@@ -16,7 +16,7 @@ import { authorizeLaunch, authorizeNamedControl } from "./authorize.js";
|
|
|
16
16
|
import { controlShutdown } from "./control-shutdown.js";
|
|
17
17
|
import { RunHosting } from "./run-hosting.js";
|
|
18
18
|
import { controlSession } from "./control-session.js";
|
|
19
|
-
import { MAX_AGENTS, parseResumeCommitArgs, parseResumeControlArgs, parseResumeFinalizeArgs } from "./resume.js";
|
|
19
|
+
import { MAX_AGENTS, parseResumeCommitArgs, parseResumeControlArgs, parseResumeFinalizeArgs, readSeatRecord, removeSeatRecord, writeSeatRecord } from "./resume.js";
|
|
20
20
|
import { TranscriptReceiver } from "./transcript-receive.js";
|
|
21
21
|
// Unit B (the static §13.1 lifecycle executor): the shared grammar/stores from core plus the
|
|
22
22
|
// manager-side adapter (transport + slot orchestration + the F1 terminal) — see static-lifecycle.ts.
|
|
@@ -219,6 +219,7 @@ const FREE_SLOT_CAUSE_TEXT = {
|
|
|
219
219
|
"resume-session-rebind-failed": "this manager stopped it: its host session could not be rebound on resume",
|
|
220
220
|
"carried-fork-refused": "this manager stopped it: it recorded no fork of the carried transcript",
|
|
221
221
|
"resume-custody-unrecorded": "this manager stopped it: its custody reference could not be recorded on resume",
|
|
222
|
+
"supervise-record-failed": "this manager stopped it: its supervise policy could not be recorded for a manager restart",
|
|
222
223
|
};
|
|
223
224
|
/** The stop family's sentences — the same job {@link FREE_SLOT_CAUSE_TEXT} does for the string
|
|
224
225
|
* causes, beside it so both stay the single place a cause becomes a sentence. A requested stop's
|
|
@@ -261,10 +262,10 @@ function parseTurnNote(raw) {
|
|
|
261
262
|
function goalKey(ref) {
|
|
262
263
|
return `${ref.caller.owner}.${ref.caller.actor}.${ref.caller.uid}.${ref.goalId}`;
|
|
263
264
|
}
|
|
264
|
-
/** A turn hold's token, DERIVED
|
|
265
|
-
*
|
|
265
|
+
/** A turn hold's token, DERIVED from the relay key, so two callers' turns under one goal id never
|
|
266
|
+
* share a hold. */
|
|
266
267
|
function turnHoldToken(key) {
|
|
267
|
-
return
|
|
268
|
+
return digestIdToken(`${key}:turn-deadline`);
|
|
268
269
|
}
|
|
269
270
|
/**
|
|
270
271
|
* Event channels in `channels` that do NOT belong to `{owner, actor}`.
|
|
@@ -464,11 +465,16 @@ export class Manager {
|
|
|
464
465
|
* credential is minted INSIDE {@link registerManagerService}'s run block and stashed here. */
|
|
465
466
|
sessionLedgerIdentity;
|
|
466
467
|
sessionLedgerCreds;
|
|
467
|
-
/** The acceptance replied for each
|
|
468
|
-
*
|
|
469
|
-
* without a second spawn.
|
|
470
|
-
*
|
|
468
|
+
/** The acceptance replied for each goal this incarnation accepted, by {@link goalKey}, so an
|
|
469
|
+
* idempotent same-goalId retry serves the IDENTICAL acceptance (same allocated name/triple)
|
|
470
|
+
* without a second spawn. An entry lives while its goal is in flight and for
|
|
471
|
+
* {@link goalAcceptanceRetentionMs} after its terminal commits; a later retry, or one at another
|
|
472
|
+
* incarnation, is answered from the goal index while the goal runs and from its committed
|
|
473
|
+
* result fact once it has ended. */
|
|
471
474
|
goalAcceptances = new Map();
|
|
475
|
+
/** How long a terminal goal's acceptance stays in {@link goalAcceptances}: the retry window its
|
|
476
|
+
* turn twin keeps a settled answer for. An instance field so a test can shorten it. */
|
|
477
|
+
goalAcceptanceRetentionMs = TURN_ANSWER_RETENTION_MS;
|
|
472
478
|
/** The boot reconcile of the durable goal index runs ONCE at start (a
|
|
473
479
|
* fresh incarnation inherits the endpoint's accepted-but-unterminal goals from any predecessor).
|
|
474
480
|
* Spawn-as-action REFUSES to accept until it completes, so the sweep never races a live goal's
|
|
@@ -982,22 +988,18 @@ export class Manager {
|
|
|
982
988
|
// THIS logical instance's own key (atomic CAS create). A DIFFERENT instance (a second manager in a
|
|
983
989
|
// second workspace root) has a distinct id ⇒ a distinct key ⇒ it coexists; the create THROWS only
|
|
984
990
|
// when the SAME instance id is already live (a same-root double-start, or a restart racing the
|
|
985
|
-
// crashed predecessor's not-yet-expired key)
|
|
986
|
-
//
|
|
991
|
+
// crashed predecessor's not-yet-expired key). A double-start REFUSES loud; a restart whose
|
|
992
|
+
// predecessor is provably dead on this host waits for that key's bucket TTL to remove it (#4092,
|
|
993
|
+
// {@link acquireInstanceLease}). Losing the key LATER ends this process only when another process holds it; otherwise
|
|
987
994
|
// the renew loop puts it back or keeps retrying ({@link renewLease}).
|
|
988
995
|
this.leaseInfo = { holder: this.ep.ref().id, instanceId: this.managerInstanceId, runtime: this.runtimeKind, root: resolve(this.workspaceRoot), pid: process.pid };
|
|
989
996
|
try {
|
|
990
|
-
this.leaseRevision = await this.
|
|
997
|
+
this.leaseRevision = await this.acquireInstanceLease(this.leaseInfo);
|
|
991
998
|
}
|
|
992
999
|
catch (e) {
|
|
993
|
-
// Our OWN instance id already holds a live key ⇒ refuse. Anything else (e.g. a KV/JS error) is a
|
|
994
|
-
// real failure to surface, not a silent "held" — keep the cause so it isn't misread as a conflict.
|
|
995
|
-
const held = (await this.ep.readOwnManagerLease(this.managerInstanceId).catch(() => undefined))?.info;
|
|
996
1000
|
await this.ep.stop();
|
|
997
1001
|
await this.attach.stop();
|
|
998
|
-
throw
|
|
999
|
-
? `manager instance ${this.managerInstanceId} already serves space "${this.space}" from ${held.root === this.leaseInfo.root ? "this" : "another"} workspace root (${held.runtime}, pid ${held.pid}, root ${held.root}) - stop it first before restarting the same instance`
|
|
1000
|
-
: `could not acquire the manager lease for space "${this.space}": ${rejectionText(e)}`);
|
|
1002
|
+
throw e;
|
|
1001
1003
|
}
|
|
1002
1004
|
this.leaseTimer = setInterval(() => { void this.renewLease(); }, MANAGER_LEASE_RENEW_MS);
|
|
1003
1005
|
this.leaseTimer.unref?.();
|
|
@@ -2021,6 +2023,13 @@ export class Manager {
|
|
|
2021
2023
|
}
|
|
2022
2024
|
}
|
|
2023
2025
|
resumeEntry(a, backfillFloor) {
|
|
2026
|
+
// A preservation cut is a stop boundary too, so a manifest launch after it reopens what a resume would.
|
|
2027
|
+
this.recordContinuityAtStop(a, "preservation");
|
|
2028
|
+
return this.resumeEntryOf(a, backfillFloor);
|
|
2029
|
+
}
|
|
2030
|
+
/** The resume entry of one live seat, with no side effect: a preservation cut and a supervised
|
|
2031
|
+
* seat's record both build it. */
|
|
2032
|
+
resumeEntryOf(a, backfillFloor) {
|
|
2024
2033
|
const principal = a.userOwner
|
|
2025
2034
|
? parsePrincipalKey(a.id)
|
|
2026
2035
|
: { owner: DEV_OWNER, actor: a.id };
|
|
@@ -2051,8 +2060,6 @@ export class Manager {
|
|
|
2051
2060
|
return { mode: "static", id: principal.actor, lifecycleUid: a.lifecycleUid, credential: { kind: "file", path: files.creds, sha256: this.fileDigestOrEmpty(files.creds) }, ...(a.issued ? { issued: a.issued } : {}) };
|
|
2052
2061
|
})()
|
|
2053
2062
|
: { mode: "open", id: principal.actor, lifecycleUid: a.lifecycleUid };
|
|
2054
|
-
// A preservation cut is a stop boundary too, so a manifest launch after it reopens what a resume would.
|
|
2055
|
-
this.recordContinuityAtStop(a, "preservation");
|
|
2056
2063
|
const dependencies = [a.launch.source.configPath];
|
|
2057
2064
|
if (a.launch.source.kind === "manifest" && a.launch.source.runId)
|
|
2058
2065
|
dependencies.unshift(join(this.workspaceRoot, ".cotal", "run", `${a.launch.source.runId}.json`));
|
|
@@ -2076,7 +2083,8 @@ export class Manager {
|
|
|
2076
2083
|
shareTools: a.launch.shareTools,
|
|
2077
2084
|
forkSource: a.launch.forkSource,
|
|
2078
2085
|
...(this.forkProvenance(a) ? { resumed: this.forkProvenance(a) } : {}),
|
|
2079
|
-
|
|
2086
|
+
// A supervised seat whose connector keeps no session state is armed with no state file to read.
|
|
2087
|
+
sessionId: a.restart?.armed && a.restart.sessionStatePath ? this.readManagedSession(a) : a.launch.sessionId,
|
|
2080
2088
|
// Additive and only when the connector supplied one. A seat with no pointer records no
|
|
2081
2089
|
// field, which is what keeps an older inventory and a fresh one the same document shape.
|
|
2082
2090
|
...(a.restart?.sessionStatePath ? { sessionStatePath: a.restart.sessionStatePath } : {}),
|
|
@@ -2105,6 +2113,7 @@ export class Manager {
|
|
|
2105
2113
|
// stop can't strand the rest, and every snapshot entry is deprovisioned below regardless.
|
|
2106
2114
|
this.recordContinuityAtStop(a, "shutdown");
|
|
2107
2115
|
this.agents.delete(a.name);
|
|
2116
|
+
this.dropSeatRecord(a);
|
|
2108
2117
|
const refused = this.stopHandle(a, false);
|
|
2109
2118
|
if (refused === undefined)
|
|
2110
2119
|
stopped.push(a);
|
|
@@ -2179,6 +2188,51 @@ export class Manager {
|
|
|
2179
2188
|
if (failures.length)
|
|
2180
2189
|
throw new Error(`manager preservation shutdown incomplete: ${failures.join("; ")}`);
|
|
2181
2190
|
}
|
|
2191
|
+
/** Create THIS instance's lease key, waiting out a dead predecessor's row first (#4092).
|
|
2192
|
+
*
|
|
2193
|
+
* A manager SIGKILLed on this host cannot release its key, so a restart inside the key's TTL
|
|
2194
|
+
* loses the create. When the row names this workspace root and a pid that is not running here
|
|
2195
|
+
* (or is this process's own pid, which a live predecessor cannot hold), the holder is dead and
|
|
2196
|
+
* the restart waits for the bucket TTL to remove the row, then creates again. It never writes
|
|
2197
|
+
* over the row: a live holder renews it inside the TTL, and the wait then refuses on that write.
|
|
2198
|
+
* Any other conflict refuses at once, as before. */
|
|
2199
|
+
async acquireInstanceLease(info) {
|
|
2200
|
+
const refuse = (held, e) => new Error(held
|
|
2201
|
+
? `manager instance ${this.managerInstanceId} already serves space "${this.space}" from ${held.root === info.root ? "this" : "another"} workspace root (${held.runtime}, pid ${held.pid}, root ${held.root}) - stop it first before restarting the same instance`
|
|
2202
|
+
: `could not acquire the manager lease for space "${this.space}": ${rejectionText(e)}`);
|
|
2203
|
+
try {
|
|
2204
|
+
return await this.ep.acquireManagerLease(info);
|
|
2205
|
+
}
|
|
2206
|
+
catch (e) {
|
|
2207
|
+
// Our OWN instance id already holds a live key ⇒ refuse. Anything else (e.g. a KV/JS error) is a
|
|
2208
|
+
// real failure to surface, not a silent "held" — keep the cause so it isn't misread as a conflict.
|
|
2209
|
+
const held = await this.ep.readOwnManagerLease(this.managerInstanceId).catch(() => undefined);
|
|
2210
|
+
if (!held || held.info.root !== info.root || (held.info.pid !== process.pid && probeLiveness(held.info.pid) !== "dead"))
|
|
2211
|
+
throw refuse(held?.info, e);
|
|
2212
|
+
console.error(`• manager instance ${this.managerInstanceId}: its lease is still held by pid ${held.info.pid}, which is not running on this host; ` +
|
|
2213
|
+
`waiting ${Math.ceil(leaseRemainingMs(held.created, MANAGER_LEASE_TTL_MS) / 1000)}s for the lease to expire`);
|
|
2214
|
+
const waited = await awaitLeaseExpiry({
|
|
2215
|
+
read: async () => {
|
|
2216
|
+
const row = await this.ep.readOwnManagerLease(this.managerInstanceId);
|
|
2217
|
+
return row && { revision: row.revision, created: row.created };
|
|
2218
|
+
},
|
|
2219
|
+
revision: held.revision,
|
|
2220
|
+
created: held.created,
|
|
2221
|
+
ttlMs: MANAGER_LEASE_TTL_MS,
|
|
2222
|
+
aborted: () => this.leaseStopping,
|
|
2223
|
+
});
|
|
2224
|
+
if (waited === "rewritten")
|
|
2225
|
+
throw new Error(`manager instance ${this.managerInstanceId}: its lease was written again while waiting for pid ${held.info.pid}'s lease to expire, so a live process holds it - stop it first before restarting the same instance`);
|
|
2226
|
+
if (waited !== "expired")
|
|
2227
|
+
throw refuse(held.info, e);
|
|
2228
|
+
try {
|
|
2229
|
+
return await this.ep.acquireManagerLease(info);
|
|
2230
|
+
}
|
|
2231
|
+
catch (again) {
|
|
2232
|
+
throw refuse((await this.ep.readOwnManagerLease(this.managerInstanceId).catch(() => undefined))?.info, again);
|
|
2233
|
+
}
|
|
2234
|
+
}
|
|
2235
|
+
}
|
|
2182
2236
|
/** Every call joins the first stop and settles with it. A call that asks for a different
|
|
2183
2237
|
* `withAgents` is refused, since the running stop has already applied its own policy. */
|
|
2184
2238
|
stop(options = {}) {
|
|
@@ -2853,7 +2907,9 @@ export class Manager {
|
|
|
2853
2907
|
return a;
|
|
2854
2908
|
};
|
|
2855
2909
|
return managerCommandDefs({
|
|
2856
|
-
|
|
2910
|
+
// A remote Manager files every seat under its own owner, so only a user mesh's shared local
|
|
2911
|
+
// Manager narrows the count, and status never waits on a remote admin check.
|
|
2912
|
+
status: (ctx) => this.serveGated(ctx, async () => this.managerStatusData(this.remoteAuthority ? undefined : await this.psOwnerFilter(ctx.subject.caller, false))),
|
|
2857
2913
|
ps: (ctx) => this.serveGated(ctx, async () => this.list(await this.psOwnerFilter(ctx.subject.caller, false))),
|
|
2858
2914
|
inspect: (ctx) => this.serveGated(ctx, async () => {
|
|
2859
2915
|
const name = String(args(ctx).name ?? "").trim();
|
|
@@ -3622,6 +3678,7 @@ export class Manager {
|
|
|
3622
3678
|
this.recordContinuityAtStop(a, "stop");
|
|
3623
3679
|
if (a.restart?.sessionStatePath)
|
|
3624
3680
|
rmSync(a.restart.sessionStatePath, { force: true });
|
|
3681
|
+
this.dropSeatRecord(a);
|
|
3625
3682
|
if (a.seatHome)
|
|
3626
3683
|
rmSync(a.seatHome, { recursive: true, force: true });
|
|
3627
3684
|
// End any live §13.6 attach session bound to THIS incarnation with the honest
|
|
@@ -4319,6 +4376,7 @@ export class Manager {
|
|
|
4319
4376
|
replacement = undefined;
|
|
4320
4377
|
restart.opts = opts;
|
|
4321
4378
|
restart.recovering = false;
|
|
4379
|
+
this.refreshSeatRecord(a);
|
|
4322
4380
|
if (continueSession !== undefined)
|
|
4323
4381
|
console.error(`! ${a.name}: recovered Pi session ${continueSession} after crash (${restart.crashes.length}/${limit})`);
|
|
4324
4382
|
else
|
|
@@ -4568,9 +4626,11 @@ export class Manager {
|
|
|
4568
4626
|
}
|
|
4569
4627
|
return { cwd, host: hostname() };
|
|
4570
4628
|
}
|
|
4571
|
-
/** Resolve a placed spawn directory on this manager's host.
|
|
4572
|
-
*
|
|
4629
|
+
/** Resolve a placed spawn directory on this manager's host. Delegated execution refuses before
|
|
4630
|
+
* filesystem lookup, since its seats cannot use host cwd. Creates nothing and never substitutes the workspace root. */
|
|
4573
4631
|
resolveSpawnCwd(value) {
|
|
4632
|
+
if (this.execution === "runtime" && this.requiredRuntime.spawnDelegated)
|
|
4633
|
+
throw new EpEnvelopeError("failed-precondition", `runtime "${this.requiredRuntime.kind}" starts seats outside this host, so it cannot resolve a directory on this host's filesystem (cwd); the seat runs in its runtime resource's own directory`);
|
|
4574
4634
|
const asked = typeof value === "string" ? value : String(value ?? "");
|
|
4575
4635
|
const refuse = (reason) => {
|
|
4576
4636
|
throw new EpEnvelopeError("failed-precondition", `cwd does not resolve on this host: ${JSON.stringify(asked)} (${reason})`);
|
|
@@ -4698,7 +4758,7 @@ export class Manager {
|
|
|
4698
4758
|
};
|
|
4699
4759
|
}
|
|
4700
4760
|
/** The refusal for a launch choice the connector cannot honour. Spawn and resume both ask before
|
|
4701
|
-
*
|
|
4761
|
+
* a seat's launch is built, so a rule changed here holds on both paths. */
|
|
4702
4762
|
capabilityRefusal(connector, launch) {
|
|
4703
4763
|
if (launch.variant && !connector.supportsModelVariant)
|
|
4704
4764
|
return `${connector.name} connector does not support model variants (variant)`;
|
|
@@ -5688,7 +5748,23 @@ export class Manager {
|
|
|
5688
5748
|
else {
|
|
5689
5749
|
managed.restart.armed = true;
|
|
5690
5750
|
}
|
|
5751
|
+
if (managed.restart.policy) {
|
|
5752
|
+
try {
|
|
5753
|
+
this.recordSeat(managed);
|
|
5754
|
+
}
|
|
5755
|
+
catch (error) {
|
|
5756
|
+
const detail = `${managed.name} joined, but supervise cannot cover a manager restart for it: ${rejectionText(error)}`;
|
|
5757
|
+
this.stopHandle(managed, false);
|
|
5758
|
+
this.freeSlot(managed, true, "supervise-record-failed");
|
|
5759
|
+
await hooks?.onOutcome?.({ kind: "failed", data: { error: detail } });
|
|
5760
|
+
return { ok: false, error: detail };
|
|
5761
|
+
}
|
|
5762
|
+
}
|
|
5691
5763
|
}
|
|
5764
|
+
// An unsupervised seat a successor could adopt keeps a record too; one that cannot be written
|
|
5765
|
+
// costs only that adoption, so the spawn stands and the line says so.
|
|
5766
|
+
if (!managed.restart?.policy)
|
|
5767
|
+
this.refreshSeatRecord(managed);
|
|
5692
5768
|
this.watchExit(managed);
|
|
5693
5769
|
// Reply with the id the slot actually carries (user-mode: the owner.actor principal —
|
|
5694
5770
|
// presence, ps, and the manifest ownership ledger all key on it; the throwaway static nkey
|
|
@@ -6075,9 +6151,10 @@ export class Manager {
|
|
|
6075
6151
|
}
|
|
6076
6152
|
return undefined;
|
|
6077
6153
|
}
|
|
6078
|
-
/** Validate/relaunch one retained inventory entry. Called
|
|
6079
|
-
*
|
|
6080
|
-
|
|
6154
|
+
/** Validate/relaunch one retained inventory entry. Called through resumePreserved so all records
|
|
6155
|
+
* pass the same preflight before the first child is exposed, and through restartSupervisedSeat
|
|
6156
|
+
* with the one record a crashed manager left (`supervised`). */
|
|
6157
|
+
async resumePreservedAgent(entry, preflightOnly = false, batchReserved = false, prepared, supervised) {
|
|
6081
6158
|
if (this.execution === "none")
|
|
6082
6159
|
return { ok: false, error: this.executionRefusal() };
|
|
6083
6160
|
const release = this.beginLifecycle(batchReserved);
|
|
@@ -6105,7 +6182,7 @@ export class Manager {
|
|
|
6105
6182
|
catch (e) {
|
|
6106
6183
|
return { ok: false, error: rejectionText(e) };
|
|
6107
6184
|
}
|
|
6108
|
-
return this.launchPreparedResume(entry, cached, batchReserved);
|
|
6185
|
+
return this.launchPreparedResume(entry, cached, batchReserved, supervised);
|
|
6109
6186
|
}
|
|
6110
6187
|
try {
|
|
6111
6188
|
const cwd = lstatSync(entry.launch.cwd);
|
|
@@ -6223,7 +6300,7 @@ export class Manager {
|
|
|
6223
6300
|
prepared?.set(entry.name, value);
|
|
6224
6301
|
if (preflightOnly)
|
|
6225
6302
|
return { ok: true, data: { name: entry.name, preflight: true } };
|
|
6226
|
-
return this.launchPreparedResume(entry, value, batchReserved);
|
|
6303
|
+
return this.launchPreparedResume(entry, value, batchReserved, supervised);
|
|
6227
6304
|
}
|
|
6228
6305
|
catch (e) {
|
|
6229
6306
|
return { ok: false, error: rejectionText(e) };
|
|
@@ -6233,7 +6310,7 @@ export class Manager {
|
|
|
6233
6310
|
release();
|
|
6234
6311
|
}
|
|
6235
6312
|
}
|
|
6236
|
-
async launchPreparedResume(entry, prepared, batchReserved) {
|
|
6313
|
+
async launchPreparedResume(entry, prepared, batchReserved, supervised) {
|
|
6237
6314
|
if (!batchReserved)
|
|
6238
6315
|
this.reserved.add(entry.name);
|
|
6239
6316
|
try {
|
|
@@ -6243,7 +6320,14 @@ export class Manager {
|
|
|
6243
6320
|
let adoptedSeed;
|
|
6244
6321
|
if (entry.identity.mode === "static") {
|
|
6245
6322
|
const stored = await this.secrets.get(agentSecretKeyForFile(resolve(entry.identity.credential.path), this.space));
|
|
6246
|
-
|
|
6323
|
+
// The core parser, which reads the creds format's six-dash END marker. A local pattern that
|
|
6324
|
+
// expected five matched no real file, so every resumed seat lost its renewal.
|
|
6325
|
+
try {
|
|
6326
|
+
adoptedSeed = stored === undefined ? undefined : identityFromCreds(stored).seed;
|
|
6327
|
+
}
|
|
6328
|
+
catch {
|
|
6329
|
+
adoptedSeed = undefined;
|
|
6330
|
+
}
|
|
6247
6331
|
if (adoptedSeed === undefined)
|
|
6248
6332
|
console.error(`! resume ${entry.name}: the adopted credential carries no readable nkey seed - the manager cannot renew it (it dies loud at its exp)`);
|
|
6249
6333
|
}
|
|
@@ -6296,9 +6380,11 @@ export class Manager {
|
|
|
6296
6380
|
...(prepared.launchOpts.resume !== undefined && prepared.spec.resumeRecordPath ? { resumeRecordPath: prepared.spec.resumeRecordPath } : {}),
|
|
6297
6381
|
sessionId: entry.launch.sessionId,
|
|
6298
6382
|
},
|
|
6299
|
-
...(
|
|
6300
|
-
? { restart: { opts: prepared.launchOpts, sessionStatePath: prepared.spec.sessionStatePath, crashes:
|
|
6301
|
-
:
|
|
6383
|
+
...(supervised
|
|
6384
|
+
? { restart: { opts: prepared.launchOpts, sessionStatePath: prepared.spec.sessionStatePath, crashes: supervised.crashes, recovering: false, armed: false, policy: supervised.policy } }
|
|
6385
|
+
: prepared.spec.sessionStatePath
|
|
6386
|
+
? { restart: { opts: prepared.launchOpts, sessionStatePath: prepared.spec.sessionStatePath, crashes: [], recovering: false, armed: false } }
|
|
6387
|
+
: {}),
|
|
6302
6388
|
suppressCleanup: true,
|
|
6303
6389
|
};
|
|
6304
6390
|
this.agents.set(entry.name, managed);
|
|
@@ -6311,7 +6397,7 @@ export class Manager {
|
|
|
6311
6397
|
catch (error) {
|
|
6312
6398
|
return await this.stopFailedResume(managed, "resume-custody-unrecorded", `${managed.name} resumed, but ${rejectionText(error)}`);
|
|
6313
6399
|
}
|
|
6314
|
-
const readiness = await this.awaitReadiness(managed, readinessTimeoutMs);
|
|
6400
|
+
const readiness = await this.awaitReadiness(managed, readinessTimeoutMs, supervised ? { joinedAfter: supervised.joinedAfter } : {});
|
|
6315
6401
|
if (!readiness.ok && !readiness.uncertain)
|
|
6316
6402
|
return { ok: false, error: readiness.detail };
|
|
6317
6403
|
if (!readiness.ok) {
|
|
@@ -6319,7 +6405,12 @@ export class Manager {
|
|
|
6319
6405
|
this.watchResumeAdoption(managed);
|
|
6320
6406
|
return { ok: false, error: readiness.detail };
|
|
6321
6407
|
}
|
|
6322
|
-
|
|
6408
|
+
// A supervised restart arms the way its spawn did: a connector with no session continuation is
|
|
6409
|
+
// armed at readiness, one with it once its session is proved below.
|
|
6410
|
+
if (supervised && managed.restart && !connector.supportsSessionContinuation && !prepared.exact) {
|
|
6411
|
+
managed.restart.armed = true;
|
|
6412
|
+
}
|
|
6413
|
+
else if (managed.restart || prepared.exact) {
|
|
6323
6414
|
try {
|
|
6324
6415
|
await this.armSessionRecovery(managed);
|
|
6325
6416
|
// As on a manifest launch, an exact seat is recorded only at the session it asked for,
|
|
@@ -6341,6 +6432,8 @@ export class Manager {
|
|
|
6341
6432
|
}
|
|
6342
6433
|
if (!this.resumeAttemptId)
|
|
6343
6434
|
managed.suppressCleanup = false;
|
|
6435
|
+
if (supervised)
|
|
6436
|
+
this.refreshSeatRecord(managed);
|
|
6344
6437
|
this.watchExit(managed);
|
|
6345
6438
|
if (this.agents.get(managed.name) !== managed)
|
|
6346
6439
|
return { ok: false, error: `${managed.name} exited immediately after same-principal readiness` };
|
|
@@ -6851,8 +6944,10 @@ export class Manager {
|
|
|
6851
6944
|
await nc.drain().catch(() => nc.close());
|
|
6852
6945
|
}
|
|
6853
6946
|
}
|
|
6854
|
-
/** The served manager-level health summary.
|
|
6855
|
-
|
|
6947
|
+
/** The served manager-level health summary. Given an owner, `agentCount` counts only that owner's
|
|
6948
|
+
* seats, as `ps` lists them ({@link psOwnerFilter}), so a person reading a shared Manager does not
|
|
6949
|
+
* learn how many seats other owners hold on it. */
|
|
6950
|
+
managerStatusData(ownerFilter) {
|
|
6856
6951
|
return {
|
|
6857
6952
|
instanceId: this.managerInstanceId,
|
|
6858
6953
|
execution: this.execution,
|
|
@@ -6860,7 +6955,7 @@ export class Manager {
|
|
|
6860
6955
|
custody: this.execution === "none" ? "none" : this.requiredRuntime.kind === "pty" && this.requiredRuntime.supportsRelease === true ? "custodied" : "legacy",
|
|
6861
6956
|
runHosting: this.execution !== "none" && (this.remoteAuthority ? this.remoteAuthority.runHosting !== undefined : !this.userMode && this.auth !== undefined),
|
|
6862
6957
|
terminalSessions: this.execution !== "none",
|
|
6863
|
-
agentCount: this.agents.size,
|
|
6958
|
+
agentCount: ownerFilter === undefined ? this.agents.size : [...this.agents.values()].filter((a) => a.userOwner === ownerFilter).length,
|
|
6864
6959
|
uptimeMs: Date.now() - this.startedAtMs,
|
|
6865
6960
|
connectors: this.connectorStatuses.map((row) => ({ ...row, binaries: { ...row.binaries } })),
|
|
6866
6961
|
classSpawn: this.classSpawnEnabled(),
|
|
@@ -7890,10 +7985,16 @@ export class Manager {
|
|
|
7890
7985
|
// name is re-allocated. Same-incarnation rides the live map; the create-only bindGoal in onAccepted
|
|
7891
7986
|
// still fences a CONCURRENT same-goalId race (the loser aborts and serves the winner's acceptance).
|
|
7892
7987
|
const prior = this.goalAcceptances.get(key);
|
|
7893
|
-
|
|
7894
|
-
|
|
7988
|
+
// A goal that already ENDED is answered from its committed terminal and never accepted again.
|
|
7989
|
+
// Running the accept for it allocates a fresh name and identity, then refuses at the index
|
|
7990
|
+
// CAS, because the entry its terminal cleared cannot be created twice, and that refusal tells
|
|
7991
|
+
// the caller to reconcile the store. A retry that outlived its entry above must not reach it.
|
|
7992
|
+
const ended = prior === undefined ? await readGoalResult(gw.ctx, ref) : undefined;
|
|
7993
|
+
const accepted = prior ?? ended;
|
|
7994
|
+
if (accepted !== undefined) {
|
|
7995
|
+
if (accepted.fingerprint !== fingerprint)
|
|
7895
7996
|
throw new EpEnvelopeError("failed-precondition", `goal "${goalId}" was accepted under a different submission; one goalId never carries two specs (SPEC 13.6)`);
|
|
7896
|
-
return prior;
|
|
7997
|
+
return prior ?? this.cachedSpawnAcceptance(ref, goalId, fingerprint, executor);
|
|
7897
7998
|
}
|
|
7898
7999
|
let resolveAccept;
|
|
7899
8000
|
let rejectAccept;
|
|
@@ -7953,6 +8054,7 @@ export class Manager {
|
|
|
7953
8054
|
const why = o.data?.reason;
|
|
7954
8055
|
({ fact } = await settleGoalUncertain(gw.ctx, { ref, now: Date.now(), committer: { instanceId: this.managerInstanceId, epoch }, ...(typeof why === "string" && why.length > 0 ? { reason: why } : {}) }));
|
|
7955
8056
|
}
|
|
8057
|
+
this.expireGoalAcceptance(ref);
|
|
7956
8058
|
this.emitGoalProgress(ref, epoch, { phase: "terminal", state: fact.state, ...(fact.data !== undefined ? { data: fact.data } : {}) });
|
|
7957
8059
|
await clearGoalIndex(gw.ctx, ref); // terminal reached - the successor never reconciles it
|
|
7958
8060
|
if (acceptance)
|
|
@@ -8085,7 +8187,7 @@ export class Manager {
|
|
|
8085
8187
|
* empty agent it can never address. The acceptance floor in the goal index exists from the moment
|
|
8086
8188
|
* of acceptance, so it answers precisely the window the terminal cannot. Where neither is
|
|
8087
8189
|
* readable the honest answer is a REFUSAL: an accepted goal whose identity nobody can name is
|
|
8088
|
-
* `unavailable
|
|
8190
|
+
* `unavailable` while it runs and its cached outcome once it has ended, never a hollow success. */
|
|
8089
8191
|
async cachedSpawnAcceptance(ref, goalId, fingerprint, executor) {
|
|
8090
8192
|
const entry = await readGoalIndex(this.goalWriter.ctx, ref);
|
|
8091
8193
|
if (entry?.allocated !== undefined)
|
|
@@ -8101,8 +8203,18 @@ export class Manager {
|
|
|
8101
8203
|
const principal = typeof d.id !== "string" || d.id.length === 0 ? null : this.userMode ? parsePrincipalKey(d.id) : { owner: DEV_OWNER, actor: d.id };
|
|
8102
8204
|
if (typeof d.name === "string" && d.name.length > 0 && principal !== null && typeof d.lifecycleUid === "string" && d.lifecycleUid.length > 0 && readinessDeadlineMs !== undefined)
|
|
8103
8205
|
return { name: d.name, owner: principal.owner, actor: principal.actor, uid: d.lifecycleUid, goalId, fingerprint, readinessDeadlineMs, executor };
|
|
8206
|
+
// Only a `succeeded` terminal names the agent. Any other ended goal has its outcome as its
|
|
8207
|
+
// whole durable answer, and no retry will make an identity readable (SPEC 13.6 item 5).
|
|
8208
|
+
if (result !== undefined)
|
|
8209
|
+
throw goalAlreadyTerminal(goalId, result);
|
|
8104
8210
|
throw new EpEnvelopeError("unavailable", `goal "${goalId}" is already accepted but its allocated identity is not readable (no acceptance floor, and no terminal carrying one); retry (SPEC 13.6)`);
|
|
8105
8211
|
}
|
|
8212
|
+
/** Drop a goal's acceptance {@link goalAcceptanceRetentionMs} after its terminal committed. A
|
|
8213
|
+
* retry after that reads the terminal, so only a goal that has one may be passed here. */
|
|
8214
|
+
expireGoalAcceptance(ref) {
|
|
8215
|
+
const key = goalKey(ref);
|
|
8216
|
+
setTimeout(() => this.goalAcceptances.delete(key), this.goalAcceptanceRetentionMs).unref?.();
|
|
8217
|
+
}
|
|
8106
8218
|
// ---- the turn relay (§8 durable actions) ------------------------------------------------------
|
|
8107
8219
|
//
|
|
8108
8220
|
// A seat is NOT an endpoint: a run's `turn(agent, payload)` rides the manager, which accepts it
|
|
@@ -8646,6 +8758,7 @@ export class Manager {
|
|
|
8646
8758
|
await this.assertGoalWriterEpochCurrent(epoch); // a superseded corpse never commits a cancel terminal either
|
|
8647
8759
|
await transitionGoal(gw.ctx, ref, "cancelling", { fields: { cancelMode: mode } });
|
|
8648
8760
|
const r = await commitGoalResult(gw.ctx, { ref, now: Date.now(), cause: "cancel", data: { cancelledBy: "despawn" }, committer: { instanceId: this.managerInstanceId, epoch } });
|
|
8761
|
+
this.expireGoalAcceptance(ref);
|
|
8649
8762
|
this.emitGoalProgress(ref, epoch, { phase: "terminal", state: r.fact.state, ...(r.fact.data !== undefined ? { data: r.fact.data } : {}) });
|
|
8650
8763
|
await clearGoalIndex(gw.ctx, ref); // terminal reached - the successor never reconciles it
|
|
8651
8764
|
}
|
|
@@ -8733,6 +8846,191 @@ export class Manager {
|
|
|
8733
8846
|
throw new Error(`runtime "${this.requiredRuntime.kind}" reserved custody ${reserved.kind}:${reserved.id} for "${name}" but spawned ` +
|
|
8734
8847
|
`${got ? `${got.kind}:${got.id}` : "a handle with no reference"}; the recorded reference would address no seat, so ${disposal}`);
|
|
8735
8848
|
}
|
|
8849
|
+
/** Persist what a successor needs to take this seat back if this process ends without stopping
|
|
8850
|
+
* it: a supervised seat, which a successor restarts (#4094), and a seat on a runtime that can adopt
|
|
8851
|
+
* it by reference, which a successor adopts while it still runs (#4095). Static meshes only: a
|
|
8852
|
+
* successor finds the seat through its active slot row, and only a static mesh keeps one. Throws
|
|
8853
|
+
* when a supervised seat's entry could not be resumed, because a policy this host cannot keep
|
|
8854
|
+
* across a manager restart is one it cannot enforce. */
|
|
8855
|
+
recordSeat(a) {
|
|
8856
|
+
if (!this.auth || this.userMode)
|
|
8857
|
+
return;
|
|
8858
|
+
const policy = a.restart?.policy;
|
|
8859
|
+
const adoptable = a.handle.reference !== undefined && typeof this.requiredRuntime.adopt === "function";
|
|
8860
|
+
if (!policy && !adoptable)
|
|
8861
|
+
return;
|
|
8862
|
+
const agent = this.resumeEntryOf(a);
|
|
8863
|
+
if (policy) {
|
|
8864
|
+
const unresumable = this.inventoryReferenceError(agent);
|
|
8865
|
+
if (unresumable)
|
|
8866
|
+
throw new Error(`a successor manager could not restart it: ${unresumable}`);
|
|
8867
|
+
}
|
|
8868
|
+
writeSeatRecord(this.workspaceRoot, { version: "cotal-seat-record/v1", instanceId: this.managerInstanceId, ...(policy ? { supervise: policy } : {}), crashes: a.restart?.crashes ?? [], agent });
|
|
8869
|
+
}
|
|
8870
|
+
/** Bring a running seat's record up to date after its credential or restart count changed. A
|
|
8871
|
+
* record that cannot be rewritten is removed, so a successor retires the seat rather than taking
|
|
8872
|
+
* it back from a stale one. */
|
|
8873
|
+
refreshSeatRecord(a) {
|
|
8874
|
+
try {
|
|
8875
|
+
this.recordSeat(a);
|
|
8876
|
+
}
|
|
8877
|
+
catch (error) {
|
|
8878
|
+
console.error(`! ${a.name}: its seat record could not be updated (${rejectionText(error)}); a manager restart now retires it`);
|
|
8879
|
+
this.dropSeatRecord(a);
|
|
8880
|
+
}
|
|
8881
|
+
}
|
|
8882
|
+
dropSeatRecord(a) {
|
|
8883
|
+
try {
|
|
8884
|
+
removeSeatRecord(this.workspaceRoot, this.space, a.name, a.lifecycleUid);
|
|
8885
|
+
}
|
|
8886
|
+
catch (error) {
|
|
8887
|
+
console.error(`! ${a.name}: its seat record could not be removed: ${rejectionText(error)}`);
|
|
8888
|
+
}
|
|
8889
|
+
}
|
|
8890
|
+
/** The seat record behind one active slot this instance owns, when it describes that exact
|
|
8891
|
+
* incarnation on this runtime. Anything else is left to the ordinary terminal. */
|
|
8892
|
+
seatRecordFor(row) {
|
|
8893
|
+
let record;
|
|
8894
|
+
try {
|
|
8895
|
+
record = readSeatRecord(this.workspaceRoot, this.space, row.alias);
|
|
8896
|
+
}
|
|
8897
|
+
catch (error) {
|
|
8898
|
+
console.error(`! ${row.alias}: ${rejectionText(error)}; the lifecycle is retired, not restarted`);
|
|
8899
|
+
return undefined;
|
|
8900
|
+
}
|
|
8901
|
+
if (record === undefined)
|
|
8902
|
+
return undefined;
|
|
8903
|
+
const identity = record.agent.identity;
|
|
8904
|
+
if (identity.mode !== "static" || identity.id !== row.actor || identity.lifecycleUid !== row.lifecycleUid)
|
|
8905
|
+
return undefined;
|
|
8906
|
+
if (record.instanceId !== this.managerInstanceId || record.agent.space !== this.space)
|
|
8907
|
+
return undefined;
|
|
8908
|
+
if (record.agent.launch.runtime !== this.requiredRuntime.kind) {
|
|
8909
|
+
console.error(`! ${row.alias}: its seat ran on runtime "${record.agent.launch.runtime}" and this manager uses "${this.requiredRuntime.kind}"; the lifecycle is retired, not restarted`);
|
|
8910
|
+
return undefined;
|
|
8911
|
+
}
|
|
8912
|
+
return record;
|
|
8913
|
+
}
|
|
8914
|
+
/** Take back a seat that outlived its manager (#4095): the runtime reattaches to the exact seat
|
|
8915
|
+
* the slot names, and this manager holds it under the record's identity and launch, with no
|
|
8916
|
+
* relaunch. Returns why it could not, and the caller then retires the lifecycle, which reaps the
|
|
8917
|
+
* seat by the same reference. */
|
|
8918
|
+
async adoptSeat(row, record) {
|
|
8919
|
+
const entry = record.agent;
|
|
8920
|
+
if (entry.identity.mode !== "static")
|
|
8921
|
+
return "its record is not a static identity";
|
|
8922
|
+
if (row.runtime === undefined)
|
|
8923
|
+
return "its slot names no seat";
|
|
8924
|
+
const runtime = this.requiredRuntime;
|
|
8925
|
+
if (typeof runtime.adopt !== "function")
|
|
8926
|
+
return `runtime "${runtime.kind}" cannot adopt a seat`;
|
|
8927
|
+
// The credential is the manager's own lifecycle-keyed file, and its seed renews it.
|
|
8928
|
+
const credsPath = resolve(entry.identity.credential.path);
|
|
8929
|
+
if (credsPath !== resolve(agentLifecycleSecretFilePaths(this.workspaceRoot, this.space, entry.name, entry.identity.lifecycleUid).creds))
|
|
8930
|
+
return `its credential reference ${credsPath} is not this lifecycle's manager-owned path`;
|
|
8931
|
+
const stored = await this.secrets.get(agentSecretKeyForFile(credsPath, this.space));
|
|
8932
|
+
if (stored === undefined)
|
|
8933
|
+
return "its credential is not in the secret store";
|
|
8934
|
+
let seed;
|
|
8935
|
+
try {
|
|
8936
|
+
const identity = identityFromCreds(stored);
|
|
8937
|
+
if (identity.id !== entry.identity.id)
|
|
8938
|
+
return `its credential is identity ${identity.id}, not ${entry.identity.id}`;
|
|
8939
|
+
seed = identity.seed;
|
|
8940
|
+
}
|
|
8941
|
+
catch (error) {
|
|
8942
|
+
return `its credential is unreadable: ${rejectionText(error)}`;
|
|
8943
|
+
}
|
|
8944
|
+
let adopted;
|
|
8945
|
+
try {
|
|
8946
|
+
adopted = runtime.adopt(row.runtime);
|
|
8947
|
+
}
|
|
8948
|
+
catch (error) {
|
|
8949
|
+
return `the seat ${row.runtime.kind}:${row.runtime.id} could not be adopted: ${rejectionText(error)}`;
|
|
8950
|
+
}
|
|
8951
|
+
const managed = {
|
|
8952
|
+
name: entry.name,
|
|
8953
|
+
role: entry.role,
|
|
8954
|
+
agent: entry.launch.connector,
|
|
8955
|
+
id: entry.identity.id,
|
|
8956
|
+
seed,
|
|
8957
|
+
...(entry.identity.issued ? { issued: entry.identity.issued } : {}),
|
|
8958
|
+
lifecycleUid: entry.identity.lifecycleUid,
|
|
8959
|
+
secretPaths: { creds: entry.identity.credential.path },
|
|
8960
|
+
spawner: entry.spawner,
|
|
8961
|
+
authorityParent: entry.authorityParent,
|
|
8962
|
+
startedAt: Date.parse(entry.startedAt),
|
|
8963
|
+
// The runtime names an adopted handle by what it holds; the record names the seat.
|
|
8964
|
+
handle: { ...adopted, name: entry.name },
|
|
8965
|
+
launch: {
|
|
8966
|
+
source: entry.launch.source,
|
|
8967
|
+
cwd: entry.launch.cwd,
|
|
8968
|
+
model: entry.launch.model,
|
|
8969
|
+
variant: entry.launch.variant,
|
|
8970
|
+
subscribe: entry.launch.subscribe,
|
|
8971
|
+
allowSubscribe: entry.launch.allowSubscribe,
|
|
8972
|
+
allowPublish: entry.launch.allowPublish,
|
|
8973
|
+
capabilities: entry.launch.capabilities,
|
|
8974
|
+
events: entry.launch.events,
|
|
8975
|
+
shareTools: entry.launch.shareTools,
|
|
8976
|
+
forkSource: entry.launch.forkSource,
|
|
8977
|
+
...(entry.launch.resumed ? { resumed: entry.launch.resumed } : {}),
|
|
8978
|
+
sessionId: entry.launch.sessionId,
|
|
8979
|
+
unresolvedLaunchOptionKeys: entry.launch.unresolvedLaunchOptionKeys,
|
|
8980
|
+
},
|
|
8981
|
+
};
|
|
8982
|
+
this.agents.set(entry.name, managed);
|
|
8983
|
+
this.watchExit(managed);
|
|
8984
|
+
console.error(`! ${entry.name}: adopted the seat its previous manager left (${row.runtime.kind}:${row.runtime.id})`);
|
|
8985
|
+
// A credential that came close to its expiry while no manager ran is renewed now, not at the
|
|
8986
|
+
// next pass.
|
|
8987
|
+
const health = inspectCredHealth(stored).state;
|
|
8988
|
+
if (health === "near-expiry" || health === "expired")
|
|
8989
|
+
await this.renewManagedStaticCred(managed).catch((error) => console.error(`! managed cred renewal ${entry.name}: ${rejectionText(error)} - the agent dies loud at this cred's expiry unless it is reminted`));
|
|
8990
|
+
return undefined;
|
|
8991
|
+
}
|
|
8992
|
+
/** Restart a supervised seat whose manager ended without stopping it (#4094), as one restart of
|
|
8993
|
+
* its own policy. It goes through the resume launch, so the seat comes back under the same name,
|
|
8994
|
+
* lifecycle uid, credential, persona and cwd, and a pending turn addressed to it is served to it.
|
|
8995
|
+
* Returns why it could not, and the caller then retires the lifecycle. */
|
|
8996
|
+
async restartSupervisedSeat(row, record) {
|
|
8997
|
+
const policy = record.supervise;
|
|
8998
|
+
if (policy === undefined)
|
|
8999
|
+
return "its spawn carried no restart policy";
|
|
9000
|
+
const now = Date.now();
|
|
9001
|
+
const crashes = [...record.crashes.filter((at) => now - at < policy.windowMs), now];
|
|
9002
|
+
if (crashes.length > policy.restarts)
|
|
9003
|
+
return `its supervise restart budget is spent (${crashes.length} deaths in ${policy.windowMs / 1000}s, ${policy.restarts} allowed)`;
|
|
9004
|
+
// A seat that outlived its manager must not run beside its replacement, so the one the slot
|
|
9005
|
+
// names is reaped first. A runtime that cannot prove it gone refuses, and the terminal retires it.
|
|
9006
|
+
if (row.runtime !== undefined) {
|
|
9007
|
+
try {
|
|
9008
|
+
const evidence = await requireRuntimeReap(this.requiredRuntime, row.runtime);
|
|
9009
|
+
console.error(`${row.alias}: the seat its previous manager left ${evidence.detail}`);
|
|
9010
|
+
}
|
|
9011
|
+
catch (error) {
|
|
9012
|
+
return `the seat its previous manager left could not be reaped: ${rejectionText(error)}`;
|
|
9013
|
+
}
|
|
9014
|
+
}
|
|
9015
|
+
await this.ep.waitForPresenceSnapshot();
|
|
9016
|
+
// Readiness counts only a heartbeat newer than any the dead seat left, as an in-place restart does.
|
|
9017
|
+
const principal = principalKey(DEV_OWNER, row.actor).key;
|
|
9018
|
+
const joinedAfter = this.ep.getRoster()
|
|
9019
|
+
.filter((p) => p.card.id === principal && p.lifecycleUid === row.lifecycleUid)
|
|
9020
|
+
.reduce((max, p) => Math.max(max, p.ts), 0) + 1;
|
|
9021
|
+
const reply = await this.resumePreservedAgent(record.agent, false, false, undefined, { policy, crashes, joinedAfter });
|
|
9022
|
+
const live = this.agents.get(row.alias);
|
|
9023
|
+
if (reply.ok) {
|
|
9024
|
+
console.error(`! ${row.alias}: restarted under the same lifecycle after its manager stopped (${crashes.length}/${policy.restarts})`);
|
|
9025
|
+
return undefined;
|
|
9026
|
+
}
|
|
9027
|
+
// An uncertain readiness keeps the seat managed and watched, as any launch does.
|
|
9028
|
+
if (live?.lifecycleUid === row.lifecycleUid) {
|
|
9029
|
+
console.error(`! ${row.alias}: restarted after its manager stopped, but readiness is uncertain: ${reply.error}`);
|
|
9030
|
+
return undefined;
|
|
9031
|
+
}
|
|
9032
|
+
return `its restart failed: ${reply.error}`;
|
|
9033
|
+
}
|
|
8736
9034
|
/** The terminal's process step: an orphan (no live handle in THIS process) is reaped by the custody
|
|
8737
9035
|
* reference the slot recorded, before its footprint goes. A successor must never retire a
|
|
8738
9036
|
* lifecycle and free its alias while the predecessor's seat process is still running outside
|
|
@@ -8882,6 +9180,8 @@ export class Manager {
|
|
|
8882
9180
|
const credsPath = a.secretPaths.creds;
|
|
8883
9181
|
await secrets.put(agentSecretKeyForFile(credsPath, this.space), creds);
|
|
8884
9182
|
await materializeSecretToFile(secrets, agentSecretKeyForFile(credsPath, this.space), credsPath);
|
|
9183
|
+
// The record pins the credential's digest, so it follows the re-signed file.
|
|
9184
|
+
this.refreshSeatRecord(a);
|
|
8885
9185
|
console.error(`managed cred renewal ${a.name}: re-signed for the same identity (exp +${MANAGED_STATIC_TTL_SEC}s); the agent endpoint's source re-read adopts it`);
|
|
8886
9186
|
}
|
|
8887
9187
|
/** The Unit B reconciliation (F3 "no active orphan"): ensure the authority stores, then sweep
|
|
@@ -9176,6 +9476,7 @@ export class Manager {
|
|
|
9176
9476
|
let preservedAdopted = 0;
|
|
9177
9477
|
let deferredSlots = 0;
|
|
9178
9478
|
const terminalRows = [];
|
|
9479
|
+
const restartRows = [];
|
|
9179
9480
|
for (const row of slotRows) {
|
|
9180
9481
|
if (row.phase === "retired") {
|
|
9181
9482
|
// A retirement is a GLOBAL refusal fact — seed the F5 index for EVERY retired incarnation
|
|
@@ -9211,6 +9512,16 @@ export class Manager {
|
|
|
9211
9512
|
deferredSlots++;
|
|
9212
9513
|
continue;
|
|
9213
9514
|
}
|
|
9515
|
+
// A boot after this instance's manager ended without stopping its seats: a seat with a record
|
|
9516
|
+
// is taken back instead of retired, restarted under its policy when supervised (#4094) and
|
|
9517
|
+
// adopted while it still runs otherwise (#4095).
|
|
9518
|
+
if (!postAdoption && row.phase === "active" && !adopted) {
|
|
9519
|
+
const record = this.seatRecordFor(row);
|
|
9520
|
+
if (record) {
|
|
9521
|
+
restartRows.push({ row, record });
|
|
9522
|
+
continue;
|
|
9523
|
+
}
|
|
9524
|
+
}
|
|
9214
9525
|
if (planStaticSlotResume(row, adopted) !== "none")
|
|
9215
9526
|
terminalRows.push(row);
|
|
9216
9527
|
}
|
|
@@ -9219,6 +9530,24 @@ export class Manager {
|
|
|
9219
9530
|
// alias, but the initial serial sweep must close every discovery-to-await gap up front.
|
|
9220
9531
|
for (const row of terminalRows)
|
|
9221
9532
|
this.reconcilingAliases.add(row.alias);
|
|
9533
|
+
for (const { row } of restartRows)
|
|
9534
|
+
this.reconcilingAliases.add(row.alias);
|
|
9535
|
+
// Restarts run first; one that cannot restart joins the terminals below.
|
|
9536
|
+
for (const { row, record } of restartRows) {
|
|
9537
|
+
if (this.staticReconcileStopping)
|
|
9538
|
+
break;
|
|
9539
|
+
// A supervised seat is restarted; any other seat is adopted while it still runs.
|
|
9540
|
+
const refusal = record.supervise ? await this.restartSupervisedSeat(row, record) : await this.adoptSeat(row, record);
|
|
9541
|
+
if (refusal === undefined) {
|
|
9542
|
+
this.reconcilingAliases.delete(row.alias);
|
|
9543
|
+
continue;
|
|
9544
|
+
}
|
|
9545
|
+
console.error(record.supervise
|
|
9546
|
+
? `! ${row.alias}: the supervised seat its previous manager left was not restarted (${refusal}); retiring the lifecycle`
|
|
9547
|
+
: `! ${row.alias}: the seat its previous manager left was not adopted (${refusal}); retiring the lifecycle`);
|
|
9548
|
+
this.dropSeatRecord({ name: row.alias, lifecycleUid: row.lifecycleUid });
|
|
9549
|
+
terminalRows.push(row);
|
|
9550
|
+
}
|
|
9222
9551
|
const sweepDeprovision = {
|
|
9223
9552
|
examined: 0,
|
|
9224
9553
|
deleted: 0,
|
|
@@ -9315,6 +9644,8 @@ export class Manager {
|
|
|
9315
9644
|
finally {
|
|
9316
9645
|
for (const row of terminalRows)
|
|
9317
9646
|
this.reconcilingAliases.delete(row.alias);
|
|
9647
|
+
for (const { row } of restartRows)
|
|
9648
|
+
this.reconcilingAliases.delete(row.alias);
|
|
9318
9649
|
}
|
|
9319
9650
|
}
|
|
9320
9651
|
finally {
|