@cotal-ai/manager 0.86.0 → 0.88.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js.map +1 -1
- package/dist/manager-service-contract.d.ts +4 -0
- package/dist/manager-service-contract.d.ts.map +1 -1
- package/dist/manager-service-contract.js +5 -1
- package/dist/manager-service-contract.js.map +1 -1
- package/dist/manager.d.ts +46 -6
- package/dist/manager.d.ts.map +1 -1
- package/dist/manager.js +350 -33
- package/dist/manager.js.map +1 -1
- package/dist/resume.d.ts +110 -0
- package/dist/resume.d.ts.map +1 -1
- package/dist/resume.js +57 -1
- package/dist/resume.js.map +1 -1
- package/package.json +12 -12
package/dist/manager.js
CHANGED
|
@@ -4,8 +4,8 @@ import { credsAuthenticator } from "@nats-io/transport-node";
|
|
|
4
4
|
import { existsSync, lstatSync, mkdirSync, readFileSync, realpathSync, rmSync, statSync } from "node:fs";
|
|
5
5
|
import { isAbsolute, join, dirname, resolve } from "node:path";
|
|
6
6
|
import Headless from "@xterm/headless";
|
|
7
|
-
import { CotalEndpoint, canonicalJson, MANAGED_HANDOFF_KIND, accountFromCreds, credsClaims, DEFAULT_SERVER, DEV_OWNER, MANAGER_LEASE_RENEW_MS, STANDING_RENEWABLE_TTL_SEC, assertLifecycleToken, discardLaunchArtifacts, divergentSecretStoreNotice, injectedSecretStoreIdentity, parseDaemonStoreAnswer, parseSecretStoreIdentity, sameSecretStoreIdentity, agentFilePath, clearSpaceHistory, dialerFor, connectorServers, spawnEnvAllow, deprovisionAgent, DeprovisionError, isConcreteChannel, deprovisionTargetPrincipal, firstFreeName, spawnNameError, idFromCreds, inspectCredHealth, credsRenewalDelayMs, composeWirePersona, loadAgentFile, listPersonaCatalog, personaCatalogDescription, personaCatalogReadable, loadCotalConfig, modelPolicyRefusal, mintCreds, mintGeneration, mintAcceptedToken, openIssuedStore, acceptedBucket, issuedBucket, mintLifecycleUid, managedRetirementOpId, mkSecretDir, newIdentity, actionContext, parsePrincipalKey, principalKey, probeConnect, provisionAgent, provisionAgentDurables, registry, resolveAuthProvider, resolveReadAcl, runAgentBearer, saveAgentFile, subjectMatches, AUTH_ENDPOINT, EP_CMD_RETIRE_LIFECYCLE, controlServiceSubject, eventChannelPrincipal, resolveService, invokeCommand, digestIdToken, } from "@cotal-ai/core";
|
|
8
|
-
import { agentAuthState, agentCredsDir, agentLifecycleSecretFilePaths, agentSecretFilePaths, agentSecretKeyForFile, authDir, claimIdentityRecord, claimManagerSiblingIdentities, connectorInstallHint, DELIVERY_CREDS_KIND, extensionConnectors, findCotalRoot, getSpaceAuth, hasUserAuthState, loadExtensionsManifest, loadManagerInstanceIdentity, loadMeshes, localTrustOfSpace, manifestExtensionNames, materializeFromManifest, materializeSecretToFile, MEMBERSHIP_RW_CREDS_KIND, mergeLaunchOptions, remintDaemonCreds, resolveAgentType, resolveOnPath, createManagerInstanceIdentity, spaceKey, spaceMaterialKey, SYSTEM_CREDS_FILES, userAuthStateDir, workspaceSecretStore, writeRenewalRecord } from "@cotal-ai/workspace";
|
|
7
|
+
import { CotalEndpoint, canonicalJson, MANAGED_HANDOFF_KIND, accountFromCreds, credsClaims, DEFAULT_SERVER, DEV_OWNER, MANAGER_LEASE_RENEW_MS, MANAGER_LEASE_TTL_MS, awaitLeaseExpiry, leaseRemainingMs, STANDING_RENEWABLE_TTL_SEC, assertLifecycleToken, discardLaunchArtifacts, divergentSecretStoreNotice, injectedSecretStoreIdentity, parseDaemonStoreAnswer, parseSecretStoreIdentity, sameSecretStoreIdentity, agentFilePath, clearSpaceHistory, dialerFor, connectorServers, spawnEnvAllow, deprovisionAgent, DeprovisionError, isConcreteChannel, deprovisionTargetPrincipal, firstFreeName, spawnNameError, idFromCreds, identityFromCreds, inspectCredHealth, credsRenewalDelayMs, composeWirePersona, loadAgentFile, listPersonaCatalog, personaCatalogDescription, personaCatalogReadable, loadCotalConfig, modelPolicyRefusal, mintCreds, mintGeneration, mintAcceptedToken, openIssuedStore, acceptedBucket, issuedBucket, mintLifecycleUid, managedRetirementOpId, mkSecretDir, newIdentity, actionContext, parsePrincipalKey, principalKey, probeConnect, provisionAgent, provisionAgentDurables, registry, resolveAuthProvider, resolveReadAcl, runAgentBearer, saveAgentFile, subjectMatches, AUTH_ENDPOINT, EP_CMD_RETIRE_LIFECYCLE, controlServiceSubject, eventChannelPrincipal, resolveService, invokeCommand, digestIdToken, } from "@cotal-ai/core";
|
|
8
|
+
import { agentAuthState, agentCredsDir, agentLifecycleSecretFilePaths, agentSecretFilePaths, agentSecretKeyForFile, authDir, claimIdentityRecord, claimManagerSiblingIdentities, connectorInstallHint, DELIVERY_CREDS_KIND, extensionConnectors, findCotalRoot, getSpaceAuth, hasUserAuthState, loadExtensionsManifest, loadManagerInstanceIdentity, loadMeshes, localTrustOfSpace, manifestExtensionNames, materializeFromManifest, materializeSecretToFile, MEMBERSHIP_RW_CREDS_KIND, mergeLaunchOptions, probeLiveness, remintDaemonCreds, resolveAgentType, resolveOnPath, createManagerInstanceIdentity, spaceKey, spaceMaterialKey, SYSTEM_CREDS_FILES, userAuthStateDir, workspaceSecretStore, writeRenewalRecord } from "@cotal-ai/workspace";
|
|
9
9
|
import { createRuntime, exitProof, isCustodialRuntime, pollProvedExit, requireRuntimeAdopt, requireRuntimeReap, RuntimeReapUnproven, watchProvedExit, } from "./runtime/index.js";
|
|
10
10
|
import { AttachEndpoint } from "./attach-endpoint.js";
|
|
11
11
|
import { DELIVERY_ADMIN_BOOT_WAIT_MS, isUnansweredDeliveryAdmin, makeManagerEndpointEvictionEvidence, makeManagerEndpointHolderEvictor, untilDeliveryAdminAnswers, withScopedEndpoint } from "./endpoint-evict.js";
|
|
@@ -16,7 +16,7 @@ import { authorizeLaunch, authorizeNamedControl } from "./authorize.js";
|
|
|
16
16
|
import { controlShutdown } from "./control-shutdown.js";
|
|
17
17
|
import { RunHosting } from "./run-hosting.js";
|
|
18
18
|
import { controlSession } from "./control-session.js";
|
|
19
|
-
import { MAX_AGENTS, parseResumeCommitArgs, parseResumeControlArgs, parseResumeFinalizeArgs } from "./resume.js";
|
|
19
|
+
import { MAX_AGENTS, parseResumeCommitArgs, parseResumeControlArgs, parseResumeFinalizeArgs, readSeatRecord, removeSeatRecord, writeSeatRecord } from "./resume.js";
|
|
20
20
|
import { TranscriptReceiver } from "./transcript-receive.js";
|
|
21
21
|
// Unit B (the static §13.1 lifecycle executor): the shared grammar/stores from core plus the
|
|
22
22
|
// manager-side adapter (transport + slot orchestration + the F1 terminal) — see static-lifecycle.ts.
|
|
@@ -33,6 +33,8 @@ import { ManagerSessionPlane, openSessionLedgerKv } from "./session/index.js";
|
|
|
33
33
|
import { provisionEndpointGateOpen, endpointRegistrationBarrier, serveIssuanceGateKv, commitSiblingIssuance, markLedgerRowRevoked, epcredRowKey, epgateKey, parseEndpointGate, readEndpointGateGeneration, registerServingInstance, deregisterServiceInstance, serveEndpoint, bindGoal, createGoal, transitionGoal, commitGoalResult, requestGoalCancel, goalAlreadyTerminal, settleGoalUncertain, readGoalResult, readGoalStatus, readGoalSpec, recordGoalIndex, readGoalIndex, clearGoalIndex, listGoalIndex, mintCheckpoint, resumeCheckpoint, readCheckpointSettle, readCheckpointSpec, expireCheckpoint, GOAL_TERMINAL_STATES, goalRefOf, goalProgressTopic, epeSubject, submissionFingerprint, EpEnvelopeError, lifecycleBlocked, lifecycleBlockedFrom, renderLifecycleBlocked, } from "@cotal-ai/core";
|
|
34
34
|
import { MANAGER_ENDPOINT, managerClusterArtifacts, managerCommandDefs, managerContractArtifactValues } from "./manager-service-contract.js";
|
|
35
35
|
import { staticLifecycleTransport, activateStaticLifecycle, runStaticTerminal, readStaticSlot, casStaticSlot, recordSlotCredential, appendStaticCredentialRow, planStaticSlotResume, ownedBySibling, observeStaticSlot, listStaticSlotObservations, walkStaticSlots, renderStaticSlotObservation, StaticSlotReadError, STATIC_SLOT_READ_FAILED_DETAIL, } from "./static-lifecycle.js";
|
|
36
|
+
// Capture the loaded package version before an installation can replace its files.
|
|
37
|
+
const MANAGER_VERSION = JSON.parse(readFileSync(new URL("../package.json", import.meta.url), "utf8")).version;
|
|
36
38
|
/** Minimum slot lifetime for rate-flooring (P4c). A slot freed (by despawn OR natural exit/reap)
|
|
37
39
|
* before living this long leaves a cooling stamp that still counts toward the ceiling until it
|
|
38
40
|
* expires — so churn (spawn↔despawn or spawn↔fast-exit) can't outrun the concurrency bound. */
|
|
@@ -219,6 +221,7 @@ const FREE_SLOT_CAUSE_TEXT = {
|
|
|
219
221
|
"resume-session-rebind-failed": "this manager stopped it: its host session could not be rebound on resume",
|
|
220
222
|
"carried-fork-refused": "this manager stopped it: it recorded no fork of the carried transcript",
|
|
221
223
|
"resume-custody-unrecorded": "this manager stopped it: its custody reference could not be recorded on resume",
|
|
224
|
+
"supervise-record-failed": "this manager stopped it: its supervise policy could not be recorded for a manager restart",
|
|
222
225
|
};
|
|
223
226
|
/** The stop family's sentences — the same job {@link FREE_SLOT_CAUSE_TEXT} does for the string
|
|
224
227
|
* causes, beside it so both stay the single place a cause becomes a sentence. A requested stop's
|
|
@@ -987,22 +990,18 @@ export class Manager {
|
|
|
987
990
|
// THIS logical instance's own key (atomic CAS create). A DIFFERENT instance (a second manager in a
|
|
988
991
|
// second workspace root) has a distinct id ⇒ a distinct key ⇒ it coexists; the create THROWS only
|
|
989
992
|
// when the SAME instance id is already live (a same-root double-start, or a restart racing the
|
|
990
|
-
// crashed predecessor's not-yet-expired key)
|
|
991
|
-
//
|
|
993
|
+
// crashed predecessor's not-yet-expired key). A double-start REFUSES loud; a restart whose
|
|
994
|
+
// predecessor is provably dead on this host waits for that key's bucket TTL to remove it (#4092,
|
|
995
|
+
// {@link acquireInstanceLease}). Losing the key LATER ends this process only when another process holds it; otherwise
|
|
992
996
|
// the renew loop puts it back or keeps retrying ({@link renewLease}).
|
|
993
997
|
this.leaseInfo = { holder: this.ep.ref().id, instanceId: this.managerInstanceId, runtime: this.runtimeKind, root: resolve(this.workspaceRoot), pid: process.pid };
|
|
994
998
|
try {
|
|
995
|
-
this.leaseRevision = await this.
|
|
999
|
+
this.leaseRevision = await this.acquireInstanceLease(this.leaseInfo);
|
|
996
1000
|
}
|
|
997
1001
|
catch (e) {
|
|
998
|
-
// Our OWN instance id already holds a live key ⇒ refuse. Anything else (e.g. a KV/JS error) is a
|
|
999
|
-
// real failure to surface, not a silent "held" — keep the cause so it isn't misread as a conflict.
|
|
1000
|
-
const held = (await this.ep.readOwnManagerLease(this.managerInstanceId).catch(() => undefined))?.info;
|
|
1001
1002
|
await this.ep.stop();
|
|
1002
1003
|
await this.attach.stop();
|
|
1003
|
-
throw
|
|
1004
|
-
? `manager instance ${this.managerInstanceId} already serves space "${this.space}" from ${held.root === this.leaseInfo.root ? "this" : "another"} workspace root (${held.runtime}, pid ${held.pid}, root ${held.root}) - stop it first before restarting the same instance`
|
|
1005
|
-
: `could not acquire the manager lease for space "${this.space}": ${rejectionText(e)}`);
|
|
1004
|
+
throw e;
|
|
1006
1005
|
}
|
|
1007
1006
|
this.leaseTimer = setInterval(() => { void this.renewLease(); }, MANAGER_LEASE_RENEW_MS);
|
|
1008
1007
|
this.leaseTimer.unref?.();
|
|
@@ -2026,6 +2025,13 @@ export class Manager {
|
|
|
2026
2025
|
}
|
|
2027
2026
|
}
|
|
2028
2027
|
resumeEntry(a, backfillFloor) {
|
|
2028
|
+
// A preservation cut is a stop boundary too, so a manifest launch after it reopens what a resume would.
|
|
2029
|
+
this.recordContinuityAtStop(a, "preservation");
|
|
2030
|
+
return this.resumeEntryOf(a, backfillFloor);
|
|
2031
|
+
}
|
|
2032
|
+
/** The resume entry of one live seat, with no side effect: a preservation cut and a supervised
|
|
2033
|
+
* seat's record both build it. */
|
|
2034
|
+
resumeEntryOf(a, backfillFloor) {
|
|
2029
2035
|
const principal = a.userOwner
|
|
2030
2036
|
? parsePrincipalKey(a.id)
|
|
2031
2037
|
: { owner: DEV_OWNER, actor: a.id };
|
|
@@ -2056,8 +2062,6 @@ export class Manager {
|
|
|
2056
2062
|
return { mode: "static", id: principal.actor, lifecycleUid: a.lifecycleUid, credential: { kind: "file", path: files.creds, sha256: this.fileDigestOrEmpty(files.creds) }, ...(a.issued ? { issued: a.issued } : {}) };
|
|
2057
2063
|
})()
|
|
2058
2064
|
: { mode: "open", id: principal.actor, lifecycleUid: a.lifecycleUid };
|
|
2059
|
-
// A preservation cut is a stop boundary too, so a manifest launch after it reopens what a resume would.
|
|
2060
|
-
this.recordContinuityAtStop(a, "preservation");
|
|
2061
2065
|
const dependencies = [a.launch.source.configPath];
|
|
2062
2066
|
if (a.launch.source.kind === "manifest" && a.launch.source.runId)
|
|
2063
2067
|
dependencies.unshift(join(this.workspaceRoot, ".cotal", "run", `${a.launch.source.runId}.json`));
|
|
@@ -2081,7 +2085,8 @@ export class Manager {
|
|
|
2081
2085
|
shareTools: a.launch.shareTools,
|
|
2082
2086
|
forkSource: a.launch.forkSource,
|
|
2083
2087
|
...(this.forkProvenance(a) ? { resumed: this.forkProvenance(a) } : {}),
|
|
2084
|
-
|
|
2088
|
+
// A supervised seat whose connector keeps no session state is armed with no state file to read.
|
|
2089
|
+
sessionId: a.restart?.armed && a.restart.sessionStatePath ? this.readManagedSession(a) : a.launch.sessionId,
|
|
2085
2090
|
// Additive and only when the connector supplied one. A seat with no pointer records no
|
|
2086
2091
|
// field, which is what keeps an older inventory and a fresh one the same document shape.
|
|
2087
2092
|
...(a.restart?.sessionStatePath ? { sessionStatePath: a.restart.sessionStatePath } : {}),
|
|
@@ -2110,6 +2115,7 @@ export class Manager {
|
|
|
2110
2115
|
// stop can't strand the rest, and every snapshot entry is deprovisioned below regardless.
|
|
2111
2116
|
this.recordContinuityAtStop(a, "shutdown");
|
|
2112
2117
|
this.agents.delete(a.name);
|
|
2118
|
+
this.dropSeatRecord(a);
|
|
2113
2119
|
const refused = this.stopHandle(a, false);
|
|
2114
2120
|
if (refused === undefined)
|
|
2115
2121
|
stopped.push(a);
|
|
@@ -2184,6 +2190,51 @@ export class Manager {
|
|
|
2184
2190
|
if (failures.length)
|
|
2185
2191
|
throw new Error(`manager preservation shutdown incomplete: ${failures.join("; ")}`);
|
|
2186
2192
|
}
|
|
2193
|
+
/** Create THIS instance's lease key, waiting out a dead predecessor's row first (#4092).
|
|
2194
|
+
*
|
|
2195
|
+
* A manager SIGKILLed on this host cannot release its key, so a restart inside the key's TTL
|
|
2196
|
+
* loses the create. When the row names this workspace root and a pid that is not running here
|
|
2197
|
+
* (or is this process's own pid, which a live predecessor cannot hold), the holder is dead and
|
|
2198
|
+
* the restart waits for the bucket TTL to remove the row, then creates again. It never writes
|
|
2199
|
+
* over the row: a live holder renews it inside the TTL, and the wait then refuses on that write.
|
|
2200
|
+
* Any other conflict refuses at once, as before. */
|
|
2201
|
+
async acquireInstanceLease(info) {
|
|
2202
|
+
const refuse = (held, e) => new Error(held
|
|
2203
|
+
? `manager instance ${this.managerInstanceId} already serves space "${this.space}" from ${held.root === info.root ? "this" : "another"} workspace root (${held.runtime}, pid ${held.pid}, root ${held.root}) - stop it first before restarting the same instance`
|
|
2204
|
+
: `could not acquire the manager lease for space "${this.space}": ${rejectionText(e)}`);
|
|
2205
|
+
try {
|
|
2206
|
+
return await this.ep.acquireManagerLease(info);
|
|
2207
|
+
}
|
|
2208
|
+
catch (e) {
|
|
2209
|
+
// Our OWN instance id already holds a live key ⇒ refuse. Anything else (e.g. a KV/JS error) is a
|
|
2210
|
+
// real failure to surface, not a silent "held" — keep the cause so it isn't misread as a conflict.
|
|
2211
|
+
const held = await this.ep.readOwnManagerLease(this.managerInstanceId).catch(() => undefined);
|
|
2212
|
+
if (!held || held.info.root !== info.root || (held.info.pid !== process.pid && probeLiveness(held.info.pid) !== "dead"))
|
|
2213
|
+
throw refuse(held?.info, e);
|
|
2214
|
+
console.error(`• manager instance ${this.managerInstanceId}: its lease is still held by pid ${held.info.pid}, which is not running on this host; ` +
|
|
2215
|
+
`waiting ${Math.ceil(leaseRemainingMs(held.created, MANAGER_LEASE_TTL_MS) / 1000)}s for the lease to expire`);
|
|
2216
|
+
const waited = await awaitLeaseExpiry({
|
|
2217
|
+
read: async () => {
|
|
2218
|
+
const row = await this.ep.readOwnManagerLease(this.managerInstanceId);
|
|
2219
|
+
return row && { revision: row.revision, created: row.created };
|
|
2220
|
+
},
|
|
2221
|
+
revision: held.revision,
|
|
2222
|
+
created: held.created,
|
|
2223
|
+
ttlMs: MANAGER_LEASE_TTL_MS,
|
|
2224
|
+
aborted: () => this.leaseStopping,
|
|
2225
|
+
});
|
|
2226
|
+
if (waited === "rewritten")
|
|
2227
|
+
throw new Error(`manager instance ${this.managerInstanceId}: its lease was written again while waiting for pid ${held.info.pid}'s lease to expire, so a live process holds it - stop it first before restarting the same instance`);
|
|
2228
|
+
if (waited !== "expired")
|
|
2229
|
+
throw refuse(held.info, e);
|
|
2230
|
+
try {
|
|
2231
|
+
return await this.ep.acquireManagerLease(info);
|
|
2232
|
+
}
|
|
2233
|
+
catch (again) {
|
|
2234
|
+
throw refuse((await this.ep.readOwnManagerLease(this.managerInstanceId).catch(() => undefined))?.info, again);
|
|
2235
|
+
}
|
|
2236
|
+
}
|
|
2237
|
+
}
|
|
2187
2238
|
/** Every call joins the first stop and settles with it. A call that asks for a different
|
|
2188
2239
|
* `withAgents` is refused, since the running stop has already applied its own policy. */
|
|
2189
2240
|
stop(options = {}) {
|
|
@@ -2858,7 +2909,9 @@ export class Manager {
|
|
|
2858
2909
|
return a;
|
|
2859
2910
|
};
|
|
2860
2911
|
return managerCommandDefs({
|
|
2861
|
-
|
|
2912
|
+
// A remote Manager files every seat under its own owner, so only a user mesh's shared local
|
|
2913
|
+
// Manager narrows the count, and status never waits on a remote admin check.
|
|
2914
|
+
status: (ctx) => this.serveGated(ctx, async () => this.managerStatusData(this.remoteAuthority ? undefined : await this.psOwnerFilter(ctx.subject.caller, false))),
|
|
2862
2915
|
ps: (ctx) => this.serveGated(ctx, async () => this.list(await this.psOwnerFilter(ctx.subject.caller, false))),
|
|
2863
2916
|
inspect: (ctx) => this.serveGated(ctx, async () => {
|
|
2864
2917
|
const name = String(args(ctx).name ?? "").trim();
|
|
@@ -3627,6 +3680,7 @@ export class Manager {
|
|
|
3627
3680
|
this.recordContinuityAtStop(a, "stop");
|
|
3628
3681
|
if (a.restart?.sessionStatePath)
|
|
3629
3682
|
rmSync(a.restart.sessionStatePath, { force: true });
|
|
3683
|
+
this.dropSeatRecord(a);
|
|
3630
3684
|
if (a.seatHome)
|
|
3631
3685
|
rmSync(a.seatHome, { recursive: true, force: true });
|
|
3632
3686
|
// End any live §13.6 attach session bound to THIS incarnation with the honest
|
|
@@ -4324,6 +4378,7 @@ export class Manager {
|
|
|
4324
4378
|
replacement = undefined;
|
|
4325
4379
|
restart.opts = opts;
|
|
4326
4380
|
restart.recovering = false;
|
|
4381
|
+
this.refreshSeatRecord(a);
|
|
4327
4382
|
if (continueSession !== undefined)
|
|
4328
4383
|
console.error(`! ${a.name}: recovered Pi session ${continueSession} after crash (${restart.crashes.length}/${limit})`);
|
|
4329
4384
|
else
|
|
@@ -4573,9 +4628,11 @@ export class Manager {
|
|
|
4573
4628
|
}
|
|
4574
4629
|
return { cwd, host: hostname() };
|
|
4575
4630
|
}
|
|
4576
|
-
/** Resolve a placed spawn directory on this manager's host.
|
|
4577
|
-
*
|
|
4631
|
+
/** Resolve a placed spawn directory on this manager's host. Delegated execution refuses before
|
|
4632
|
+
* filesystem lookup, since its seats cannot use host cwd. Creates nothing and never substitutes the workspace root. */
|
|
4578
4633
|
resolveSpawnCwd(value) {
|
|
4634
|
+
if (this.execution === "runtime" && this.requiredRuntime.spawnDelegated)
|
|
4635
|
+
throw new EpEnvelopeError("failed-precondition", `runtime "${this.requiredRuntime.kind}" starts seats outside this host, so it cannot resolve a directory on this host's filesystem (cwd); the seat runs in its runtime resource's own directory`);
|
|
4579
4636
|
const asked = typeof value === "string" ? value : String(value ?? "");
|
|
4580
4637
|
const refuse = (reason) => {
|
|
4581
4638
|
throw new EpEnvelopeError("failed-precondition", `cwd does not resolve on this host: ${JSON.stringify(asked)} (${reason})`);
|
|
@@ -4892,6 +4949,11 @@ export class Manager {
|
|
|
4892
4949
|
* `resolve-agent` read: a detached resume locates the session it carries with the connector this
|
|
4893
4950
|
* names, so the two doors must not drift (#2899). */
|
|
4894
4951
|
async spawnPersona(opts, spawner, authenticatedCaller) {
|
|
4952
|
+
// Both config-taking wire commands and direct/roster starts enter here before interpreting
|
|
4953
|
+
// a path. A resolved manifest is host-built authority over its own materialized resource,
|
|
4954
|
+
// never a wire-supplied config override (the public schemas accept no `resolved` field).
|
|
4955
|
+
if (this.execution === "runtime" && this.requiredRuntime.spawnDelegated && opts.config !== undefined && !opts.resolved)
|
|
4956
|
+
return { error: "delegated execution cannot use a manager-host persona config path; select a named persona" };
|
|
4895
4957
|
// The spawn argument is a persona REF — a filename in `.cotal/agents` (the unique spawn KEY), or
|
|
4896
4958
|
// a path via `--config`. It is NOT the mesh identity: the identity comes from inside the file
|
|
4897
4959
|
// (`name:`), so a persona can be filed descriptively (review-critic.md) yet present under a
|
|
@@ -5693,7 +5755,23 @@ export class Manager {
|
|
|
5693
5755
|
else {
|
|
5694
5756
|
managed.restart.armed = true;
|
|
5695
5757
|
}
|
|
5758
|
+
if (managed.restart.policy) {
|
|
5759
|
+
try {
|
|
5760
|
+
this.recordSeat(managed);
|
|
5761
|
+
}
|
|
5762
|
+
catch (error) {
|
|
5763
|
+
const detail = `${managed.name} joined, but supervise cannot cover a manager restart for it: ${rejectionText(error)}`;
|
|
5764
|
+
this.stopHandle(managed, false);
|
|
5765
|
+
this.freeSlot(managed, true, "supervise-record-failed");
|
|
5766
|
+
await hooks?.onOutcome?.({ kind: "failed", data: { error: detail } });
|
|
5767
|
+
return { ok: false, error: detail };
|
|
5768
|
+
}
|
|
5769
|
+
}
|
|
5696
5770
|
}
|
|
5771
|
+
// An unsupervised seat a successor could adopt keeps a record too; one that cannot be written
|
|
5772
|
+
// costs only that adoption, so the spawn stands and the line says so.
|
|
5773
|
+
if (!managed.restart?.policy)
|
|
5774
|
+
this.refreshSeatRecord(managed);
|
|
5697
5775
|
this.watchExit(managed);
|
|
5698
5776
|
// Reply with the id the slot actually carries (user-mode: the owner.actor principal —
|
|
5699
5777
|
// presence, ps, and the manifest ownership ledger all key on it; the throwaway static nkey
|
|
@@ -6080,9 +6158,10 @@ export class Manager {
|
|
|
6080
6158
|
}
|
|
6081
6159
|
return undefined;
|
|
6082
6160
|
}
|
|
6083
|
-
/** Validate/relaunch one retained inventory entry. Called
|
|
6084
|
-
*
|
|
6085
|
-
|
|
6161
|
+
/** Validate/relaunch one retained inventory entry. Called through resumePreserved so all records
|
|
6162
|
+
* pass the same preflight before the first child is exposed, and through restartSupervisedSeat
|
|
6163
|
+
* with the one record a crashed manager left (`supervised`). */
|
|
6164
|
+
async resumePreservedAgent(entry, preflightOnly = false, batchReserved = false, prepared, supervised) {
|
|
6086
6165
|
if (this.execution === "none")
|
|
6087
6166
|
return { ok: false, error: this.executionRefusal() };
|
|
6088
6167
|
const release = this.beginLifecycle(batchReserved);
|
|
@@ -6110,7 +6189,7 @@ export class Manager {
|
|
|
6110
6189
|
catch (e) {
|
|
6111
6190
|
return { ok: false, error: rejectionText(e) };
|
|
6112
6191
|
}
|
|
6113
|
-
return this.launchPreparedResume(entry, cached, batchReserved);
|
|
6192
|
+
return this.launchPreparedResume(entry, cached, batchReserved, supervised);
|
|
6114
6193
|
}
|
|
6115
6194
|
try {
|
|
6116
6195
|
const cwd = lstatSync(entry.launch.cwd);
|
|
@@ -6228,7 +6307,7 @@ export class Manager {
|
|
|
6228
6307
|
prepared?.set(entry.name, value);
|
|
6229
6308
|
if (preflightOnly)
|
|
6230
6309
|
return { ok: true, data: { name: entry.name, preflight: true } };
|
|
6231
|
-
return this.launchPreparedResume(entry, value, batchReserved);
|
|
6310
|
+
return this.launchPreparedResume(entry, value, batchReserved, supervised);
|
|
6232
6311
|
}
|
|
6233
6312
|
catch (e) {
|
|
6234
6313
|
return { ok: false, error: rejectionText(e) };
|
|
@@ -6238,7 +6317,7 @@ export class Manager {
|
|
|
6238
6317
|
release();
|
|
6239
6318
|
}
|
|
6240
6319
|
}
|
|
6241
|
-
async launchPreparedResume(entry, prepared, batchReserved) {
|
|
6320
|
+
async launchPreparedResume(entry, prepared, batchReserved, supervised) {
|
|
6242
6321
|
if (!batchReserved)
|
|
6243
6322
|
this.reserved.add(entry.name);
|
|
6244
6323
|
try {
|
|
@@ -6248,7 +6327,14 @@ export class Manager {
|
|
|
6248
6327
|
let adoptedSeed;
|
|
6249
6328
|
if (entry.identity.mode === "static") {
|
|
6250
6329
|
const stored = await this.secrets.get(agentSecretKeyForFile(resolve(entry.identity.credential.path), this.space));
|
|
6251
|
-
|
|
6330
|
+
// The core parser, which reads the creds format's six-dash END marker. A local pattern that
|
|
6331
|
+
// expected five matched no real file, so every resumed seat lost its renewal.
|
|
6332
|
+
try {
|
|
6333
|
+
adoptedSeed = stored === undefined ? undefined : identityFromCreds(stored).seed;
|
|
6334
|
+
}
|
|
6335
|
+
catch {
|
|
6336
|
+
adoptedSeed = undefined;
|
|
6337
|
+
}
|
|
6252
6338
|
if (adoptedSeed === undefined)
|
|
6253
6339
|
console.error(`! resume ${entry.name}: the adopted credential carries no readable nkey seed - the manager cannot renew it (it dies loud at its exp)`);
|
|
6254
6340
|
}
|
|
@@ -6301,9 +6387,11 @@ export class Manager {
|
|
|
6301
6387
|
...(prepared.launchOpts.resume !== undefined && prepared.spec.resumeRecordPath ? { resumeRecordPath: prepared.spec.resumeRecordPath } : {}),
|
|
6302
6388
|
sessionId: entry.launch.sessionId,
|
|
6303
6389
|
},
|
|
6304
|
-
...(
|
|
6305
|
-
? { restart: { opts: prepared.launchOpts, sessionStatePath: prepared.spec.sessionStatePath, crashes:
|
|
6306
|
-
:
|
|
6390
|
+
...(supervised
|
|
6391
|
+
? { restart: { opts: prepared.launchOpts, sessionStatePath: prepared.spec.sessionStatePath, crashes: supervised.crashes, recovering: false, armed: false, policy: supervised.policy } }
|
|
6392
|
+
: prepared.spec.sessionStatePath
|
|
6393
|
+
? { restart: { opts: prepared.launchOpts, sessionStatePath: prepared.spec.sessionStatePath, crashes: [], recovering: false, armed: false } }
|
|
6394
|
+
: {}),
|
|
6307
6395
|
suppressCleanup: true,
|
|
6308
6396
|
};
|
|
6309
6397
|
this.agents.set(entry.name, managed);
|
|
@@ -6316,7 +6404,7 @@ export class Manager {
|
|
|
6316
6404
|
catch (error) {
|
|
6317
6405
|
return await this.stopFailedResume(managed, "resume-custody-unrecorded", `${managed.name} resumed, but ${rejectionText(error)}`);
|
|
6318
6406
|
}
|
|
6319
|
-
const readiness = await this.awaitReadiness(managed, readinessTimeoutMs);
|
|
6407
|
+
const readiness = await this.awaitReadiness(managed, readinessTimeoutMs, supervised ? { joinedAfter: supervised.joinedAfter } : {});
|
|
6320
6408
|
if (!readiness.ok && !readiness.uncertain)
|
|
6321
6409
|
return { ok: false, error: readiness.detail };
|
|
6322
6410
|
if (!readiness.ok) {
|
|
@@ -6324,7 +6412,12 @@ export class Manager {
|
|
|
6324
6412
|
this.watchResumeAdoption(managed);
|
|
6325
6413
|
return { ok: false, error: readiness.detail };
|
|
6326
6414
|
}
|
|
6327
|
-
|
|
6415
|
+
// A supervised restart arms the way its spawn did: a connector with no session continuation is
|
|
6416
|
+
// armed at readiness, one with it once its session is proved below.
|
|
6417
|
+
if (supervised && managed.restart && !connector.supportsSessionContinuation && !prepared.exact) {
|
|
6418
|
+
managed.restart.armed = true;
|
|
6419
|
+
}
|
|
6420
|
+
else if (managed.restart || prepared.exact) {
|
|
6328
6421
|
try {
|
|
6329
6422
|
await this.armSessionRecovery(managed);
|
|
6330
6423
|
// As on a manifest launch, an exact seat is recorded only at the session it asked for,
|
|
@@ -6346,6 +6439,8 @@ export class Manager {
|
|
|
6346
6439
|
}
|
|
6347
6440
|
if (!this.resumeAttemptId)
|
|
6348
6441
|
managed.suppressCleanup = false;
|
|
6442
|
+
if (supervised)
|
|
6443
|
+
this.refreshSeatRecord(managed);
|
|
6349
6444
|
this.watchExit(managed);
|
|
6350
6445
|
if (this.agents.get(managed.name) !== managed)
|
|
6351
6446
|
return { ok: false, error: `${managed.name} exited immediately after same-principal readiness` };
|
|
@@ -6856,16 +6951,20 @@ export class Manager {
|
|
|
6856
6951
|
await nc.drain().catch(() => nc.close());
|
|
6857
6952
|
}
|
|
6858
6953
|
}
|
|
6859
|
-
/** The served manager-level health summary.
|
|
6860
|
-
|
|
6954
|
+
/** The served manager-level health summary. Given an owner, `agentCount` counts only that owner's
|
|
6955
|
+
* seats, as `ps` lists them ({@link psOwnerFilter}), so a person reading a shared Manager does not
|
|
6956
|
+
* learn how many seats other owners hold on it. */
|
|
6957
|
+
managerStatusData(ownerFilter) {
|
|
6861
6958
|
return {
|
|
6862
6959
|
instanceId: this.managerInstanceId,
|
|
6960
|
+
host: hostname(),
|
|
6961
|
+
version: MANAGER_VERSION,
|
|
6863
6962
|
execution: this.execution,
|
|
6864
6963
|
runtime: this.runtimeKind,
|
|
6865
6964
|
custody: this.execution === "none" ? "none" : this.requiredRuntime.kind === "pty" && this.requiredRuntime.supportsRelease === true ? "custodied" : "legacy",
|
|
6866
6965
|
runHosting: this.execution !== "none" && (this.remoteAuthority ? this.remoteAuthority.runHosting !== undefined : !this.userMode && this.auth !== undefined),
|
|
6867
6966
|
terminalSessions: this.execution !== "none",
|
|
6868
|
-
agentCount: this.agents.size,
|
|
6967
|
+
agentCount: ownerFilter === undefined ? this.agents.size : [...this.agents.values()].filter((a) => a.userOwner === ownerFilter).length,
|
|
6869
6968
|
uptimeMs: Date.now() - this.startedAtMs,
|
|
6870
6969
|
connectors: this.connectorStatuses.map((row) => ({ ...row, binaries: { ...row.binaries } })),
|
|
6871
6970
|
classSpawn: this.classSpawnEnabled(),
|
|
@@ -8756,6 +8855,191 @@ export class Manager {
|
|
|
8756
8855
|
throw new Error(`runtime "${this.requiredRuntime.kind}" reserved custody ${reserved.kind}:${reserved.id} for "${name}" but spawned ` +
|
|
8757
8856
|
`${got ? `${got.kind}:${got.id}` : "a handle with no reference"}; the recorded reference would address no seat, so ${disposal}`);
|
|
8758
8857
|
}
|
|
8858
|
+
/** Persist what a successor needs to take this seat back if this process ends without stopping
|
|
8859
|
+
* it: a supervised seat, which a successor restarts (#4094), and a seat on a runtime that can adopt
|
|
8860
|
+
* it by reference, which a successor adopts while it still runs (#4095). Static meshes only: a
|
|
8861
|
+
* successor finds the seat through its active slot row, and only a static mesh keeps one. Throws
|
|
8862
|
+
* when a supervised seat's entry could not be resumed, because a policy this host cannot keep
|
|
8863
|
+
* across a manager restart is one it cannot enforce. */
|
|
8864
|
+
recordSeat(a) {
|
|
8865
|
+
if (!this.auth || this.userMode)
|
|
8866
|
+
return;
|
|
8867
|
+
const policy = a.restart?.policy;
|
|
8868
|
+
const adoptable = a.handle.reference !== undefined && typeof this.requiredRuntime.adopt === "function";
|
|
8869
|
+
if (!policy && !adoptable)
|
|
8870
|
+
return;
|
|
8871
|
+
const agent = this.resumeEntryOf(a);
|
|
8872
|
+
if (policy) {
|
|
8873
|
+
const unresumable = this.inventoryReferenceError(agent);
|
|
8874
|
+
if (unresumable)
|
|
8875
|
+
throw new Error(`a successor manager could not restart it: ${unresumable}`);
|
|
8876
|
+
}
|
|
8877
|
+
writeSeatRecord(this.workspaceRoot, { version: "cotal-seat-record/v1", instanceId: this.managerInstanceId, ...(policy ? { supervise: policy } : {}), crashes: a.restart?.crashes ?? [], agent });
|
|
8878
|
+
}
|
|
8879
|
+
/** Bring a running seat's record up to date after its credential or restart count changed. A
|
|
8880
|
+
* record that cannot be rewritten is removed, so a successor retires the seat rather than taking
|
|
8881
|
+
* it back from a stale one. */
|
|
8882
|
+
refreshSeatRecord(a) {
|
|
8883
|
+
try {
|
|
8884
|
+
this.recordSeat(a);
|
|
8885
|
+
}
|
|
8886
|
+
catch (error) {
|
|
8887
|
+
console.error(`! ${a.name}: its seat record could not be updated (${rejectionText(error)}); a manager restart now retires it`);
|
|
8888
|
+
this.dropSeatRecord(a);
|
|
8889
|
+
}
|
|
8890
|
+
}
|
|
8891
|
+
dropSeatRecord(a) {
|
|
8892
|
+
try {
|
|
8893
|
+
removeSeatRecord(this.workspaceRoot, this.space, a.name, a.lifecycleUid);
|
|
8894
|
+
}
|
|
8895
|
+
catch (error) {
|
|
8896
|
+
console.error(`! ${a.name}: its seat record could not be removed: ${rejectionText(error)}`);
|
|
8897
|
+
}
|
|
8898
|
+
}
|
|
8899
|
+
/** The seat record behind one active slot this instance owns, when it describes that exact
|
|
8900
|
+
* incarnation on this runtime. Anything else is left to the ordinary terminal. */
|
|
8901
|
+
seatRecordFor(row) {
|
|
8902
|
+
let record;
|
|
8903
|
+
try {
|
|
8904
|
+
record = readSeatRecord(this.workspaceRoot, this.space, row.alias);
|
|
8905
|
+
}
|
|
8906
|
+
catch (error) {
|
|
8907
|
+
console.error(`! ${row.alias}: ${rejectionText(error)}; the lifecycle is retired, not restarted`);
|
|
8908
|
+
return undefined;
|
|
8909
|
+
}
|
|
8910
|
+
if (record === undefined)
|
|
8911
|
+
return undefined;
|
|
8912
|
+
const identity = record.agent.identity;
|
|
8913
|
+
if (identity.mode !== "static" || identity.id !== row.actor || identity.lifecycleUid !== row.lifecycleUid)
|
|
8914
|
+
return undefined;
|
|
8915
|
+
if (record.instanceId !== this.managerInstanceId || record.agent.space !== this.space)
|
|
8916
|
+
return undefined;
|
|
8917
|
+
if (record.agent.launch.runtime !== this.requiredRuntime.kind) {
|
|
8918
|
+
console.error(`! ${row.alias}: its seat ran on runtime "${record.agent.launch.runtime}" and this manager uses "${this.requiredRuntime.kind}"; the lifecycle is retired, not restarted`);
|
|
8919
|
+
return undefined;
|
|
8920
|
+
}
|
|
8921
|
+
return record;
|
|
8922
|
+
}
|
|
8923
|
+
/** Take back a seat that outlived its manager (#4095): the runtime reattaches to the exact seat
|
|
8924
|
+
* the slot names, and this manager holds it under the record's identity and launch, with no
|
|
8925
|
+
* relaunch. Returns why it could not, and the caller then retires the lifecycle, which reaps the
|
|
8926
|
+
* seat by the same reference. */
|
|
8927
|
+
async adoptSeat(row, record) {
|
|
8928
|
+
const entry = record.agent;
|
|
8929
|
+
if (entry.identity.mode !== "static")
|
|
8930
|
+
return "its record is not a static identity";
|
|
8931
|
+
if (row.runtime === undefined)
|
|
8932
|
+
return "its slot names no seat";
|
|
8933
|
+
const runtime = this.requiredRuntime;
|
|
8934
|
+
if (typeof runtime.adopt !== "function")
|
|
8935
|
+
return `runtime "${runtime.kind}" cannot adopt a seat`;
|
|
8936
|
+
// The credential is the manager's own lifecycle-keyed file, and its seed renews it.
|
|
8937
|
+
const credsPath = resolve(entry.identity.credential.path);
|
|
8938
|
+
if (credsPath !== resolve(agentLifecycleSecretFilePaths(this.workspaceRoot, this.space, entry.name, entry.identity.lifecycleUid).creds))
|
|
8939
|
+
return `its credential reference ${credsPath} is not this lifecycle's manager-owned path`;
|
|
8940
|
+
const stored = await this.secrets.get(agentSecretKeyForFile(credsPath, this.space));
|
|
8941
|
+
if (stored === undefined)
|
|
8942
|
+
return "its credential is not in the secret store";
|
|
8943
|
+
let seed;
|
|
8944
|
+
try {
|
|
8945
|
+
const identity = identityFromCreds(stored);
|
|
8946
|
+
if (identity.id !== entry.identity.id)
|
|
8947
|
+
return `its credential is identity ${identity.id}, not ${entry.identity.id}`;
|
|
8948
|
+
seed = identity.seed;
|
|
8949
|
+
}
|
|
8950
|
+
catch (error) {
|
|
8951
|
+
return `its credential is unreadable: ${rejectionText(error)}`;
|
|
8952
|
+
}
|
|
8953
|
+
let adopted;
|
|
8954
|
+
try {
|
|
8955
|
+
adopted = runtime.adopt(row.runtime);
|
|
8956
|
+
}
|
|
8957
|
+
catch (error) {
|
|
8958
|
+
return `the seat ${row.runtime.kind}:${row.runtime.id} could not be adopted: ${rejectionText(error)}`;
|
|
8959
|
+
}
|
|
8960
|
+
const managed = {
|
|
8961
|
+
name: entry.name,
|
|
8962
|
+
role: entry.role,
|
|
8963
|
+
agent: entry.launch.connector,
|
|
8964
|
+
id: entry.identity.id,
|
|
8965
|
+
seed,
|
|
8966
|
+
...(entry.identity.issued ? { issued: entry.identity.issued } : {}),
|
|
8967
|
+
lifecycleUid: entry.identity.lifecycleUid,
|
|
8968
|
+
secretPaths: { creds: entry.identity.credential.path },
|
|
8969
|
+
spawner: entry.spawner,
|
|
8970
|
+
authorityParent: entry.authorityParent,
|
|
8971
|
+
startedAt: Date.parse(entry.startedAt),
|
|
8972
|
+
// The runtime names an adopted handle by what it holds; the record names the seat.
|
|
8973
|
+
handle: { ...adopted, name: entry.name },
|
|
8974
|
+
launch: {
|
|
8975
|
+
source: entry.launch.source,
|
|
8976
|
+
cwd: entry.launch.cwd,
|
|
8977
|
+
model: entry.launch.model,
|
|
8978
|
+
variant: entry.launch.variant,
|
|
8979
|
+
subscribe: entry.launch.subscribe,
|
|
8980
|
+
allowSubscribe: entry.launch.allowSubscribe,
|
|
8981
|
+
allowPublish: entry.launch.allowPublish,
|
|
8982
|
+
capabilities: entry.launch.capabilities,
|
|
8983
|
+
events: entry.launch.events,
|
|
8984
|
+
shareTools: entry.launch.shareTools,
|
|
8985
|
+
forkSource: entry.launch.forkSource,
|
|
8986
|
+
...(entry.launch.resumed ? { resumed: entry.launch.resumed } : {}),
|
|
8987
|
+
sessionId: entry.launch.sessionId,
|
|
8988
|
+
unresolvedLaunchOptionKeys: entry.launch.unresolvedLaunchOptionKeys,
|
|
8989
|
+
},
|
|
8990
|
+
};
|
|
8991
|
+
this.agents.set(entry.name, managed);
|
|
8992
|
+
this.watchExit(managed);
|
|
8993
|
+
console.error(`! ${entry.name}: adopted the seat its previous manager left (${row.runtime.kind}:${row.runtime.id})`);
|
|
8994
|
+
// A credential that came close to its expiry while no manager ran is renewed now, not at the
|
|
8995
|
+
// next pass.
|
|
8996
|
+
const health = inspectCredHealth(stored).state;
|
|
8997
|
+
if (health === "near-expiry" || health === "expired")
|
|
8998
|
+
await this.renewManagedStaticCred(managed).catch((error) => console.error(`! managed cred renewal ${entry.name}: ${rejectionText(error)} - the agent dies loud at this cred's expiry unless it is reminted`));
|
|
8999
|
+
return undefined;
|
|
9000
|
+
}
|
|
9001
|
+
/** Restart a supervised seat whose manager ended without stopping it (#4094), as one restart of
|
|
9002
|
+
* its own policy. It goes through the resume launch, so the seat comes back under the same name,
|
|
9003
|
+
* lifecycle uid, credential, persona and cwd, and a pending turn addressed to it is served to it.
|
|
9004
|
+
* Returns why it could not, and the caller then retires the lifecycle. */
|
|
9005
|
+
async restartSupervisedSeat(row, record) {
|
|
9006
|
+
const policy = record.supervise;
|
|
9007
|
+
if (policy === undefined)
|
|
9008
|
+
return "its spawn carried no restart policy";
|
|
9009
|
+
const now = Date.now();
|
|
9010
|
+
const crashes = [...record.crashes.filter((at) => now - at < policy.windowMs), now];
|
|
9011
|
+
if (crashes.length > policy.restarts)
|
|
9012
|
+
return `its supervise restart budget is spent (${crashes.length} deaths in ${policy.windowMs / 1000}s, ${policy.restarts} allowed)`;
|
|
9013
|
+
// A seat that outlived its manager must not run beside its replacement, so the one the slot
|
|
9014
|
+
// names is reaped first. A runtime that cannot prove it gone refuses, and the terminal retires it.
|
|
9015
|
+
if (row.runtime !== undefined) {
|
|
9016
|
+
try {
|
|
9017
|
+
const evidence = await requireRuntimeReap(this.requiredRuntime, row.runtime);
|
|
9018
|
+
console.error(`${row.alias}: the seat its previous manager left ${evidence.detail}`);
|
|
9019
|
+
}
|
|
9020
|
+
catch (error) {
|
|
9021
|
+
return `the seat its previous manager left could not be reaped: ${rejectionText(error)}`;
|
|
9022
|
+
}
|
|
9023
|
+
}
|
|
9024
|
+
await this.ep.waitForPresenceSnapshot();
|
|
9025
|
+
// Readiness counts only a heartbeat newer than any the dead seat left, as an in-place restart does.
|
|
9026
|
+
const principal = principalKey(DEV_OWNER, row.actor).key;
|
|
9027
|
+
const joinedAfter = this.ep.getRoster()
|
|
9028
|
+
.filter((p) => p.card.id === principal && p.lifecycleUid === row.lifecycleUid)
|
|
9029
|
+
.reduce((max, p) => Math.max(max, p.ts), 0) + 1;
|
|
9030
|
+
const reply = await this.resumePreservedAgent(record.agent, false, false, undefined, { policy, crashes, joinedAfter });
|
|
9031
|
+
const live = this.agents.get(row.alias);
|
|
9032
|
+
if (reply.ok) {
|
|
9033
|
+
console.error(`! ${row.alias}: restarted under the same lifecycle after its manager stopped (${crashes.length}/${policy.restarts})`);
|
|
9034
|
+
return undefined;
|
|
9035
|
+
}
|
|
9036
|
+
// An uncertain readiness keeps the seat managed and watched, as any launch does.
|
|
9037
|
+
if (live?.lifecycleUid === row.lifecycleUid) {
|
|
9038
|
+
console.error(`! ${row.alias}: restarted after its manager stopped, but readiness is uncertain: ${reply.error}`);
|
|
9039
|
+
return undefined;
|
|
9040
|
+
}
|
|
9041
|
+
return `its restart failed: ${reply.error}`;
|
|
9042
|
+
}
|
|
8759
9043
|
/** The terminal's process step: an orphan (no live handle in THIS process) is reaped by the custody
|
|
8760
9044
|
* reference the slot recorded, before its footprint goes. A successor must never retire a
|
|
8761
9045
|
* lifecycle and free its alias while the predecessor's seat process is still running outside
|
|
@@ -8905,6 +9189,8 @@ export class Manager {
|
|
|
8905
9189
|
const credsPath = a.secretPaths.creds;
|
|
8906
9190
|
await secrets.put(agentSecretKeyForFile(credsPath, this.space), creds);
|
|
8907
9191
|
await materializeSecretToFile(secrets, agentSecretKeyForFile(credsPath, this.space), credsPath);
|
|
9192
|
+
// The record pins the credential's digest, so it follows the re-signed file.
|
|
9193
|
+
this.refreshSeatRecord(a);
|
|
8908
9194
|
console.error(`managed cred renewal ${a.name}: re-signed for the same identity (exp +${MANAGED_STATIC_TTL_SEC}s); the agent endpoint's source re-read adopts it`);
|
|
8909
9195
|
}
|
|
8910
9196
|
/** The Unit B reconciliation (F3 "no active orphan"): ensure the authority stores, then sweep
|
|
@@ -9199,6 +9485,7 @@ export class Manager {
|
|
|
9199
9485
|
let preservedAdopted = 0;
|
|
9200
9486
|
let deferredSlots = 0;
|
|
9201
9487
|
const terminalRows = [];
|
|
9488
|
+
const restartRows = [];
|
|
9202
9489
|
for (const row of slotRows) {
|
|
9203
9490
|
if (row.phase === "retired") {
|
|
9204
9491
|
// A retirement is a GLOBAL refusal fact — seed the F5 index for EVERY retired incarnation
|
|
@@ -9234,6 +9521,16 @@ export class Manager {
|
|
|
9234
9521
|
deferredSlots++;
|
|
9235
9522
|
continue;
|
|
9236
9523
|
}
|
|
9524
|
+
// A boot after this instance's manager ended without stopping its seats: a seat with a record
|
|
9525
|
+
// is taken back instead of retired, restarted under its policy when supervised (#4094) and
|
|
9526
|
+
// adopted while it still runs otherwise (#4095).
|
|
9527
|
+
if (!postAdoption && row.phase === "active" && !adopted) {
|
|
9528
|
+
const record = this.seatRecordFor(row);
|
|
9529
|
+
if (record) {
|
|
9530
|
+
restartRows.push({ row, record });
|
|
9531
|
+
continue;
|
|
9532
|
+
}
|
|
9533
|
+
}
|
|
9237
9534
|
if (planStaticSlotResume(row, adopted) !== "none")
|
|
9238
9535
|
terminalRows.push(row);
|
|
9239
9536
|
}
|
|
@@ -9242,6 +9539,24 @@ export class Manager {
|
|
|
9242
9539
|
// alias, but the initial serial sweep must close every discovery-to-await gap up front.
|
|
9243
9540
|
for (const row of terminalRows)
|
|
9244
9541
|
this.reconcilingAliases.add(row.alias);
|
|
9542
|
+
for (const { row } of restartRows)
|
|
9543
|
+
this.reconcilingAliases.add(row.alias);
|
|
9544
|
+
// Restarts run first; one that cannot restart joins the terminals below.
|
|
9545
|
+
for (const { row, record } of restartRows) {
|
|
9546
|
+
if (this.staticReconcileStopping)
|
|
9547
|
+
break;
|
|
9548
|
+
// A supervised seat is restarted; any other seat is adopted while it still runs.
|
|
9549
|
+
const refusal = record.supervise ? await this.restartSupervisedSeat(row, record) : await this.adoptSeat(row, record);
|
|
9550
|
+
if (refusal === undefined) {
|
|
9551
|
+
this.reconcilingAliases.delete(row.alias);
|
|
9552
|
+
continue;
|
|
9553
|
+
}
|
|
9554
|
+
console.error(record.supervise
|
|
9555
|
+
? `! ${row.alias}: the supervised seat its previous manager left was not restarted (${refusal}); retiring the lifecycle`
|
|
9556
|
+
: `! ${row.alias}: the seat its previous manager left was not adopted (${refusal}); retiring the lifecycle`);
|
|
9557
|
+
this.dropSeatRecord({ name: row.alias, lifecycleUid: row.lifecycleUid });
|
|
9558
|
+
terminalRows.push(row);
|
|
9559
|
+
}
|
|
9245
9560
|
const sweepDeprovision = {
|
|
9246
9561
|
examined: 0,
|
|
9247
9562
|
deleted: 0,
|
|
@@ -9338,6 +9653,8 @@ export class Manager {
|
|
|
9338
9653
|
finally {
|
|
9339
9654
|
for (const row of terminalRows)
|
|
9340
9655
|
this.reconcilingAliases.delete(row.alias);
|
|
9656
|
+
for (const { row } of restartRows)
|
|
9657
|
+
this.reconcilingAliases.delete(row.alias);
|
|
9341
9658
|
}
|
|
9342
9659
|
}
|
|
9343
9660
|
finally {
|