@cotal-ai/manager 0.24.0 → 0.26.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/control-session.d.ts +11 -0
- package/dist/control-session.d.ts.map +1 -0
- package/dist/control-session.js +75 -0
- package/dist/control-session.js.map +1 -0
- package/dist/manager-service-contract.d.ts.map +1 -1
- package/dist/manager-service-contract.js +15 -0
- package/dist/manager-service-contract.js.map +1 -1
- package/dist/manager.d.ts +21 -3
- package/dist/manager.d.ts.map +1 -1
- package/dist/manager.js +312 -20
- package/dist/manager.js.map +1 -1
- package/dist/resume.d.ts.map +1 -1
- package/dist/resume.js +1 -0
- package/dist/resume.js.map +1 -1
- package/package.json +6 -6
package/dist/manager.js
CHANGED
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
import { execFile } from "node:child_process";
|
|
2
2
|
import { createHash, randomUUID, randomBytes } from "node:crypto";
|
|
3
|
+
import { hostname } from "node:os";
|
|
3
4
|
import { connect, credsAuthenticator } from "@nats-io/transport-node";
|
|
4
5
|
import { existsSync, lstatSync, readFileSync, rmSync } from "node:fs";
|
|
5
6
|
import { join, dirname, resolve } from "node:path";
|
|
6
|
-
import { CotalEndpoint, DEFAULT_SERVER, DEV_OWNER, MANAGER_LEASE_TTL_MS, MANAGER_LEASE_RENEW_MS, STANDING_RENEWABLE_TTL_SEC, agentFilePath, clearSpaceHistory, connectorServers, deprovisionAgent, firstFreeName, idFromCreds, inspectCredHealth, loadAgentFile, loadCotalConfig, mintCreds, mintLifecycleUid, mkSecretDir, newIdentity, actionContext, parsePrincipalKey, parseShareSelection, principalKey, probeConnect, provisionAgent, provisionAgentDurables, registry, resolveAuthProvider, saveAgentFile, subjectMatches, AUTH_ENDPOINT, EP_CMD_RETIRE_LIFECYCLE, epRequestSubject, epCallerReplyFilter, parseEpSubject, controlServiceSubject, eventChannelPrincipal, } from "@cotal-ai/core";
|
|
7
|
+
import { CotalEndpoint, DEFAULT_SERVER, DEV_OWNER, MANAGER_LEASE_TTL_MS, MANAGER_LEASE_RENEW_MS, STANDING_RENEWABLE_TTL_SEC, agentFilePath, clearSpaceHistory, connectorServers, spawnEnvAllow, deprovisionAgent, firstFreeName, idFromCreds, inspectCredHealth, loadAgentFile, loadCotalConfig, mintCreds, mintLifecycleUid, mkSecretDir, newIdentity, actionContext, parsePrincipalKey, parseShareSelection, principalKey, probeConnect, provisionAgent, provisionAgentDurables, registry, resolveAuthProvider, saveAgentFile, subjectMatches, AUTH_ENDPOINT, EP_CMD_RETIRE_LIFECYCLE, epRequestSubject, epCallerReplyFilter, parseEpSubject, controlServiceSubject, eventChannelPrincipal, } from "@cotal-ai/core";
|
|
7
8
|
import { agentAuthState, agentCredsDir, agentLifecycleSecretFilePaths, agentSecretFilePaths, agentSecretKeyForFile, authDir, connectorInstallHint, DEFAULT_CONNECTOR, defaultAgentType, DELIVERY_CREDS_KEY, findCotalRoot, getSpaceAuth, hasUserAuthState, loadManagerInstanceIdentity, loadMeshes, manifestExtensionNames, materializeFromManifest, materializeSecretToFile, MEMBERSHIP_RW_CREDS_KEY, mergeLaunchOptions, remintDaemonCreds, resolveOnPath, saveManagerInstanceIdentity, SYSTEM_CREDS_FILES, userAuthStateDir, workspaceSecretStore, writeRenewalRecord } from "@cotal-ai/workspace";
|
|
8
9
|
import { createRuntime, } from "./runtime/index.js";
|
|
9
10
|
import { AttachEndpoint } from "./attach-endpoint.js";
|
|
@@ -11,6 +12,7 @@ import { makeManagerEndpointEvictor } from "./endpoint-evict.js";
|
|
|
11
12
|
import { launchSpecForRun, materializePersona, launchAgentToStartOpts, parseLaunchSpec, persistLaunchSpec } from "./launch.js";
|
|
12
13
|
import { authorizeLaunch, authorizeNamedControl } from "./authorize.js";
|
|
13
14
|
import { controlShutdown } from "./control-shutdown.js";
|
|
15
|
+
import { controlSession } from "./control-session.js";
|
|
14
16
|
import { parseResumeCommitArgs, parseResumeControlArgs, parseResumeFinalizeArgs } from "./resume.js";
|
|
15
17
|
// Unit B (the static §13.1 lifecycle executor): the shared grammar/stores from core plus the
|
|
16
18
|
// manager-side adapter (transport + slot orchestration + the F1 terminal) — see static-lifecycle.ts.
|
|
@@ -43,6 +45,10 @@ const MIN_LIFETIME = 10_000;
|
|
|
43
45
|
* launch-parity smoke can assert every launch client's request timeout OUTLIVES this window — the tier
|
|
44
46
|
* rule forbids the clients importing it directly. */
|
|
45
47
|
export const READINESS_TIMEOUT_MS = 30_000;
|
|
48
|
+
/** Managed same-session crash recovery follows the Codex host precedent: three restarts are allowed
|
|
49
|
+
* inside a rolling two-minute window; the fourth crash is a loop and retires the seat loud. */
|
|
50
|
+
const SESSION_RESTART_LIMIT = 3;
|
|
51
|
+
const SESSION_RESTART_WINDOW_MS = 120_000;
|
|
46
52
|
/** Upper bound on a detached agent-exit deprovision (#159 B2). A wedged broker must not leave the
|
|
47
53
|
* fire-and-forget teardown pending forever with no log — past this it rejects into freeSlot's fail-loud
|
|
48
54
|
* `.catch`. Generous over the helper's 5s connect timeout to allow the two consumer-deletes + ACL purge
|
|
@@ -1118,6 +1124,7 @@ export class Manager {
|
|
|
1118
1124
|
events: a.launch.events,
|
|
1119
1125
|
shareTools: a.launch.shareTools,
|
|
1120
1126
|
forkSource: a.launch.forkSource,
|
|
1127
|
+
sessionId: a.restart?.armed ? this.readManagedSession(a) : a.launch.sessionId,
|
|
1121
1128
|
unresolvedLaunchOptionKeys: a.launch.unresolvedLaunchOptionKeys,
|
|
1122
1129
|
},
|
|
1123
1130
|
dependencies,
|
|
@@ -2032,6 +2039,8 @@ export class Manager {
|
|
|
2032
2039
|
return; // already freed (exit raced despawn, etc.)
|
|
2033
2040
|
a.terminalizing = true; // F5 latch (Unit B): also covers exit/reap paths that never rode stopHandle
|
|
2034
2041
|
this.agents.delete(a.name);
|
|
2042
|
+
if (a.restart?.sessionStatePath)
|
|
2043
|
+
rmSync(a.restart.sessionStatePath, { force: true });
|
|
2035
2044
|
// P2 item 6 (pin 4): end any live §13.6 attach session bound to THIS incarnation with the honest
|
|
2036
2045
|
// `target-despawn` reason. Fires once per agent on every free path (despawn / self-stop / reap /
|
|
2037
2046
|
// exit) via the `agents` guard above; a no-op when no plane or no live session for the target.
|
|
@@ -2204,11 +2213,23 @@ export class Manager {
|
|
|
2204
2213
|
if (!this.auth)
|
|
2205
2214
|
return; // guaranteed by requestRetirement; re-checked for the type narrowing below
|
|
2206
2215
|
const held = this.retiring.get(a.name);
|
|
2207
|
-
const me = parsePrincipalKey(this.ep.ref().id);
|
|
2208
2216
|
const target = parsePrincipalKey(a.id);
|
|
2209
|
-
if (!
|
|
2217
|
+
if (!target) {
|
|
2210
2218
|
if (held)
|
|
2211
|
-
held.lastError = "the
|
|
2219
|
+
held.lastError = "the target principal could not be derived; the retirement was not requested";
|
|
2220
|
+
return;
|
|
2221
|
+
}
|
|
2222
|
+
// The SERVE identity, not the endpoint identity, is who this request speaks as (#549). The field
|
|
2223
|
+
// is declared `!:`, which asserts to the type system what the ordering happens to provide, so it
|
|
2224
|
+
// is read here as what it actually is. On today's ordering this guard cannot fire: `start()`
|
|
2225
|
+
// assigns the identity before it connects anything, and a retirement needs an agent that only a
|
|
2226
|
+
// started manager can hold. It is kept as a fail-closed assertion rather than a live face,
|
|
2227
|
+
// because the alternative is carrying `undefined` into the caller triple and surfacing the
|
|
2228
|
+
// result as "the rail could not be reached", which would name the wrong cause.
|
|
2229
|
+
const serveIdentity = this.managerServeIdentity;
|
|
2230
|
+
if (!serveIdentity?.id) {
|
|
2231
|
+
if (held)
|
|
2232
|
+
held.lastError = `the retirement was NOT requested: this manager has no serve identity yet, so it cannot speak as the registered serving instance. The despawn stopped "${a.name}" and the name stays held. NEXT: let the manager finish registering, then re-attempt the same-name spawn to re-drive the teardown.`;
|
|
2212
2233
|
return;
|
|
2213
2234
|
}
|
|
2214
2235
|
const uncertain = (why) => `the despawn stopped "${a.name}", but the retirement's completion could NOT be confirmed (${why}). The name stays held - not failed, not done - and a same-name spawn re-drives the same teardown; the auth service also finishes any started retirement on its next boot. NEXT: if the auth rail stays unreachable, recover the stack (\`cotal supervise\`), then re-attempt the same-name spawn.`;
|
|
@@ -2216,7 +2237,22 @@ export class Manager {
|
|
|
2216
2237
|
// The caller triple and the TARGET are both grant-pinned now (#350): the `handle` target
|
|
2217
2238
|
// rides the subject, so this ephemeral credential can ask to retire exactly this
|
|
2218
2239
|
// incarnation and nothing else.
|
|
2219
|
-
|
|
2240
|
+
//
|
|
2241
|
+
// THE TRIPLE IS THE SERVE PRINCIPAL, and it has to be (#549). The auth rail authorizes this
|
|
2242
|
+
// request by comparing the caller's `<owner>.<actor>` against the serve issuance gate's bound
|
|
2243
|
+
// principal, and that gate is opened with `principalKey(DEV_OWNER, serveIdentity.id)` (see the
|
|
2244
|
+
// registration block). Deriving the caller from `ep.ref().id` instead put the manager's
|
|
2245
|
+
// ENDPOINT identity nkey on the wire, which is a different, equally real identity of the same
|
|
2246
|
+
// manager, so the comparison was unsatisfiable and EVERY user-mesh retirement was refused as a
|
|
2247
|
+
// full no-op. Measured before the fix: 8 refusals in one suite run across 5 agents, with the
|
|
2248
|
+
// epoch and the instance id both matching and only the principal disagreeing.
|
|
2249
|
+
//
|
|
2250
|
+
// Both halves come from the gate's own sources rather than from `ep.ref()`: `DEV_OWNER` is
|
|
2251
|
+
// hard-coded at the gate site, so taking the owner from `ep.ref().id` would re-open the same
|
|
2252
|
+
// mismatch in the owner half the moment a manager ran under a user-shaped identity. This is
|
|
2253
|
+
// also the more honest attribution: the authority being exercised is "I am the registered
|
|
2254
|
+
// serving instance", which is exactly what the gate records.
|
|
2255
|
+
const caller = { owner: DEV_OWNER, actor: serveIdentity.id, uid: this.managerLifecycleUid };
|
|
2220
2256
|
const creds = await mintCreds(this.auth, newIdentity(), "retirement-requester", {
|
|
2221
2257
|
retirementRequester: { ...caller, target: { owner: target.owner, actor: target.actor, lifecycleUid: a.lifecycleUid } },
|
|
2222
2258
|
});
|
|
@@ -2325,14 +2361,191 @@ export class Manager {
|
|
|
2325
2361
|
this.trackStoppedHandle(child, true, true);
|
|
2326
2362
|
}
|
|
2327
2363
|
}
|
|
2328
|
-
|
|
2329
|
-
|
|
2330
|
-
|
|
2364
|
+
readSessionStatePath(path) {
|
|
2365
|
+
let parsed;
|
|
2366
|
+
try {
|
|
2367
|
+
parsed = JSON.parse(readFileSync(path, "utf8"));
|
|
2368
|
+
}
|
|
2369
|
+
catch (error) {
|
|
2370
|
+
throw new Error(`cannot read connector session state ${path}: ${error.message}`);
|
|
2371
|
+
}
|
|
2372
|
+
const state = parsed;
|
|
2373
|
+
if (state.version !== 1 || typeof state.sessionId !== "string" || !state.sessionId.trim() || state.sessionId.length > 4096 ||
|
|
2374
|
+
(state.status !== "running" && state.status !== "quit"))
|
|
2375
|
+
throw new Error(`connector session state ${path} is malformed`);
|
|
2376
|
+
return { sessionId: state.sessionId, status: state.status };
|
|
2377
|
+
}
|
|
2378
|
+
readManagedSessionState(a) {
|
|
2379
|
+
const path = a.restart?.sessionStatePath;
|
|
2380
|
+
if (!path)
|
|
2381
|
+
throw new Error("connector supplied no session state path");
|
|
2382
|
+
return this.readSessionStatePath(path);
|
|
2383
|
+
}
|
|
2384
|
+
/** Upgrade bridge: inventories written before sessionId existed can still resume exactly after
|
|
2385
|
+
* the old Pi seat has loaded the new extension once and written its lifecycle-keyed state file. */
|
|
2386
|
+
retainedSessionId(entry, connector) {
|
|
2387
|
+
if (entry.launch.sessionId)
|
|
2388
|
+
return entry.launch.sessionId;
|
|
2389
|
+
if (!connector.supportsSessionContinuation)
|
|
2390
|
+
return undefined;
|
|
2391
|
+
const path = join(this.workspaceRoot, ".cotal", "pi-sessions", `${entry.name}-${entry.identity.lifecycleUid}.json`);
|
|
2392
|
+
try {
|
|
2393
|
+
return this.readSessionStatePath(path).sessionId;
|
|
2394
|
+
}
|
|
2395
|
+
catch (error) {
|
|
2396
|
+
throw new Error(`retained ${connector.name} agent ${entry.name} has no sessionId in its older inventory and no usable upgrade state at ${path} ` +
|
|
2397
|
+
`(${error.message}). Reload the live Pi seat before the preservation cut; refusing to resume fresh and lose its context.`);
|
|
2398
|
+
}
|
|
2399
|
+
}
|
|
2400
|
+
readManagedSession(a) {
|
|
2401
|
+
return this.readManagedSessionState(a).sessionId;
|
|
2402
|
+
}
|
|
2403
|
+
async awaitManagedSessionState(a) {
|
|
2404
|
+
const deadline = Date.now() + 15_000;
|
|
2405
|
+
let last = "session state not written yet";
|
|
2406
|
+
while (Date.now() < deadline) {
|
|
2407
|
+
if (a.handle.status() === "exited")
|
|
2408
|
+
throw new Error("process exited before writing session state");
|
|
2409
|
+
try {
|
|
2410
|
+
return this.readManagedSessionState(a);
|
|
2411
|
+
}
|
|
2412
|
+
catch (error) {
|
|
2413
|
+
last = error.message;
|
|
2414
|
+
}
|
|
2415
|
+
await new Promise((resolve) => setTimeout(resolve, 100));
|
|
2416
|
+
}
|
|
2417
|
+
throw new Error(`connector did not write session state (${last})`);
|
|
2418
|
+
}
|
|
2419
|
+
/** Bind a continuation-capable connector only after its process is ready. The file proves the
|
|
2420
|
+
* latest in-process session (including Pi /resume); the authenticated socket proves this process
|
|
2421
|
+
* owns that session now. Presence may lead session_start by milliseconds, so both proofs are
|
|
2422
|
+
* awaited within a bounded readiness window rather than read once. */
|
|
2423
|
+
async armSessionRecovery(a) {
|
|
2424
|
+
if (!a.restart || !a.control)
|
|
2425
|
+
return;
|
|
2426
|
+
const state = await this.awaitManagedSessionState(a);
|
|
2427
|
+
if (state.status !== "running")
|
|
2428
|
+
throw new Error("connector reported a deliberate quit before readiness completed");
|
|
2429
|
+
await this.awaitRecoveredSession(a, state.sessionId);
|
|
2430
|
+
a.restart.armed = true;
|
|
2431
|
+
}
|
|
2432
|
+
async awaitRecoveredSession(a, expected, handle = a.handle, control = a.control) {
|
|
2433
|
+
const deadline = Date.now() + 15_000;
|
|
2434
|
+
let last = "control endpoint not ready";
|
|
2435
|
+
while (Date.now() < deadline) {
|
|
2436
|
+
if (handle.status() === "exited")
|
|
2437
|
+
throw new Error("replacement process exited before reporting its session");
|
|
2438
|
+
if (control) {
|
|
2439
|
+
try {
|
|
2440
|
+
const reported = await controlSession(control);
|
|
2441
|
+
if (reported !== expected)
|
|
2442
|
+
throw new Error(`replacement reported session ${reported}, expected ${expected}`);
|
|
2443
|
+
return;
|
|
2444
|
+
}
|
|
2445
|
+
catch (error) {
|
|
2446
|
+
last = error.message;
|
|
2447
|
+
}
|
|
2448
|
+
}
|
|
2449
|
+
await new Promise((resolve) => setTimeout(resolve, 100));
|
|
2450
|
+
}
|
|
2451
|
+
throw new Error(`replacement did not prove session ${expected} (${last})`);
|
|
2452
|
+
}
|
|
2453
|
+
/** Restart one continuation-capable managed process in place. Identity, lifecycle, credentials,
|
|
2454
|
+
* durables, children, and the manager row remain owned; only the process handle/control endpoint
|
|
2455
|
+
* change. A fourth crash inside two minutes is a loop and falls through to normal retirement. */
|
|
2456
|
+
recoverManagedSession(a) {
|
|
2457
|
+
const restart = a.restart;
|
|
2458
|
+
if (!restart || !restart.armed || restart.recovering || a.terminalizing)
|
|
2459
|
+
return;
|
|
2460
|
+
const release = this.beginLifecycle();
|
|
2461
|
+
if (!release)
|
|
2462
|
+
return; // preservation owns the cut once the lifecycle fence closes
|
|
2463
|
+
const now = Date.now();
|
|
2464
|
+
restart.crashes = restart.crashes.filter((at) => now - at < SESSION_RESTART_WINDOW_MS);
|
|
2465
|
+
restart.crashes.push(now);
|
|
2466
|
+
if (restart.crashes.length > SESSION_RESTART_LIMIT) {
|
|
2467
|
+
console.error(`! ${a.name}: Pi crash loop (${restart.crashes.length} crashes in ${SESSION_RESTART_WINDOW_MS / 1000}s) - retiring the managed seat`);
|
|
2468
|
+
restart.armed = false;
|
|
2469
|
+
this.freeSlot(a, true);
|
|
2470
|
+
this.reapChildrenOf(this.managedPrincipal(a));
|
|
2471
|
+
release();
|
|
2472
|
+
return;
|
|
2473
|
+
}
|
|
2474
|
+
restart.recovering = true;
|
|
2475
|
+
void (async () => {
|
|
2476
|
+
let replacement;
|
|
2477
|
+
try {
|
|
2478
|
+
const sessionId = this.readManagedSession(a);
|
|
2479
|
+
const connector = await this.resolveConnector(a.agent);
|
|
2480
|
+
if (!connector.supportsSessionContinuation)
|
|
2481
|
+
throw new Error(`connector ${connector.name} no longer declares same-session continuation`);
|
|
2482
|
+
const opts = {
|
|
2483
|
+
...restart.opts,
|
|
2484
|
+
resume: undefined,
|
|
2485
|
+
prompt: undefined,
|
|
2486
|
+
continueSession: sessionId,
|
|
2487
|
+
};
|
|
2488
|
+
const spec = connector.buildLaunch(opts);
|
|
2489
|
+
const handle = this.runtime.spawn(a.name, spec, a.launch.cwd);
|
|
2490
|
+
replacement = handle;
|
|
2491
|
+
restart.sessionStatePath = spec.sessionStatePath ?? restart.sessionStatePath;
|
|
2492
|
+
await this.awaitRecoveredSession(a, sessionId, handle, spec.control);
|
|
2493
|
+
if (this.agents.get(a.name) !== a || a.terminalizing) {
|
|
2494
|
+
try {
|
|
2495
|
+
handle.stop({ graceful: false });
|
|
2496
|
+
}
|
|
2497
|
+
catch { /* terminal path owns cleanup */ }
|
|
2498
|
+
return;
|
|
2499
|
+
}
|
|
2500
|
+
a.handle = handle;
|
|
2501
|
+
a.control = spec.control;
|
|
2502
|
+
replacement = undefined;
|
|
2503
|
+
restart.opts = opts;
|
|
2504
|
+
restart.recovering = false;
|
|
2505
|
+
console.error(`! ${a.name}: recovered Pi session ${sessionId} after crash (${restart.crashes.length}/${SESSION_RESTART_LIMIT})`);
|
|
2506
|
+
this.watchExit(a);
|
|
2507
|
+
}
|
|
2508
|
+
catch (error) {
|
|
2509
|
+
restart.recovering = false;
|
|
2510
|
+
restart.armed = false;
|
|
2511
|
+
let tail = "";
|
|
2512
|
+
try {
|
|
2513
|
+
tail = this.tail(await (replacement ?? a.handle).attach().backlog());
|
|
2514
|
+
}
|
|
2515
|
+
catch { /* runtime has no readable tail */ }
|
|
2516
|
+
console.error(`! ${a.name}: Pi session recovery failed: ${error.message}${tail ? ` - last output: ${tail}` : ""} - retiring the managed seat`);
|
|
2517
|
+
// The replacement may be alive but unable to prove the expected session. Stop it BEFORE
|
|
2518
|
+
// retiring credentials/durables; otherwise an untracked process survives under torn auth.
|
|
2519
|
+
try {
|
|
2520
|
+
replacement?.stop({ graceful: false });
|
|
2521
|
+
}
|
|
2522
|
+
catch { /* terminal cleanup continues */ }
|
|
2523
|
+
this.freeSlot(a, true);
|
|
2524
|
+
this.reapChildrenOf(this.managedPrincipal(a));
|
|
2525
|
+
}
|
|
2526
|
+
finally {
|
|
2527
|
+
release();
|
|
2528
|
+
}
|
|
2529
|
+
})();
|
|
2530
|
+
}
|
|
2531
|
+
/** A managed agent's process exited on its own (crash, /exit, finished). Continuation-capable Pi
|
|
2532
|
+
* seats restart in place after readiness; every other exit follows the existing terminal path. */
|
|
2331
2533
|
onAgentExit(a) {
|
|
2332
2534
|
// Preservation owns the child-stop snapshot. Exit watchers must neither delete that snapshot nor
|
|
2333
2535
|
// trigger normal deprovision/reap while the cut is being formed.
|
|
2334
2536
|
if (this.maintenanceState !== "active")
|
|
2335
2537
|
return;
|
|
2538
|
+
if (a.restart?.armed && !a.terminalizing) {
|
|
2539
|
+
try {
|
|
2540
|
+
if (this.readManagedSessionState(a).status === "running") {
|
|
2541
|
+
this.recoverManagedSession(a);
|
|
2542
|
+
return;
|
|
2543
|
+
}
|
|
2544
|
+
}
|
|
2545
|
+
catch (error) {
|
|
2546
|
+
console.error(`! ${a.name}: cannot classify Pi process exit for recovery: ${error.message} - retiring the seat`);
|
|
2547
|
+
}
|
|
2548
|
+
}
|
|
2336
2549
|
this.freeSlot(a, true);
|
|
2337
2550
|
this.reapChildrenOf(this.managedPrincipal(a));
|
|
2338
2551
|
}
|
|
@@ -2745,9 +2958,20 @@ export class Manager {
|
|
|
2745
2958
|
allowSubscribe = opts.allowSubscribe ?? def.allowSubscribe ?? subscribe ?? ["general"];
|
|
2746
2959
|
allowPublish = opts.allowPublish ?? def.allowPublish;
|
|
2747
2960
|
capabilities = def.capabilities;
|
|
2961
|
+
// #651: fold the persona's model into the launch record, mirroring the variant line below
|
|
2962
|
+
// and the manifest branch above. Without this, a persona-file model (the common pin source)
|
|
2963
|
+
// never reaches `launch.model`, so the connector runs the seat on it while `ps --wide`/`--json`
|
|
2964
|
+
// reports the model ABSENT - a false "no model pinned" for a seat that has one.
|
|
2965
|
+
model = opts.model ?? def.model;
|
|
2748
2966
|
variant = opts.variant ?? def.variant;
|
|
2749
2967
|
launchOptions = mergeLaunchOptions(def.launchOptions, opts.launchOptions);
|
|
2750
2968
|
}
|
|
2969
|
+
// #651: an empty or whitespace-only model string is not a pin. Coerce it to undefined here, at
|
|
2970
|
+
// the single point every path (persona, manifest, imperative) has resolved `model`, so it
|
|
2971
|
+
// serializes ABSENT rather than present-but-empty (`"model": ""`), which a key-presence consumer
|
|
2972
|
+
// would misread as "a pin was recorded".
|
|
2973
|
+
if (model !== undefined && model.trim() === "")
|
|
2974
|
+
model = undefined;
|
|
2751
2975
|
const idErr = this.nameError(identityName);
|
|
2752
2976
|
if (idErr)
|
|
2753
2977
|
return { ok: false, error: opts.resolved ? `launch agent: ${idErr}` : `persona ${configPath}: ${idErr}` };
|
|
@@ -2970,7 +3194,11 @@ export class Manager {
|
|
|
2970
3194
|
// Personal MCP servers the operator opted to share with manager-spawned agents of this type
|
|
2971
3195
|
// (cotal config; default none → isolated, the memory-safe default this guards), narrowed by
|
|
2972
3196
|
// an optional --share-tools selection (absent → all declared, the pre-merge behavior).
|
|
2973
|
-
const
|
|
3197
|
+
const cotalConfig = loadCotalConfig(this.workspaceRoot);
|
|
3198
|
+
const mcpServers = connectorServers(cotalConfig, agent, parseShareSelection(opts.shareTools));
|
|
3199
|
+
// The operator's spawn-env policy travels the same route: absent means the child inherits
|
|
3200
|
+
// their environment, present means containment. A connector never reads the config itself.
|
|
3201
|
+
const envAllow = spawnEnvAllow(cotalConfig);
|
|
2974
3202
|
// Per-agent cwd overrides the manager's shared workspace root, so agents can be rooted at
|
|
2975
3203
|
// arbitrary folders/repos. A relative path resolves against the workspace root; omitted → the
|
|
2976
3204
|
// agent shares the workspace root (the prior, unchanged behavior).
|
|
@@ -2980,7 +3208,7 @@ export class Manager {
|
|
|
2980
3208
|
? join(this.workspaceRoot, ".cotal", "run", `${opts.launchRef.runId}.json`)
|
|
2981
3209
|
: undefined;
|
|
2982
3210
|
const manifestSha256 = manifestPath ? this.fileDigest(manifestPath) : undefined;
|
|
2983
|
-
const
|
|
3211
|
+
const launchOpts = {
|
|
2984
3212
|
space: this.space,
|
|
2985
3213
|
name,
|
|
2986
3214
|
role,
|
|
@@ -3014,10 +3242,12 @@ export class Manager {
|
|
|
3014
3242
|
capabilities,
|
|
3015
3243
|
events,
|
|
3016
3244
|
mcpServers,
|
|
3245
|
+
envAllow,
|
|
3017
3246
|
// So a connector that keeps per-agent local state can root it at the workspace, not the
|
|
3018
3247
|
// (possibly per-agent) launch cwd below. The cwd itself rides runtime.spawn, not the launch.
|
|
3019
3248
|
workspaceRoot: this.workspaceRoot,
|
|
3020
|
-
}
|
|
3249
|
+
};
|
|
3250
|
+
const spec = connector.buildLaunch(launchOpts);
|
|
3021
3251
|
const handle = this.runtime.spawn(name, spec, cwd);
|
|
3022
3252
|
hooks?.onLaunched?.(); // P2 item 2: the "launched" progress edge (process spawned, pre-presence)
|
|
3023
3253
|
const managed = {
|
|
@@ -3063,6 +3293,9 @@ export class Manager {
|
|
|
3063
3293
|
? Object.keys(opts.launchOptions).sort()
|
|
3064
3294
|
: undefined,
|
|
3065
3295
|
},
|
|
3296
|
+
...(connector.supportsSessionContinuation
|
|
3297
|
+
? { restart: { opts: launchOpts, sessionStatePath: spec.sessionStatePath, crashes: [], recovering: false, armed: false } }
|
|
3298
|
+
: {}),
|
|
3066
3299
|
};
|
|
3067
3300
|
// Unit B: the DURABLE slot takes the `active` phase before the in-memory row takes the
|
|
3068
3301
|
// name — a crash between the two leaves an active-but-unadopted slot the boot sweep
|
|
@@ -3098,11 +3331,25 @@ export class Manager {
|
|
|
3098
3331
|
} // failed → already reaped
|
|
3099
3332
|
// Started OR uncertain: the agent stays managed, so wire the ongoing exit reaper (it reaps a later
|
|
3100
3333
|
// death — including one that follows an `uncertain` verdict, which deliberately does NOT deprovision).
|
|
3101
|
-
this.watchExit(managed);
|
|
3102
3334
|
if (!readiness.ok) {
|
|
3103
|
-
|
|
3335
|
+
this.watchExit(managed);
|
|
3336
|
+
await hooks?.onOutcome?.({ kind: "uncertain", data: { reason: readiness.detail } });
|
|
3104
3337
|
return { ok: false, error: readiness.detail };
|
|
3105
|
-
}
|
|
3338
|
+
}
|
|
3339
|
+
if (managed.restart) {
|
|
3340
|
+
try {
|
|
3341
|
+
await this.armSessionRecovery(managed);
|
|
3342
|
+
managed.launch.sessionId = this.readManagedSession(managed);
|
|
3343
|
+
}
|
|
3344
|
+
catch (error) {
|
|
3345
|
+
const detail = `${managed.name} joined, but its exact host session could not be bound for supervised recovery: ${error.message}`;
|
|
3346
|
+
this.stopHandle(managed, false);
|
|
3347
|
+
this.freeSlot(managed, true);
|
|
3348
|
+
await hooks?.onOutcome?.({ kind: "failed", data: { error: detail } });
|
|
3349
|
+
return { ok: false, error: detail };
|
|
3350
|
+
}
|
|
3351
|
+
}
|
|
3352
|
+
this.watchExit(managed);
|
|
3106
3353
|
// Reply with the id the slot actually carries (user-mode: the owner.actor principal —
|
|
3107
3354
|
// presence, ps, and the manifest ownership ledger all key on it; the throwaway static nkey
|
|
3108
3355
|
// would never match and down -f would treat the agent as foreign).
|
|
@@ -3418,6 +3665,17 @@ export class Manager {
|
|
|
3418
3665
|
return { ok: false, error: `${connector.name} harness needs ${missing.join(", ")} on PATH - not found` };
|
|
3419
3666
|
if (entry.launch.variant && !connector.supportsModelVariant)
|
|
3420
3667
|
return { ok: false, error: `${connector.name} connector does not support model variants (variant)` };
|
|
3668
|
+
let retainedSession;
|
|
3669
|
+
try {
|
|
3670
|
+
retainedSession = this.retainedSessionId(entry, connector);
|
|
3671
|
+
}
|
|
3672
|
+
catch (error) {
|
|
3673
|
+
return { ok: false, error: error.message };
|
|
3674
|
+
}
|
|
3675
|
+
if (entry.launch.forkSource && !retainedSession && !connector.supportsResume)
|
|
3676
|
+
return { ok: false, error: `${connector.name} connector does not support session fork (resume)` };
|
|
3677
|
+
if (retainedSession && !connector.supportsSessionContinuation)
|
|
3678
|
+
return { ok: false, error: `${connector.name} connector does not support exact-session continuation` };
|
|
3421
3679
|
let launchOptions;
|
|
3422
3680
|
if (entry.launch.source.kind === "manifest") {
|
|
3423
3681
|
const launchSource = entry.launch.source;
|
|
@@ -3453,8 +3711,10 @@ export class Manager {
|
|
|
3453
3711
|
return { ok: false, error: e.message };
|
|
3454
3712
|
}
|
|
3455
3713
|
try {
|
|
3456
|
-
const
|
|
3457
|
-
const
|
|
3714
|
+
const resumeConfig = loadCotalConfig(this.workspaceRoot);
|
|
3715
|
+
const mcpServers = connectorServers(resumeConfig, entry.launch.connector, parseShareSelection(entry.launch.shareTools));
|
|
3716
|
+
const envAllow = spawnEnvAllow(resumeConfig);
|
|
3717
|
+
const launchOpts = {
|
|
3458
3718
|
space: this.space,
|
|
3459
3719
|
name: entry.name,
|
|
3460
3720
|
role: entry.role,
|
|
@@ -3472,16 +3732,19 @@ export class Manager {
|
|
|
3472
3732
|
model: entry.launch.model,
|
|
3473
3733
|
variant: entry.launch.variant,
|
|
3474
3734
|
launchOptions,
|
|
3475
|
-
resume: entry.launch.forkSource,
|
|
3735
|
+
resume: retainedSession ? undefined : entry.launch.forkSource,
|
|
3736
|
+
continueSession: retainedSession,
|
|
3476
3737
|
subscribe: entry.launch.subscribe,
|
|
3477
3738
|
allowSubscribe: entry.launch.allowSubscribe,
|
|
3478
3739
|
allowPublish: entry.launch.allowPublish,
|
|
3479
3740
|
capabilities: entry.launch.capabilities,
|
|
3480
3741
|
events: entry.launch.events,
|
|
3481
3742
|
mcpServers,
|
|
3743
|
+
envAllow,
|
|
3482
3744
|
workspaceRoot: this.workspaceRoot,
|
|
3483
|
-
}
|
|
3484
|
-
const
|
|
3745
|
+
};
|
|
3746
|
+
const spec = connector.buildLaunch(launchOpts);
|
|
3747
|
+
const value = { spec, launchOpts, ...authority };
|
|
3485
3748
|
prepared?.set(entry.name, value);
|
|
3486
3749
|
if (preflightOnly)
|
|
3487
3750
|
return { ok: true, data: { name: entry.name, preflight: true } };
|
|
@@ -3544,7 +3807,11 @@ export class Manager {
|
|
|
3544
3807
|
events: entry.launch.events,
|
|
3545
3808
|
shareTools: entry.launch.shareTools,
|
|
3546
3809
|
forkSource: entry.launch.forkSource,
|
|
3810
|
+
sessionId: entry.launch.sessionId,
|
|
3547
3811
|
},
|
|
3812
|
+
...(prepared.spec.sessionStatePath
|
|
3813
|
+
? { restart: { opts: prepared.launchOpts, sessionStatePath: prepared.spec.sessionStatePath, crashes: [], recovering: false, armed: false } }
|
|
3814
|
+
: {}),
|
|
3548
3815
|
suppressCleanup: true,
|
|
3549
3816
|
};
|
|
3550
3817
|
this.agents.set(entry.name, managed);
|
|
@@ -3558,6 +3825,17 @@ export class Manager {
|
|
|
3558
3825
|
this.watchResumeAdoption(managed);
|
|
3559
3826
|
return { ok: false, error: readiness.detail };
|
|
3560
3827
|
}
|
|
3828
|
+
if (managed.restart) {
|
|
3829
|
+
try {
|
|
3830
|
+
await this.armSessionRecovery(managed);
|
|
3831
|
+
managed.launch.sessionId = this.readManagedSession(managed);
|
|
3832
|
+
}
|
|
3833
|
+
catch (error) {
|
|
3834
|
+
this.stopHandle(managed, false);
|
|
3835
|
+
this.freeSlot(managed, true, true);
|
|
3836
|
+
return { ok: false, error: `${managed.name} resumed, but its exact host session could not be rebound: ${error.message}` };
|
|
3837
|
+
}
|
|
3838
|
+
}
|
|
3561
3839
|
if (!this.resumeAttemptId)
|
|
3562
3840
|
managed.suppressCleanup = false;
|
|
3563
3841
|
this.watchExit(managed);
|
|
@@ -4615,7 +4893,10 @@ export class Manager {
|
|
|
4615
4893
|
({ fact } = await commitGoalResult(gw.ctx, { ref, now: Date.now(), cause: "complete", state: "failed", data: o.data, committer: { instanceId: this.managerInstanceId, epoch } }));
|
|
4616
4894
|
}
|
|
4617
4895
|
else {
|
|
4618
|
-
|
|
4896
|
+
// Forward the readiness detail as the terminal's reason: this manager owns the deadline,
|
|
4897
|
+
// so it owns what elapsing it MEANS. Absent, core commits its own generic line (#605).
|
|
4898
|
+
const why = o.data?.reason;
|
|
4899
|
+
({ fact } = await settleGoalUncertain(gw.ctx, { ref, now: Date.now(), committer: { instanceId: this.managerInstanceId, epoch }, ...(typeof why === "string" && why.length > 0 ? { reason: why } : {}) }));
|
|
4619
4900
|
}
|
|
4620
4901
|
this.emitGoalProgress(ref, epoch, { phase: "terminal", state: fact.state, ...(fact.data !== undefined ? { data: fact.data } : {}) });
|
|
4621
4902
|
await clearGoalIndex(gw.ctx, ref); // must-5 Q-B: terminal reached - the successor never reconciles it
|
|
@@ -5227,6 +5508,17 @@ export class Manager {
|
|
|
5227
5508
|
// The incarnation coordinate (SPEC 13.1) — with `id`, exactly what a v0.4 caller needs to
|
|
5228
5509
|
// build a targeted (`despawn`/`attach`) request against THIS incarnation.
|
|
5229
5510
|
lifecycleUid: a.lifecycleUid,
|
|
5511
|
+
// #651 enrichment: per-seat facts the manager ALREADY holds, carried on the row so `ps
|
|
5512
|
+
// --wide`/`--json` can surface them without a new collection path. All optional in the row
|
|
5513
|
+
// schema: a fact this backend did not record serializes absent, never fabricated (the
|
|
5514
|
+
// pid is absent on runtimes that do not own a real process; a launch may pin no model).
|
|
5515
|
+
model: a.launch.model,
|
|
5516
|
+
variant: a.launch.variant,
|
|
5517
|
+
cwd: a.launch.cwd,
|
|
5518
|
+
pid: a.handle.pid,
|
|
5519
|
+
spawner: a.spawner,
|
|
5520
|
+
instanceId: this.managerInstanceId,
|
|
5521
|
+
host: hostname(),
|
|
5230
5522
|
...(health && health.state !== "ok" ? { authHealth: health.state, authReason: health.reason } : {}),
|
|
5231
5523
|
};
|
|
5232
5524
|
});
|