@cotal-ai/manager 0.23.0 → 0.25.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/control-session.d.ts +11 -0
- package/dist/control-session.d.ts.map +1 -0
- package/dist/control-session.js +75 -0
- package/dist/control-session.js.map +1 -0
- package/dist/manager-service-contract.d.ts.map +1 -1
- package/dist/manager-service-contract.js +15 -0
- package/dist/manager-service-contract.js.map +1 -1
- package/dist/manager.d.ts +17 -3
- package/dist/manager.d.ts.map +1 -1
- package/dist/manager.js +286 -20
- package/dist/manager.js.map +1 -1
- package/dist/resume.d.ts.map +1 -1
- package/dist/resume.js +1 -0
- package/dist/resume.js.map +1 -1
- package/package.json +6 -6
package/dist/manager.js
CHANGED
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
import { execFile } from "node:child_process";
|
|
2
2
|
import { createHash, randomUUID, randomBytes } from "node:crypto";
|
|
3
|
+
import { hostname } from "node:os";
|
|
3
4
|
import { connect, credsAuthenticator } from "@nats-io/transport-node";
|
|
4
5
|
import { existsSync, lstatSync, readFileSync, rmSync } from "node:fs";
|
|
5
6
|
import { join, dirname, resolve } from "node:path";
|
|
6
|
-
import { CotalEndpoint, DEFAULT_SERVER, DEV_OWNER, MANAGER_LEASE_TTL_MS, MANAGER_LEASE_RENEW_MS, STANDING_RENEWABLE_TTL_SEC, agentFilePath, clearSpaceHistory, connectorServers, deprovisionAgent, firstFreeName, idFromCreds, inspectCredHealth, loadAgentFile, loadCotalConfig, mintCreds, mintLifecycleUid, mkSecretDir, newIdentity, actionContext, parsePrincipalKey, parseShareSelection, principalKey, probeConnect, provisionAgent, provisionAgentDurables, registry, resolveAuthProvider, saveAgentFile, subjectMatches, AUTH_ENDPOINT, EP_CMD_RETIRE_LIFECYCLE, epRequestSubject, epCallerReplyFilter, parseEpSubject, controlServiceSubject, eventChannelPrincipal, } from "@cotal-ai/core";
|
|
7
|
+
import { CotalEndpoint, DEFAULT_SERVER, DEV_OWNER, MANAGER_LEASE_TTL_MS, MANAGER_LEASE_RENEW_MS, STANDING_RENEWABLE_TTL_SEC, agentFilePath, clearSpaceHistory, connectorServers, spawnEnvAllow, deprovisionAgent, firstFreeName, idFromCreds, inspectCredHealth, loadAgentFile, loadCotalConfig, mintCreds, mintLifecycleUid, mkSecretDir, newIdentity, actionContext, parsePrincipalKey, parseShareSelection, principalKey, probeConnect, provisionAgent, provisionAgentDurables, registry, resolveAuthProvider, saveAgentFile, subjectMatches, AUTH_ENDPOINT, EP_CMD_RETIRE_LIFECYCLE, epRequestSubject, epCallerReplyFilter, parseEpSubject, controlServiceSubject, eventChannelPrincipal, } from "@cotal-ai/core";
|
|
7
8
|
import { agentAuthState, agentCredsDir, agentLifecycleSecretFilePaths, agentSecretFilePaths, agentSecretKeyForFile, authDir, connectorInstallHint, DEFAULT_CONNECTOR, defaultAgentType, DELIVERY_CREDS_KEY, findCotalRoot, getSpaceAuth, hasUserAuthState, loadManagerInstanceIdentity, loadMeshes, manifestExtensionNames, materializeFromManifest, materializeSecretToFile, MEMBERSHIP_RW_CREDS_KEY, mergeLaunchOptions, remintDaemonCreds, resolveOnPath, saveManagerInstanceIdentity, SYSTEM_CREDS_FILES, userAuthStateDir, workspaceSecretStore, writeRenewalRecord } from "@cotal-ai/workspace";
|
|
8
9
|
import { createRuntime, } from "./runtime/index.js";
|
|
9
10
|
import { AttachEndpoint } from "./attach-endpoint.js";
|
|
@@ -11,6 +12,7 @@ import { makeManagerEndpointEvictor } from "./endpoint-evict.js";
|
|
|
11
12
|
import { launchSpecForRun, materializePersona, launchAgentToStartOpts, parseLaunchSpec, persistLaunchSpec } from "./launch.js";
|
|
12
13
|
import { authorizeLaunch, authorizeNamedControl } from "./authorize.js";
|
|
13
14
|
import { controlShutdown } from "./control-shutdown.js";
|
|
15
|
+
import { controlSession } from "./control-session.js";
|
|
14
16
|
import { parseResumeCommitArgs, parseResumeControlArgs, parseResumeFinalizeArgs } from "./resume.js";
|
|
15
17
|
// Unit B (the static §13.1 lifecycle executor): the shared grammar/stores from core plus the
|
|
16
18
|
// manager-side adapter (transport + slot orchestration + the F1 terminal) — see static-lifecycle.ts.
|
|
@@ -43,6 +45,10 @@ const MIN_LIFETIME = 10_000;
|
|
|
43
45
|
* launch-parity smoke can assert every launch client's request timeout OUTLIVES this window — the tier
|
|
44
46
|
* rule forbids the clients importing it directly. */
|
|
45
47
|
export const READINESS_TIMEOUT_MS = 30_000;
|
|
48
|
+
/** Managed same-session crash recovery follows the Codex host precedent: three restarts are allowed
|
|
49
|
+
* inside a rolling two-minute window; the fourth crash is a loop and retires the seat loud. */
|
|
50
|
+
const SESSION_RESTART_LIMIT = 3;
|
|
51
|
+
const SESSION_RESTART_WINDOW_MS = 120_000;
|
|
46
52
|
/** Upper bound on a detached agent-exit deprovision (#159 B2). A wedged broker must not leave the
|
|
47
53
|
* fire-and-forget teardown pending forever with no log — past this it rejects into freeSlot's fail-loud
|
|
48
54
|
* `.catch`. Generous over the helper's 5s connect timeout to allow the two consumer-deletes + ACL purge
|
|
@@ -1118,6 +1124,7 @@ export class Manager {
|
|
|
1118
1124
|
events: a.launch.events,
|
|
1119
1125
|
shareTools: a.launch.shareTools,
|
|
1120
1126
|
forkSource: a.launch.forkSource,
|
|
1127
|
+
sessionId: a.restart?.armed ? this.readManagedSession(a) : a.launch.sessionId,
|
|
1121
1128
|
unresolvedLaunchOptionKeys: a.launch.unresolvedLaunchOptionKeys,
|
|
1122
1129
|
},
|
|
1123
1130
|
dependencies,
|
|
@@ -2032,6 +2039,8 @@ export class Manager {
|
|
|
2032
2039
|
return; // already freed (exit raced despawn, etc.)
|
|
2033
2040
|
a.terminalizing = true; // F5 latch (Unit B): also covers exit/reap paths that never rode stopHandle
|
|
2034
2041
|
this.agents.delete(a.name);
|
|
2042
|
+
if (a.restart?.sessionStatePath)
|
|
2043
|
+
rmSync(a.restart.sessionStatePath, { force: true });
|
|
2035
2044
|
// P2 item 6 (pin 4): end any live §13.6 attach session bound to THIS incarnation with the honest
|
|
2036
2045
|
// `target-despawn` reason. Fires once per agent on every free path (despawn / self-stop / reap /
|
|
2037
2046
|
// exit) via the `agents` guard above; a no-op when no plane or no live session for the target.
|
|
@@ -2204,11 +2213,23 @@ export class Manager {
|
|
|
2204
2213
|
if (!this.auth)
|
|
2205
2214
|
return; // guaranteed by requestRetirement; re-checked for the type narrowing below
|
|
2206
2215
|
const held = this.retiring.get(a.name);
|
|
2207
|
-
const me = parsePrincipalKey(this.ep.ref().id);
|
|
2208
2216
|
const target = parsePrincipalKey(a.id);
|
|
2209
|
-
if (!
|
|
2217
|
+
if (!target) {
|
|
2210
2218
|
if (held)
|
|
2211
|
-
held.lastError = "the
|
|
2219
|
+
held.lastError = "the target principal could not be derived; the retirement was not requested";
|
|
2220
|
+
return;
|
|
2221
|
+
}
|
|
2222
|
+
// The SERVE identity, not the endpoint identity, is who this request speaks as (#549). The field
|
|
2223
|
+
// is declared `!:`, which asserts to the type system what the ordering happens to provide, so it
|
|
2224
|
+
// is read here as what it actually is. On today's ordering this guard cannot fire: `start()`
|
|
2225
|
+
// assigns the identity before it connects anything, and a retirement needs an agent that only a
|
|
2226
|
+
// started manager can hold. It is kept as a fail-closed assertion rather than a live face,
|
|
2227
|
+
// because the alternative is carrying `undefined` into the caller triple and surfacing the
|
|
2228
|
+
// result as "the rail could not be reached", which would name the wrong cause.
|
|
2229
|
+
const serveIdentity = this.managerServeIdentity;
|
|
2230
|
+
if (!serveIdentity?.id) {
|
|
2231
|
+
if (held)
|
|
2232
|
+
held.lastError = `the retirement was NOT requested: this manager has no serve identity yet, so it cannot speak as the registered serving instance. The despawn stopped "${a.name}" and the name stays held. NEXT: let the manager finish registering, then re-attempt the same-name spawn to re-drive the teardown.`;
|
|
2212
2233
|
return;
|
|
2213
2234
|
}
|
|
2214
2235
|
const uncertain = (why) => `the despawn stopped "${a.name}", but the retirement's completion could NOT be confirmed (${why}). The name stays held - not failed, not done - and a same-name spawn re-drives the same teardown; the auth service also finishes any started retirement on its next boot. NEXT: if the auth rail stays unreachable, recover the stack (\`cotal supervise\`), then re-attempt the same-name spawn.`;
|
|
@@ -2216,7 +2237,22 @@ export class Manager {
|
|
|
2216
2237
|
// The caller triple and the TARGET are both grant-pinned now (#350): the `handle` target
|
|
2217
2238
|
// rides the subject, so this ephemeral credential can ask to retire exactly this
|
|
2218
2239
|
// incarnation and nothing else.
|
|
2219
|
-
|
|
2240
|
+
//
|
|
2241
|
+
// THE TRIPLE IS THE SERVE PRINCIPAL, and it has to be (#549). The auth rail authorizes this
|
|
2242
|
+
// request by comparing the caller's `<owner>.<actor>` against the serve issuance gate's bound
|
|
2243
|
+
// principal, and that gate is opened with `principalKey(DEV_OWNER, serveIdentity.id)` (see the
|
|
2244
|
+
// registration block). Deriving the caller from `ep.ref().id` instead put the manager's
|
|
2245
|
+
// ENDPOINT identity nkey on the wire, which is a different, equally real identity of the same
|
|
2246
|
+
// manager, so the comparison was unsatisfiable and EVERY user-mesh retirement was refused as a
|
|
2247
|
+
// full no-op. Measured before the fix: 8 refusals in one suite run across 5 agents, with the
|
|
2248
|
+
// epoch and the instance id both matching and only the principal disagreeing.
|
|
2249
|
+
//
|
|
2250
|
+
// Both halves come from the gate's own sources rather than from `ep.ref()`: `DEV_OWNER` is
|
|
2251
|
+
// hard-coded at the gate site, so taking the owner from `ep.ref().id` would re-open the same
|
|
2252
|
+
// mismatch in the owner half the moment a manager ran under a user-shaped identity. This is
|
|
2253
|
+
// also the more honest attribution: the authority being exercised is "I am the registered
|
|
2254
|
+
// serving instance", which is exactly what the gate records.
|
|
2255
|
+
const caller = { owner: DEV_OWNER, actor: serveIdentity.id, uid: this.managerLifecycleUid };
|
|
2220
2256
|
const creds = await mintCreds(this.auth, newIdentity(), "retirement-requester", {
|
|
2221
2257
|
retirementRequester: { ...caller, target: { owner: target.owner, actor: target.actor, lifecycleUid: a.lifecycleUid } },
|
|
2222
2258
|
});
|
|
@@ -2325,14 +2361,172 @@ export class Manager {
|
|
|
2325
2361
|
this.trackStoppedHandle(child, true, true);
|
|
2326
2362
|
}
|
|
2327
2363
|
}
|
|
2328
|
-
|
|
2329
|
-
|
|
2330
|
-
|
|
2364
|
+
readManagedSessionState(a) {
|
|
2365
|
+
const path = a.restart?.sessionStatePath;
|
|
2366
|
+
if (!path)
|
|
2367
|
+
throw new Error("connector supplied no session state path");
|
|
2368
|
+
let parsed;
|
|
2369
|
+
try {
|
|
2370
|
+
parsed = JSON.parse(readFileSync(path, "utf8"));
|
|
2371
|
+
}
|
|
2372
|
+
catch (error) {
|
|
2373
|
+
throw new Error(`cannot read connector session state ${path}: ${error.message}`);
|
|
2374
|
+
}
|
|
2375
|
+
const state = parsed;
|
|
2376
|
+
if (state.version !== 1 || typeof state.sessionId !== "string" || !state.sessionId.trim() || state.sessionId.length > 4096 ||
|
|
2377
|
+
(state.status !== "running" && state.status !== "quit"))
|
|
2378
|
+
throw new Error(`connector session state ${path} is malformed`);
|
|
2379
|
+
return { sessionId: state.sessionId, status: state.status };
|
|
2380
|
+
}
|
|
2381
|
+
readManagedSession(a) {
|
|
2382
|
+
return this.readManagedSessionState(a).sessionId;
|
|
2383
|
+
}
|
|
2384
|
+
async awaitManagedSessionState(a) {
|
|
2385
|
+
const deadline = Date.now() + 15_000;
|
|
2386
|
+
let last = "session state not written yet";
|
|
2387
|
+
while (Date.now() < deadline) {
|
|
2388
|
+
if (a.handle.status() === "exited")
|
|
2389
|
+
throw new Error("process exited before writing session state");
|
|
2390
|
+
try {
|
|
2391
|
+
return this.readManagedSessionState(a);
|
|
2392
|
+
}
|
|
2393
|
+
catch (error) {
|
|
2394
|
+
last = error.message;
|
|
2395
|
+
}
|
|
2396
|
+
await new Promise((resolve) => setTimeout(resolve, 100));
|
|
2397
|
+
}
|
|
2398
|
+
throw new Error(`connector did not write session state (${last})`);
|
|
2399
|
+
}
|
|
2400
|
+
/** Bind a continuation-capable connector only after its process is ready. The file proves the
|
|
2401
|
+
* latest in-process session (including Pi /resume); the authenticated socket proves this process
|
|
2402
|
+
* owns that session now. Presence may lead session_start by milliseconds, so both proofs are
|
|
2403
|
+
* awaited within a bounded readiness window rather than read once. */
|
|
2404
|
+
async armSessionRecovery(a) {
|
|
2405
|
+
if (!a.restart || !a.control)
|
|
2406
|
+
return;
|
|
2407
|
+
const state = await this.awaitManagedSessionState(a);
|
|
2408
|
+
if (state.status !== "running")
|
|
2409
|
+
throw new Error("connector reported a deliberate quit before readiness completed");
|
|
2410
|
+
await this.awaitRecoveredSession(a, state.sessionId);
|
|
2411
|
+
a.restart.armed = true;
|
|
2412
|
+
}
|
|
2413
|
+
async awaitRecoveredSession(a, expected, handle = a.handle, control = a.control) {
|
|
2414
|
+
const deadline = Date.now() + 15_000;
|
|
2415
|
+
let last = "control endpoint not ready";
|
|
2416
|
+
while (Date.now() < deadline) {
|
|
2417
|
+
if (handle.status() === "exited")
|
|
2418
|
+
throw new Error("replacement process exited before reporting its session");
|
|
2419
|
+
if (control) {
|
|
2420
|
+
try {
|
|
2421
|
+
const reported = await controlSession(control);
|
|
2422
|
+
if (reported !== expected)
|
|
2423
|
+
throw new Error(`replacement reported session ${reported}, expected ${expected}`);
|
|
2424
|
+
return;
|
|
2425
|
+
}
|
|
2426
|
+
catch (error) {
|
|
2427
|
+
last = error.message;
|
|
2428
|
+
}
|
|
2429
|
+
}
|
|
2430
|
+
await new Promise((resolve) => setTimeout(resolve, 100));
|
|
2431
|
+
}
|
|
2432
|
+
throw new Error(`replacement did not prove session ${expected} (${last})`);
|
|
2433
|
+
}
|
|
2434
|
+
/** Restart one continuation-capable managed process in place. Identity, lifecycle, credentials,
|
|
2435
|
+
* durables, children, and the manager row remain owned; only the process handle/control endpoint
|
|
2436
|
+
* change. A fourth crash inside two minutes is a loop and falls through to normal retirement. */
|
|
2437
|
+
recoverManagedSession(a) {
|
|
2438
|
+
const restart = a.restart;
|
|
2439
|
+
if (!restart || !restart.armed || restart.recovering || a.terminalizing)
|
|
2440
|
+
return;
|
|
2441
|
+
const release = this.beginLifecycle();
|
|
2442
|
+
if (!release)
|
|
2443
|
+
return; // preservation owns the cut once the lifecycle fence closes
|
|
2444
|
+
const now = Date.now();
|
|
2445
|
+
restart.crashes = restart.crashes.filter((at) => now - at < SESSION_RESTART_WINDOW_MS);
|
|
2446
|
+
restart.crashes.push(now);
|
|
2447
|
+
if (restart.crashes.length > SESSION_RESTART_LIMIT) {
|
|
2448
|
+
console.error(`! ${a.name}: Pi crash loop (${restart.crashes.length} crashes in ${SESSION_RESTART_WINDOW_MS / 1000}s) - retiring the managed seat`);
|
|
2449
|
+
restart.armed = false;
|
|
2450
|
+
this.freeSlot(a, true);
|
|
2451
|
+
this.reapChildrenOf(this.managedPrincipal(a));
|
|
2452
|
+
release();
|
|
2453
|
+
return;
|
|
2454
|
+
}
|
|
2455
|
+
restart.recovering = true;
|
|
2456
|
+
void (async () => {
|
|
2457
|
+
let replacement;
|
|
2458
|
+
try {
|
|
2459
|
+
const sessionId = this.readManagedSession(a);
|
|
2460
|
+
const connector = await this.resolveConnector(a.agent);
|
|
2461
|
+
if (!connector.supportsSessionContinuation)
|
|
2462
|
+
throw new Error(`connector ${connector.name} no longer declares same-session continuation`);
|
|
2463
|
+
const opts = {
|
|
2464
|
+
...restart.opts,
|
|
2465
|
+
resume: undefined,
|
|
2466
|
+
prompt: undefined,
|
|
2467
|
+
continueSession: sessionId,
|
|
2468
|
+
};
|
|
2469
|
+
const spec = connector.buildLaunch(opts);
|
|
2470
|
+
const handle = this.runtime.spawn(a.name, spec, a.launch.cwd);
|
|
2471
|
+
replacement = handle;
|
|
2472
|
+
restart.sessionStatePath = spec.sessionStatePath ?? restart.sessionStatePath;
|
|
2473
|
+
await this.awaitRecoveredSession(a, sessionId, handle, spec.control);
|
|
2474
|
+
if (this.agents.get(a.name) !== a || a.terminalizing) {
|
|
2475
|
+
try {
|
|
2476
|
+
handle.stop({ graceful: false });
|
|
2477
|
+
}
|
|
2478
|
+
catch { /* terminal path owns cleanup */ }
|
|
2479
|
+
return;
|
|
2480
|
+
}
|
|
2481
|
+
a.handle = handle;
|
|
2482
|
+
a.control = spec.control;
|
|
2483
|
+
replacement = undefined;
|
|
2484
|
+
restart.opts = opts;
|
|
2485
|
+
restart.recovering = false;
|
|
2486
|
+
console.error(`! ${a.name}: recovered Pi session ${sessionId} after crash (${restart.crashes.length}/${SESSION_RESTART_LIMIT})`);
|
|
2487
|
+
this.watchExit(a);
|
|
2488
|
+
}
|
|
2489
|
+
catch (error) {
|
|
2490
|
+
restart.recovering = false;
|
|
2491
|
+
restart.armed = false;
|
|
2492
|
+
let tail = "";
|
|
2493
|
+
try {
|
|
2494
|
+
tail = this.tail(await (replacement ?? a.handle).attach().backlog());
|
|
2495
|
+
}
|
|
2496
|
+
catch { /* runtime has no readable tail */ }
|
|
2497
|
+
console.error(`! ${a.name}: Pi session recovery failed: ${error.message}${tail ? ` - last output: ${tail}` : ""} - retiring the managed seat`);
|
|
2498
|
+
// The replacement may be alive but unable to prove the expected session. Stop it BEFORE
|
|
2499
|
+
// retiring credentials/durables; otherwise an untracked process survives under torn auth.
|
|
2500
|
+
try {
|
|
2501
|
+
replacement?.stop({ graceful: false });
|
|
2502
|
+
}
|
|
2503
|
+
catch { /* terminal cleanup continues */ }
|
|
2504
|
+
this.freeSlot(a, true);
|
|
2505
|
+
this.reapChildrenOf(this.managedPrincipal(a));
|
|
2506
|
+
}
|
|
2507
|
+
finally {
|
|
2508
|
+
release();
|
|
2509
|
+
}
|
|
2510
|
+
})();
|
|
2511
|
+
}
|
|
2512
|
+
/** A managed agent's process exited on its own (crash, /exit, finished). Continuation-capable Pi
|
|
2513
|
+
* seats restart in place after readiness; every other exit follows the existing terminal path. */
|
|
2331
2514
|
onAgentExit(a) {
|
|
2332
2515
|
// Preservation owns the child-stop snapshot. Exit watchers must neither delete that snapshot nor
|
|
2333
2516
|
// trigger normal deprovision/reap while the cut is being formed.
|
|
2334
2517
|
if (this.maintenanceState !== "active")
|
|
2335
2518
|
return;
|
|
2519
|
+
if (a.restart?.armed && !a.terminalizing) {
|
|
2520
|
+
try {
|
|
2521
|
+
if (this.readManagedSessionState(a).status === "running") {
|
|
2522
|
+
this.recoverManagedSession(a);
|
|
2523
|
+
return;
|
|
2524
|
+
}
|
|
2525
|
+
}
|
|
2526
|
+
catch (error) {
|
|
2527
|
+
console.error(`! ${a.name}: cannot classify Pi process exit for recovery: ${error.message} - retiring the seat`);
|
|
2528
|
+
}
|
|
2529
|
+
}
|
|
2336
2530
|
this.freeSlot(a, true);
|
|
2337
2531
|
this.reapChildrenOf(this.managedPrincipal(a));
|
|
2338
2532
|
}
|
|
@@ -2745,9 +2939,20 @@ export class Manager {
|
|
|
2745
2939
|
allowSubscribe = opts.allowSubscribe ?? def.allowSubscribe ?? subscribe ?? ["general"];
|
|
2746
2940
|
allowPublish = opts.allowPublish ?? def.allowPublish;
|
|
2747
2941
|
capabilities = def.capabilities;
|
|
2942
|
+
// #651: fold the persona's model into the launch record, mirroring the variant line below
|
|
2943
|
+
// and the manifest branch above. Without this, a persona-file model (the common pin source)
|
|
2944
|
+
// never reaches `launch.model`, so the connector runs the seat on it while `ps --wide`/`--json`
|
|
2945
|
+
// reports the model ABSENT - a false "no model pinned" for a seat that has one.
|
|
2946
|
+
model = opts.model ?? def.model;
|
|
2748
2947
|
variant = opts.variant ?? def.variant;
|
|
2749
2948
|
launchOptions = mergeLaunchOptions(def.launchOptions, opts.launchOptions);
|
|
2750
2949
|
}
|
|
2950
|
+
// #651: an empty or whitespace-only model string is not a pin. Coerce it to undefined here, at
|
|
2951
|
+
// the single point every path (persona, manifest, imperative) has resolved `model`, so it
|
|
2952
|
+
// serializes ABSENT rather than present-but-empty (`"model": ""`), which a key-presence consumer
|
|
2953
|
+
// would misread as "a pin was recorded".
|
|
2954
|
+
if (model !== undefined && model.trim() === "")
|
|
2955
|
+
model = undefined;
|
|
2751
2956
|
const idErr = this.nameError(identityName);
|
|
2752
2957
|
if (idErr)
|
|
2753
2958
|
return { ok: false, error: opts.resolved ? `launch agent: ${idErr}` : `persona ${configPath}: ${idErr}` };
|
|
@@ -2970,7 +3175,11 @@ export class Manager {
|
|
|
2970
3175
|
// Personal MCP servers the operator opted to share with manager-spawned agents of this type
|
|
2971
3176
|
// (cotal config; default none → isolated, the memory-safe default this guards), narrowed by
|
|
2972
3177
|
// an optional --share-tools selection (absent → all declared, the pre-merge behavior).
|
|
2973
|
-
const
|
|
3178
|
+
const cotalConfig = loadCotalConfig(this.workspaceRoot);
|
|
3179
|
+
const mcpServers = connectorServers(cotalConfig, agent, parseShareSelection(opts.shareTools));
|
|
3180
|
+
// The operator's spawn-env policy travels the same route: absent means the child inherits
|
|
3181
|
+
// their environment, present means containment. A connector never reads the config itself.
|
|
3182
|
+
const envAllow = spawnEnvAllow(cotalConfig);
|
|
2974
3183
|
// Per-agent cwd overrides the manager's shared workspace root, so agents can be rooted at
|
|
2975
3184
|
// arbitrary folders/repos. A relative path resolves against the workspace root; omitted → the
|
|
2976
3185
|
// agent shares the workspace root (the prior, unchanged behavior).
|
|
@@ -2980,7 +3189,7 @@ export class Manager {
|
|
|
2980
3189
|
? join(this.workspaceRoot, ".cotal", "run", `${opts.launchRef.runId}.json`)
|
|
2981
3190
|
: undefined;
|
|
2982
3191
|
const manifestSha256 = manifestPath ? this.fileDigest(manifestPath) : undefined;
|
|
2983
|
-
const
|
|
3192
|
+
const launchOpts = {
|
|
2984
3193
|
space: this.space,
|
|
2985
3194
|
name,
|
|
2986
3195
|
role,
|
|
@@ -3014,10 +3223,12 @@ export class Manager {
|
|
|
3014
3223
|
capabilities,
|
|
3015
3224
|
events,
|
|
3016
3225
|
mcpServers,
|
|
3226
|
+
envAllow,
|
|
3017
3227
|
// So a connector that keeps per-agent local state can root it at the workspace, not the
|
|
3018
3228
|
// (possibly per-agent) launch cwd below. The cwd itself rides runtime.spawn, not the launch.
|
|
3019
3229
|
workspaceRoot: this.workspaceRoot,
|
|
3020
|
-
}
|
|
3230
|
+
};
|
|
3231
|
+
const spec = connector.buildLaunch(launchOpts);
|
|
3021
3232
|
const handle = this.runtime.spawn(name, spec, cwd);
|
|
3022
3233
|
hooks?.onLaunched?.(); // P2 item 2: the "launched" progress edge (process spawned, pre-presence)
|
|
3023
3234
|
const managed = {
|
|
@@ -3063,6 +3274,9 @@ export class Manager {
|
|
|
3063
3274
|
? Object.keys(opts.launchOptions).sort()
|
|
3064
3275
|
: undefined,
|
|
3065
3276
|
},
|
|
3277
|
+
...(connector.supportsSessionContinuation
|
|
3278
|
+
? { restart: { opts: launchOpts, sessionStatePath: spec.sessionStatePath, crashes: [], recovering: false, armed: false } }
|
|
3279
|
+
: {}),
|
|
3066
3280
|
};
|
|
3067
3281
|
// Unit B: the DURABLE slot takes the `active` phase before the in-memory row takes the
|
|
3068
3282
|
// name — a crash between the two leaves an active-but-unadopted slot the boot sweep
|
|
@@ -3098,11 +3312,25 @@ export class Manager {
|
|
|
3098
3312
|
} // failed → already reaped
|
|
3099
3313
|
// Started OR uncertain: the agent stays managed, so wire the ongoing exit reaper (it reaps a later
|
|
3100
3314
|
// death — including one that follows an `uncertain` verdict, which deliberately does NOT deprovision).
|
|
3101
|
-
this.watchExit(managed);
|
|
3102
3315
|
if (!readiness.ok) {
|
|
3103
|
-
|
|
3316
|
+
this.watchExit(managed);
|
|
3317
|
+
await hooks?.onOutcome?.({ kind: "uncertain", data: { reason: readiness.detail } });
|
|
3104
3318
|
return { ok: false, error: readiness.detail };
|
|
3105
|
-
}
|
|
3319
|
+
}
|
|
3320
|
+
if (managed.restart) {
|
|
3321
|
+
try {
|
|
3322
|
+
await this.armSessionRecovery(managed);
|
|
3323
|
+
managed.launch.sessionId = this.readManagedSession(managed);
|
|
3324
|
+
}
|
|
3325
|
+
catch (error) {
|
|
3326
|
+
const detail = `${managed.name} joined, but its exact host session could not be bound for supervised recovery: ${error.message}`;
|
|
3327
|
+
this.stopHandle(managed, false);
|
|
3328
|
+
this.freeSlot(managed, true);
|
|
3329
|
+
await hooks?.onOutcome?.({ kind: "failed", data: { error: detail } });
|
|
3330
|
+
return { ok: false, error: detail };
|
|
3331
|
+
}
|
|
3332
|
+
}
|
|
3333
|
+
this.watchExit(managed);
|
|
3106
3334
|
// Reply with the id the slot actually carries (user-mode: the owner.actor principal —
|
|
3107
3335
|
// presence, ps, and the manifest ownership ledger all key on it; the throwaway static nkey
|
|
3108
3336
|
// would never match and down -f would treat the agent as foreign).
|
|
@@ -3418,6 +3646,10 @@ export class Manager {
|
|
|
3418
3646
|
return { ok: false, error: `${connector.name} harness needs ${missing.join(", ")} on PATH - not found` };
|
|
3419
3647
|
if (entry.launch.variant && !connector.supportsModelVariant)
|
|
3420
3648
|
return { ok: false, error: `${connector.name} connector does not support model variants (variant)` };
|
|
3649
|
+
if (entry.launch.forkSource && !entry.launch.sessionId && !connector.supportsResume)
|
|
3650
|
+
return { ok: false, error: `${connector.name} connector does not support session fork (resume)` };
|
|
3651
|
+
if (entry.launch.sessionId && !connector.supportsSessionContinuation)
|
|
3652
|
+
return { ok: false, error: `${connector.name} connector does not support exact-session continuation` };
|
|
3421
3653
|
let launchOptions;
|
|
3422
3654
|
if (entry.launch.source.kind === "manifest") {
|
|
3423
3655
|
const launchSource = entry.launch.source;
|
|
@@ -3453,8 +3685,10 @@ export class Manager {
|
|
|
3453
3685
|
return { ok: false, error: e.message };
|
|
3454
3686
|
}
|
|
3455
3687
|
try {
|
|
3456
|
-
const
|
|
3457
|
-
const
|
|
3688
|
+
const resumeConfig = loadCotalConfig(this.workspaceRoot);
|
|
3689
|
+
const mcpServers = connectorServers(resumeConfig, entry.launch.connector, parseShareSelection(entry.launch.shareTools));
|
|
3690
|
+
const envAllow = spawnEnvAllow(resumeConfig);
|
|
3691
|
+
const launchOpts = {
|
|
3458
3692
|
space: this.space,
|
|
3459
3693
|
name: entry.name,
|
|
3460
3694
|
role: entry.role,
|
|
@@ -3472,16 +3706,19 @@ export class Manager {
|
|
|
3472
3706
|
model: entry.launch.model,
|
|
3473
3707
|
variant: entry.launch.variant,
|
|
3474
3708
|
launchOptions,
|
|
3475
|
-
resume: entry.launch.forkSource,
|
|
3709
|
+
resume: entry.launch.sessionId ? undefined : entry.launch.forkSource,
|
|
3710
|
+
continueSession: entry.launch.sessionId,
|
|
3476
3711
|
subscribe: entry.launch.subscribe,
|
|
3477
3712
|
allowSubscribe: entry.launch.allowSubscribe,
|
|
3478
3713
|
allowPublish: entry.launch.allowPublish,
|
|
3479
3714
|
capabilities: entry.launch.capabilities,
|
|
3480
3715
|
events: entry.launch.events,
|
|
3481
3716
|
mcpServers,
|
|
3717
|
+
envAllow,
|
|
3482
3718
|
workspaceRoot: this.workspaceRoot,
|
|
3483
|
-
}
|
|
3484
|
-
const
|
|
3719
|
+
};
|
|
3720
|
+
const spec = connector.buildLaunch(launchOpts);
|
|
3721
|
+
const value = { spec, launchOpts, ...authority };
|
|
3485
3722
|
prepared?.set(entry.name, value);
|
|
3486
3723
|
if (preflightOnly)
|
|
3487
3724
|
return { ok: true, data: { name: entry.name, preflight: true } };
|
|
@@ -3544,7 +3781,11 @@ export class Manager {
|
|
|
3544
3781
|
events: entry.launch.events,
|
|
3545
3782
|
shareTools: entry.launch.shareTools,
|
|
3546
3783
|
forkSource: entry.launch.forkSource,
|
|
3784
|
+
sessionId: entry.launch.sessionId,
|
|
3547
3785
|
},
|
|
3786
|
+
...(prepared.spec.sessionStatePath
|
|
3787
|
+
? { restart: { opts: prepared.launchOpts, sessionStatePath: prepared.spec.sessionStatePath, crashes: [], recovering: false, armed: false } }
|
|
3788
|
+
: {}),
|
|
3548
3789
|
suppressCleanup: true,
|
|
3549
3790
|
};
|
|
3550
3791
|
this.agents.set(entry.name, managed);
|
|
@@ -3558,6 +3799,17 @@ export class Manager {
|
|
|
3558
3799
|
this.watchResumeAdoption(managed);
|
|
3559
3800
|
return { ok: false, error: readiness.detail };
|
|
3560
3801
|
}
|
|
3802
|
+
if (managed.restart) {
|
|
3803
|
+
try {
|
|
3804
|
+
await this.armSessionRecovery(managed);
|
|
3805
|
+
managed.launch.sessionId = this.readManagedSession(managed);
|
|
3806
|
+
}
|
|
3807
|
+
catch (error) {
|
|
3808
|
+
this.stopHandle(managed, false);
|
|
3809
|
+
this.freeSlot(managed, true, true);
|
|
3810
|
+
return { ok: false, error: `${managed.name} resumed, but its exact host session could not be rebound: ${error.message}` };
|
|
3811
|
+
}
|
|
3812
|
+
}
|
|
3561
3813
|
if (!this.resumeAttemptId)
|
|
3562
3814
|
managed.suppressCleanup = false;
|
|
3563
3815
|
this.watchExit(managed);
|
|
@@ -4615,7 +4867,10 @@ export class Manager {
|
|
|
4615
4867
|
({ fact } = await commitGoalResult(gw.ctx, { ref, now: Date.now(), cause: "complete", state: "failed", data: o.data, committer: { instanceId: this.managerInstanceId, epoch } }));
|
|
4616
4868
|
}
|
|
4617
4869
|
else {
|
|
4618
|
-
|
|
4870
|
+
// Forward the readiness detail as the terminal's reason: this manager owns the deadline,
|
|
4871
|
+
// so it owns what elapsing it MEANS. Absent, core commits its own generic line (#605).
|
|
4872
|
+
const why = o.data?.reason;
|
|
4873
|
+
({ fact } = await settleGoalUncertain(gw.ctx, { ref, now: Date.now(), committer: { instanceId: this.managerInstanceId, epoch }, ...(typeof why === "string" && why.length > 0 ? { reason: why } : {}) }));
|
|
4619
4874
|
}
|
|
4620
4875
|
this.emitGoalProgress(ref, epoch, { phase: "terminal", state: fact.state, ...(fact.data !== undefined ? { data: fact.data } : {}) });
|
|
4621
4876
|
await clearGoalIndex(gw.ctx, ref); // must-5 Q-B: terminal reached - the successor never reconciles it
|
|
@@ -5227,6 +5482,17 @@ export class Manager {
|
|
|
5227
5482
|
// The incarnation coordinate (SPEC 13.1) — with `id`, exactly what a v0.4 caller needs to
|
|
5228
5483
|
// build a targeted (`despawn`/`attach`) request against THIS incarnation.
|
|
5229
5484
|
lifecycleUid: a.lifecycleUid,
|
|
5485
|
+
// #651 enrichment: per-seat facts the manager ALREADY holds, carried on the row so `ps
|
|
5486
|
+
// --wide`/`--json` can surface them without a new collection path. All optional in the row
|
|
5487
|
+
// schema: a fact this backend did not record serializes absent, never fabricated (the
|
|
5488
|
+
// pid is absent on runtimes that do not own a real process; a launch may pin no model).
|
|
5489
|
+
model: a.launch.model,
|
|
5490
|
+
variant: a.launch.variant,
|
|
5491
|
+
cwd: a.launch.cwd,
|
|
5492
|
+
pid: a.handle.pid,
|
|
5493
|
+
spawner: a.spawner,
|
|
5494
|
+
instanceId: this.managerInstanceId,
|
|
5495
|
+
host: hostname(),
|
|
5230
5496
|
...(health && health.state !== "ok" ? { authHealth: health.state, authReason: health.reason } : {}),
|
|
5231
5497
|
};
|
|
5232
5498
|
});
|