@cotal-ai/manager 0.29.2 → 0.30.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands.d.ts +19 -0
- package/dist/commands.d.ts.map +1 -1
- package/dist/commands.js +154 -6
- package/dist/commands.js.map +1 -1
- package/dist/manager-service-contract.d.ts +6 -0
- package/dist/manager-service-contract.d.ts.map +1 -1
- package/dist/manager-service-contract.js +5 -0
- package/dist/manager-service-contract.js.map +1 -1
- package/dist/manager.d.ts +78 -1
- package/dist/manager.d.ts.map +1 -1
- package/dist/manager.js +214 -50
- package/dist/manager.js.map +1 -1
- package/dist/remote-authority.d.ts +22 -0
- package/dist/remote-authority.d.ts.map +1 -0
- package/dist/remote-authority.js +102 -0
- package/dist/remote-authority.js.map +1 -0
- package/dist/remote-register.d.ts +19 -0
- package/dist/remote-register.d.ts.map +1 -0
- package/dist/remote-register.js +73 -0
- package/dist/remote-register.js.map +1 -0
- package/dist/runtime/pty.d.ts.map +1 -1
- package/dist/runtime/pty.js +10 -1
- package/dist/runtime/pty.js.map +1 -1
- package/dist/session/bridge.d.ts +11 -6
- package/dist/session/bridge.d.ts.map +1 -1
- package/dist/session/bridge.js +192 -41
- package/dist/session/bridge.js.map +1 -1
- package/package.json +4 -4
package/dist/manager.js
CHANGED
|
@@ -101,6 +101,16 @@ function withTimeout(p, ms, msg) {
|
|
|
101
101
|
function sameStrings(a, b) {
|
|
102
102
|
return JSON.stringify([...(a ?? [])].sort()) === JSON.stringify([...(b ?? [])].sort());
|
|
103
103
|
}
|
|
104
|
+
/** Operator-facing phrasing per cause. Kept beside the union so adding a member without a sentence
|
|
105
|
+
* is a type error rather than a blank in the log. */
|
|
106
|
+
const FREE_SLOT_CAUSE_TEXT = {
|
|
107
|
+
"stopped": "this manager stopped it (despawn or shutdown)",
|
|
108
|
+
"process-exit": "its own process exited and this manager did not stop it",
|
|
109
|
+
"pi-crash-loop": "this manager retired it after a Pi crash loop",
|
|
110
|
+
"pi-recovery-failed": "this manager retired it after Pi session recovery failed",
|
|
111
|
+
"session-bind-failed": "this manager stopped it: its host session could not be bound at launch",
|
|
112
|
+
"resume-session-rebind-failed": "this manager stopped it: its host session could not be rebound on resume",
|
|
113
|
+
};
|
|
104
114
|
/** One ep request/reply round-trip on the caller's OWN reply-plane filter (§13.2). The responder
|
|
105
115
|
* derives the reply subject from the authenticated request, so there is no caller-selected reply
|
|
106
116
|
* target to honour; the caller binds the answer off the reply SUBJECT — endpoint and nonce, both
|
|
@@ -367,12 +377,16 @@ export class Manager {
|
|
|
367
377
|
resumeFinalized = false;
|
|
368
378
|
resumeDurableCommitToken;
|
|
369
379
|
resumedAgentNames = new Set();
|
|
380
|
+
remoteAuthority;
|
|
370
381
|
constructor(opts) {
|
|
371
382
|
this.space = opts.space;
|
|
372
383
|
this.servers = opts.servers;
|
|
373
384
|
this.name = opts.name ?? "manager";
|
|
374
385
|
this.workspaceRoot = opts.workspaceRoot ?? findCotalRoot();
|
|
375
386
|
this.maxSessions = opts.maxSessions;
|
|
387
|
+
this.remoteAuthority = opts.remoteAuthority;
|
|
388
|
+
if (opts.remoteAuthority)
|
|
389
|
+
this.managerLifecycleUid = opts.remoteAuthority.lifecycleUid;
|
|
376
390
|
this.secrets = opts.secretStore ?? workspaceSecretStore(this.workspaceRoot);
|
|
377
391
|
this.installedExtensions = opts.installedExtensions ?? false;
|
|
378
392
|
this.runtime = createRuntime(opts.runtime ?? "auto", `cotal-${this.space}`);
|
|
@@ -454,17 +468,19 @@ export class Manager {
|
|
|
454
468
|
// the local `.cotal/auth/auth.json` FS default), so a HOSTED composition mints from its KMS/Vault
|
|
455
469
|
// and no signing seed is ever read from the hosted disk. `this.space` cross-checks the bundle.
|
|
456
470
|
this.auth = await getSpaceAuth(this.secrets, this.space);
|
|
471
|
+
if (this.remoteAuthority && this.auth)
|
|
472
|
+
throw new Error("remote manager-service authority cannot be combined with local space signing trust - choose one authority path, never a fallback");
|
|
457
473
|
// USER-MODE detection is FAIL-CLOSED on the on-disk marker (the space-scoped state dir), never
|
|
458
474
|
// on the mutable mesh registry alone — registry drift/tamper must not let a user-auth space
|
|
459
475
|
// take the static self-mint branch. A marker/registry disagreement is a refused start with the
|
|
460
476
|
// repair, not a guess.
|
|
461
|
-
this.userMode = hasUserAuthState(this.workspaceRoot, this.space);
|
|
477
|
+
this.userMode = this.remoteAuthority !== undefined || hasUserAuthState(this.workspaceRoot, this.space);
|
|
462
478
|
const recorded = loadMeshes().find((m) => m.space === this.space);
|
|
463
|
-
if (recorded && (recorded.mode === "user") !== this.userMode)
|
|
479
|
+
if (!this.remoteAuthority && recorded && (recorded.mode === "user") !== this.userMode)
|
|
464
480
|
throw new Error(`mesh registry says space "${this.space}" is ${recorded.mode}-mode but the on-disk user-auth marker ${this.userMode ? "exists" : "is missing"} (${userAuthStateDir(this.workspaceRoot, this.space)}) - \`cotal down\` and re-\`cotal up\` this space to reconcile before running a manager`);
|
|
465
|
-
if (this.userMode && !recorded)
|
|
481
|
+
if (!this.remoteAuthority && this.userMode && !recorded)
|
|
466
482
|
throw new Error(`space "${this.space}" has user-auth state on disk but no mesh registry entry - a user-mode manager needs the authoritative record (\`cotal up\` writes it before the control plane); \`cotal up --user-auth\` this space, or remove the stale ${userAuthStateDir(this.workspaceRoot, this.space)}`);
|
|
467
|
-
if (this.userMode && !this.auth)
|
|
483
|
+
if (!this.remoteAuthority && this.userMode && !this.auth)
|
|
468
484
|
throw new Error(`space "${this.space}" has user-auth state but no auth.json under ${authDir(this.workspaceRoot)} - the pre-flip manager still needs the space trust bundle; re-run \`cotal up --user-auth\` here`);
|
|
469
485
|
// P2 item 3 (SPEC 13.6 item 7): the LOGICAL instance id + serve identity PERSIST across restart
|
|
470
486
|
// (a space-scoped manager identity file under .cotal). A restart re-registers the SAME id with an
|
|
@@ -473,20 +489,31 @@ export class Manager {
|
|
|
473
489
|
// manager in a DIFFERENT workspace root is a DIFFERENT logical id by construction (its own state
|
|
474
490
|
// dir) - two managers in ONE space are two workspace roots.
|
|
475
491
|
{
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
this.
|
|
479
|
-
this.managerServeIdentity = persisted.serveIdentity;
|
|
492
|
+
if (this.remoteAuthority) {
|
|
493
|
+
this.managerInstanceId = this.remoteAuthority.instanceId;
|
|
494
|
+
this.managerServeIdentity = this.remoteAuthority.identities.serve;
|
|
480
495
|
}
|
|
481
496
|
else {
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
497
|
+
const persisted = loadManagerInstanceIdentity(this.workspaceRoot, this.space);
|
|
498
|
+
if (persisted !== undefined) {
|
|
499
|
+
this.managerInstanceId = persisted.instanceId;
|
|
500
|
+
this.managerServeIdentity = persisted.serveIdentity;
|
|
501
|
+
}
|
|
502
|
+
else {
|
|
503
|
+
this.managerInstanceId = mintLifecycleUid();
|
|
504
|
+
this.managerServeIdentity = newIdentity();
|
|
505
|
+
saveManagerInstanceIdentity(this.workspaceRoot, this.space, { instanceId: this.managerInstanceId, serveIdentity: this.managerServeIdentity });
|
|
506
|
+
}
|
|
485
507
|
}
|
|
486
508
|
}
|
|
487
509
|
let creds;
|
|
488
510
|
let id;
|
|
489
|
-
if (this.
|
|
511
|
+
if (this.remoteAuthority) {
|
|
512
|
+
const remote = this.remoteAuthority;
|
|
513
|
+
id = remote.identities.supervisor.id;
|
|
514
|
+
creds = async () => remote.supervisorCreds;
|
|
515
|
+
}
|
|
516
|
+
else if (this.auth) {
|
|
490
517
|
const identity = newIdentity();
|
|
491
518
|
const auth = this.auth;
|
|
492
519
|
id = identity.id;
|
|
@@ -511,7 +538,7 @@ export class Manager {
|
|
|
511
538
|
// own launch chain (the operator command IS its launcher): its incarnation uid is the
|
|
512
539
|
// per-process `managerLifecycleUid` field (also the `managerInstance` audit coordinate on
|
|
513
540
|
// every static activation, Unit B).
|
|
514
|
-
lifecycleUid: this.managerLifecycleUid,
|
|
541
|
+
lifecycleUid: this.remoteAuthority?.lifecycleUid ?? this.managerLifecycleUid,
|
|
515
542
|
// The supervisor serves control + watches presence; it never consumes chat/dm/task
|
|
516
543
|
// (no message handler). consume:false avoids binding consumers it doesn't use — and
|
|
517
544
|
// under auth avoids trying to bind its own DM/task durables that nothing pre-created.
|
|
@@ -521,7 +548,7 @@ export class Manager {
|
|
|
521
548
|
// pull/display), so skip the channel-registry watch — the supervisor cred (residual 2) then
|
|
522
549
|
// holds no channel-KV read grant. Presence (the roster) is still watched.
|
|
523
550
|
watchChannels: false,
|
|
524
|
-
card: { id, name: this.name, role: "manager", kind: "endpoint" },
|
|
551
|
+
card: { id, ...(this.remoteAuthority ? { owner: this.remoteAuthority.owner, actor: this.remoteAuthority.actors.supervisor } : {}), name: this.name, role: "manager", kind: "endpoint" },
|
|
525
552
|
});
|
|
526
553
|
// Surface endpoint errors (incl. NATS permission denials) — without a listener an
|
|
527
554
|
// emitted "error" would crash the supervisor.
|
|
@@ -1375,16 +1402,57 @@ export class Manager {
|
|
|
1375
1402
|
return { kind: "taken", by: `pid ${current.info.pid} (${current.info.runtime}, root ${current.info.root})` };
|
|
1376
1403
|
return { kind: "held", revision: current.revision };
|
|
1377
1404
|
}
|
|
1405
|
+
/**
|
|
1406
|
+
* Release ownership of every managed agent WITHOUT stopping it and WITHOUT deprovisioning its
|
|
1407
|
+
* footprint — the child disposition for an exit this instance did not choose.
|
|
1408
|
+
*
|
|
1409
|
+
* WHY THIS IS NOT {@link teardownManagedAgents}. Losing the argument about who may SERVE a space
|
|
1410
|
+
* is not a finding about whether these agents should die. The two were one act on the lease-loss
|
|
1411
|
+
* path, so a supervisor that could not reach the broker for a lease TTL killed every seat it held
|
|
1412
|
+
* and revoked their credentials. That is the wrong conclusion drawn from a connectivity fact, and
|
|
1413
|
+
* on a live mesh it is reached over a timeout the very next retry would have cleared.
|
|
1414
|
+
*
|
|
1415
|
+
* WHAT IT LEAVES BEHIND, AND WHY THAT IS SAFE. Each agent is marked `suppressCleanup` before it
|
|
1416
|
+
* leaves the map, so no later call site in this process can select it for deprovision, and its
|
|
1417
|
+
* credential, durables and broker footprint outlive us. That is EXACTLY the state an abrupt death
|
|
1418
|
+
* leaves (SIGKILL, OOM, power loss), which the next manager's static reconcile sweep already
|
|
1419
|
+
* recovers by terminalizing an active slot with no live managed owner. The difference is that
|
|
1420
|
+
* this one is announced.
|
|
1421
|
+
*
|
|
1422
|
+
* WHAT IT DOES NOT CLAIM. Detaching does not make a child outlive the process. A `node-pty` child
|
|
1423
|
+
* dies when its spawning process exits, measured, and no manager-side policy changes that; only a
|
|
1424
|
+
* runtime whose child is owned elsewhere (tmux/herdr/cmux) actually survives. This method removes
|
|
1425
|
+
* the manager's DELIBERATE kill and revoke, which is the half the manager controls.
|
|
1426
|
+
*/
|
|
1427
|
+
detachManagedAgents(reason) {
|
|
1428
|
+
const managed = [...this.agents.values()];
|
|
1429
|
+
for (const a of managed) {
|
|
1430
|
+
// Set BEFORE the map delete: the flag is what every deprovision call site filters on, and an
|
|
1431
|
+
// agent that left the map without it is one a concurrent path could still select.
|
|
1432
|
+
a.suppressCleanup = true;
|
|
1433
|
+
this.agents.delete(a.name);
|
|
1434
|
+
}
|
|
1435
|
+
if (managed.length === 0)
|
|
1436
|
+
return;
|
|
1437
|
+
const seats = managed.map((a) => `${a.name} (${a.id}${a.handle.pid !== undefined ? `, pid ${a.handle.pid}` : ""})`).join(", ");
|
|
1438
|
+
console.error(`! manager instance ${this.managerInstanceId} detached ${managed.length} managed agent(s) - ${reason}\n` +
|
|
1439
|
+
` Left running and NOT stopped by this manager; their credentials and durables are RETAINED, not revoked: ${seats}\n` +
|
|
1440
|
+
` A child owned by this process (the pty runtime) still dies with it; a child owned elsewhere (tmux/herdr/cmux) keeps running.\n` +
|
|
1441
|
+
` NEXT: start a manager for space "${this.space}" - its reconcile sweep recovers any seat whose child did not survive.`);
|
|
1442
|
+
}
|
|
1378
1443
|
/** Stop serving and end this process, naming what was PROVED rather than what merely failed. */
|
|
1379
1444
|
async failClosedOnLeaseLoss(proof) {
|
|
1380
1445
|
console.error(`! manager instance ${this.managerInstanceId} lost its liveness lease for space "${this.space}": ${proof} - shutting down THIS instance (its serving only; siblings keep the space)`);
|
|
1381
1446
|
if (this.leaseTimer)
|
|
1382
1447
|
clearInterval(this.leaseTimer);
|
|
1383
|
-
//
|
|
1384
|
-
//
|
|
1448
|
+
// DETACH rather than tear down — but ONLY from the active state. A cut that has committed and
|
|
1449
|
+
// not finalized still has an inventory the successor will replay, and children left running
|
|
1450
|
+
// would be spawned a SECOND time under the same identities; that window keeps the retained stop
|
|
1451
|
+
// (which stops the child but does not deprovision it). Do NOT release the lease key (it may
|
|
1452
|
+
// belong to the replacement holder). Best-effort, like ep/attach.
|
|
1385
1453
|
try {
|
|
1386
1454
|
if (this.maintenanceState === "active" && !this.resumeRequired)
|
|
1387
|
-
|
|
1455
|
+
this.detachManagedAgents(`lease loss: ${proof}`);
|
|
1388
1456
|
else
|
|
1389
1457
|
await this.stopRetainedAgentsOnExit();
|
|
1390
1458
|
}
|
|
@@ -1919,14 +1987,14 @@ export class Manager {
|
|
|
1919
1987
|
});
|
|
1920
1988
|
if (requireAuthoritativeExit)
|
|
1921
1989
|
return;
|
|
1922
|
-
this.freeSlot(a, floor, true);
|
|
1990
|
+
this.freeSlot(a, floor, "stopped", true);
|
|
1923
1991
|
return;
|
|
1924
1992
|
}
|
|
1925
1993
|
if (!requireAuthoritativeExit)
|
|
1926
|
-
this.freeSlot(a, floor, true);
|
|
1994
|
+
this.freeSlot(a, floor, "stopped", true);
|
|
1927
1995
|
this.lifecycleInFlight++;
|
|
1928
1996
|
void this.awaitHandleExit(a.handle)
|
|
1929
|
-
.then(() => this.freeSlot(a, floor, true)) // no-op once an accepted stop already freed it
|
|
1997
|
+
.then(() => this.freeSlot(a, floor, "stopped", true)) // no-op once an accepted stop already freed it
|
|
1930
1998
|
.catch((e) => {
|
|
1931
1999
|
this.unverifiedStops.push({
|
|
1932
2000
|
name: a.name,
|
|
@@ -2047,9 +2115,40 @@ export class Manager {
|
|
|
2047
2115
|
* expires — flooring the RECYCLE, not the call, so both free paths (despawn + exit/reap) are
|
|
2048
2116
|
* covered (P4c). Floor self + own-child despawn and natural exit; NEVER admin despawn (operator
|
|
2049
2117
|
* emergency-kill stays unthrottled) and NEVER the reserved-rollback path (no cold-start paid). */
|
|
2050
|
-
|
|
2118
|
+
/**
|
|
2119
|
+
* WHY A SEAT LEFT — one line, at the one place every free path passes through.
|
|
2120
|
+
*
|
|
2121
|
+
* A manager that had held twelve seats could not say why any of them had died: the log carried no
|
|
2122
|
+
* per-seat exit line at all, so "the supervisor reaped them" and "they died on their own" were
|
|
2123
|
+
* indistinguishable after the fact and the incident was unattributable. `freeSlot` is the single
|
|
2124
|
+
* chokepoint for despawn, self-stop, reap and exit, which is why the line lives here rather than
|
|
2125
|
+
* being remembered at each of the six callers.
|
|
2126
|
+
*
|
|
2127
|
+
* THE RUNTIME'S EXIT DETAIL IS OPTIONAL AND ITS ABSENCE IS SAID OUT LOUD. `AgentHandle.exitInfo`
|
|
2128
|
+
* is only implemented by backends that own the child process; the rest genuinely cannot see how
|
|
2129
|
+
* it ended. Printing `code 0` for those would be a fabricated clean exit on exactly the seats
|
|
2130
|
+
* whose death nobody can explain, so an absent answer prints as unavailable and names the runtime
|
|
2131
|
+
* that could not provide it.
|
|
2132
|
+
*/
|
|
2133
|
+
logSeatReaped(a, cause) {
|
|
2134
|
+
let detail;
|
|
2135
|
+
try {
|
|
2136
|
+
const info = a.handle.exitInfo?.();
|
|
2137
|
+
detail = info === undefined
|
|
2138
|
+
? `exit detail unavailable from runtime "${a.handle.kind}"`
|
|
2139
|
+
: `exit code ${info.code ?? "unknown"}${info.signal === undefined ? "" : `, signal ${info.signal}`}`;
|
|
2140
|
+
}
|
|
2141
|
+
catch (e) {
|
|
2142
|
+
// A runtime that throws while being asked has told us something real; it must not take the
|
|
2143
|
+
// log line (or the free path it sits on) down with it.
|
|
2144
|
+
detail = `exit detail unreadable from runtime "${a.handle.kind}": ${e.message}`;
|
|
2145
|
+
}
|
|
2146
|
+
console.error(`seat reaped: ${a.name} (${a.id}, uid ${a.lifecycleUid}${a.handle.pid !== undefined ? `, pid ${a.handle.pid}` : ""}) - ${FREE_SLOT_CAUSE_TEXT[cause]}; ${detail}`);
|
|
2147
|
+
}
|
|
2148
|
+
freeSlot(a, floor, cause, acceptedBeforeFence = false) {
|
|
2051
2149
|
if (this.agents.get(a.name) !== a)
|
|
2052
2150
|
return; // already freed (exit raced despawn, etc.)
|
|
2151
|
+
this.logSeatReaped(a, cause);
|
|
2053
2152
|
a.terminalizing = true; // F5 latch (Unit B): also covers exit/reap paths that never rode stopHandle
|
|
2054
2153
|
this.agents.delete(a.name);
|
|
2055
2154
|
if (a.restart?.sessionStatePath)
|
|
@@ -2479,7 +2578,7 @@ export class Manager {
|
|
|
2479
2578
|
if (restart.crashes.length > SESSION_RESTART_LIMIT) {
|
|
2480
2579
|
console.error(`! ${a.name}: Pi crash loop (${restart.crashes.length} crashes in ${SESSION_RESTART_WINDOW_MS / 1000}s) - retiring the managed seat`);
|
|
2481
2580
|
restart.armed = false;
|
|
2482
|
-
this.freeSlot(a, true);
|
|
2581
|
+
this.freeSlot(a, true, "pi-crash-loop");
|
|
2483
2582
|
this.reapChildrenOf(this.managedPrincipal(a));
|
|
2484
2583
|
release();
|
|
2485
2584
|
return;
|
|
@@ -2533,7 +2632,7 @@ export class Manager {
|
|
|
2533
2632
|
replacement?.stop({ graceful: false });
|
|
2534
2633
|
}
|
|
2535
2634
|
catch { /* terminal cleanup continues */ }
|
|
2536
|
-
this.freeSlot(a, true);
|
|
2635
|
+
this.freeSlot(a, true, "pi-recovery-failed");
|
|
2537
2636
|
this.reapChildrenOf(this.managedPrincipal(a));
|
|
2538
2637
|
}
|
|
2539
2638
|
finally {
|
|
@@ -2559,7 +2658,7 @@ export class Manager {
|
|
|
2559
2658
|
console.error(`! ${a.name}: cannot classify Pi process exit for recovery: ${error.message} - retiring the seat`);
|
|
2560
2659
|
}
|
|
2561
2660
|
}
|
|
2562
|
-
this.freeSlot(a, true);
|
|
2661
|
+
this.freeSlot(a, true, "process-exit");
|
|
2563
2662
|
this.reapChildrenOf(this.managedPrincipal(a));
|
|
2564
2663
|
}
|
|
2565
2664
|
/** Agent names become `.cotal/agents/<name>.md` paths and mesh identities, so they must be bare
|
|
@@ -2884,6 +2983,9 @@ export class Manager {
|
|
|
2884
2983
|
catch (e) {
|
|
2885
2984
|
return { ok: false, error: e.message };
|
|
2886
2985
|
}
|
|
2986
|
+
const readinessTimeoutMs = connector.readinessTimeoutMs ?? this.readinessTimeoutMs;
|
|
2987
|
+
if (!Number.isSafeInteger(readinessTimeoutMs) || readinessTimeoutMs <= 0)
|
|
2988
|
+
return { ok: false, error: `connector ${connector.name} declares invalid readinessTimeoutMs ${JSON.stringify(connector.readinessTimeoutMs)}; expected a positive safe integer` };
|
|
2887
2989
|
// Capacity check first (cheap, fail-fast). Everything from here to the reserve below is
|
|
2888
2990
|
// SYNCHRONOUS (existsSync / registry / accessSync / readFileSync — no await), so the gate stays
|
|
2889
2991
|
// atomic: the capacity snapshot and the reserve land in one tick (P4a/P4c), and two concurrent
|
|
@@ -3134,6 +3236,7 @@ export class Manager {
|
|
|
3134
3236
|
}
|
|
3135
3237
|
if (events)
|
|
3136
3238
|
allowPublish = [...(allowPublish ?? []), connector.eventChannel({ owner: agentTriple.owner, actor: agentTriple.actor })];
|
|
3239
|
+
await hooks?.onReadinessWindow?.(readinessTimeoutMs);
|
|
3137
3240
|
await hooks?.onAccepted?.({ name, identity, lifecycleUid, agentTriple });
|
|
3138
3241
|
// In auth mode, mint the agent's creds from the space signing key and write them where the
|
|
3139
3242
|
// spawned session reads them (COTAL_CREDS path). Open mesh → no creds. Scope = the resolved
|
|
@@ -3332,7 +3435,7 @@ export class Manager {
|
|
|
3332
3435
|
// (presence) → started, the child to exit → failed (with its last output; already reaped), or
|
|
3333
3436
|
// neither in time → uncertain. `✓ started` therefore means "it joined", never just "a process
|
|
3334
3437
|
// launched".
|
|
3335
|
-
const readiness = await this.awaitReadiness(managed);
|
|
3438
|
+
const readiness = await this.awaitReadiness(managed, readinessTimeoutMs);
|
|
3336
3439
|
// Deliberately stopped mid-launch: reaped by onExit, and the despawn/stop path owns the
|
|
3337
3440
|
// goal terminal. Return BEFORE the failed/uncertain arms so this emits no competing
|
|
3338
3441
|
// outcome and does not re-arm an exit watcher on an agent already gone.
|
|
@@ -3359,7 +3462,7 @@ export class Manager {
|
|
|
3359
3462
|
catch (error) {
|
|
3360
3463
|
const detail = `${managed.name} joined, but its exact host session could not be bound for supervised recovery: ${error.message}`;
|
|
3361
3464
|
this.stopHandle(managed, false);
|
|
3362
|
-
this.freeSlot(managed, true);
|
|
3465
|
+
this.freeSlot(managed, true, "session-bind-failed");
|
|
3363
3466
|
await hooks?.onOutcome?.({ kind: "failed", data: { error: detail } });
|
|
3364
3467
|
return { ok: false, error: detail };
|
|
3365
3468
|
}
|
|
@@ -3787,6 +3890,10 @@ export class Manager {
|
|
|
3787
3890
|
if (adoptedSeed === undefined)
|
|
3788
3891
|
console.error(`! resume ${entry.name}: the adopted credential carries no readable nkey seed - the manager cannot renew it (it dies loud at its exp)`);
|
|
3789
3892
|
}
|
|
3893
|
+
const connector = await this.resolveConnector(entry.launch.connector);
|
|
3894
|
+
const readinessTimeoutMs = connector.readinessTimeoutMs ?? this.readinessTimeoutMs;
|
|
3895
|
+
if (!Number.isSafeInteger(readinessTimeoutMs) || readinessTimeoutMs <= 0)
|
|
3896
|
+
return { ok: false, error: `connector ${connector.name} declares invalid readinessTimeoutMs ${JSON.stringify(connector.readinessTimeoutMs)}; expected a positive safe integer` };
|
|
3790
3897
|
const handle = this.runtime.spawn(entry.name, prepared.spec, entry.launch.cwd);
|
|
3791
3898
|
const managed = {
|
|
3792
3899
|
name: entry.name,
|
|
@@ -3832,7 +3939,7 @@ export class Manager {
|
|
|
3832
3939
|
this.agents.set(entry.name, managed);
|
|
3833
3940
|
if (this.resumeAttemptId)
|
|
3834
3941
|
this.resumedAgentNames.add(entry.name);
|
|
3835
|
-
const readiness = await this.awaitReadiness(managed);
|
|
3942
|
+
const readiness = await this.awaitReadiness(managed, readinessTimeoutMs);
|
|
3836
3943
|
if (!readiness.ok && !readiness.uncertain)
|
|
3837
3944
|
return { ok: false, error: readiness.detail };
|
|
3838
3945
|
if (!readiness.ok) {
|
|
@@ -3847,7 +3954,7 @@ export class Manager {
|
|
|
3847
3954
|
}
|
|
3848
3955
|
catch (error) {
|
|
3849
3956
|
this.stopHandle(managed, false);
|
|
3850
|
-
this.freeSlot(managed, true, true);
|
|
3957
|
+
this.freeSlot(managed, true, "resume-session-rebind-failed", true);
|
|
3851
3958
|
return { ok: false, error: `${managed.name} resumed, but its exact host session could not be rebound: ${error.message}` };
|
|
3852
3959
|
}
|
|
3853
3960
|
}
|
|
@@ -3904,7 +4011,7 @@ export class Manager {
|
|
|
3904
4011
|
* `"presence"` event is only a wake; the roster is re-read as the source of truth (subscribe-then-check
|
|
3905
4012
|
* catches a join/exit that landed before we subscribed). Runtimes that stream no exit signal (external surfaces,
|
|
3906
4013
|
* whose `attach()` throws) race presence-vs-backstop only — better than the old "assume up". */
|
|
3907
|
-
async awaitReadiness(a) {
|
|
4014
|
+
async awaitReadiness(a, readinessTimeoutMs) {
|
|
3908
4015
|
let session;
|
|
3909
4016
|
try {
|
|
3910
4017
|
session = a.handle.attach();
|
|
@@ -3997,8 +4104,8 @@ export class Manager {
|
|
|
3997
4104
|
timer = setTimeout(() => finish({
|
|
3998
4105
|
ok: false,
|
|
3999
4106
|
uncertain: true,
|
|
4000
|
-
detail: `${a.name} (${a.id}): launch status uncertain - no process exit and no mesh presence within ${Math.round(
|
|
4001
|
-
}),
|
|
4107
|
+
detail: `${a.name} (${a.id}): launch status uncertain - no process exit and no mesh presence within ${Math.round(readinessTimeoutMs / 1000)}s; it may still be booting or stuck before connector startup. Inspect with \`cotal attach ${a.name}\` / \`cotal ps\`; do not stop it solely because this bounded wait elapsed.`,
|
|
4108
|
+
}), readinessTimeoutMs);
|
|
4002
4109
|
unsubExit = s ? s.onExit(onExit) : () => { };
|
|
4003
4110
|
this.ep.on("presence", onPresence);
|
|
4004
4111
|
// Subscribe-then-check (TOCTOU): a join or an exit that already landed before we subscribed.
|
|
@@ -4116,12 +4223,14 @@ export class Manager {
|
|
|
4116
4223
|
* gate CAS, the mint fence, and the spec/governance writes ride a one-shot scoped authority —
|
|
4117
4224
|
* NEVER the manager's standing seed/supervisor connection (the panel's "no seed shortcut"). */
|
|
4118
4225
|
async withEndpointServeExecutor(fn) {
|
|
4119
|
-
|
|
4120
|
-
|
|
4121
|
-
|
|
4122
|
-
|
|
4123
|
-
|
|
4124
|
-
|
|
4226
|
+
const identity = this.remoteAuthority?.identities.executor ?? newIdentity();
|
|
4227
|
+
const creds = this.remoteAuthority?.executorCreds ?? (this.auth
|
|
4228
|
+
? await mintCreds(this.auth, identity, "endpoint-serve-executor", {
|
|
4229
|
+
endpointServeExecutor: { endpoint: MANAGER_ENDPOINT, instanceId: this.managerInstanceId },
|
|
4230
|
+
})
|
|
4231
|
+
: undefined);
|
|
4232
|
+
if (!creds)
|
|
4233
|
+
throw new Error("withEndpointServeExecutor: no scoped executor authority (an open mesh must use the bare path)");
|
|
4125
4234
|
const nc = await connect({ servers: this.servers ?? DEFAULT_SERVER, ...standaloneConnectOpts({ creds, /* not yet wired to a recorded transport */ tls: false }), maxReconnectAttempts: 0 });
|
|
4126
4235
|
try {
|
|
4127
4236
|
const kvm = new Kvm(nc);
|
|
@@ -4176,6 +4285,46 @@ export class Manager {
|
|
|
4176
4285
|
* old open-mesh ctl trust ("open = single-trusted-host"). */
|
|
4177
4286
|
async registerManagerService() {
|
|
4178
4287
|
const auth = this.auth;
|
|
4288
|
+
if (this.remoteAuthority) {
|
|
4289
|
+
const remote = this.remoteAuthority;
|
|
4290
|
+
this.goalWriterIdentity = remote.identities.goalWriter;
|
|
4291
|
+
this.sessionLedgerIdentity = remote.identities.sessionLedger;
|
|
4292
|
+
this.goalWriterCreds = remote.goalWriterCreds;
|
|
4293
|
+
this.sessionLedgerCreds = remote.sessionLedgerCreds;
|
|
4294
|
+
const state = {
|
|
4295
|
+
handle: undefined,
|
|
4296
|
+
nc: undefined,
|
|
4297
|
+
identity: remote.identities.serve,
|
|
4298
|
+
grant: remote.serveGrant,
|
|
4299
|
+
creds: remote.serveCreds,
|
|
4300
|
+
};
|
|
4301
|
+
const enc = new TextEncoder();
|
|
4302
|
+
const nc = await connect({
|
|
4303
|
+
servers: this.servers ?? DEFAULT_SERVER,
|
|
4304
|
+
authenticator: (nonce) => credsAuthenticator(enc.encode(state.creds))(nonce),
|
|
4305
|
+
inboxPrefix: `_INBOX_${state.identity.id}`,
|
|
4306
|
+
maxReconnectAttempts: -1,
|
|
4307
|
+
});
|
|
4308
|
+
try {
|
|
4309
|
+
state.handle = serveEndpoint(nc, this.space, state.grant, this.managerServiceDefs(), { public: true }, {
|
|
4310
|
+
resolveTarget: (t) => {
|
|
4311
|
+
const key = principalKey(t.owner, t.actor).key;
|
|
4312
|
+
for (const a of this.agents.values())
|
|
4313
|
+
if (a.id === key)
|
|
4314
|
+
return { lifecycleUid: a.lifecycleUid, mappingRevision: 0 };
|
|
4315
|
+
return undefined;
|
|
4316
|
+
},
|
|
4317
|
+
});
|
|
4318
|
+
}
|
|
4319
|
+
catch (e) {
|
|
4320
|
+
await nc.drain().catch(() => nc.close());
|
|
4321
|
+
throw e;
|
|
4322
|
+
}
|
|
4323
|
+
state.nc = nc;
|
|
4324
|
+
this.serviceServe = state;
|
|
4325
|
+
console.error(`remote manager service endpoint activated: ${MANAGER_ENDPOINT}/${this.managerInstanceId} (epoch ${state.grant.epoch})`);
|
|
4326
|
+
return;
|
|
4327
|
+
}
|
|
4179
4328
|
// The §13.7 contract store is REGISTRATION's dependency, ensured here MODE-NEUTRALLY (1c.2c):
|
|
4180
4329
|
// it used to ride the static-only lifecycle reconcile, so a USER-mode manager registered
|
|
4181
4330
|
// against an absent stream and its artifact publish died no-responders (live-repro'd). A
|
|
@@ -4407,14 +4556,14 @@ export class Manager {
|
|
|
4407
4556
|
* KV + JS + JSM to this one connection and space (SPEC 13.4), so a composition mixup cannot splice
|
|
4408
4557
|
* goal state across brokers. */
|
|
4409
4558
|
async startGoalWriter() {
|
|
4410
|
-
const identity = this.auth ? this.goalWriterIdentity : newIdentity();
|
|
4559
|
+
const identity = (this.auth || this.remoteAuthority) ? this.goalWriterIdentity : newIdentity();
|
|
4411
4560
|
const enc = new TextEncoder();
|
|
4412
4561
|
// The mutable holder captured by the authenticator (mirrors the serve connection): a half-TTL
|
|
4413
4562
|
// renewal updates `gw.creds` and the next (re)connect presents the refreshed credential.
|
|
4414
|
-
const gw = { nc: undefined, ctx: undefined, creds: this.auth ? this.goalWriterCreds : undefined, identity };
|
|
4563
|
+
const gw = { nc: undefined, ctx: undefined, creds: (this.auth || this.remoteAuthority) ? this.goalWriterCreds : undefined, identity };
|
|
4415
4564
|
const nc = await connect({
|
|
4416
4565
|
servers: this.servers ?? DEFAULT_SERVER,
|
|
4417
|
-
...(this.auth ? { authenticator: (nonce) => credsAuthenticator(enc.encode(gw.creds))(nonce) } : {}),
|
|
4566
|
+
...((this.auth || this.remoteAuthority) ? { authenticator: (nonce) => credsAuthenticator(enc.encode(gw.creds))(nonce) } : {}),
|
|
4418
4567
|
inboxPrefix: `_INBOX_${identity.id}`,
|
|
4419
4568
|
maxReconnectAttempts: -1,
|
|
4420
4569
|
});
|
|
@@ -4446,7 +4595,7 @@ export class Manager {
|
|
|
4446
4595
|
// cluster-verified eviction BEFORE the epoch advance); an open mesh gets none.
|
|
4447
4596
|
gw.gate = serveIssuanceGateKv(await new Kvm(nc).open(epAuthBucket(this.space)), this.space, { endpoint: MANAGER_ENDPOINT, instanceId: this.managerInstanceId });
|
|
4448
4597
|
this.goalWriter = gw;
|
|
4449
|
-
console.error(`manager goal-writer standing (endpoint ${MANAGER_ENDPOINT}, ${this.auth ? "scoped cred, §13.1 family-staged" : "open/bare"})`);
|
|
4598
|
+
console.error(`manager goal-writer standing (endpoint ${MANAGER_ENDPOINT}, ${(this.auth || this.remoteAuthority) ? "scoped cred, §13.1 family-staged" : "open/bare"})`);
|
|
4450
4599
|
}
|
|
4451
4600
|
/** Drain the goal-writer connection (best-effort, both exit paths). */
|
|
4452
4601
|
async stopGoalWriter() {
|
|
@@ -4513,14 +4662,14 @@ export class Manager {
|
|
|
4513
4662
|
* per-session caller cred is the holder's real fence). The plane's ledger lives in the DEDICATED
|
|
4514
4663
|
* sessions bucket (createEndpointStreams provisioned it at registration). */
|
|
4515
4664
|
async startSessionPlane() {
|
|
4516
|
-
const identity = this.auth ? this.sessionLedgerIdentity : newIdentity();
|
|
4665
|
+
const identity = (this.auth || this.remoteAuthority) ? this.sessionLedgerIdentity : newIdentity();
|
|
4517
4666
|
const enc = new TextEncoder();
|
|
4518
4667
|
// The mutable holder captured by the authenticator (mirrors the goal-writer): a half-TTL renewal
|
|
4519
4668
|
// updates `sw.creds` and the next (re)connect presents the refreshed credential.
|
|
4520
|
-
const sw = { nc: undefined, creds: this.auth ? this.sessionLedgerCreds : undefined };
|
|
4669
|
+
const sw = { nc: undefined, creds: (this.auth || this.remoteAuthority) ? this.sessionLedgerCreds : undefined };
|
|
4521
4670
|
const nc = await connect({
|
|
4522
4671
|
servers: this.servers ?? DEFAULT_SERVER,
|
|
4523
|
-
...(this.auth ? { authenticator: (nonce) => credsAuthenticator(enc.encode(sw.creds))(nonce) } : {}),
|
|
4672
|
+
...((this.auth || this.remoteAuthority) ? { authenticator: (nonce) => credsAuthenticator(enc.encode(sw.creds))(nonce) } : {}),
|
|
4524
4673
|
inboxPrefix: `_INBOX_${identity.id}`,
|
|
4525
4674
|
maxReconnectAttempts: -1,
|
|
4526
4675
|
});
|
|
@@ -4588,7 +4737,7 @@ export class Manager {
|
|
|
4588
4737
|
}, 15 * 60 * 1000);
|
|
4589
4738
|
renewTimer.unref?.();
|
|
4590
4739
|
this.sessionKeyRenewTimer = renewTimer;
|
|
4591
|
-
console.error(`manager session plane standing (endpoint ${MANAGER_ENDPOINT}, epoch ${serveEpoch}, ${this.auth ? "scoped session-ledger cred, §13.1 family-staged" : "open/bare"})`);
|
|
4740
|
+
console.error(`manager session plane standing (endpoint ${MANAGER_ENDPOINT}, epoch ${serveEpoch}, ${(this.auth || this.remoteAuthority) ? "scoped session-ledger cred, §13.1 family-staged" : "open/bare"})`);
|
|
4592
4741
|
}
|
|
4593
4742
|
/**
|
|
4594
4743
|
* The per-session SERVING credential seam (P2 item 6, SPEC 13.6): the manager mints, gate-stages,
|
|
@@ -4614,6 +4763,19 @@ export class Manager {
|
|
|
4614
4763
|
mint: async (grant) => {
|
|
4615
4764
|
// Open mesh: no auth to mint from. The id still names the session so the ledger row and the
|
|
4616
4765
|
// teardown path are identical in both modes.
|
|
4766
|
+
if (this.remoteAuthority) {
|
|
4767
|
+
const identity = newIdentity();
|
|
4768
|
+
const creds = await this.remoteAuthority.mintSessionServing({
|
|
4769
|
+
identity,
|
|
4770
|
+
endpoint: grant.endpoint,
|
|
4771
|
+
sessionId: grant.sessionId,
|
|
4772
|
+
epoch: grant.serving.epoch,
|
|
4773
|
+
exp: Math.floor(grant.exp / 1000),
|
|
4774
|
+
});
|
|
4775
|
+
const id = rawDigest(creds).replace("sha256:", "sha256-");
|
|
4776
|
+
this.sessionServingKeys.set(id, identity.id);
|
|
4777
|
+
return { id, creds, exp: grant.exp };
|
|
4778
|
+
}
|
|
4617
4779
|
if (!this.auth)
|
|
4618
4780
|
return { id: `${grant.sessionId}.s`, creds: "", exp: grant.exp };
|
|
4619
4781
|
const identity = newIdentity();
|
|
@@ -4630,7 +4792,7 @@ export class Manager {
|
|
|
4630
4792
|
observeGate: async (_endpoint, instanceId) => {
|
|
4631
4793
|
// Open mesh: nothing is minted and nothing is staged, so there is no gate to pin (see
|
|
4632
4794
|
// ServingGatePin.gate — the stage refuses loudly if this is ever missing on an auth mesh).
|
|
4633
|
-
if (!this.auth)
|
|
4795
|
+
if (!this.auth && !this.remoteAuthority)
|
|
4634
4796
|
return { key: epgateKey(MANAGER_ENDPOINT, instanceId), revision: 0 };
|
|
4635
4797
|
return gate(async (authKv) => {
|
|
4636
4798
|
const observed = await serveIssuanceGateKv(authKv, this.space, { endpoint: MANAGER_ENDPOINT, instanceId }).observe();
|
|
@@ -4642,7 +4804,7 @@ export class Manager {
|
|
|
4642
4804
|
});
|
|
4643
4805
|
},
|
|
4644
4806
|
stage: async (grant, cred, pin) => {
|
|
4645
|
-
if (!this.auth)
|
|
4807
|
+
if (!this.auth && !this.remoteAuthority)
|
|
4646
4808
|
return; // open mesh: nothing minted, so nothing to make revocable
|
|
4647
4809
|
const key = this.sessionServingKeys.get(cred.id);
|
|
4648
4810
|
if (key === undefined)
|
|
@@ -4676,11 +4838,11 @@ export class Manager {
|
|
|
4676
4838
|
open: async (cred) => {
|
|
4677
4839
|
// FAIL LOUD: there is deliberately no shared connection to fall back to. Serving a session
|
|
4678
4840
|
// without its own credential is exactly the standing-writer shape this design removes.
|
|
4679
|
-
const opts = this.auth ? standaloneConnectOpts({ creds: cred.creds, /* not yet wired to a recorded transport */ tls: false }) : {};
|
|
4841
|
+
const opts = (this.auth || this.remoteAuthority) ? standaloneConnectOpts({ creds: cred.creds, /* not yet wired to a recorded transport */ tls: false }) : {};
|
|
4680
4842
|
return connect({ servers: this.servers ?? DEFAULT_SERVER, ...opts, maxReconnectAttempts: -1 });
|
|
4681
4843
|
},
|
|
4682
4844
|
revoke: async (credentialId) => {
|
|
4683
|
-
if (!this.auth)
|
|
4845
|
+
if (!this.auth && !this.remoteAuthority)
|
|
4684
4846
|
return; // open mesh: nothing was minted
|
|
4685
4847
|
await gate(async (authKv) => {
|
|
4686
4848
|
await markLedgerRowRevoked(authKv, epcredRowKey(MANAGER_ENDPOINT, iid, credentialId));
|
|
@@ -4898,6 +5060,7 @@ export class Manager {
|
|
|
4898
5060
|
let rejectAccept;
|
|
4899
5061
|
const acceptP = new Promise((res, rej) => { resolveAccept = res; rejectAccept = rej; });
|
|
4900
5062
|
let acceptance;
|
|
5063
|
+
let readinessTimeoutMs = this.readinessTimeoutMs;
|
|
4901
5064
|
// H1: set the instant the terminal path is ENTERED, not when it succeeds — the post-accept
|
|
4902
5065
|
// fallback below must fire only when `onOutcome` never ran at all, never as a second attempt
|
|
4903
5066
|
// behind a commit that threw.
|
|
@@ -5020,7 +5183,7 @@ export class Manager {
|
|
|
5020
5183
|
requestId: goalId,
|
|
5021
5184
|
sourceSeq: 0,
|
|
5022
5185
|
acceptedAt,
|
|
5023
|
-
readinessDeadlineMs:
|
|
5186
|
+
readinessDeadlineMs: readinessTimeoutMs,
|
|
5024
5187
|
});
|
|
5025
5188
|
acceptance = { name, owner: agentTriple.owner, actor: agentTriple.actor, uid: agentTriple.uid, goalId, fingerprint, executor };
|
|
5026
5189
|
this.goalAcceptances.set(goalId, acceptance);
|
|
@@ -5029,6 +5192,7 @@ export class Manager {
|
|
|
5029
5192
|
this.emitGoalProgress(ref, epoch, { phase: "handoff" });
|
|
5030
5193
|
},
|
|
5031
5194
|
onLaunched: () => this.emitGoalProgress(ref, epoch, { phase: "launched" }),
|
|
5195
|
+
onReadinessWindow: (value) => { readinessTimeoutMs = value; },
|
|
5032
5196
|
onOutcome,
|
|
5033
5197
|
// Claim the terminal WITHOUT committing one: the despawn/stop that ended this launch owns
|
|
5034
5198
|
// it and commits `cancel`. This only stops the non-ok reply below from manufacturing a
|