@cotal-ai/manager 0.58.0 → 0.59.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands.d.ts.map +1 -1
- package/dist/commands.js +41 -12
- package/dist/commands.js.map +1 -1
- package/dist/endpoint-evict.d.ts +9 -4
- package/dist/endpoint-evict.d.ts.map +1 -1
- package/dist/endpoint-evict.js +82 -26
- package/dist/endpoint-evict.js.map +1 -1
- package/dist/holder-liveness.d.ts +4 -0
- package/dist/holder-liveness.d.ts.map +1 -1
- package/dist/holder-liveness.js +95 -3
- package/dist/holder-liveness.js.map +1 -1
- package/dist/manager-service-contract.d.ts +15 -6
- package/dist/manager-service-contract.d.ts.map +1 -1
- package/dist/manager-service-contract.js +84 -13
- package/dist/manager-service-contract.js.map +1 -1
- package/dist/manager.d.ts +117 -51
- package/dist/manager.d.ts.map +1 -1
- package/dist/manager.js +355 -105
- package/dist/manager.js.map +1 -1
- package/dist/remote-authority.d.ts +12 -1
- package/dist/remote-authority.d.ts.map +1 -1
- package/dist/remote-authority.js +29 -2
- package/dist/remote-authority.js.map +1 -1
- package/dist/resume.d.ts.map +1 -1
- package/dist/resume.js +3 -0
- package/dist/resume.js.map +1 -1
- package/dist/run-hosting.d.ts +12 -1
- package/dist/run-hosting.d.ts.map +1 -1
- package/dist/run-hosting.js +28 -0
- package/dist/run-hosting.js.map +1 -1
- package/dist/runtime/custodial-pty.d.ts +8 -5
- package/dist/runtime/custodial-pty.d.ts.map +1 -1
- package/dist/runtime/custodial-pty.js +23 -12
- package/dist/runtime/custodial-pty.js.map +1 -1
- package/dist/runtime/index.d.ts +2 -0
- package/dist/runtime/index.d.ts.map +1 -1
- package/dist/runtime/index.js +148 -5
- package/dist/runtime/index.js.map +1 -1
- package/dist/runtime/pty.d.ts +3 -4
- package/dist/runtime/pty.d.ts.map +1 -1
- package/dist/runtime/pty.js +9 -6
- package/dist/runtime/pty.js.map +1 -1
- package/package.json +12 -12
- package/dist/ui.d.ts +0 -10
- package/dist/ui.d.ts.map +0 -1
- package/dist/ui.js +0 -10
- package/dist/ui.js.map +0 -1
package/dist/manager.js
CHANGED
|
@@ -4,11 +4,11 @@ import { hostname } from "node:os";
|
|
|
4
4
|
import { credsAuthenticator } from "@nats-io/transport-node";
|
|
5
5
|
import { existsSync, lstatSync, readFileSync, realpathSync, rmSync, statSync } from "node:fs";
|
|
6
6
|
import { isAbsolute, join, dirname, resolve } from "node:path";
|
|
7
|
-
import { CotalEndpoint, accountFromCreds, credsClaims, DEFAULT_SERVER, DEV_OWNER, MANAGER_LEASE_RENEW_MS, STANDING_RENEWABLE_TTL_SEC, assertLifecycleToken, divergentSecretStoreNotice, parseDaemonStoreAnswer, parseSecretStoreIdentity, sameSecretStoreIdentity, agentFilePath, clearSpaceHistory, dialerFor, connectorServers, spawnEnvAllow, deprovisionAgent, DeprovisionError, isConcreteChannel, deprovisionTargetPrincipal, firstFreeName, spawnNameError, idFromCreds, inspectCredHealth, credsRenewalDelayMs, composeWirePersona, loadAgentFile, listPersonaCatalog, personaCatalogDescription, personaCatalogReadable, loadCotalConfig, mintCreds, mintGeneration, mintAcceptedToken, openIssuedStore, acceptedBucket, issuedBucket, mintLifecycleUid, managedRetirementOpId, mkSecretDir, newIdentity, actionContext, parsePrincipalKey, parseShareSelection, principalKey, probeConnect, provisionAgent, provisionAgentDurables, registry, resolveAuthProvider, saveAgentFile, subjectMatches, AUTH_ENDPOINT, EP_CMD_RETIRE_LIFECYCLE, controlServiceSubject, eventChannelPrincipal, resolveService, invokeCommand, } from "@cotal-ai/core";
|
|
7
|
+
import { CotalEndpoint, accountFromCreds, credsClaims, DEFAULT_SERVER, DEV_OWNER, MANAGER_LEASE_RENEW_MS, STANDING_RENEWABLE_TTL_SEC, assertLifecycleToken, discardLaunchArtifacts, divergentSecretStoreNotice, parseDaemonStoreAnswer, parseSecretStoreIdentity, sameSecretStoreIdentity, agentFilePath, clearSpaceHistory, dialerFor, connectorServers, spawnEnvAllow, deprovisionAgent, DeprovisionError, isConcreteChannel, deprovisionTargetPrincipal, firstFreeName, spawnNameError, idFromCreds, inspectCredHealth, credsRenewalDelayMs, composeWirePersona, loadAgentFile, listPersonaCatalog, personaCatalogDescription, personaCatalogReadable, loadCotalConfig, mintCreds, mintGeneration, mintAcceptedToken, openIssuedStore, acceptedBucket, issuedBucket, mintLifecycleUid, managedRetirementOpId, mkSecretDir, newIdentity, actionContext, parsePrincipalKey, parseShareSelection, principalKey, probeConnect, provisionAgent, provisionAgentDurables, registry, resolveAuthProvider, saveAgentFile, subjectMatches, AUTH_ENDPOINT, EP_CMD_RETIRE_LIFECYCLE, controlServiceSubject, eventChannelPrincipal, resolveService, invokeCommand, } from "@cotal-ai/core";
|
|
8
8
|
import { agentAuthState, agentCredsDir, agentLifecycleSecretFilePaths, agentSecretFilePaths, agentSecretKeyForFile, authDir, connectorInstallHint, DEFAULT_CONNECTOR, defaultAgentType, DELIVERY_CREDS_KIND, extensionConnectors, findCotalRoot, getSpaceAuth, hasUserAuthState, loadExtensionsManifest, loadManagerInstanceIdentity, loadMeshes, localTrustOfSpace, manifestExtensionNames, materializeFromManifest, materializeSecretToFile, MEMBERSHIP_RW_CREDS_KIND, mergeLaunchOptions, remintDaemonCreds, resolveOnPath, createManagerInstanceIdentity, spaceMaterialKey, SYSTEM_CREDS_FILES, userAuthStateDir, workspaceSecretStore, writeRenewalRecord } from "@cotal-ai/workspace";
|
|
9
9
|
import { createRuntime, isCustodialRuntime, requireRuntimeAdopt, requireRuntimeReap, RuntimeReapUnproven, } from "./runtime/index.js";
|
|
10
10
|
import { AttachEndpoint } from "./attach-endpoint.js";
|
|
11
|
-
import { makeManagerEndpointEvictionEvidence, makeManagerEndpointEvictor } from "./endpoint-evict.js";
|
|
11
|
+
import { makeManagerEndpointEvictionEvidence, makeManagerEndpointEvictor, makeManagerEndpointHolderEvictor } from "./endpoint-evict.js";
|
|
12
12
|
import { makeManagerHolderLivenessProbe } from "./holder-liveness.js";
|
|
13
13
|
import { GateReconcileRefused, reconcileEndpointGate } from "./reconcile-gate.js";
|
|
14
14
|
import { launchSpecForRun, materializePersona, launchAgentToStartOpts, parseLaunchSpec, persistLaunchSpec } from "./launch.js";
|
|
@@ -197,12 +197,6 @@ function parseTurnNote(raw) {
|
|
|
197
197
|
function turnHoldToken(goalId) {
|
|
198
198
|
return createHash("sha256").update(`${goalId}:turn-deadline`, "utf8").digest("base64url").slice(0, 43);
|
|
199
199
|
}
|
|
200
|
-
/**
|
|
201
|
-
* The agent supervisor: a long-lived mesh node that owns agent process lifecycle.
|
|
202
|
-
* It serves control requests on the "manager" service and spawns/kills agents
|
|
203
|
-
* through a pluggable {@link Runtime} (pty by default). It does NOT proxy agent
|
|
204
|
-
* mesh traffic — terminal I/O streams over its own attach endpoint instead.
|
|
205
|
-
*/
|
|
206
200
|
/**
|
|
207
201
|
* Event channels in `channels` that do NOT belong to `{owner, actor}`.
|
|
208
202
|
*
|
|
@@ -243,6 +237,12 @@ function isAbsentDeliveryAdmin(msg) {
|
|
|
243
237
|
return false;
|
|
244
238
|
return /no responders|\b503\b/i.test(msg);
|
|
245
239
|
}
|
|
240
|
+
/**
|
|
241
|
+
* The agent supervisor: a long-lived mesh node that owns agent process lifecycle.
|
|
242
|
+
* It serves control requests on the "manager" service and spawns/kills agents
|
|
243
|
+
* through a pluggable {@link Runtime} (pty by default). It does NOT proxy agent
|
|
244
|
+
* mesh traffic — terminal I/O streams over its own attach endpoint instead.
|
|
245
|
+
*/
|
|
246
246
|
export class Manager {
|
|
247
247
|
space;
|
|
248
248
|
servers;
|
|
@@ -268,8 +268,13 @@ export class Manager {
|
|
|
268
268
|
endpointServeExecutorExpiresInSeconds;
|
|
269
269
|
agents = new Map();
|
|
270
270
|
/** Names whose spawn is in flight (reserved synchronously before the provision await) — counted
|
|
271
|
-
* toward the ceiling
|
|
271
|
+
* toward the ceiling (once per name, see {@link Manager.occupancy}) so two concurrent same-name
|
|
272
|
+
* spawns can't both pass the gate (P4a). */
|
|
272
273
|
reserved = new Set();
|
|
274
|
+
/** Reserved names whose launch already put its seat in `agents`. The name stays reserved until
|
|
275
|
+
* the launch settles, so no other spawn can take it, but the ceiling counts that seat through
|
|
276
|
+
* `agents` and, if it frees young, its cooling stamp, never through the reservation (#906). */
|
|
277
|
+
reservedLive = new Set();
|
|
273
278
|
/** Expiry stamps (`startedAt + MIN_LIFETIME`) for slots that freed while still young — a
|
|
274
279
|
* count-only, lazily-pruned recycle floor (P4c). Pruned + summed into the ceiling gate. */
|
|
275
280
|
cooling = [];
|
|
@@ -321,6 +326,11 @@ export class Manager {
|
|
|
321
326
|
* every (re)connect, so a renewal is adopted without re-registration). Absent on open meshes,
|
|
322
327
|
* in user mode (the named 1a follow-up), and before registration completes. */
|
|
323
328
|
serviceServe;
|
|
329
|
+
/** #1577: serve connections whose transport is down while the client reconnects them. Such a
|
|
330
|
+
* connection is open (`isClosed()` is false under unbounded reconnects) yet the broker holds
|
|
331
|
+
* none of its rails, so the liveness responder must not grade it `bound`. Keyed by the
|
|
332
|
+
* connection, so a re-dial that swaps in a fresh one starts from its own state. */
|
|
333
|
+
serveTransportDown = new WeakSet();
|
|
324
334
|
/** P2 item 2 (spawn-as-action): the SELF-MEDIATED goal-writer connection + ActionContext — a
|
|
325
335
|
* standing connection DISJOINT from the serve credential (Q2), scoped to exactly this endpoint's
|
|
326
336
|
* goal bind/terminal facts + goal-record writes ({@link goalWriterGrants}). Auth mode mints the
|
|
@@ -375,10 +385,11 @@ export class Manager {
|
|
|
375
385
|
/** P2 item 2 (M4): the live spawn goal ref for each managed agent name, so a despawn MID-GOAL
|
|
376
386
|
* drives the cancel path (transition -> cancel terminal). Cleared when the goal terminalizes. */
|
|
377
387
|
agentGoals = new Map();
|
|
378
|
-
/** The turn relay's same-incarnation idempotency map ({@link goalAcceptances}'s twin): a
|
|
379
|
-
* same-goalId retry serves the identical acceptance; cross-incarnation retries rebuild from
|
|
380
|
-
* the goal-index entry (its acceptance floor + note). */
|
|
381
388
|
/**
|
|
389
|
+
* The turn relay's same-incarnation idempotency map ({@link goalAcceptances}'s twin): a
|
|
390
|
+
* same-goalId retry serves the identical acceptance; cross-incarnation retries rebuild from
|
|
391
|
+
* the goal-index entry (its acceptance floor + note).
|
|
392
|
+
*
|
|
382
393
|
* One entry per turn this incarnation accepted: the acceptance a duplicate submission is served
|
|
383
394
|
* from, and — once the turn settles — the answer a RETRIED yield is served from.
|
|
384
395
|
*
|
|
@@ -656,8 +667,8 @@ export class Manager {
|
|
|
656
667
|
case "user":
|
|
657
668
|
return (`\`cotal actor grant <reader> --owner ${owner} --scope '' --allow-subscribe '${channel}' ` +
|
|
658
669
|
`--allow-publish ''\`, with every field spelled out: \`actor grant\` is an upsert of the ` +
|
|
659
|
-
`WHOLE row and an omitted flag
|
|
660
|
-
`scope)
|
|
670
|
+
`WHOLE row, and an omitted flag is refused, or with \`--full\` is the WIDE default (\`>\` read, \`>\` post, \`spawn,role:default\` ` +
|
|
671
|
+
`scope).`);
|
|
661
672
|
case "static":
|
|
662
673
|
return (`\`cotal mint <reader> --profile agent --allow-subscribe ${channel} --provision\`, where there ` +
|
|
663
674
|
`is no actor ledger for \`actor grant\` to write to. The AGENT profile, not the observer ` +
|
|
@@ -804,6 +815,10 @@ export class Manager {
|
|
|
804
815
|
const reportEndpoint = (e) => console.error(`! manager endpoint: ${e.message}`);
|
|
805
816
|
this.ep.on("error", reportEndpoint);
|
|
806
817
|
this.ep.on("warning", reportEndpoint);
|
|
818
|
+
// Every roster change and every view transition, so a seat that is offline in a fresh snapshot
|
|
819
|
+
// (recorded with no presence event) is logged the moment the view can support the verdict.
|
|
820
|
+
this.ep.on("roster", () => this.logSeatMeshTransitions());
|
|
821
|
+
this.ep.on("presence-view", () => this.logSeatMeshTransitions());
|
|
807
822
|
await this.ep.start();
|
|
808
823
|
await this.ep.setActivity(`supervisor (${this.runtime.kind})`);
|
|
809
824
|
// Per-instance liveness lease (P2 item 3 — the old per-space singleton is DEMOTED per D9). Acquire
|
|
@@ -829,6 +844,31 @@ export class Manager {
|
|
|
829
844
|
}
|
|
830
845
|
this.leaseTimer = setInterval(() => { void this.renewLease(); }, MANAGER_LEASE_RENEW_MS);
|
|
831
846
|
this.leaseTimer.unref?.();
|
|
847
|
+
// THE PEER-READABLE LIVENESS RESPONDER for the manager plane (#1577). Bound HERE — after the
|
|
848
|
+
// lease, before registration — precisely so it is up during the window in which the manager is
|
|
849
|
+
// NOT yet serving. A responder that only appeared once everything worked could never report the
|
|
850
|
+
// broken state it exists to report, which is the reported defect in a new place.
|
|
851
|
+
//
|
|
852
|
+
// WHAT IT GRADES is `serviceServe`, the v0.4 service endpoint's serve handle, which is assigned
|
|
853
|
+
// only after `serveEndpoint` has bound this instance's rails, AND that handle's connection. A
|
|
854
|
+
// closed serve connection holds no subscriptions, while `onServeConnectionClosed` keeps the
|
|
855
|
+
// handle through its re-dial waits, so the handle alone would answer `bound` with every rail
|
|
856
|
+
// gone. The re-dial swaps in the fresh connection and handle together, so `bound` returns only
|
|
857
|
+
// once the rails are served again. A DISCONNECTED serve connection is the same fault without the
|
|
858
|
+
// close: it reconnects without limit, so it stays open while the broker holds none of its rails,
|
|
859
|
+
// and this probe arrives on the separate supervisor connection that is still up. So the grade
|
|
860
|
+
// also requires the serve transport to be connected. That is a RESPONDER fact. It is
|
|
861
|
+
// deliberately NOT "this process is alive": this process is trivially alive whenever it answers
|
|
862
|
+
// at all, so grading on that would make the surface a `pgrep` that also lies in both directions
|
|
863
|
+
// — the exact instrument the reporter's evidence table indicts. Between lease acquisition and a
|
|
864
|
+
// successful registration the honest answer is `unbound`, and that is what a peer is told.
|
|
865
|
+
//
|
|
866
|
+
// The lease row is never consulted and never sent: presence is the whole ask, and the row
|
|
867
|
+
// carries the operator's workspace root and pid.
|
|
868
|
+
this.ep.serveLiveness("manager", () => {
|
|
869
|
+
const s = this.serviceServe;
|
|
870
|
+
return s?.handle && !s.nc.isClosed() && !this.serveTransportDown.has(s.nc) ? "bound" : "unbound";
|
|
871
|
+
});
|
|
832
872
|
// Unit B (static §13.1): after this instance holds its lease, collect the durable static rows
|
|
833
873
|
// now, but do not let their exact-op terminals make the whole space unreachable. The service
|
|
834
874
|
// comes up below, then the sweep overlaps the remaining registration work. Two-window
|
|
@@ -1839,6 +1879,7 @@ export class Manager {
|
|
|
1839
1879
|
events: a.launch.events,
|
|
1840
1880
|
shareTools: a.launch.shareTools,
|
|
1841
1881
|
forkSource: a.launch.forkSource,
|
|
1882
|
+
...(this.forkProvenance(a) ? { resumed: this.forkProvenance(a) } : {}),
|
|
1842
1883
|
sessionId: a.restart?.armed ? this.readManagedSession(a) : a.launch.sessionId,
|
|
1843
1884
|
// Additive and only when the connector supplied one. A seat with no pointer records no
|
|
1844
1885
|
// field, which is what keeps an older inventory and a fresh one the same document shape.
|
|
@@ -1964,38 +2005,64 @@ export class Manager {
|
|
|
1964
2005
|
// revision it has since moved is a CAS the broker correctly refuses, silently, forever.
|
|
1965
2006
|
await this.leaseRenewTask?.catch(() => { });
|
|
1966
2007
|
let releaseFailures = [];
|
|
1967
|
-
|
|
1968
|
-
|
|
1969
|
-
|
|
1970
|
-
|
|
1971
|
-
|
|
2008
|
+
let seatFailure;
|
|
2009
|
+
try {
|
|
2010
|
+
if (this.maintenanceState === "active" && !this.resumeRequired) {
|
|
2011
|
+
if (options.withAgents)
|
|
2012
|
+
await this.teardownManagedAgents();
|
|
2013
|
+
else
|
|
2014
|
+
releaseFailures = this.releaseManagedAgents();
|
|
2015
|
+
}
|
|
2016
|
+
else {
|
|
2017
|
+
// A signal after a partial preservation must never fall back into destructive teardown.
|
|
2018
|
+
await this.stopRetainedAgentsOnExit();
|
|
2019
|
+
}
|
|
1972
2020
|
}
|
|
1973
|
-
|
|
1974
|
-
|
|
1975
|
-
|
|
2021
|
+
catch (e) {
|
|
2022
|
+
seatFailure = e;
|
|
2023
|
+
}
|
|
2024
|
+
// Every step below runs even when an earlier one fails, so a failed stop still closes this
|
|
2025
|
+
// process's connections and listener (#2290). Failures are collected and thrown at the end.
|
|
2026
|
+
const failures = [];
|
|
2027
|
+
const step = async (name, run) => {
|
|
2028
|
+
try {
|
|
2029
|
+
await run();
|
|
2030
|
+
}
|
|
2031
|
+
catch (e) {
|
|
2032
|
+
failures.push(`${name}: ${e.message}`);
|
|
2033
|
+
}
|
|
2034
|
+
};
|
|
2035
|
+
// A seat that may still be alive keeps this instance's leases and registration (see
|
|
2036
|
+
// teardownManagedAgents): they lapse on their TTL instead of being handed to a successor.
|
|
2037
|
+
if (!seatFailure) {
|
|
2038
|
+
await step("release manager lease", () => this.releaseOwnManagerLease());
|
|
2039
|
+
// #1634: hand the renewal lease back on a clean stop so a sibling picks it up on its next pass
|
|
2040
|
+
// rather than waiting out the bucket TTL. A non-owner holds no revision and this is a no-op.
|
|
2041
|
+
await step("release renewal lease", () => this.ep.releaseDaemonRenewalLease());
|
|
1976
2042
|
}
|
|
1977
|
-
await this.releaseOwnManagerLease();
|
|
1978
|
-
// #1634: hand the renewal lease back on a clean stop so a sibling picks it up on its next pass
|
|
1979
|
-
// rather than waiting out the bucket TTL. A non-owner holds no revision and this is a no-op.
|
|
1980
|
-
await this.ep.releaseDaemonRenewalLease();
|
|
1981
2043
|
// Capture BEFORE the serve loop is torn down: `stopServiceServe` clears the state, and what is
|
|
1982
2044
|
// being asked here is "did this process register a service instance", which only the state
|
|
1983
2045
|
// before teardown can answer. Deregistration runs AFTER the serve loop has drained, so no
|
|
1984
2046
|
// in-flight command can write a status back onto the record it just removed.
|
|
1985
2047
|
const registered = this.serviceServe !== undefined;
|
|
1986
|
-
await this.stopServiceServe();
|
|
2048
|
+
await step("stop service serve", () => this.stopServiceServe());
|
|
1987
2049
|
// The drives AFTER the serve loop (no new `run-start` can land) and BEFORE deregistration: a
|
|
1988
2050
|
// released run's status write is the last thing this incarnation says about it.
|
|
1989
|
-
await this.runHosting?.stop();
|
|
2051
|
+
await step("stop run hosting", () => this.runHosting?.stop());
|
|
1990
2052
|
this.runHosting = undefined;
|
|
1991
|
-
if (registered)
|
|
1992
|
-
await this.deregisterServiceOnStop();
|
|
1993
|
-
await this.stopGoalWriter();
|
|
1994
|
-
await this.stopSessionPlane();
|
|
1995
|
-
await this.ep.stop();
|
|
1996
|
-
await this.attach.stop();
|
|
2053
|
+
if (registered && !seatFailure)
|
|
2054
|
+
await step("deregister service", () => this.deregisterServiceOnStop());
|
|
2055
|
+
await step("stop goal writer", () => this.stopGoalWriter());
|
|
2056
|
+
await step("stop session plane", () => this.stopSessionPlane());
|
|
2057
|
+
await step("stop endpoint", () => this.ep.stop());
|
|
2058
|
+
await step("stop attach endpoint", () => this.attach.stop());
|
|
2059
|
+
const also = failures.length ? `; teardown also failed: ${failures.join("; ")}` : "";
|
|
2060
|
+
if (seatFailure)
|
|
2061
|
+
throw new Error(`${seatFailure.message}${also}`);
|
|
1997
2062
|
if (releaseFailures.length)
|
|
1998
|
-
throw new Error(`manager shutdown could not release every detachable seat: ${releaseFailures.join("; ")}`);
|
|
2063
|
+
throw new Error(`manager shutdown could not release every detachable seat: ${releaseFailures.join("; ")}${also}`);
|
|
2064
|
+
if (failures.length)
|
|
2065
|
+
throw new Error(`manager shutdown incomplete: ${failures.join("; ")}`);
|
|
1999
2066
|
}
|
|
2000
2067
|
/** Release this instance's liveness lease key on a clean stop, at the broker's own revision
|
|
2001
2068
|
* rather than a cached one that a renew still in flight moments ago may have already moved past.
|
|
@@ -2432,33 +2499,6 @@ export class Manager {
|
|
|
2432
2499
|
throw new EpEnvelopeError("permission-denied", `an any-mode ${ctx.subject.command} is operator reach; the caller's current ledger grant does not carry "admin" (SPEC 13.2)`);
|
|
2433
2500
|
return true;
|
|
2434
2501
|
}
|
|
2435
|
-
/** The v0.4 typed command table (P2 item 1, slice 1b): every ordinary handler runs the SHARED
|
|
2436
|
-
* admission chokepoint ({@link serveGated}) and then delegates to the SAME op core the ctl
|
|
2437
|
-
* door dispatches (checklist 8: one core, two thin doors). The resume/preservation family
|
|
2438
|
-
* deliberately BYPASSES serveGated — exactly as it sits before {@link admitControl} on the ctl
|
|
2439
|
-
* door (those ops must run while `resumeRequired` fences ordinary work) — riding its own state
|
|
2440
|
-
* fences; its ep gate is the admin-grade `manager.admin` capability grant (the 1b rule: static
|
|
2441
|
-
* admin-class commands are capability-gated + untargeted, never a fabricated ledger mode).
|
|
2442
|
-
*
|
|
2443
|
-
* TIER SEMANTICS on the ep door (the 1c grant-migration table): the tier lives in the CALLER'S
|
|
2444
|
-
* GRANT, refined per-op exactly as the ctl doors refine their subject tier. Owner-mode
|
|
2445
|
-
* `despawn`/`attach` keep the privileged semantics (`admin=false`, own-domain via
|
|
2446
|
-
* {@link authorizeNamed}) — every spawn-capable agent holds those rows. ANY-mode requests are
|
|
2447
|
-
* the operator instrument's cross-agent reach (rev 3): the any-mode subject row is mintable
|
|
2448
|
-
* only under operator policy (§13.2), so on a static mesh holding it IS the admin tier, and in
|
|
2449
|
-
* user mode the caller's CURRENT ledger scope must still carry `admin`
|
|
2450
|
-
* ({@link epAdminReach}, the same fresh-read authority `psOwnerFilter` consults). The
|
|
2451
|
-
* `manager.admin` family (purge + the resume/preservation ops) is capability-gated at mint AND
|
|
2452
|
-
* re-checked at serve time via {@link epAdminReach} (the `adminGated` wrapper) so a user's
|
|
2453
|
-
* revoked scope demotes the next call. `launch` is OWNER-EQUALITY on this door for everyone
|
|
2454
|
-
* (freelance HIGH #2): the deploy path is its only consumer and stamps the caller's own owner,
|
|
2455
|
-
* so cross-owner launch was a ctl-tier incidental never exercised, and keying it on the actor's
|
|
2456
|
-
* ledger scope broke the deployer-view attenuation - uniform owner-equality is the safe tier.
|
|
2457
|
-
* TWO DELIBERATE NARROWINGS vs the ctl doors (NOT bit-exact parity, panel-accepted): (1)
|
|
2458
|
-
* `define-persona` is `admin=false` for everyone (own-persona discipline; no ep consumer needs
|
|
2459
|
-
* cross-owner persona writes - an operator redefines via config, not the wire), where the ctl
|
|
2460
|
-
* admin tier allowed operator cross-owner redefine; (2) launch is owner-equality-only, above.
|
|
2461
|
-
* Both are least-privilege reductions, never widenings. */
|
|
2462
2502
|
/** The run host, or one of three refusals. A remote-authority manager holds no space signer, so
|
|
2463
2503
|
* it cannot mint the per-run driver credential SPEC 14.6 requires without all four closed
|
|
2464
2504
|
* host callbacks. A local user-auth mesh is `unimplemented`: a hosted run's seats
|
|
@@ -2515,6 +2555,44 @@ export class Manager {
|
|
|
2515
2555
|
if (p.run !== args.runId || p.step !== args.stepKey || addressedToken !== open.token)
|
|
2516
2556
|
throw new EpEnvelopeError("permission-denied", `run-answer is allowed to ${agent.name} only for the open pause named by its pending ask or escalation relay; ${args.runId} ${args.stepKey} is not that pause`);
|
|
2517
2557
|
}
|
|
2558
|
+
/** A baseline seat may amend only an answer recorded under its own name: it changes its own
|
|
2559
|
+
* position, never another answerer's. Explicit `run` holders and unmanaged operator instruments
|
|
2560
|
+
* keep the reach they have to answer, as {@link authorizeRunAnswer} does. */
|
|
2561
|
+
authorizeRunAmend(ctx, accepted) {
|
|
2562
|
+
const caller = principalKey(ctx.subject.caller.owner, ctx.subject.caller.actor).key;
|
|
2563
|
+
const agent = [...this.agents.values()].find((a) => this.managedPrincipal(a) === caller && a.lifecycleUid === ctx.subject.caller.uid);
|
|
2564
|
+
if (agent === undefined || agent.launch.capabilities?.includes("run"))
|
|
2565
|
+
return;
|
|
2566
|
+
if (accepted.by !== agent.name)
|
|
2567
|
+
throw new EpEnvelopeError("permission-denied", `run-answer amend is allowed to ${agent.name} only for an answer recorded under its own name; this step's accepted answer is ${JSON.stringify(accepted.by)}'s`);
|
|
2568
|
+
}
|
|
2569
|
+
/** The v0.4 typed command table (P2 item 1, slice 1b): every ordinary handler runs the SHARED
|
|
2570
|
+
* admission chokepoint ({@link serveGated}) and then delegates to the SAME op core the ctl
|
|
2571
|
+
* door dispatches (checklist 8: one core, two thin doors). The resume/preservation family
|
|
2572
|
+
* deliberately BYPASSES serveGated — exactly as it sits before {@link admitControl} on the ctl
|
|
2573
|
+
* door (those ops must run while `resumeRequired` fences ordinary work) — riding its own state
|
|
2574
|
+
* fences; its ep gate is the admin-grade `manager.admin` capability grant (the 1b rule: static
|
|
2575
|
+
* admin-class commands are capability-gated + untargeted, never a fabricated ledger mode).
|
|
2576
|
+
*
|
|
2577
|
+
* TIER SEMANTICS on the ep door (the 1c grant-migration table): the tier lives in the CALLER'S
|
|
2578
|
+
* GRANT, refined per-op exactly as the ctl doors refine their subject tier. Owner-mode
|
|
2579
|
+
* `despawn`/`attach` keep the privileged semantics (`admin=false`, own-domain via
|
|
2580
|
+
* {@link authorizeNamed}) — every spawn-capable agent holds those rows. ANY-mode requests are
|
|
2581
|
+
* the operator instrument's cross-agent reach (rev 3): the any-mode subject row is mintable
|
|
2582
|
+
* only under operator policy (§13.2), so on a static mesh holding it IS the admin tier, and in
|
|
2583
|
+
* user mode the caller's CURRENT ledger scope must still carry `admin`
|
|
2584
|
+
* ({@link epAdminReach}, the same fresh-read authority `psOwnerFilter` consults). The
|
|
2585
|
+
* `manager.admin` family (purge + the resume/preservation ops) is capability-gated at mint AND
|
|
2586
|
+
* re-checked at serve time via {@link epAdminReach} (the `adminGated` wrapper) so a user's
|
|
2587
|
+
* revoked scope demotes the next call. `launch` is OWNER-EQUALITY on this door for everyone
|
|
2588
|
+
* (freelance HIGH #2): the deploy path is its only consumer and stamps the caller's own owner,
|
|
2589
|
+
* so cross-owner launch was a ctl-tier incidental never exercised, and keying it on the actor's
|
|
2590
|
+
* ledger scope broke the deployer-view attenuation - uniform owner-equality is the safe tier.
|
|
2591
|
+
* TWO DELIBERATE NARROWINGS vs the ctl doors (NOT bit-exact parity, panel-accepted): (1)
|
|
2592
|
+
* `define-persona` is `admin=false` for everyone (own-persona discipline; no ep consumer needs
|
|
2593
|
+
* cross-owner persona writes - an operator redefines via config, not the wire), where the ctl
|
|
2594
|
+
* admin tier allowed operator cross-owner redefine; (2) launch is owner-equality-only, above.
|
|
2595
|
+
* Both are least-privilege reductions, never widenings. */
|
|
2518
2596
|
managerServiceDefs() {
|
|
2519
2597
|
const args = (ctx) => (ctx.request.args ?? {});
|
|
2520
2598
|
const callerOf = (ctx) => principalKey(ctx.subject.caller.owner, ctx.subject.caller.actor).key;
|
|
@@ -2703,6 +2781,8 @@ export class Manager {
|
|
|
2703
2781
|
runResume: (ctx) => this.serveGated(ctx, () => this.runHost().resume(args(ctx))),
|
|
2704
2782
|
runAnswer: (ctx) => this.serveGated(ctx, () => {
|
|
2705
2783
|
const input = args(ctx);
|
|
2784
|
+
if (input.amend === true)
|
|
2785
|
+
return this.runHost().amend(input, this.runAnswerer(ctx), (accepted) => this.authorizeRunAmend(ctx, accepted));
|
|
2706
2786
|
return this.runHost().answer(input, this.runAnswerer(ctx), (open) => this.authorizeRunAnswer(ctx, input, open));
|
|
2707
2787
|
}),
|
|
2708
2788
|
runStatus: (ctx) => this.serveGated(ctx, () => this.runHost().status(args(ctx))),
|
|
@@ -3137,11 +3217,6 @@ export class Manager {
|
|
|
3137
3217
|
return { error: `agent auth preflight failed for "${name}": ${e.message}` };
|
|
3138
3218
|
}
|
|
3139
3219
|
}
|
|
3140
|
-
/** Drop a live agent's slot. When `floor` is set and the agent died young (lived less than
|
|
3141
|
-
* MIN_LIFETIME), push a cooling stamp so the freed slot still counts toward the ceiling until it
|
|
3142
|
-
* expires — flooring the RECYCLE, not the call, so both free paths (despawn + exit/reap) are
|
|
3143
|
-
* covered (P4c). Floor self + own-child despawn and natural exit; NEVER admin despawn (operator
|
|
3144
|
-
* emergency-kill stays unthrottled) and NEVER the reserved-rollback path (no cold-start paid). */
|
|
3145
3220
|
/**
|
|
3146
3221
|
* WHY A SEAT LEFT — one line, at the one place every free path passes through.
|
|
3147
3222
|
*
|
|
@@ -3164,7 +3239,7 @@ export class Manager {
|
|
|
3164
3239
|
const info = a.handle.exitInfo?.();
|
|
3165
3240
|
detail = info === undefined
|
|
3166
3241
|
? `exit detail unavailable from runtime "${a.handle.kind}"`
|
|
3167
|
-
: `exit code ${info.code ?? "unknown"}${info.signal === undefined ? "" : `, signal ${info.signal}`}`;
|
|
3242
|
+
: `exit code ${info.code ?? "unknown"}${info.signal === undefined ? "" : `, signal ${info.signal}`}${info.diagnostic ? `; last connector diagnostic: ${info.diagnostic}` : ""}`;
|
|
3168
3243
|
}
|
|
3169
3244
|
catch (e) {
|
|
3170
3245
|
// A runtime that throws while being asked has told us something real; it must not take the
|
|
@@ -3173,6 +3248,49 @@ export class Manager {
|
|
|
3173
3248
|
}
|
|
3174
3249
|
console.error(`seat reaped: ${a.name} (${a.id}, uid ${a.lifecycleUid}${a.handle.pid !== undefined ? `, pid ${a.handle.pid}` : ""}) - ${causeText}; ${detail}`);
|
|
3175
3250
|
}
|
|
3251
|
+
/**
|
|
3252
|
+
* A SEAT THAT LEAVES THE MESH WHILE ITS SLOT IS HELD GETS A LINE, AND SO DOES ITS RETURN.
|
|
3253
|
+
*
|
|
3254
|
+
* Seats once read `running · mesh offline` for many hours with no manager log line, so a watchdog
|
|
3255
|
+
* keyed on process liveness saw nothing wrong and nobody could say when each seat dropped (#1208).
|
|
3256
|
+
* The line follows the same state `ps` reads, checked on every roster change and view transition,
|
|
3257
|
+
* because not every offline record arrives as a presence event: after a reconnect the fresh snapshot
|
|
3258
|
+
* records an already-offline seat quietly. Only a current view is a verdict, so a stale or
|
|
3259
|
+
* unpopulated one logs nothing. The match is {@link isOwnPresence}. A seat being stopped is left to
|
|
3260
|
+
* the reap line. The heartbeat is the seat's own record and nothing validated it, so an unreadable
|
|
3261
|
+
* one prints as unknown instead of throwing inside the watch.
|
|
3262
|
+
*/
|
|
3263
|
+
logSeatMeshTransitions() {
|
|
3264
|
+
if (typeof this.ep.presenceView === "function" && this.ep.presenceView().state !== "current")
|
|
3265
|
+
return;
|
|
3266
|
+
const roster = new Map(this.ep.getRoster().map((p) => [p.card.id, p]));
|
|
3267
|
+
for (const a of this.agents.values()) {
|
|
3268
|
+
const seen = roster.get(this.managedPrincipal(a));
|
|
3269
|
+
if (a.terminalizing || !this.isOwnPresence(a, seen))
|
|
3270
|
+
continue;
|
|
3271
|
+
const who = `${a.name} (${a.id}, uid ${a.lifecycleUid}${a.handle.pid !== undefined ? `, pid ${a.handle.pid}` : ""})`;
|
|
3272
|
+
if (seen.status === "offline" && !a.offMesh) {
|
|
3273
|
+
a.offMesh = true;
|
|
3274
|
+
const ts = seen.ts;
|
|
3275
|
+
const last = typeof ts === "number" && !Number.isNaN(new Date(ts).getTime()) ? new Date(ts).toISOString() : "unknown";
|
|
3276
|
+
console.error(`seat offline on the mesh: ${who} - last heartbeat ${last}; process ${a.handle.status()}`);
|
|
3277
|
+
}
|
|
3278
|
+
else if (seen.status !== "offline" && a.offMesh) {
|
|
3279
|
+
a.offMesh = false;
|
|
3280
|
+
console.error(`seat back on the mesh: ${who}`);
|
|
3281
|
+
}
|
|
3282
|
+
}
|
|
3283
|
+
}
|
|
3284
|
+
/** Whether a roster record is this seat's own: the readiness fence's exact principal and
|
|
3285
|
+
* incarnation, so a same-named foreign peer or an older incarnation never speaks for it (#1208). */
|
|
3286
|
+
isOwnPresence(a, p) {
|
|
3287
|
+
return p !== undefined && p.card.id === this.managedPrincipal(a) && p.lifecycleUid === a.lifecycleUid;
|
|
3288
|
+
}
|
|
3289
|
+
/** Drop a live agent's slot. When `floor` is set and the agent died young (lived less than
|
|
3290
|
+
* MIN_LIFETIME), push a cooling stamp so the freed slot still counts toward the ceiling until it
|
|
3291
|
+
* expires — flooring the RECYCLE, not the call, so both free paths (despawn + exit/reap) are
|
|
3292
|
+
* covered (P4c). Floor self + own-child despawn and natural exit; NEVER admin despawn (operator
|
|
3293
|
+
* emergency-kill stays unthrottled) and NEVER the reserved-rollback path (no cold-start paid). */
|
|
3176
3294
|
freeSlot(a, floor, cause, acceptedBeforeFence = false) {
|
|
3177
3295
|
if (this.agents.get(a.name) !== a)
|
|
3178
3296
|
return; // already freed (exit raced despawn, etc.)
|
|
@@ -3631,6 +3749,28 @@ export class Manager {
|
|
|
3631
3749
|
`(${error.message}). Reload the live Pi seat before the preservation cut; refusing to resume fresh and lose its context.`);
|
|
3632
3750
|
}
|
|
3633
3751
|
}
|
|
3752
|
+
/** A resumed seat's fork provenance (#1500). The seat forks after it launches, so this reads the
|
|
3753
|
+
* record its connector named once the seat has written it, and keeps it on the launch so the
|
|
3754
|
+
* resume document carries it across a manager restart. Until the seat has written it, or when the
|
|
3755
|
+
* record cannot be read or does not name this seat's source, it is absent: `ps` then shows the
|
|
3756
|
+
* source id alone rather than failing the listing for every other seat. */
|
|
3757
|
+
forkProvenance(a) {
|
|
3758
|
+
if (a.launch.resumed || !a.launch.resumeRecordPath || !a.launch.forkSource)
|
|
3759
|
+
return a.launch.resumed;
|
|
3760
|
+
let record;
|
|
3761
|
+
try {
|
|
3762
|
+
record = JSON.parse(readFileSync(a.launch.resumeRecordPath, "utf8"));
|
|
3763
|
+
}
|
|
3764
|
+
catch {
|
|
3765
|
+
return undefined;
|
|
3766
|
+
}
|
|
3767
|
+
const r = (typeof record === "object" && record !== null ? record : {});
|
|
3768
|
+
if (r.source !== a.launch.forkSource || typeof r.transcriptSha256 !== "string" || !/^[a-f0-9]{64}$/.test(r.transcriptSha256) ||
|
|
3769
|
+
!(r.title === undefined || r.title === null || (typeof r.title === "string" && r.title.length <= 1024)))
|
|
3770
|
+
return undefined;
|
|
3771
|
+
a.launch.resumed = { source: r.source, ...(typeof r.title === "string" && r.title ? { title: r.title } : {}), transcriptSha256: r.transcriptSha256 };
|
|
3772
|
+
return a.launch.resumed;
|
|
3773
|
+
}
|
|
3634
3774
|
readManagedSession(a) {
|
|
3635
3775
|
return this.readManagedSessionState(a).sessionId;
|
|
3636
3776
|
}
|
|
@@ -4341,9 +4481,9 @@ export class Manager {
|
|
|
4341
4481
|
// SYNCHRONOUS (existsSync / registry / accessSync / readFileSync — no await), so the gate stays
|
|
4342
4482
|
// atomic: the capacity snapshot and the reserve land in one tick (P4a/P4c), and two concurrent
|
|
4343
4483
|
// spawns can't overshoot the ceiling or pick the same name.
|
|
4344
|
-
const
|
|
4345
|
-
if (
|
|
4346
|
-
return { ok: false, error:
|
|
4484
|
+
const occupancy = this.occupancy(); // prunes expired cooling stamps; counts each name once
|
|
4485
|
+
if (occupancy.used >= MAX_AGENTS)
|
|
4486
|
+
return { ok: false, error: this.capacityRefusal(occupancy) };
|
|
4347
4487
|
// Harness preflight before reserving a slot or minting — a missing `claude`/`opencode` binary
|
|
4348
4488
|
// fails here with a clear name, not obscurely at process spawn. No fallback. All synchronous, so
|
|
4349
4489
|
// the reserve gate stays atomic. (The connector itself was resolved up top, before the capacity gate.)
|
|
@@ -4833,6 +4973,7 @@ export class Manager {
|
|
|
4833
4973
|
events,
|
|
4834
4974
|
shareTools: opts.shareTools,
|
|
4835
4975
|
forkSource: opts.resume,
|
|
4976
|
+
...(opts.resume !== undefined && spec.resumeRecordPath ? { resumeRecordPath: spec.resumeRecordPath } : {}),
|
|
4836
4977
|
// Opaque values may contain secrets. Preserve only their keys and require the referenced
|
|
4837
4978
|
// persona/manifest to resolve the values again; imperative overrides have no safe payload.
|
|
4838
4979
|
unresolvedLaunchOptionKeys: opts.launchOptions && Object.keys(opts.launchOptions).length
|
|
@@ -4867,6 +5008,7 @@ export class Manager {
|
|
|
4867
5008
|
});
|
|
4868
5009
|
}
|
|
4869
5010
|
this.agents.set(name, managed);
|
|
5011
|
+
this.reservedLive.add(name);
|
|
4870
5012
|
// The live slot now owns teardown — freeSlot deprovisions this identity on exit — so the
|
|
4871
5013
|
// orphan-rollback in `finally` no longer applies to it.
|
|
4872
5014
|
provisioned = undefined;
|
|
@@ -4936,6 +5078,7 @@ export class Manager {
|
|
|
4936
5078
|
}
|
|
4937
5079
|
finally {
|
|
4938
5080
|
this.reserved.delete(name);
|
|
5081
|
+
this.reservedLive.delete(name);
|
|
4939
5082
|
// Minted but never handed to a live slot (buildLaunch / runtime.spawn threw after mint) → tear the
|
|
4940
5083
|
// orphan down (detached, fail-loud) so a failed spawn leaves no creds/durables behind (#159 B).
|
|
4941
5084
|
if (provisioned) {
|
|
@@ -4952,6 +5095,7 @@ export class Manager {
|
|
|
4952
5095
|
if (!release)
|
|
4953
5096
|
return { ok: false, agents: [], error: this.maintenanceError() };
|
|
4954
5097
|
const batchReservations = [];
|
|
5098
|
+
const prepared = new Map();
|
|
4955
5099
|
try {
|
|
4956
5100
|
if (inventory.version !== "cotal-manager-resume/v1")
|
|
4957
5101
|
return { ok: false, agents: [], error: `unsupported manager resume inventory version ${String(inventory.version)}` };
|
|
@@ -4961,8 +5105,9 @@ export class Manager {
|
|
|
4961
5105
|
const principals = new Set();
|
|
4962
5106
|
await this.ep.waitForPresenceSnapshot();
|
|
4963
5107
|
const liveRoster = this.ep.getRoster().filter((presence) => presence.status !== "offline");
|
|
4964
|
-
|
|
4965
|
-
|
|
5108
|
+
const occupancy = this.occupancy();
|
|
5109
|
+
if (occupancy.used + inventory.agents.length > MAX_AGENTS)
|
|
5110
|
+
return { ok: false, agents: [], error: `resume inventory of ${inventory.agents.length} would exceed manager capacity (${this.occupancyText(occupancy)})` };
|
|
4966
5111
|
for (const entry of inventory.agents) {
|
|
4967
5112
|
if (seen.has(entry.name))
|
|
4968
5113
|
return { ok: false, agents: [], error: `resume inventory contains duplicate agent name "${entry.name}"` };
|
|
@@ -5018,7 +5163,6 @@ export class Manager {
|
|
|
5018
5163
|
this.reserved.add(entry.name);
|
|
5019
5164
|
batchReservations.push(entry.name);
|
|
5020
5165
|
}
|
|
5021
|
-
const prepared = new Map();
|
|
5022
5166
|
const preflight = [];
|
|
5023
5167
|
for (const entry of inventory.agents) {
|
|
5024
5168
|
const reply = await this.resumePreservedAgent(entry, true, true, prepared);
|
|
@@ -5047,8 +5191,22 @@ export class Manager {
|
|
|
5047
5191
|
return { ok: true, agents };
|
|
5048
5192
|
}
|
|
5049
5193
|
finally {
|
|
5050
|
-
for (const name of batchReservations)
|
|
5194
|
+
for (const name of batchReservations) {
|
|
5051
5195
|
this.reserved.delete(name);
|
|
5196
|
+
this.reservedLive.delete(name);
|
|
5197
|
+
}
|
|
5198
|
+
// Preflight builds every launch before any child starts, so a refused batch, a failed launch,
|
|
5199
|
+
// or the agents skipped after it leave built specs no runtime ever received.
|
|
5200
|
+
for (const { spec, spawned } of prepared.values()) {
|
|
5201
|
+
if (spawned)
|
|
5202
|
+
continue;
|
|
5203
|
+
try {
|
|
5204
|
+
discardLaunchArtifacts(spec.artifacts);
|
|
5205
|
+
}
|
|
5206
|
+
catch (e) {
|
|
5207
|
+
console.error(`! resume: ${e.message}`);
|
|
5208
|
+
}
|
|
5209
|
+
}
|
|
5052
5210
|
release();
|
|
5053
5211
|
}
|
|
5054
5212
|
}
|
|
@@ -5232,8 +5390,9 @@ export class Manager {
|
|
|
5232
5390
|
return { ok: false, error: nameErr };
|
|
5233
5391
|
if (this.agents.has(entry.name) || (!batchReserved && this.reserved.has(entry.name)))
|
|
5234
5392
|
return { ok: false, error: `retained agent "${entry.name}" is already managed or reserved; same-principal resume never auto-numbers` };
|
|
5235
|
-
|
|
5236
|
-
|
|
5393
|
+
const occupancy = batchReserved ? undefined : this.occupancy();
|
|
5394
|
+
if (occupancy && occupancy.used >= MAX_AGENTS)
|
|
5395
|
+
return { ok: false, error: `${this.capacityRefusal(occupancy)}; same-principal resume refused` };
|
|
5237
5396
|
const cached = prepared?.get(entry.name);
|
|
5238
5397
|
if (!preflightOnly && cached) {
|
|
5239
5398
|
try {
|
|
@@ -5392,6 +5551,7 @@ export class Manager {
|
|
|
5392
5551
|
const custody = this.reserveCustody();
|
|
5393
5552
|
if (custody)
|
|
5394
5553
|
await this.recordSlotCustody({ name: entry.name, id: resumeId, lifecycleUid: entry.identity.lifecycleUid }, custody);
|
|
5554
|
+
prepared.spawned = true;
|
|
5395
5555
|
const handle = await this.spawnCustodied(entry.name, prepared.spec, entry.launch.cwd, custody);
|
|
5396
5556
|
const managed = {
|
|
5397
5557
|
name: entry.name,
|
|
@@ -5428,6 +5588,8 @@ export class Manager {
|
|
|
5428
5588
|
events: entry.launch.events,
|
|
5429
5589
|
shareTools: entry.launch.shareTools,
|
|
5430
5590
|
forkSource: entry.launch.forkSource,
|
|
5591
|
+
...(entry.launch.resumed ? { resumed: entry.launch.resumed } : {}),
|
|
5592
|
+
...(prepared.launchOpts.resume !== undefined && prepared.spec.resumeRecordPath ? { resumeRecordPath: prepared.spec.resumeRecordPath } : {}),
|
|
5431
5593
|
sessionId: entry.launch.sessionId,
|
|
5432
5594
|
},
|
|
5433
5595
|
...(prepared.spec.sessionStatePath
|
|
@@ -5436,6 +5598,7 @@ export class Manager {
|
|
|
5436
5598
|
suppressCleanup: true,
|
|
5437
5599
|
};
|
|
5438
5600
|
this.agents.set(entry.name, managed);
|
|
5601
|
+
this.reservedLive.add(entry.name);
|
|
5439
5602
|
if (this.resumeAttemptId)
|
|
5440
5603
|
this.resumedAgentNames.add(entry.name);
|
|
5441
5604
|
await this.recordSlotRuntime(managed);
|
|
@@ -5472,8 +5635,10 @@ export class Manager {
|
|
|
5472
5635
|
return { ok: false, error: e.message };
|
|
5473
5636
|
}
|
|
5474
5637
|
finally {
|
|
5475
|
-
if (!batchReserved)
|
|
5638
|
+
if (!batchReserved) {
|
|
5476
5639
|
this.reserved.delete(entry.name);
|
|
5640
|
+
this.reservedLive.delete(entry.name);
|
|
5641
|
+
}
|
|
5477
5642
|
}
|
|
5478
5643
|
}
|
|
5479
5644
|
probeStaticCredential(creds) {
|
|
@@ -5677,6 +5842,48 @@ export class Manager {
|
|
|
5677
5842
|
this.cooling = this.cooling.filter((stamp) => stamp > now);
|
|
5678
5843
|
return this.cooling.length;
|
|
5679
5844
|
}
|
|
5845
|
+
/** What holds the ceiling's slots, one count per seat (#906). A launching seat sits in `reserved`
|
|
5846
|
+
* from accept until its launch settles, in `agents` once its process exists, and in `cooling`
|
|
5847
|
+
* if it then frees young, so a reservation stops counting once it reaches `agents`. `managed` is
|
|
5848
|
+
* what this manager's `ps` lists, `reserved` is the accepted launches that have no process yet,
|
|
5849
|
+
* and `launching` is every unsettled launch that still holds a slot through `reserved` or
|
|
5850
|
+
* `agents`. A launch whose seat already freed holds no slot but its cooling stamp, so its
|
|
5851
|
+
* failure can free nothing. */
|
|
5852
|
+
occupancy() {
|
|
5853
|
+
const cooling = this.coolingCount();
|
|
5854
|
+
let reserved = 0;
|
|
5855
|
+
let launching = 0;
|
|
5856
|
+
for (const name of this.reserved) {
|
|
5857
|
+
const placed = this.reservedLive.has(name);
|
|
5858
|
+
if (!placed)
|
|
5859
|
+
reserved++;
|
|
5860
|
+
if (!placed || this.agents.has(name))
|
|
5861
|
+
launching++;
|
|
5862
|
+
}
|
|
5863
|
+
const managed = this.agents.size;
|
|
5864
|
+
return {
|
|
5865
|
+
managed,
|
|
5866
|
+
reserved,
|
|
5867
|
+
cooling,
|
|
5868
|
+
launching,
|
|
5869
|
+
used: managed + reserved + cooling,
|
|
5870
|
+
nextCoolingMs: cooling ? Math.min(...this.cooling) - Date.now() : undefined,
|
|
5871
|
+
};
|
|
5872
|
+
}
|
|
5873
|
+
occupancyText(o) {
|
|
5874
|
+
return `${o.used} of ${MAX_AGENTS} slots: ${o.managed} managed, ${o.reserved} reserved, ${o.cooling} cooling`;
|
|
5875
|
+
}
|
|
5876
|
+
/** The capacity refusal states the split it enforces and whether waiting can free a slot: a
|
|
5877
|
+
* cooling slot frees at a known time, an unsettled launch frees one only if it fails, and a
|
|
5878
|
+
* managed seat frees one only when it is stopped. */
|
|
5879
|
+
capacityRefusal(o) {
|
|
5880
|
+
const remedy = o.nextCoolingMs !== undefined
|
|
5881
|
+
? `waiting frees a cooling slot in ${Math.ceil(o.nextCoolingMs / 1000)}s, or despawn one`
|
|
5882
|
+
: o.launching
|
|
5883
|
+
? `waiting frees a slot only if an unsettled launch fails (${o.launching} pending); otherwise despawn one`
|
|
5884
|
+
: "waiting frees no slot; despawn one";
|
|
5885
|
+
return `at capacity (${this.occupancyText(o)}); ${remedy}`;
|
|
5886
|
+
}
|
|
5680
5887
|
async opStop(args, caller, admin) {
|
|
5681
5888
|
const name = String(args.name ?? "").trim();
|
|
5682
5889
|
const a = this.agents.get(name);
|
|
@@ -5836,7 +6043,7 @@ export class Manager {
|
|
|
5836
6043
|
return {
|
|
5837
6044
|
instanceId: this.managerInstanceId,
|
|
5838
6045
|
runtime: this.runtime.kind,
|
|
5839
|
-
custody:
|
|
6046
|
+
custody: this.runtime.kind === "pty" && this.runtime.supportsRelease === true ? "custodied" : "legacy",
|
|
5840
6047
|
agentCount: this.agents.size,
|
|
5841
6048
|
uptimeMs: Date.now() - this.startedAtMs,
|
|
5842
6049
|
connectors: this.connectorStatuses.map((row) => ({ ...row, binaries: { ...row.binaries } })),
|
|
@@ -5887,7 +6094,7 @@ export class Manager {
|
|
|
5887
6094
|
report = await reconcileEndpointGate({
|
|
5888
6095
|
kv: authKv, space: this.space, endpoint: MANAGER_ENDPOINT, instanceId,
|
|
5889
6096
|
probeHolder: makeManagerHolderLivenessProbe({ space: this.space, servers, auth, log }),
|
|
5890
|
-
|
|
6097
|
+
evictHolders: makeManagerEndpointHolderEvictor({ space: this.space, servers, auth, log }),
|
|
5891
6098
|
log,
|
|
5892
6099
|
recordsKv,
|
|
5893
6100
|
});
|
|
@@ -5921,6 +6128,17 @@ export class Manager {
|
|
|
5921
6128
|
inboxPrefix: `_INBOX_${state.identity.id}`,
|
|
5922
6129
|
maxReconnectAttempts: -1,
|
|
5923
6130
|
});
|
|
6131
|
+
// Track the transport from the first moment: nats.js reports `disconnect` when the socket
|
|
6132
|
+
// drops and `reconnect` only after it has re-sent this connection's subscriptions. If the
|
|
6133
|
+
// status stream itself fails, the transport can no longer be observed, so it counts as down.
|
|
6134
|
+
void (async () => {
|
|
6135
|
+
for await (const st of nc.status()) {
|
|
6136
|
+
if (st.type === "disconnect")
|
|
6137
|
+
this.serveTransportDown.add(nc);
|
|
6138
|
+
else if (st.type === "reconnect")
|
|
6139
|
+
this.serveTransportDown.delete(nc);
|
|
6140
|
+
}
|
|
6141
|
+
})().catch(() => this.serveTransportDown.add(nc));
|
|
5924
6142
|
let handle;
|
|
5925
6143
|
try {
|
|
5926
6144
|
// The 1b typed surface + the derived `describe`. The descriptor stays PUBLIC in static
|
|
@@ -6294,15 +6512,6 @@ export class Manager {
|
|
|
6294
6512
|
});
|
|
6295
6513
|
return creds;
|
|
6296
6514
|
}
|
|
6297
|
-
/** P2 item 2 (spawn-as-action): stand up the standing self-mediated goal-writer connection +
|
|
6298
|
-
* ActionContext. Mode-dual, mirroring {@link registerManagerService}: an AUTH mesh uses the
|
|
6299
|
-
* scoped `goal-writer` credential already minted + family-STAGED inside registration's run block
|
|
6300
|
-
* ({@link mintAndStageGoalWriter} — DISJOINT grant from the serve cred, SHARED §13.1 revocation
|
|
6301
|
-
* family); an OPEN mesh uses a bare connection (no credential system to mint from - the broker
|
|
6302
|
-
* enforces nothing). The connection presents the CURRENT credential on every (re)connect (a
|
|
6303
|
-
* renewal is adopted without reconnecting the whole endpoint); the ActionContext bonds its
|
|
6304
|
-
* KV + JS + JSM to this one connection and space (SPEC 13.4), so a composition mixup cannot splice
|
|
6305
|
-
* goal state across brokers. */
|
|
6306
6515
|
/** #2073: dial the goal-writer's ONE standing connection and rebuild everything derived from it
|
|
6307
6516
|
* (the fence-resolving action context, the issuance-gate reader) — shared by the initial start
|
|
6308
6517
|
* and the closed-connection recovery ({@link onGoalWriterConnectionClosed}), so a redial never
|
|
@@ -6370,6 +6579,15 @@ export class Manager {
|
|
|
6370
6579
|
return;
|
|
6371
6580
|
console.error("! manager goal-writer could not be re-dialed; leaving the credential to the next renewal pass");
|
|
6372
6581
|
}
|
|
6582
|
+
/** P2 item 2 (spawn-as-action): stand up the standing self-mediated goal-writer connection +
|
|
6583
|
+
* ActionContext. Mode-dual, mirroring {@link registerManagerService}: an AUTH mesh uses the
|
|
6584
|
+
* scoped `goal-writer` credential already minted + family-STAGED inside registration's run block
|
|
6585
|
+
* ({@link mintAndStageGoalWriter} — DISJOINT grant from the serve cred, SHARED §13.1 revocation
|
|
6586
|
+
* family); an OPEN mesh uses a bare connection (no credential system to mint from - the broker
|
|
6587
|
+
* enforces nothing). The connection presents the CURRENT credential on every (re)connect (a
|
|
6588
|
+
* renewal is adopted without reconnecting the whole endpoint); the ActionContext bonds its
|
|
6589
|
+
* KV + JS + JSM to this one connection and space (SPEC 13.4), so a composition mixup cannot splice
|
|
6590
|
+
* goal state across brokers. */
|
|
6373
6591
|
async startGoalWriter() {
|
|
6374
6592
|
const identity = (this.auth || this.remoteAuthority) ? this.goalWriterIdentity : newIdentity();
|
|
6375
6593
|
// The mutable holder captured by the authenticator (mirrors the serve connection): a half-TTL
|
|
@@ -6431,18 +6649,6 @@ export class Manager {
|
|
|
6431
6649
|
});
|
|
6432
6650
|
return creds;
|
|
6433
6651
|
}
|
|
6434
|
-
/** P2 item 6: stand up the ONE §13.6 session plane on its own standing connection. Mode-dual,
|
|
6435
|
-
* mirroring {@link startGoalWriter}: an AUTH mesh presents the scoped `session-ledger` cred
|
|
6436
|
-
* already minted + family-staged inside registration's run block ({@link mintAndStageSessionLedger});
|
|
6437
|
-
* an OPEN mesh uses a bare connection (no credential system to mint from). The connection presents
|
|
6438
|
-
* the CURRENT credential on every (re)connect, so a half-TTL renewal is adopted without reconnecting.
|
|
6439
|
-
*
|
|
6440
|
-
* The offer SIGNER is a per-incarnation in-memory keypair: the static collapsed path mints AND
|
|
6441
|
-
* redeems the offer in one call ({@link ManagerSessionPlane.establishAttach}), so the manager
|
|
6442
|
-
* self-signs and self-verifies its own §13.6 grants and the keypair never leaves the process — a
|
|
6443
|
-
* holder never verifies the signature (it presents the grant back over the rail; the broker's
|
|
6444
|
-
* per-session caller cred is the holder's real fence). The plane's ledger lives in the DEDICATED
|
|
6445
|
-
* sessions bucket (createEndpointStreams provisioned it at registration). */
|
|
6446
6652
|
/** #2073: dial the session-ledger's ONE standing connection — shared by the initial start and the
|
|
6447
6653
|
* closed-connection recovery ({@link onSessionLedgerConnectionClosed}), the same shape {@link
|
|
6448
6654
|
* dialGoalWriter} uses for its rail. The KV/JetStream clients the session plane opened from the
|
|
@@ -6487,6 +6693,18 @@ export class Manager {
|
|
|
6487
6693
|
return;
|
|
6488
6694
|
console.error("! manager session-ledger could not be re-dialed; leaving the credential to the next renewal pass");
|
|
6489
6695
|
}
|
|
6696
|
+
/** P2 item 6: stand up the ONE §13.6 session plane on its own standing connection. Mode-dual,
|
|
6697
|
+
* mirroring {@link startGoalWriter}: an AUTH mesh presents the scoped `session-ledger` cred
|
|
6698
|
+
* already minted + family-staged inside registration's run block ({@link mintAndStageSessionLedger});
|
|
6699
|
+
* an OPEN mesh uses a bare connection (no credential system to mint from). The connection presents
|
|
6700
|
+
* the CURRENT credential on every (re)connect, so a half-TTL renewal is adopted without reconnecting.
|
|
6701
|
+
*
|
|
6702
|
+
* The offer SIGNER is a per-incarnation in-memory keypair: the static collapsed path mints AND
|
|
6703
|
+
* redeems the offer in one call ({@link ManagerSessionPlane.establishAttach}), so the manager
|
|
6704
|
+
* self-signs and self-verifies its own §13.6 grants and the keypair never leaves the process — a
|
|
6705
|
+
* holder never verifies the signature (it presents the grant back over the rail; the broker's
|
|
6706
|
+
* per-session caller cred is the holder's real fence). The plane's ledger lives in the DEDICATED
|
|
6707
|
+
* sessions bucket (createEndpointStreams provisioned it at registration). */
|
|
6490
6708
|
async startSessionPlane() {
|
|
6491
6709
|
const identity = (this.auth || this.remoteAuthority) ? this.sessionLedgerIdentity : newIdentity();
|
|
6492
6710
|
// The mutable holder captured by the authenticator (mirrors the goal-writer): a half-TTL renewal
|
|
@@ -8627,8 +8845,9 @@ export class Manager {
|
|
|
8627
8845
|
throw e;
|
|
8628
8846
|
}
|
|
8629
8847
|
}
|
|
8630
|
-
/** Managed agents cross-referenced with live presence (the manager sees the roster).
|
|
8631
|
-
|
|
8848
|
+
/** Managed agents cross-referenced with live presence (the manager sees the roster).
|
|
8849
|
+
*
|
|
8850
|
+
* `ownerFilter`: restrict to agents whose spawn-time stored `userOwner` equals it (the ps/status
|
|
8632
8851
|
* owner-domain bound); undefined = unbounded. {@link NO_OWNER_MATCHES} matches nothing. */
|
|
8633
8852
|
list(ownerFilter) {
|
|
8634
8853
|
const roster = new Map(this.ep.getRoster().map((p) => [p.card.name, p]));
|
|
@@ -8650,8 +8869,33 @@ export class Manager {
|
|
|
8650
8869
|
: undefined;
|
|
8651
8870
|
// The connector-reported provider from presence (#785): the live roster card's `meta.provider`,
|
|
8652
8871
|
// when it is a non-empty string. Absent when the connector reported none — never fabricated.
|
|
8653
|
-
const
|
|
8872
|
+
const seen = roster.get(a.name);
|
|
8873
|
+
const providerMeta = seen?.card.meta?.provider;
|
|
8654
8874
|
const provider = typeof providerMeta === "string" && providerMeta ? providerMeta : undefined;
|
|
8875
|
+
// The harness-reported condition beside the mesh status (#618): a turn that died upstream reads
|
|
8876
|
+
// `waiting` with `rate_limit` here, not a bare `waiting`. Absent when the connector reported none.
|
|
8877
|
+
// Projected field by field like `provider`: a malformed peer row must not fail this reply's schema.
|
|
8878
|
+
const reported = roster.get(a.name)?.condition;
|
|
8879
|
+
const condition = reported && typeof reported.code === "string" && reported.code
|
|
8880
|
+
? {
|
|
8881
|
+
code: reported.code,
|
|
8882
|
+
...(typeof reported.source === "string" ? { source: reported.source } : {}),
|
|
8883
|
+
...(typeof reported.message === "string" ? { message: reported.message } : {}),
|
|
8884
|
+
...(typeof reported.since === "number" && Number.isFinite(reported.since) ? { since: reported.since } : {}),
|
|
8885
|
+
}
|
|
8886
|
+
: undefined;
|
|
8887
|
+
// The seat's last harness-reported work progress (#618). `ts` is only its heartbeat, so a turn
|
|
8888
|
+
// that stopped advancing reads as an old `activeAt` beside a live process. Absent when none was reported.
|
|
8889
|
+
const activeAtReported = roster.get(a.name)?.activeAt;
|
|
8890
|
+
const activeAt = typeof activeAtReported === "number" && Number.isFinite(activeAtReported) ? activeAtReported : undefined;
|
|
8891
|
+
// How long an offline verdict has stood (#1208): the seat's last heartbeat before it went
|
|
8892
|
+
// offline. Only beside a verdict, since a stale or unpopulated view dates nothing, only from the
|
|
8893
|
+
// seat's own record, since `seen` is matched by name and a same-named foreign peer or an older
|
|
8894
|
+
// incarnation must not date this seat, and only when the seat's unvalidated `ts` fits the row
|
|
8895
|
+
// schema, so one odd record cannot fail the reply.
|
|
8896
|
+
const offlineSince = view.state === "current" && this.isOwnPresence(a, seen) && seen.status === "offline" && Number.isSafeInteger(seen.ts) && seen.ts >= 0
|
|
8897
|
+
? seen.ts
|
|
8898
|
+
: undefined;
|
|
8655
8899
|
return {
|
|
8656
8900
|
name: a.name,
|
|
8657
8901
|
// The spawned agent's id (nkey, or the user-mode principal) — lets an operator tool (e.g.
|
|
@@ -8664,10 +8908,13 @@ export class Manager {
|
|
|
8664
8908
|
mode: a.handle.kind,
|
|
8665
8909
|
status: a.handle.status(),
|
|
8666
8910
|
uptimeMs: Date.now() - a.startedAt,
|
|
8667
|
-
mesh:
|
|
8911
|
+
mesh: seen?.status ?? "absent",
|
|
8912
|
+
...(condition ? { condition } : {}),
|
|
8913
|
+
...(activeAt !== undefined ? { activeAt } : {}),
|
|
8668
8914
|
// `current` is the only state in which `mesh` is a verdict; the other two are the
|
|
8669
8915
|
// observer's own condition and travel on the row (older CLIs ignore the field).
|
|
8670
8916
|
meshView: view.state,
|
|
8917
|
+
...(offlineSince !== undefined ? { offlineSince } : {}),
|
|
8671
8918
|
// The incarnation coordinate (SPEC 13.1) — with `id`, exactly what a v0.4 caller needs to
|
|
8672
8919
|
// build a targeted (`despawn`/`attach`) request against THIS incarnation.
|
|
8673
8920
|
lifecycleUid: a.lifecycleUid,
|
|
@@ -8683,6 +8930,9 @@ export class Manager {
|
|
|
8683
8930
|
spawner: a.spawner,
|
|
8684
8931
|
instanceId: this.managerInstanceId,
|
|
8685
8932
|
host: hostname(),
|
|
8933
|
+
// #1500: the session a `--resume` seat forked, with its title and transcript hash once the
|
|
8934
|
+
// seat has recorded them.
|
|
8935
|
+
...(a.launch.forkSource ? { resume: this.forkProvenance(a) ?? { source: a.launch.forkSource } } : {}),
|
|
8686
8936
|
...(health && health.state !== "ok" ? { authHealth: health.state, authReason: health.reason } : {}),
|
|
8687
8937
|
};
|
|
8688
8938
|
});
|