@cotal-ai/manager 0.58.0 → 0.60.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands.d.ts.map +1 -1
- package/dist/commands.js +41 -12
- package/dist/commands.js.map +1 -1
- package/dist/endpoint-evict.d.ts +9 -4
- package/dist/endpoint-evict.d.ts.map +1 -1
- package/dist/endpoint-evict.js +82 -26
- package/dist/endpoint-evict.js.map +1 -1
- package/dist/holder-liveness.d.ts +4 -0
- package/dist/holder-liveness.d.ts.map +1 -1
- package/dist/holder-liveness.js +95 -3
- package/dist/holder-liveness.js.map +1 -1
- package/dist/launch.d.ts +16 -0
- package/dist/launch.d.ts.map +1 -1
- package/dist/launch.js +49 -0
- package/dist/launch.js.map +1 -1
- package/dist/manager-service-contract.d.ts +15 -6
- package/dist/manager-service-contract.d.ts.map +1 -1
- package/dist/manager-service-contract.js +84 -13
- package/dist/manager-service-contract.js.map +1 -1
- package/dist/manager.d.ts +123 -51
- package/dist/manager.d.ts.map +1 -1
- package/dist/manager.js +441 -117
- package/dist/manager.js.map +1 -1
- package/dist/remote-authority.d.ts +12 -1
- package/dist/remote-authority.d.ts.map +1 -1
- package/dist/remote-authority.js +29 -2
- package/dist/remote-authority.js.map +1 -1
- package/dist/resume.d.ts.map +1 -1
- package/dist/resume.js +3 -0
- package/dist/resume.js.map +1 -1
- package/dist/run-hosting.d.ts +12 -1
- package/dist/run-hosting.d.ts.map +1 -1
- package/dist/run-hosting.js +28 -0
- package/dist/run-hosting.js.map +1 -1
- package/dist/runtime/custodial-pty.d.ts +8 -5
- package/dist/runtime/custodial-pty.d.ts.map +1 -1
- package/dist/runtime/custodial-pty.js +23 -12
- package/dist/runtime/custodial-pty.js.map +1 -1
- package/dist/runtime/index.d.ts +2 -0
- package/dist/runtime/index.d.ts.map +1 -1
- package/dist/runtime/index.js +148 -5
- package/dist/runtime/index.js.map +1 -1
- package/dist/runtime/pty.d.ts +3 -4
- package/dist/runtime/pty.d.ts.map +1 -1
- package/dist/runtime/pty.js +9 -6
- package/dist/runtime/pty.js.map +1 -1
- package/package.json +12 -12
- package/dist/ui.d.ts +0 -10
- package/dist/ui.d.ts.map +0 -1
- package/dist/ui.js +0 -10
- package/dist/ui.js.map +0 -1
package/dist/manager.js
CHANGED
|
@@ -4,14 +4,14 @@ import { hostname } from "node:os";
|
|
|
4
4
|
import { credsAuthenticator } from "@nats-io/transport-node";
|
|
5
5
|
import { existsSync, lstatSync, readFileSync, realpathSync, rmSync, statSync } from "node:fs";
|
|
6
6
|
import { isAbsolute, join, dirname, resolve } from "node:path";
|
|
7
|
-
import { CotalEndpoint, accountFromCreds, credsClaims, DEFAULT_SERVER, DEV_OWNER, MANAGER_LEASE_RENEW_MS, STANDING_RENEWABLE_TTL_SEC, assertLifecycleToken, divergentSecretStoreNotice, parseDaemonStoreAnswer, parseSecretStoreIdentity, sameSecretStoreIdentity, agentFilePath, clearSpaceHistory, dialerFor, connectorServers, spawnEnvAllow, deprovisionAgent, DeprovisionError, isConcreteChannel, deprovisionTargetPrincipal, firstFreeName, spawnNameError, idFromCreds, inspectCredHealth, credsRenewalDelayMs, composeWirePersona, loadAgentFile, listPersonaCatalog, personaCatalogDescription, personaCatalogReadable, loadCotalConfig, mintCreds, mintGeneration, mintAcceptedToken, openIssuedStore, acceptedBucket, issuedBucket, mintLifecycleUid, managedRetirementOpId, mkSecretDir, newIdentity, actionContext, parsePrincipalKey, parseShareSelection, principalKey, probeConnect, provisionAgent, provisionAgentDurables, registry, resolveAuthProvider, saveAgentFile, subjectMatches, AUTH_ENDPOINT, EP_CMD_RETIRE_LIFECYCLE, controlServiceSubject, eventChannelPrincipal, resolveService, invokeCommand, } from "@cotal-ai/core";
|
|
7
|
+
import { CotalEndpoint, accountFromCreds, credsClaims, DEFAULT_SERVER, DEV_OWNER, MANAGER_LEASE_RENEW_MS, STANDING_RENEWABLE_TTL_SEC, assertLifecycleToken, discardLaunchArtifacts, divergentSecretStoreNotice, parseDaemonStoreAnswer, parseSecretStoreIdentity, sameSecretStoreIdentity, agentFilePath, clearSpaceHistory, dialerFor, connectorServers, spawnEnvAllow, deprovisionAgent, DeprovisionError, isConcreteChannel, deprovisionTargetPrincipal, firstFreeName, spawnNameError, idFromCreds, inspectCredHealth, credsRenewalDelayMs, composeWirePersona, loadAgentFile, listPersonaCatalog, personaCatalogDescription, personaCatalogReadable, loadCotalConfig, mintCreds, mintGeneration, mintAcceptedToken, openIssuedStore, acceptedBucket, issuedBucket, mintLifecycleUid, managedRetirementOpId, mkSecretDir, newIdentity, actionContext, parsePrincipalKey, parseShareSelection, principalKey, probeConnect, provisionAgent, provisionAgentDurables, registry, resolveAuthProvider, saveAgentFile, subjectMatches, AUTH_ENDPOINT, EP_CMD_RETIRE_LIFECYCLE, controlServiceSubject, eventChannelPrincipal, resolveService, invokeCommand, } from "@cotal-ai/core";
|
|
8
8
|
import { agentAuthState, agentCredsDir, agentLifecycleSecretFilePaths, agentSecretFilePaths, agentSecretKeyForFile, authDir, connectorInstallHint, DEFAULT_CONNECTOR, defaultAgentType, DELIVERY_CREDS_KIND, extensionConnectors, findCotalRoot, getSpaceAuth, hasUserAuthState, loadExtensionsManifest, loadManagerInstanceIdentity, loadMeshes, localTrustOfSpace, manifestExtensionNames, materializeFromManifest, materializeSecretToFile, MEMBERSHIP_RW_CREDS_KIND, mergeLaunchOptions, remintDaemonCreds, resolveOnPath, createManagerInstanceIdentity, spaceMaterialKey, SYSTEM_CREDS_FILES, userAuthStateDir, workspaceSecretStore, writeRenewalRecord } from "@cotal-ai/workspace";
|
|
9
9
|
import { createRuntime, isCustodialRuntime, requireRuntimeAdopt, requireRuntimeReap, RuntimeReapUnproven, } from "./runtime/index.js";
|
|
10
10
|
import { AttachEndpoint } from "./attach-endpoint.js";
|
|
11
|
-
import { makeManagerEndpointEvictionEvidence, makeManagerEndpointEvictor } from "./endpoint-evict.js";
|
|
11
|
+
import { makeManagerEndpointEvictionEvidence, makeManagerEndpointEvictor, makeManagerEndpointHolderEvictor } from "./endpoint-evict.js";
|
|
12
12
|
import { makeManagerHolderLivenessProbe } from "./holder-liveness.js";
|
|
13
13
|
import { GateReconcileRefused, reconcileEndpointGate } from "./reconcile-gate.js";
|
|
14
|
-
import { launchSpecForRun, materializePersona, launchAgentToStartOpts, parseLaunchSpec, persistLaunchSpec } from "./launch.js";
|
|
14
|
+
import { launchSpecForRun, materializePersona, launchAgentToStartOpts, parseLaunchSpec, persistLaunchSpec, readContinuityAssignment, writeContinuityAssignment } from "./launch.js";
|
|
15
15
|
import { authorizeLaunch, authorizeNamedControl } from "./authorize.js";
|
|
16
16
|
import { controlShutdown } from "./control-shutdown.js";
|
|
17
17
|
import { RunHosting } from "./run-hosting.js";
|
|
@@ -197,12 +197,6 @@ function parseTurnNote(raw) {
|
|
|
197
197
|
function turnHoldToken(goalId) {
|
|
198
198
|
return createHash("sha256").update(`${goalId}:turn-deadline`, "utf8").digest("base64url").slice(0, 43);
|
|
199
199
|
}
|
|
200
|
-
/**
|
|
201
|
-
* The agent supervisor: a long-lived mesh node that owns agent process lifecycle.
|
|
202
|
-
* It serves control requests on the "manager" service and spawns/kills agents
|
|
203
|
-
* through a pluggable {@link Runtime} (pty by default). It does NOT proxy agent
|
|
204
|
-
* mesh traffic — terminal I/O streams over its own attach endpoint instead.
|
|
205
|
-
*/
|
|
206
200
|
/**
|
|
207
201
|
* Event channels in `channels` that do NOT belong to `{owner, actor}`.
|
|
208
202
|
*
|
|
@@ -243,6 +237,12 @@ function isAbsentDeliveryAdmin(msg) {
|
|
|
243
237
|
return false;
|
|
244
238
|
return /no responders|\b503\b/i.test(msg);
|
|
245
239
|
}
|
|
240
|
+
/**
|
|
241
|
+
* The agent supervisor: a long-lived mesh node that owns agent process lifecycle.
|
|
242
|
+
* It serves control requests on the "manager" service and spawns/kills agents
|
|
243
|
+
* through a pluggable {@link Runtime} (pty by default). It does NOT proxy agent
|
|
244
|
+
* mesh traffic — terminal I/O streams over its own attach endpoint instead.
|
|
245
|
+
*/
|
|
246
246
|
export class Manager {
|
|
247
247
|
space;
|
|
248
248
|
servers;
|
|
@@ -268,8 +268,13 @@ export class Manager {
|
|
|
268
268
|
endpointServeExecutorExpiresInSeconds;
|
|
269
269
|
agents = new Map();
|
|
270
270
|
/** Names whose spawn is in flight (reserved synchronously before the provision await) — counted
|
|
271
|
-
* toward the ceiling
|
|
271
|
+
* toward the ceiling (once per name, see {@link Manager.occupancy}) so two concurrent same-name
|
|
272
|
+
* spawns can't both pass the gate (P4a). */
|
|
272
273
|
reserved = new Set();
|
|
274
|
+
/** Reserved names whose launch already put its seat in `agents`. The name stays reserved until
|
|
275
|
+
* the launch settles, so no other spawn can take it, but the ceiling counts that seat through
|
|
276
|
+
* `agents` and, if it frees young, its cooling stamp, never through the reservation (#906). */
|
|
277
|
+
reservedLive = new Set();
|
|
273
278
|
/** Expiry stamps (`startedAt + MIN_LIFETIME`) for slots that freed while still young — a
|
|
274
279
|
* count-only, lazily-pruned recycle floor (P4c). Pruned + summed into the ceiling gate. */
|
|
275
280
|
cooling = [];
|
|
@@ -321,6 +326,11 @@ export class Manager {
|
|
|
321
326
|
* every (re)connect, so a renewal is adopted without re-registration). Absent on open meshes,
|
|
322
327
|
* in user mode (the named 1a follow-up), and before registration completes. */
|
|
323
328
|
serviceServe;
|
|
329
|
+
/** #1577: serve connections whose transport is down while the client reconnects them. Such a
|
|
330
|
+
* connection is open (`isClosed()` is false under unbounded reconnects) yet the broker holds
|
|
331
|
+
* none of its rails, so the liveness responder must not grade it `bound`. Keyed by the
|
|
332
|
+
* connection, so a re-dial that swaps in a fresh one starts from its own state. */
|
|
333
|
+
serveTransportDown = new WeakSet();
|
|
324
334
|
/** P2 item 2 (spawn-as-action): the SELF-MEDIATED goal-writer connection + ActionContext — a
|
|
325
335
|
* standing connection DISJOINT from the serve credential (Q2), scoped to exactly this endpoint's
|
|
326
336
|
* goal bind/terminal facts + goal-record writes ({@link goalWriterGrants}). Auth mode mints the
|
|
@@ -375,10 +385,11 @@ export class Manager {
|
|
|
375
385
|
/** P2 item 2 (M4): the live spawn goal ref for each managed agent name, so a despawn MID-GOAL
|
|
376
386
|
* drives the cancel path (transition -> cancel terminal). Cleared when the goal terminalizes. */
|
|
377
387
|
agentGoals = new Map();
|
|
378
|
-
/** The turn relay's same-incarnation idempotency map ({@link goalAcceptances}'s twin): a
|
|
379
|
-
* same-goalId retry serves the identical acceptance; cross-incarnation retries rebuild from
|
|
380
|
-
* the goal-index entry (its acceptance floor + note). */
|
|
381
388
|
/**
|
|
389
|
+
* The turn relay's same-incarnation idempotency map ({@link goalAcceptances}'s twin): a
|
|
390
|
+
* same-goalId retry serves the identical acceptance; cross-incarnation retries rebuild from
|
|
391
|
+
* the goal-index entry (its acceptance floor + note).
|
|
392
|
+
*
|
|
382
393
|
* One entry per turn this incarnation accepted: the acceptance a duplicate submission is served
|
|
383
394
|
* from, and — once the turn settles — the answer a RETRIED yield is served from.
|
|
384
395
|
*
|
|
@@ -656,8 +667,8 @@ export class Manager {
|
|
|
656
667
|
case "user":
|
|
657
668
|
return (`\`cotal actor grant <reader> --owner ${owner} --scope '' --allow-subscribe '${channel}' ` +
|
|
658
669
|
`--allow-publish ''\`, with every field spelled out: \`actor grant\` is an upsert of the ` +
|
|
659
|
-
`WHOLE row and an omitted flag
|
|
660
|
-
`scope)
|
|
670
|
+
`WHOLE row, and an omitted flag is refused, or with \`--full\` is the WIDE default (\`>\` read, \`>\` post, \`spawn,role:default\` ` +
|
|
671
|
+
`scope).`);
|
|
661
672
|
case "static":
|
|
662
673
|
return (`\`cotal mint <reader> --profile agent --allow-subscribe ${channel} --provision\`, where there ` +
|
|
663
674
|
`is no actor ledger for \`actor grant\` to write to. The AGENT profile, not the observer ` +
|
|
@@ -804,6 +815,10 @@ export class Manager {
|
|
|
804
815
|
const reportEndpoint = (e) => console.error(`! manager endpoint: ${e.message}`);
|
|
805
816
|
this.ep.on("error", reportEndpoint);
|
|
806
817
|
this.ep.on("warning", reportEndpoint);
|
|
818
|
+
// Every roster change and every view transition, so a seat that is offline in a fresh snapshot
|
|
819
|
+
// (recorded with no presence event) is logged the moment the view can support the verdict.
|
|
820
|
+
this.ep.on("roster", () => this.logSeatMeshTransitions());
|
|
821
|
+
this.ep.on("presence-view", () => this.logSeatMeshTransitions());
|
|
807
822
|
await this.ep.start();
|
|
808
823
|
await this.ep.setActivity(`supervisor (${this.runtime.kind})`);
|
|
809
824
|
// Per-instance liveness lease (P2 item 3 — the old per-space singleton is DEMOTED per D9). Acquire
|
|
@@ -829,6 +844,31 @@ export class Manager {
|
|
|
829
844
|
}
|
|
830
845
|
this.leaseTimer = setInterval(() => { void this.renewLease(); }, MANAGER_LEASE_RENEW_MS);
|
|
831
846
|
this.leaseTimer.unref?.();
|
|
847
|
+
// THE PEER-READABLE LIVENESS RESPONDER for the manager plane (#1577). Bound HERE — after the
|
|
848
|
+
// lease, before registration — precisely so it is up during the window in which the manager is
|
|
849
|
+
// NOT yet serving. A responder that only appeared once everything worked could never report the
|
|
850
|
+
// broken state it exists to report, which is the reported defect in a new place.
|
|
851
|
+
//
|
|
852
|
+
// WHAT IT GRADES is `serviceServe`, the v0.4 service endpoint's serve handle, which is assigned
|
|
853
|
+
// only after `serveEndpoint` has bound this instance's rails, AND that handle's connection. A
|
|
854
|
+
// closed serve connection holds no subscriptions, while `onServeConnectionClosed` keeps the
|
|
855
|
+
// handle through its re-dial waits, so the handle alone would answer `bound` with every rail
|
|
856
|
+
// gone. The re-dial swaps in the fresh connection and handle together, so `bound` returns only
|
|
857
|
+
// once the rails are served again. A DISCONNECTED serve connection is the same fault without the
|
|
858
|
+
// close: it reconnects without limit, so it stays open while the broker holds none of its rails,
|
|
859
|
+
// and this probe arrives on the separate supervisor connection that is still up. So the grade
|
|
860
|
+
// also requires the serve transport to be connected. That is a RESPONDER fact. It is
|
|
861
|
+
// deliberately NOT "this process is alive": this process is trivially alive whenever it answers
|
|
862
|
+
// at all, so grading on that would make the surface a `pgrep` that also lies in both directions
|
|
863
|
+
// — the exact instrument the reporter's evidence table indicts. Between lease acquisition and a
|
|
864
|
+
// successful registration the honest answer is `unbound`, and that is what a peer is told.
|
|
865
|
+
//
|
|
866
|
+
// The lease row is never consulted and never sent: presence is the whole ask, and the row
|
|
867
|
+
// carries the operator's workspace root and pid.
|
|
868
|
+
this.ep.serveLiveness("manager", () => {
|
|
869
|
+
const s = this.serviceServe;
|
|
870
|
+
return s?.handle && !s.nc.isClosed() && !this.serveTransportDown.has(s.nc) ? "bound" : "unbound";
|
|
871
|
+
});
|
|
832
872
|
// Unit B (static §13.1): after this instance holds its lease, collect the durable static rows
|
|
833
873
|
// now, but do not let their exact-op terminals make the whole space unreachable. The service
|
|
834
874
|
// comes up below, then the sweep overlaps the remaining registration work. Two-window
|
|
@@ -1817,6 +1857,8 @@ export class Manager {
|
|
|
1817
1857
|
return { mode: "static", id: principal.actor, lifecycleUid: a.lifecycleUid, credential: { kind: "file", path: files.creds, sha256: this.fileDigestOrEmpty(files.creds) }, ...(a.issued ? { issued: a.issued } : {}) };
|
|
1818
1858
|
})()
|
|
1819
1859
|
: { mode: "open", id: principal.actor, lifecycleUid: a.lifecycleUid };
|
|
1860
|
+
// A preservation cut is a stop boundary too, so a manifest launch after it reopens what a resume would.
|
|
1861
|
+
this.recordContinuityAtStop(a, "preservation");
|
|
1820
1862
|
const dependencies = [a.launch.source.configPath];
|
|
1821
1863
|
if (a.launch.source.kind === "manifest" && a.launch.source.runId)
|
|
1822
1864
|
dependencies.unshift(join(this.workspaceRoot, ".cotal", "run", `${a.launch.source.runId}.json`));
|
|
@@ -1839,6 +1881,7 @@ export class Manager {
|
|
|
1839
1881
|
events: a.launch.events,
|
|
1840
1882
|
shareTools: a.launch.shareTools,
|
|
1841
1883
|
forkSource: a.launch.forkSource,
|
|
1884
|
+
...(this.forkProvenance(a) ? { resumed: this.forkProvenance(a) } : {}),
|
|
1842
1885
|
sessionId: a.restart?.armed ? this.readManagedSession(a) : a.launch.sessionId,
|
|
1843
1886
|
// Additive and only when the connector supplied one. A seat with no pointer records no
|
|
1844
1887
|
// field, which is what keeps an older inventory and a fresh one the same document shape.
|
|
@@ -1865,6 +1908,7 @@ export class Manager {
|
|
|
1865
1908
|
for (const a of managed) {
|
|
1866
1909
|
// Free the slot + hard-stop each; `stopHandle` is best-effort (never throws — see it), so one bad
|
|
1867
1910
|
// stop can't strand the rest, and every snapshot entry is deprovisioned below regardless.
|
|
1911
|
+
this.recordContinuityAtStop(a, "shutdown");
|
|
1868
1912
|
this.agents.delete(a.name);
|
|
1869
1913
|
this.stopHandle(a, false);
|
|
1870
1914
|
}
|
|
@@ -1964,38 +2008,64 @@ export class Manager {
|
|
|
1964
2008
|
// revision it has since moved is a CAS the broker correctly refuses, silently, forever.
|
|
1965
2009
|
await this.leaseRenewTask?.catch(() => { });
|
|
1966
2010
|
let releaseFailures = [];
|
|
1967
|
-
|
|
1968
|
-
|
|
1969
|
-
|
|
1970
|
-
|
|
1971
|
-
|
|
2011
|
+
let seatFailure;
|
|
2012
|
+
try {
|
|
2013
|
+
if (this.maintenanceState === "active" && !this.resumeRequired) {
|
|
2014
|
+
if (options.withAgents)
|
|
2015
|
+
await this.teardownManagedAgents();
|
|
2016
|
+
else
|
|
2017
|
+
releaseFailures = this.releaseManagedAgents();
|
|
2018
|
+
}
|
|
2019
|
+
else {
|
|
2020
|
+
// A signal after a partial preservation must never fall back into destructive teardown.
|
|
2021
|
+
await this.stopRetainedAgentsOnExit();
|
|
2022
|
+
}
|
|
1972
2023
|
}
|
|
1973
|
-
|
|
1974
|
-
|
|
1975
|
-
|
|
2024
|
+
catch (e) {
|
|
2025
|
+
seatFailure = e;
|
|
2026
|
+
}
|
|
2027
|
+
// Every step below runs even when an earlier one fails, so a failed stop still closes this
|
|
2028
|
+
// process's connections and listener (#2290). Failures are collected and thrown at the end.
|
|
2029
|
+
const failures = [];
|
|
2030
|
+
const step = async (name, run) => {
|
|
2031
|
+
try {
|
|
2032
|
+
await run();
|
|
2033
|
+
}
|
|
2034
|
+
catch (e) {
|
|
2035
|
+
failures.push(`${name}: ${e.message}`);
|
|
2036
|
+
}
|
|
2037
|
+
};
|
|
2038
|
+
// A seat that may still be alive keeps this instance's leases and registration (see
|
|
2039
|
+
// teardownManagedAgents): they lapse on their TTL instead of being handed to a successor.
|
|
2040
|
+
if (!seatFailure) {
|
|
2041
|
+
await step("release manager lease", () => this.releaseOwnManagerLease());
|
|
2042
|
+
// #1634: hand the renewal lease back on a clean stop so a sibling picks it up on its next pass
|
|
2043
|
+
// rather than waiting out the bucket TTL. A non-owner holds no revision and this is a no-op.
|
|
2044
|
+
await step("release renewal lease", () => this.ep.releaseDaemonRenewalLease());
|
|
1976
2045
|
}
|
|
1977
|
-
await this.releaseOwnManagerLease();
|
|
1978
|
-
// #1634: hand the renewal lease back on a clean stop so a sibling picks it up on its next pass
|
|
1979
|
-
// rather than waiting out the bucket TTL. A non-owner holds no revision and this is a no-op.
|
|
1980
|
-
await this.ep.releaseDaemonRenewalLease();
|
|
1981
2046
|
// Capture BEFORE the serve loop is torn down: `stopServiceServe` clears the state, and what is
|
|
1982
2047
|
// being asked here is "did this process register a service instance", which only the state
|
|
1983
2048
|
// before teardown can answer. Deregistration runs AFTER the serve loop has drained, so no
|
|
1984
2049
|
// in-flight command can write a status back onto the record it just removed.
|
|
1985
2050
|
const registered = this.serviceServe !== undefined;
|
|
1986
|
-
await this.stopServiceServe();
|
|
2051
|
+
await step("stop service serve", () => this.stopServiceServe());
|
|
1987
2052
|
// The drives AFTER the serve loop (no new `run-start` can land) and BEFORE deregistration: a
|
|
1988
2053
|
// released run's status write is the last thing this incarnation says about it.
|
|
1989
|
-
await this.runHosting?.stop();
|
|
2054
|
+
await step("stop run hosting", () => this.runHosting?.stop());
|
|
1990
2055
|
this.runHosting = undefined;
|
|
1991
|
-
if (registered)
|
|
1992
|
-
await this.deregisterServiceOnStop();
|
|
1993
|
-
await this.stopGoalWriter();
|
|
1994
|
-
await this.stopSessionPlane();
|
|
1995
|
-
await this.ep.stop();
|
|
1996
|
-
await this.attach.stop();
|
|
2056
|
+
if (registered && !seatFailure)
|
|
2057
|
+
await step("deregister service", () => this.deregisterServiceOnStop());
|
|
2058
|
+
await step("stop goal writer", () => this.stopGoalWriter());
|
|
2059
|
+
await step("stop session plane", () => this.stopSessionPlane());
|
|
2060
|
+
await step("stop endpoint", () => this.ep.stop());
|
|
2061
|
+
await step("stop attach endpoint", () => this.attach.stop());
|
|
2062
|
+
const also = failures.length ? `; teardown also failed: ${failures.join("; ")}` : "";
|
|
2063
|
+
if (seatFailure)
|
|
2064
|
+
throw new Error(`${seatFailure.message}${also}`);
|
|
1997
2065
|
if (releaseFailures.length)
|
|
1998
|
-
throw new Error(`manager shutdown could not release every detachable seat: ${releaseFailures.join("; ")}`);
|
|
2066
|
+
throw new Error(`manager shutdown could not release every detachable seat: ${releaseFailures.join("; ")}${also}`);
|
|
2067
|
+
if (failures.length)
|
|
2068
|
+
throw new Error(`manager shutdown incomplete: ${failures.join("; ")}`);
|
|
1999
2069
|
}
|
|
2000
2070
|
/** Release this instance's liveness lease key on a clean stop, at the broker's own revision
|
|
2001
2071
|
* rather than a cached one that a renew still in flight moments ago may have already moved past.
|
|
@@ -2432,33 +2502,6 @@ export class Manager {
|
|
|
2432
2502
|
throw new EpEnvelopeError("permission-denied", `an any-mode ${ctx.subject.command} is operator reach; the caller's current ledger grant does not carry "admin" (SPEC 13.2)`);
|
|
2433
2503
|
return true;
|
|
2434
2504
|
}
|
|
2435
|
-
/** The v0.4 typed command table (P2 item 1, slice 1b): every ordinary handler runs the SHARED
|
|
2436
|
-
* admission chokepoint ({@link serveGated}) and then delegates to the SAME op core the ctl
|
|
2437
|
-
* door dispatches (checklist 8: one core, two thin doors). The resume/preservation family
|
|
2438
|
-
* deliberately BYPASSES serveGated — exactly as it sits before {@link admitControl} on the ctl
|
|
2439
|
-
* door (those ops must run while `resumeRequired` fences ordinary work) — riding its own state
|
|
2440
|
-
* fences; its ep gate is the admin-grade `manager.admin` capability grant (the 1b rule: static
|
|
2441
|
-
* admin-class commands are capability-gated + untargeted, never a fabricated ledger mode).
|
|
2442
|
-
*
|
|
2443
|
-
* TIER SEMANTICS on the ep door (the 1c grant-migration table): the tier lives in the CALLER'S
|
|
2444
|
-
* GRANT, refined per-op exactly as the ctl doors refine their subject tier. Owner-mode
|
|
2445
|
-
* `despawn`/`attach` keep the privileged semantics (`admin=false`, own-domain via
|
|
2446
|
-
* {@link authorizeNamed}) — every spawn-capable agent holds those rows. ANY-mode requests are
|
|
2447
|
-
* the operator instrument's cross-agent reach (rev 3): the any-mode subject row is mintable
|
|
2448
|
-
* only under operator policy (§13.2), so on a static mesh holding it IS the admin tier, and in
|
|
2449
|
-
* user mode the caller's CURRENT ledger scope must still carry `admin`
|
|
2450
|
-
* ({@link epAdminReach}, the same fresh-read authority `psOwnerFilter` consults). The
|
|
2451
|
-
* `manager.admin` family (purge + the resume/preservation ops) is capability-gated at mint AND
|
|
2452
|
-
* re-checked at serve time via {@link epAdminReach} (the `adminGated` wrapper) so a user's
|
|
2453
|
-
* revoked scope demotes the next call. `launch` is OWNER-EQUALITY on this door for everyone
|
|
2454
|
-
* (freelance HIGH #2): the deploy path is its only consumer and stamps the caller's own owner,
|
|
2455
|
-
* so cross-owner launch was a ctl-tier incidental never exercised, and keying it on the actor's
|
|
2456
|
-
* ledger scope broke the deployer-view attenuation - uniform owner-equality is the safe tier.
|
|
2457
|
-
* TWO DELIBERATE NARROWINGS vs the ctl doors (NOT bit-exact parity, panel-accepted): (1)
|
|
2458
|
-
* `define-persona` is `admin=false` for everyone (own-persona discipline; no ep consumer needs
|
|
2459
|
-
* cross-owner persona writes - an operator redefines via config, not the wire), where the ctl
|
|
2460
|
-
* admin tier allowed operator cross-owner redefine; (2) launch is owner-equality-only, above.
|
|
2461
|
-
* Both are least-privilege reductions, never widenings. */
|
|
2462
2505
|
/** The run host, or one of three refusals. A remote-authority manager holds no space signer, so
|
|
2463
2506
|
* it cannot mint the per-run driver credential SPEC 14.6 requires without all four closed
|
|
2464
2507
|
* host callbacks. A local user-auth mesh is `unimplemented`: a hosted run's seats
|
|
@@ -2515,6 +2558,44 @@ export class Manager {
|
|
|
2515
2558
|
if (p.run !== args.runId || p.step !== args.stepKey || addressedToken !== open.token)
|
|
2516
2559
|
throw new EpEnvelopeError("permission-denied", `run-answer is allowed to ${agent.name} only for the open pause named by its pending ask or escalation relay; ${args.runId} ${args.stepKey} is not that pause`);
|
|
2517
2560
|
}
|
|
2561
|
+
/** A baseline seat may amend only an answer recorded under its own name: it changes its own
|
|
2562
|
+
* position, never another answerer's. Explicit `run` holders and unmanaged operator instruments
|
|
2563
|
+
* keep the reach they have to answer, as {@link authorizeRunAnswer} does. */
|
|
2564
|
+
authorizeRunAmend(ctx, accepted) {
|
|
2565
|
+
const caller = principalKey(ctx.subject.caller.owner, ctx.subject.caller.actor).key;
|
|
2566
|
+
const agent = [...this.agents.values()].find((a) => this.managedPrincipal(a) === caller && a.lifecycleUid === ctx.subject.caller.uid);
|
|
2567
|
+
if (agent === undefined || agent.launch.capabilities?.includes("run"))
|
|
2568
|
+
return;
|
|
2569
|
+
if (accepted.by !== agent.name)
|
|
2570
|
+
throw new EpEnvelopeError("permission-denied", `run-answer amend is allowed to ${agent.name} only for an answer recorded under its own name; this step's accepted answer is ${JSON.stringify(accepted.by)}'s`);
|
|
2571
|
+
}
|
|
2572
|
+
/** The v0.4 typed command table (P2 item 1, slice 1b): every ordinary handler runs the SHARED
|
|
2573
|
+
* admission chokepoint ({@link serveGated}) and then delegates to the SAME op core the ctl
|
|
2574
|
+
* door dispatches (checklist 8: one core, two thin doors). The resume/preservation family
|
|
2575
|
+
* deliberately BYPASSES serveGated — exactly as it sits before {@link admitControl} on the ctl
|
|
2576
|
+
* door (those ops must run while `resumeRequired` fences ordinary work) — riding its own state
|
|
2577
|
+
* fences; its ep gate is the admin-grade `manager.admin` capability grant (the 1b rule: static
|
|
2578
|
+
* admin-class commands are capability-gated + untargeted, never a fabricated ledger mode).
|
|
2579
|
+
*
|
|
2580
|
+
* TIER SEMANTICS on the ep door (the 1c grant-migration table): the tier lives in the CALLER'S
|
|
2581
|
+
* GRANT, refined per-op exactly as the ctl doors refine their subject tier. Owner-mode
|
|
2582
|
+
* `despawn`/`attach` keep the privileged semantics (`admin=false`, own-domain via
|
|
2583
|
+
* {@link authorizeNamed}) — every spawn-capable agent holds those rows. ANY-mode requests are
|
|
2584
|
+
* the operator instrument's cross-agent reach (rev 3): the any-mode subject row is mintable
|
|
2585
|
+
* only under operator policy (§13.2), so on a static mesh holding it IS the admin tier, and in
|
|
2586
|
+
* user mode the caller's CURRENT ledger scope must still carry `admin`
|
|
2587
|
+
* ({@link epAdminReach}, the same fresh-read authority `psOwnerFilter` consults). The
|
|
2588
|
+
* `manager.admin` family (purge + the resume/preservation ops) is capability-gated at mint AND
|
|
2589
|
+
* re-checked at serve time via {@link epAdminReach} (the `adminGated` wrapper) so a user's
|
|
2590
|
+
* revoked scope demotes the next call. `launch` is OWNER-EQUALITY on this door for everyone
|
|
2591
|
+
* (freelance HIGH #2): the deploy path is its only consumer and stamps the caller's own owner,
|
|
2592
|
+
* so cross-owner launch was a ctl-tier incidental never exercised, and keying it on the actor's
|
|
2593
|
+
* ledger scope broke the deployer-view attenuation - uniform owner-equality is the safe tier.
|
|
2594
|
+
* TWO DELIBERATE NARROWINGS vs the ctl doors (NOT bit-exact parity, panel-accepted): (1)
|
|
2595
|
+
* `define-persona` is `admin=false` for everyone (own-persona discipline; no ep consumer needs
|
|
2596
|
+
* cross-owner persona writes - an operator redefines via config, not the wire), where the ctl
|
|
2597
|
+
* admin tier allowed operator cross-owner redefine; (2) launch is owner-equality-only, above.
|
|
2598
|
+
* Both are least-privilege reductions, never widenings. */
|
|
2518
2599
|
managerServiceDefs() {
|
|
2519
2600
|
const args = (ctx) => (ctx.request.args ?? {});
|
|
2520
2601
|
const callerOf = (ctx) => principalKey(ctx.subject.caller.owner, ctx.subject.caller.actor).key;
|
|
@@ -2703,6 +2784,8 @@ export class Manager {
|
|
|
2703
2784
|
runResume: (ctx) => this.serveGated(ctx, () => this.runHost().resume(args(ctx))),
|
|
2704
2785
|
runAnswer: (ctx) => this.serveGated(ctx, () => {
|
|
2705
2786
|
const input = args(ctx);
|
|
2787
|
+
if (input.amend === true)
|
|
2788
|
+
return this.runHost().amend(input, this.runAnswerer(ctx), (accepted) => this.authorizeRunAmend(ctx, accepted));
|
|
2706
2789
|
return this.runHost().answer(input, this.runAnswerer(ctx), (open) => this.authorizeRunAnswer(ctx, input, open));
|
|
2707
2790
|
}),
|
|
2708
2791
|
runStatus: (ctx) => this.serveGated(ctx, () => this.runHost().status(args(ctx))),
|
|
@@ -3137,11 +3220,6 @@ export class Manager {
|
|
|
3137
3220
|
return { error: `agent auth preflight failed for "${name}": ${e.message}` };
|
|
3138
3221
|
}
|
|
3139
3222
|
}
|
|
3140
|
-
/** Drop a live agent's slot. When `floor` is set and the agent died young (lived less than
|
|
3141
|
-
* MIN_LIFETIME), push a cooling stamp so the freed slot still counts toward the ceiling until it
|
|
3142
|
-
* expires — flooring the RECYCLE, not the call, so both free paths (despawn + exit/reap) are
|
|
3143
|
-
* covered (P4c). Floor self + own-child despawn and natural exit; NEVER admin despawn (operator
|
|
3144
|
-
* emergency-kill stays unthrottled) and NEVER the reserved-rollback path (no cold-start paid). */
|
|
3145
3223
|
/**
|
|
3146
3224
|
* WHY A SEAT LEFT — one line, at the one place every free path passes through.
|
|
3147
3225
|
*
|
|
@@ -3164,7 +3242,7 @@ export class Manager {
|
|
|
3164
3242
|
const info = a.handle.exitInfo?.();
|
|
3165
3243
|
detail = info === undefined
|
|
3166
3244
|
? `exit detail unavailable from runtime "${a.handle.kind}"`
|
|
3167
|
-
: `exit code ${info.code ?? "unknown"}${info.signal === undefined ? "" : `, signal ${info.signal}`}`;
|
|
3245
|
+
: `exit code ${info.code ?? "unknown"}${info.signal === undefined ? "" : `, signal ${info.signal}`}${info.diagnostic ? `; last connector diagnostic: ${info.diagnostic}` : ""}`;
|
|
3168
3246
|
}
|
|
3169
3247
|
catch (e) {
|
|
3170
3248
|
// A runtime that throws while being asked has told us something real; it must not take the
|
|
@@ -3173,6 +3251,49 @@ export class Manager {
|
|
|
3173
3251
|
}
|
|
3174
3252
|
console.error(`seat reaped: ${a.name} (${a.id}, uid ${a.lifecycleUid}${a.handle.pid !== undefined ? `, pid ${a.handle.pid}` : ""}) - ${causeText}; ${detail}`);
|
|
3175
3253
|
}
|
|
3254
|
+
/**
|
|
3255
|
+
* A SEAT THAT LEAVES THE MESH WHILE ITS SLOT IS HELD GETS A LINE, AND SO DOES ITS RETURN.
|
|
3256
|
+
*
|
|
3257
|
+
* Seats once read `running · mesh offline` for many hours with no manager log line, so a watchdog
|
|
3258
|
+
* keyed on process liveness saw nothing wrong and nobody could say when each seat dropped (#1208).
|
|
3259
|
+
* The line follows the same state `ps` reads, checked on every roster change and view transition,
|
|
3260
|
+
* because not every offline record arrives as a presence event: after a reconnect the fresh snapshot
|
|
3261
|
+
* records an already-offline seat quietly. Only a current view is a verdict, so a stale or
|
|
3262
|
+
* unpopulated one logs nothing. The match is {@link isOwnPresence}. A seat being stopped is left to
|
|
3263
|
+
* the reap line. The heartbeat is the seat's own record and nothing validated it, so an unreadable
|
|
3264
|
+
* one prints as unknown instead of throwing inside the watch.
|
|
3265
|
+
*/
|
|
3266
|
+
logSeatMeshTransitions() {
|
|
3267
|
+
if (typeof this.ep.presenceView === "function" && this.ep.presenceView().state !== "current")
|
|
3268
|
+
return;
|
|
3269
|
+
const roster = new Map(this.ep.getRoster().map((p) => [p.card.id, p]));
|
|
3270
|
+
for (const a of this.agents.values()) {
|
|
3271
|
+
const seen = roster.get(this.managedPrincipal(a));
|
|
3272
|
+
if (a.terminalizing || !this.isOwnPresence(a, seen))
|
|
3273
|
+
continue;
|
|
3274
|
+
const who = `${a.name} (${a.id}, uid ${a.lifecycleUid}${a.handle.pid !== undefined ? `, pid ${a.handle.pid}` : ""})`;
|
|
3275
|
+
if (seen.status === "offline" && !a.offMesh) {
|
|
3276
|
+
a.offMesh = true;
|
|
3277
|
+
const ts = seen.ts;
|
|
3278
|
+
const last = typeof ts === "number" && !Number.isNaN(new Date(ts).getTime()) ? new Date(ts).toISOString() : "unknown";
|
|
3279
|
+
console.error(`seat offline on the mesh: ${who} - last heartbeat ${last}; process ${a.handle.status()}`);
|
|
3280
|
+
}
|
|
3281
|
+
else if (seen.status !== "offline" && a.offMesh) {
|
|
3282
|
+
a.offMesh = false;
|
|
3283
|
+
console.error(`seat back on the mesh: ${who}`);
|
|
3284
|
+
}
|
|
3285
|
+
}
|
|
3286
|
+
}
|
|
3287
|
+
/** Whether a roster record is this seat's own: the readiness fence's exact principal and
|
|
3288
|
+
* incarnation, so a same-named foreign peer or an older incarnation never speaks for it (#1208). */
|
|
3289
|
+
isOwnPresence(a, p) {
|
|
3290
|
+
return p !== undefined && p.card.id === this.managedPrincipal(a) && p.lifecycleUid === a.lifecycleUid;
|
|
3291
|
+
}
|
|
3292
|
+
/** Drop a live agent's slot. When `floor` is set and the agent died young (lived less than
|
|
3293
|
+
* MIN_LIFETIME), push a cooling stamp so the freed slot still counts toward the ceiling until it
|
|
3294
|
+
* expires — flooring the RECYCLE, not the call, so both free paths (despawn + exit/reap) are
|
|
3295
|
+
* covered (P4c). Floor self + own-child despawn and natural exit; NEVER admin despawn (operator
|
|
3296
|
+
* emergency-kill stays unthrottled) and NEVER the reserved-rollback path (no cold-start paid). */
|
|
3176
3297
|
freeSlot(a, floor, cause, acceptedBeforeFence = false) {
|
|
3177
3298
|
if (this.agents.get(a.name) !== a)
|
|
3178
3299
|
return; // already freed (exit raced despawn, etc.)
|
|
@@ -3180,6 +3301,7 @@ export class Manager {
|
|
|
3180
3301
|
this.logSeatReaped(a, cause);
|
|
3181
3302
|
a.terminalizing = true;
|
|
3182
3303
|
this.agents.delete(a.name);
|
|
3304
|
+
this.recordContinuityAtStop(a, "stop");
|
|
3183
3305
|
if (a.restart?.sessionStatePath)
|
|
3184
3306
|
rmSync(a.restart.sessionStatePath, { force: true });
|
|
3185
3307
|
// P2 item 6 (pin 4): end any live §13.6 attach session bound to THIS incarnation with the honest
|
|
@@ -3631,9 +3753,49 @@ export class Manager {
|
|
|
3631
3753
|
`(${error.message}). Reload the live Pi seat before the preservation cut; refusing to resume fresh and lose its context.`);
|
|
3632
3754
|
}
|
|
3633
3755
|
}
|
|
3756
|
+
/** A resumed seat's fork provenance (#1500). The seat forks after it launches, so this reads the
|
|
3757
|
+
* record its connector named once the seat has written it, and keeps it on the launch so the
|
|
3758
|
+
* resume document carries it across a manager restart. Until the seat has written it, or when the
|
|
3759
|
+
* record cannot be read or does not name this seat's source, it is absent: `ps` then shows the
|
|
3760
|
+
* source id alone rather than failing the listing for every other seat. */
|
|
3761
|
+
forkProvenance(a) {
|
|
3762
|
+
if (a.launch.resumed || !a.launch.resumeRecordPath || !a.launch.forkSource)
|
|
3763
|
+
return a.launch.resumed;
|
|
3764
|
+
let record;
|
|
3765
|
+
try {
|
|
3766
|
+
record = JSON.parse(readFileSync(a.launch.resumeRecordPath, "utf8"));
|
|
3767
|
+
}
|
|
3768
|
+
catch {
|
|
3769
|
+
return undefined;
|
|
3770
|
+
}
|
|
3771
|
+
const r = (typeof record === "object" && record !== null ? record : {});
|
|
3772
|
+
if (r.source !== a.launch.forkSource || typeof r.transcriptSha256 !== "string" || !/^[a-f0-9]{64}$/.test(r.transcriptSha256) ||
|
|
3773
|
+
!(r.title === undefined || r.title === null || (typeof r.title === "string" && r.title.length <= 1024)))
|
|
3774
|
+
return undefined;
|
|
3775
|
+
a.launch.resumed = { source: r.source, ...(typeof r.title === "string" && r.title ? { title: r.title } : {}), transcriptSha256: r.transcriptSha256 };
|
|
3776
|
+
return a.launch.resumed;
|
|
3777
|
+
}
|
|
3634
3778
|
readManagedSession(a) {
|
|
3635
3779
|
return this.readManagedSessionState(a).sessionId;
|
|
3636
3780
|
}
|
|
3781
|
+
/** Re-record an exact seat's assignment at a proven session, so crash recovery, a preserved resume
|
|
3782
|
+
* and the next manifest launch never pick different sessions. */
|
|
3783
|
+
recordContinuity(a, sessionId) {
|
|
3784
|
+
if (a.continuity)
|
|
3785
|
+
writeContinuityAssignment(this.workspaceRoot, { ...a.continuity, sessionId });
|
|
3786
|
+
}
|
|
3787
|
+
/** At a stop boundary the connector's session record is the last proof of the session an exact
|
|
3788
|
+
* seat ran, including one it switched to itself. A failed write is logged; the stop goes on. */
|
|
3789
|
+
recordContinuityAtStop(a, boundary) {
|
|
3790
|
+
if (!a.continuity || !a.restart?.armed)
|
|
3791
|
+
return;
|
|
3792
|
+
try {
|
|
3793
|
+
this.recordContinuity(a, this.readManagedSession(a));
|
|
3794
|
+
}
|
|
3795
|
+
catch (error) {
|
|
3796
|
+
console.error(`! ${a.name}: continuity: exact assignment not updated at ${boundary}: ${error.message}`);
|
|
3797
|
+
}
|
|
3798
|
+
}
|
|
3637
3799
|
async awaitManagedSessionState(a) {
|
|
3638
3800
|
const deadline = Date.now() + 15_000;
|
|
3639
3801
|
let last = "session state not written yet";
|
|
@@ -3716,8 +3878,8 @@ export class Manager {
|
|
|
3716
3878
|
const connector = await this.resolveConnector(a.agent);
|
|
3717
3879
|
const continueSession = connector.supportsSessionContinuation ? this.readManagedSession(a) : undefined;
|
|
3718
3880
|
const opts = continueSession !== undefined
|
|
3719
|
-
? { ...restart.opts, resume: undefined, prompt: undefined, continueSession }
|
|
3720
|
-
: { ...restart.opts, resume: undefined, prompt: undefined };
|
|
3881
|
+
? { ...restart.opts, resume: undefined, reopenSession: undefined, prompt: undefined, continueSession }
|
|
3882
|
+
: { ...restart.opts, resume: undefined, reopenSession: undefined, prompt: undefined };
|
|
3721
3883
|
const spec = connector.buildLaunch(opts);
|
|
3722
3884
|
spec.env = { ...spec.env, COTAL_MANAGER_INSTANCE: this.managerInstanceId };
|
|
3723
3885
|
const wanted = this.managedPrincipal(a);
|
|
@@ -3732,8 +3894,10 @@ export class Manager {
|
|
|
3732
3894
|
const handle = await this.spawnCustodied(a.name, spec, a.launch.cwd, custody);
|
|
3733
3895
|
replacement = handle;
|
|
3734
3896
|
restart.sessionStatePath = spec.sessionStatePath ?? restart.sessionStatePath;
|
|
3735
|
-
if (continueSession !== undefined)
|
|
3897
|
+
if (continueSession !== undefined) {
|
|
3736
3898
|
await this.awaitRecoveredSession(a, continueSession, handle, spec.control);
|
|
3899
|
+
this.recordContinuity(a, continueSession);
|
|
3900
|
+
}
|
|
3737
3901
|
else {
|
|
3738
3902
|
const previousHandle = a.handle;
|
|
3739
3903
|
const previousControl = a.control;
|
|
@@ -4341,9 +4505,9 @@ export class Manager {
|
|
|
4341
4505
|
// SYNCHRONOUS (existsSync / registry / accessSync / readFileSync — no await), so the gate stays
|
|
4342
4506
|
// atomic: the capacity snapshot and the reserve land in one tick (P4a/P4c), and two concurrent
|
|
4343
4507
|
// spawns can't overshoot the ceiling or pick the same name.
|
|
4344
|
-
const
|
|
4345
|
-
if (
|
|
4346
|
-
return { ok: false, error:
|
|
4508
|
+
const occupancy = this.occupancy(); // prunes expired cooling stamps; counts each name once
|
|
4509
|
+
if (occupancy.used >= MAX_AGENTS)
|
|
4510
|
+
return { ok: false, error: this.capacityRefusal(occupancy) };
|
|
4347
4511
|
// Harness preflight before reserving a slot or minting — a missing `claude`/`opencode` binary
|
|
4348
4512
|
// fails here with a clear name, not obscurely at process spawn. No fallback. All synchronous, so
|
|
4349
4513
|
// the reserve gate stays atomic. (The connector itself was resolved up top, before the capacity gate.)
|
|
@@ -4496,6 +4660,21 @@ export class Manager {
|
|
|
4496
4660
|
return { ok: false, error: `${agent} connector does not support model variants (variant)` };
|
|
4497
4661
|
if (prompt !== undefined && !connector.supportsPrompt)
|
|
4498
4662
|
return { ok: false, error: `${agent} connector does not support an initial prompt (prompt)` };
|
|
4663
|
+
// A manifest `continuity: exact` agent reopens the session this manager last bound to its
|
|
4664
|
+
// declared name. Refused before any reserve or mint when the connector cannot reopen an exact
|
|
4665
|
+
// session or the recorded assignment does not match this declaration.
|
|
4666
|
+
const exact = opts.resolved?.continuity === "exact";
|
|
4667
|
+
let reopenSession;
|
|
4668
|
+
if (exact) {
|
|
4669
|
+
if (!connector.supportsSessionReopen || !connector.supportsSessionContinuation)
|
|
4670
|
+
return { ok: false, error: `${agent} connector does not support exact session continuity (continuity: exact)` };
|
|
4671
|
+
try {
|
|
4672
|
+
reopenSession = readContinuityAssignment(this.workspaceRoot, { space: this.space, name: identityName, connector: agent, cwd: resolvedCwd ?? this.workspaceRoot });
|
|
4673
|
+
}
|
|
4674
|
+
catch (e) {
|
|
4675
|
+
return { ok: false, error: e.message };
|
|
4676
|
+
}
|
|
4677
|
+
}
|
|
4499
4678
|
// #4 A4 (panel): the roster the allocation consults must reflect the initial presence snapshot,
|
|
4500
4679
|
// or a spawn immediately after manager boot races an already-live unmanaged peer and re-opens the
|
|
4501
4680
|
// very collision black-hole this closes. Await the snapshot (bounded internally, fail-safe on an
|
|
@@ -4769,8 +4948,10 @@ export class Manager {
|
|
|
4769
4948
|
// control arg), never from `opts.resolved` — so the manifest launch path carries no resume by
|
|
4770
4949
|
// construction. An unsupported connector throws here before any process is spawned.
|
|
4771
4950
|
resume: opts.resume,
|
|
4951
|
+
reopenSession,
|
|
4772
4952
|
// Initial prompt: the `--prompt` flag, or the manifest entry's `prompt:` on a resolved launch.
|
|
4773
|
-
|
|
4953
|
+
// A reopened session already had its kickoff turn, so it is not submitted again.
|
|
4954
|
+
prompt: reopenSession === undefined ? prompt : undefined,
|
|
4774
4955
|
// The SAME access set the creds were minted from (above) — forwarded so the session's
|
|
4775
4956
|
// runtime read/post set matches its credentials. Without this a manifest-spawned agent
|
|
4776
4957
|
// (materialized persona has no access frontmatter) has no channel set to read and joins
|
|
@@ -4833,6 +5014,7 @@ export class Manager {
|
|
|
4833
5014
|
events,
|
|
4834
5015
|
shareTools: opts.shareTools,
|
|
4835
5016
|
forkSource: opts.resume,
|
|
5017
|
+
...(opts.resume !== undefined && spec.resumeRecordPath ? { resumeRecordPath: spec.resumeRecordPath } : {}),
|
|
4836
5018
|
// Opaque values may contain secrets. Preserve only their keys and require the referenced
|
|
4837
5019
|
// persona/manifest to resolve the values again; imperative overrides have no safe payload.
|
|
4838
5020
|
unresolvedLaunchOptionKeys: opts.launchOptions && Object.keys(opts.launchOptions).length
|
|
@@ -4867,6 +5049,7 @@ export class Manager {
|
|
|
4867
5049
|
});
|
|
4868
5050
|
}
|
|
4869
5051
|
this.agents.set(name, managed);
|
|
5052
|
+
this.reservedLive.add(name);
|
|
4870
5053
|
// The live slot now owns teardown — freeSlot deprovisions this identity on exit — so the
|
|
4871
5054
|
// orphan-rollback in `finally` no longer applies to it.
|
|
4872
5055
|
provisioned = undefined;
|
|
@@ -4882,10 +5065,15 @@ export class Manager {
|
|
|
4882
5065
|
hooks?.onTerminalDeferred?.();
|
|
4883
5066
|
return { ok: false, error: readiness.detail };
|
|
4884
5067
|
}
|
|
4885
|
-
if (!readiness.ok && !readiness.uncertain) {
|
|
4886
|
-
|
|
4887
|
-
|
|
4888
|
-
|
|
5068
|
+
if (!readiness.ok && !readiness.uncertain) { // failed → already reaped
|
|
5069
|
+
// A reopen fails before readiness when the host no longer has the recorded session, which
|
|
5070
|
+
// is refused rather than replaced by an empty one; name the record that pins it.
|
|
5071
|
+
const error = reopenSession === undefined ? readiness.detail
|
|
5072
|
+
: `${readiness.detail} (continuity: exact was reopening recorded session ${reopenSession}; if the harness no longer has it, ` +
|
|
5073
|
+
`remove ${join(this.workspaceRoot, ".cotal", "continuity", `${name}.json`)} to start a new session)`;
|
|
5074
|
+
await hooks?.onOutcome?.({ kind: "failed", data: { error } });
|
|
5075
|
+
return { ok: false, error };
|
|
5076
|
+
}
|
|
4889
5077
|
// Started OR uncertain: the agent stays managed, so wire the ongoing exit reaper (it reaps a later
|
|
4890
5078
|
// death — including one that follows an `uncertain` verdict, which deliberately does NOT deprovision).
|
|
4891
5079
|
if (!readiness.ok) {
|
|
@@ -4897,7 +5085,17 @@ export class Manager {
|
|
|
4897
5085
|
if (connector.supportsSessionContinuation) {
|
|
4898
5086
|
try {
|
|
4899
5087
|
await this.armSessionRecovery(managed);
|
|
5088
|
+
// The capability flags are only a claim. An exact seat is recorded once the connector
|
|
5089
|
+
// has proved its session over its authenticated control endpoint, never before.
|
|
5090
|
+
if (exact && !managed.restart.armed)
|
|
5091
|
+
throw new Error("continuity: exact needs the connector to prove its session over an authenticated control endpoint, and it supplied none");
|
|
4900
5092
|
managed.launch.sessionId = this.readManagedSession(managed);
|
|
5093
|
+
if (reopenSession !== undefined && managed.launch.sessionId !== reopenSession)
|
|
5094
|
+
throw new Error(`continuity: exact asked for session ${reopenSession}, the connector bound ${managed.launch.sessionId}`);
|
|
5095
|
+
if (exact) {
|
|
5096
|
+
managed.continuity = { space: this.space, name, connector: agent, cwd };
|
|
5097
|
+
this.recordContinuity(managed, managed.launch.sessionId);
|
|
5098
|
+
}
|
|
4901
5099
|
}
|
|
4902
5100
|
catch (error) {
|
|
4903
5101
|
const detail = `${managed.name} joined, but its exact host session could not be bound for supervised recovery: ${error.message}`;
|
|
@@ -4936,6 +5134,7 @@ export class Manager {
|
|
|
4936
5134
|
}
|
|
4937
5135
|
finally {
|
|
4938
5136
|
this.reserved.delete(name);
|
|
5137
|
+
this.reservedLive.delete(name);
|
|
4939
5138
|
// Minted but never handed to a live slot (buildLaunch / runtime.spawn threw after mint) → tear the
|
|
4940
5139
|
// orphan down (detached, fail-loud) so a failed spawn leaves no creds/durables behind (#159 B).
|
|
4941
5140
|
if (provisioned) {
|
|
@@ -4952,6 +5151,7 @@ export class Manager {
|
|
|
4952
5151
|
if (!release)
|
|
4953
5152
|
return { ok: false, agents: [], error: this.maintenanceError() };
|
|
4954
5153
|
const batchReservations = [];
|
|
5154
|
+
const prepared = new Map();
|
|
4955
5155
|
try {
|
|
4956
5156
|
if (inventory.version !== "cotal-manager-resume/v1")
|
|
4957
5157
|
return { ok: false, agents: [], error: `unsupported manager resume inventory version ${String(inventory.version)}` };
|
|
@@ -4961,8 +5161,9 @@ export class Manager {
|
|
|
4961
5161
|
const principals = new Set();
|
|
4962
5162
|
await this.ep.waitForPresenceSnapshot();
|
|
4963
5163
|
const liveRoster = this.ep.getRoster().filter((presence) => presence.status !== "offline");
|
|
4964
|
-
|
|
4965
|
-
|
|
5164
|
+
const occupancy = this.occupancy();
|
|
5165
|
+
if (occupancy.used + inventory.agents.length > MAX_AGENTS)
|
|
5166
|
+
return { ok: false, agents: [], error: `resume inventory of ${inventory.agents.length} would exceed manager capacity (${this.occupancyText(occupancy)})` };
|
|
4966
5167
|
for (const entry of inventory.agents) {
|
|
4967
5168
|
if (seen.has(entry.name))
|
|
4968
5169
|
return { ok: false, agents: [], error: `resume inventory contains duplicate agent name "${entry.name}"` };
|
|
@@ -5018,7 +5219,6 @@ export class Manager {
|
|
|
5018
5219
|
this.reserved.add(entry.name);
|
|
5019
5220
|
batchReservations.push(entry.name);
|
|
5020
5221
|
}
|
|
5021
|
-
const prepared = new Map();
|
|
5022
5222
|
const preflight = [];
|
|
5023
5223
|
for (const entry of inventory.agents) {
|
|
5024
5224
|
const reply = await this.resumePreservedAgent(entry, true, true, prepared);
|
|
@@ -5047,8 +5247,22 @@ export class Manager {
|
|
|
5047
5247
|
return { ok: true, agents };
|
|
5048
5248
|
}
|
|
5049
5249
|
finally {
|
|
5050
|
-
for (const name of batchReservations)
|
|
5250
|
+
for (const name of batchReservations) {
|
|
5051
5251
|
this.reserved.delete(name);
|
|
5252
|
+
this.reservedLive.delete(name);
|
|
5253
|
+
}
|
|
5254
|
+
// Preflight builds every launch before any child starts, so a refused batch, a failed launch,
|
|
5255
|
+
// or the agents skipped after it leave built specs no runtime ever received.
|
|
5256
|
+
for (const { spec, spawned } of prepared.values()) {
|
|
5257
|
+
if (spawned)
|
|
5258
|
+
continue;
|
|
5259
|
+
try {
|
|
5260
|
+
discardLaunchArtifacts(spec.artifacts);
|
|
5261
|
+
}
|
|
5262
|
+
catch (e) {
|
|
5263
|
+
console.error(`! resume: ${e.message}`);
|
|
5264
|
+
}
|
|
5265
|
+
}
|
|
5052
5266
|
release();
|
|
5053
5267
|
}
|
|
5054
5268
|
}
|
|
@@ -5232,8 +5446,9 @@ export class Manager {
|
|
|
5232
5446
|
return { ok: false, error: nameErr };
|
|
5233
5447
|
if (this.agents.has(entry.name) || (!batchReserved && this.reserved.has(entry.name)))
|
|
5234
5448
|
return { ok: false, error: `retained agent "${entry.name}" is already managed or reserved; same-principal resume never auto-numbers` };
|
|
5235
|
-
|
|
5236
|
-
|
|
5449
|
+
const occupancy = batchReserved ? undefined : this.occupancy();
|
|
5450
|
+
if (occupancy && occupancy.used >= MAX_AGENTS)
|
|
5451
|
+
return { ok: false, error: `${this.capacityRefusal(occupancy)}; same-principal resume refused` };
|
|
5237
5452
|
const cached = prepared?.get(entry.name);
|
|
5238
5453
|
if (!preflightOnly && cached) {
|
|
5239
5454
|
try {
|
|
@@ -5286,6 +5501,7 @@ export class Manager {
|
|
|
5286
5501
|
if (retainedSession && !connector.supportsSessionContinuation)
|
|
5287
5502
|
return { ok: false, error: `${connector.name} connector does not support exact-session continuation` };
|
|
5288
5503
|
let launchOptions;
|
|
5504
|
+
let exact = false;
|
|
5289
5505
|
if (entry.launch.source.kind === "manifest") {
|
|
5290
5506
|
const launchSource = entry.launch.source;
|
|
5291
5507
|
if (!launchSource.runId)
|
|
@@ -5303,6 +5519,11 @@ export class Manager {
|
|
|
5303
5519
|
if (!spec || spec.hash !== launchSource.hash)
|
|
5304
5520
|
return { ok: false, error: `retained manifest agent ${launchSource.requested} is missing or its hash changed; refusing same-principal resume` };
|
|
5305
5521
|
launchOptions = spec.launchOptions;
|
|
5522
|
+
exact = spec.continuity === "exact";
|
|
5523
|
+
// An exact seat reopens its retained session or fails, like its manifest launch, so the
|
|
5524
|
+
// connector must honor reopenSession before the batch starts any child.
|
|
5525
|
+
if (exact && (!connector.supportsSessionReopen || !connector.supportsSessionContinuation))
|
|
5526
|
+
return { ok: false, error: `${connector.name} connector does not support exact session continuity (continuity: exact)` };
|
|
5306
5527
|
}
|
|
5307
5528
|
else {
|
|
5308
5529
|
try {
|
|
@@ -5344,7 +5565,8 @@ export class Manager {
|
|
|
5344
5565
|
variant: entry.launch.variant,
|
|
5345
5566
|
launchOptions,
|
|
5346
5567
|
resume: retainedSession ? undefined : entry.launch.forkSource,
|
|
5347
|
-
continueSession: retainedSession,
|
|
5568
|
+
continueSession: exact ? undefined : retainedSession,
|
|
5569
|
+
reopenSession: exact ? retainedSession : undefined,
|
|
5348
5570
|
subscribe: entry.launch.subscribe,
|
|
5349
5571
|
allowSubscribe: entry.launch.allowSubscribe,
|
|
5350
5572
|
allowPublish: entry.launch.allowPublish,
|
|
@@ -5356,7 +5578,7 @@ export class Manager {
|
|
|
5356
5578
|
};
|
|
5357
5579
|
const spec = connector.buildLaunch(launchOpts);
|
|
5358
5580
|
spec.env = { ...spec.env, COTAL_MANAGER_INSTANCE: this.managerInstanceId };
|
|
5359
|
-
const value = { spec, launchOpts, ...authority };
|
|
5581
|
+
const value = { spec, launchOpts, ...authority, exact };
|
|
5360
5582
|
prepared?.set(entry.name, value);
|
|
5361
5583
|
if (preflightOnly)
|
|
5362
5584
|
return { ok: true, data: { name: entry.name, preflight: true } };
|
|
@@ -5392,6 +5614,7 @@ export class Manager {
|
|
|
5392
5614
|
const custody = this.reserveCustody();
|
|
5393
5615
|
if (custody)
|
|
5394
5616
|
await this.recordSlotCustody({ name: entry.name, id: resumeId, lifecycleUid: entry.identity.lifecycleUid }, custody);
|
|
5617
|
+
prepared.spawned = true;
|
|
5395
5618
|
const handle = await this.spawnCustodied(entry.name, prepared.spec, entry.launch.cwd, custody);
|
|
5396
5619
|
const managed = {
|
|
5397
5620
|
name: entry.name,
|
|
@@ -5428,6 +5651,8 @@ export class Manager {
|
|
|
5428
5651
|
events: entry.launch.events,
|
|
5429
5652
|
shareTools: entry.launch.shareTools,
|
|
5430
5653
|
forkSource: entry.launch.forkSource,
|
|
5654
|
+
...(entry.launch.resumed ? { resumed: entry.launch.resumed } : {}),
|
|
5655
|
+
...(prepared.launchOpts.resume !== undefined && prepared.spec.resumeRecordPath ? { resumeRecordPath: prepared.spec.resumeRecordPath } : {}),
|
|
5431
5656
|
sessionId: entry.launch.sessionId,
|
|
5432
5657
|
},
|
|
5433
5658
|
...(prepared.spec.sessionStatePath
|
|
@@ -5436,6 +5661,7 @@ export class Manager {
|
|
|
5436
5661
|
suppressCleanup: true,
|
|
5437
5662
|
};
|
|
5438
5663
|
this.agents.set(entry.name, managed);
|
|
5664
|
+
this.reservedLive.add(entry.name);
|
|
5439
5665
|
if (this.resumeAttemptId)
|
|
5440
5666
|
this.resumedAgentNames.add(entry.name);
|
|
5441
5667
|
await this.recordSlotRuntime(managed);
|
|
@@ -5447,10 +5673,21 @@ export class Manager {
|
|
|
5447
5673
|
this.watchResumeAdoption(managed);
|
|
5448
5674
|
return { ok: false, error: readiness.detail };
|
|
5449
5675
|
}
|
|
5450
|
-
if (managed.restart) {
|
|
5676
|
+
if (managed.restart || prepared.exact) {
|
|
5451
5677
|
try {
|
|
5452
5678
|
await this.armSessionRecovery(managed);
|
|
5679
|
+
// As on a manifest launch, an exact seat is recorded only at the session it asked for,
|
|
5680
|
+
// proved over the connector's authenticated control endpoint.
|
|
5681
|
+
if (prepared.exact && !managed.restart?.armed)
|
|
5682
|
+
throw new Error("continuity: exact needs the connector to prove its session over an authenticated control endpoint, and it supplied none");
|
|
5453
5683
|
managed.launch.sessionId = this.readManagedSession(managed);
|
|
5684
|
+
const asked = prepared.launchOpts.reopenSession;
|
|
5685
|
+
if (asked !== undefined && managed.launch.sessionId !== asked)
|
|
5686
|
+
throw new Error(`continuity: exact asked for session ${asked}, the connector bound ${managed.launch.sessionId}`);
|
|
5687
|
+
if (prepared.exact) {
|
|
5688
|
+
managed.continuity = { space: this.space, name: managed.name, connector: managed.agent, cwd: managed.launch.cwd };
|
|
5689
|
+
this.recordContinuity(managed, managed.launch.sessionId);
|
|
5690
|
+
}
|
|
5454
5691
|
}
|
|
5455
5692
|
catch (error) {
|
|
5456
5693
|
this.stopHandle(managed, false);
|
|
@@ -5472,8 +5709,10 @@ export class Manager {
|
|
|
5472
5709
|
return { ok: false, error: e.message };
|
|
5473
5710
|
}
|
|
5474
5711
|
finally {
|
|
5475
|
-
if (!batchReserved)
|
|
5712
|
+
if (!batchReserved) {
|
|
5476
5713
|
this.reserved.delete(entry.name);
|
|
5714
|
+
this.reservedLive.delete(entry.name);
|
|
5715
|
+
}
|
|
5477
5716
|
}
|
|
5478
5717
|
}
|
|
5479
5718
|
probeStaticCredential(creds) {
|
|
@@ -5677,6 +5916,48 @@ export class Manager {
|
|
|
5677
5916
|
this.cooling = this.cooling.filter((stamp) => stamp > now);
|
|
5678
5917
|
return this.cooling.length;
|
|
5679
5918
|
}
|
|
5919
|
+
/** What holds the ceiling's slots, one count per seat (#906). A launching seat sits in `reserved`
|
|
5920
|
+
* from accept until its launch settles, in `agents` once its process exists, and in `cooling`
|
|
5921
|
+
* if it then frees young, so a reservation stops counting once it reaches `agents`. `managed` is
|
|
5922
|
+
* what this manager's `ps` lists, `reserved` is the accepted launches that have no process yet,
|
|
5923
|
+
* and `launching` is every unsettled launch that still holds a slot through `reserved` or
|
|
5924
|
+
* `agents`. A launch whose seat already freed holds no slot but its cooling stamp, so its
|
|
5925
|
+
* failure can free nothing. */
|
|
5926
|
+
occupancy() {
|
|
5927
|
+
const cooling = this.coolingCount();
|
|
5928
|
+
let reserved = 0;
|
|
5929
|
+
let launching = 0;
|
|
5930
|
+
for (const name of this.reserved) {
|
|
5931
|
+
const placed = this.reservedLive.has(name);
|
|
5932
|
+
if (!placed)
|
|
5933
|
+
reserved++;
|
|
5934
|
+
if (!placed || this.agents.has(name))
|
|
5935
|
+
launching++;
|
|
5936
|
+
}
|
|
5937
|
+
const managed = this.agents.size;
|
|
5938
|
+
return {
|
|
5939
|
+
managed,
|
|
5940
|
+
reserved,
|
|
5941
|
+
cooling,
|
|
5942
|
+
launching,
|
|
5943
|
+
used: managed + reserved + cooling,
|
|
5944
|
+
nextCoolingMs: cooling ? Math.min(...this.cooling) - Date.now() : undefined,
|
|
5945
|
+
};
|
|
5946
|
+
}
|
|
5947
|
+
occupancyText(o) {
|
|
5948
|
+
return `${o.used} of ${MAX_AGENTS} slots: ${o.managed} managed, ${o.reserved} reserved, ${o.cooling} cooling`;
|
|
5949
|
+
}
|
|
5950
|
+
/** The capacity refusal states the split it enforces and whether waiting can free a slot: a
|
|
5951
|
+
* cooling slot frees at a known time, an unsettled launch frees one only if it fails, and a
|
|
5952
|
+
* managed seat frees one only when it is stopped. */
|
|
5953
|
+
capacityRefusal(o) {
|
|
5954
|
+
const remedy = o.nextCoolingMs !== undefined
|
|
5955
|
+
? `waiting frees a cooling slot in ${Math.ceil(o.nextCoolingMs / 1000)}s, or despawn one`
|
|
5956
|
+
: o.launching
|
|
5957
|
+
? `waiting frees a slot only if an unsettled launch fails (${o.launching} pending); otherwise despawn one`
|
|
5958
|
+
: "waiting frees no slot; despawn one";
|
|
5959
|
+
return `at capacity (${this.occupancyText(o)}); ${remedy}`;
|
|
5960
|
+
}
|
|
5680
5961
|
async opStop(args, caller, admin) {
|
|
5681
5962
|
const name = String(args.name ?? "").trim();
|
|
5682
5963
|
const a = this.agents.get(name);
|
|
@@ -5836,7 +6117,7 @@ export class Manager {
|
|
|
5836
6117
|
return {
|
|
5837
6118
|
instanceId: this.managerInstanceId,
|
|
5838
6119
|
runtime: this.runtime.kind,
|
|
5839
|
-
custody:
|
|
6120
|
+
custody: this.runtime.kind === "pty" && this.runtime.supportsRelease === true ? "custodied" : "legacy",
|
|
5840
6121
|
agentCount: this.agents.size,
|
|
5841
6122
|
uptimeMs: Date.now() - this.startedAtMs,
|
|
5842
6123
|
connectors: this.connectorStatuses.map((row) => ({ ...row, binaries: { ...row.binaries } })),
|
|
@@ -5887,7 +6168,7 @@ export class Manager {
|
|
|
5887
6168
|
report = await reconcileEndpointGate({
|
|
5888
6169
|
kv: authKv, space: this.space, endpoint: MANAGER_ENDPOINT, instanceId,
|
|
5889
6170
|
probeHolder: makeManagerHolderLivenessProbe({ space: this.space, servers, auth, log }),
|
|
5890
|
-
|
|
6171
|
+
evictHolders: makeManagerEndpointHolderEvictor({ space: this.space, servers, auth, log }),
|
|
5891
6172
|
log,
|
|
5892
6173
|
recordsKv,
|
|
5893
6174
|
});
|
|
@@ -5921,6 +6202,17 @@ export class Manager {
|
|
|
5921
6202
|
inboxPrefix: `_INBOX_${state.identity.id}`,
|
|
5922
6203
|
maxReconnectAttempts: -1,
|
|
5923
6204
|
});
|
|
6205
|
+
// Track the transport from the first moment: nats.js reports `disconnect` when the socket
|
|
6206
|
+
// drops and `reconnect` only after it has re-sent this connection's subscriptions. If the
|
|
6207
|
+
// status stream itself fails, the transport can no longer be observed, so it counts as down.
|
|
6208
|
+
void (async () => {
|
|
6209
|
+
for await (const st of nc.status()) {
|
|
6210
|
+
if (st.type === "disconnect")
|
|
6211
|
+
this.serveTransportDown.add(nc);
|
|
6212
|
+
else if (st.type === "reconnect")
|
|
6213
|
+
this.serveTransportDown.delete(nc);
|
|
6214
|
+
}
|
|
6215
|
+
})().catch(() => this.serveTransportDown.add(nc));
|
|
5924
6216
|
let handle;
|
|
5925
6217
|
try {
|
|
5926
6218
|
// The 1b typed surface + the derived `describe`. The descriptor stays PUBLIC in static
|
|
@@ -6294,15 +6586,6 @@ export class Manager {
|
|
|
6294
6586
|
});
|
|
6295
6587
|
return creds;
|
|
6296
6588
|
}
|
|
6297
|
-
/** P2 item 2 (spawn-as-action): stand up the standing self-mediated goal-writer connection +
|
|
6298
|
-
* ActionContext. Mode-dual, mirroring {@link registerManagerService}: an AUTH mesh uses the
|
|
6299
|
-
* scoped `goal-writer` credential already minted + family-STAGED inside registration's run block
|
|
6300
|
-
* ({@link mintAndStageGoalWriter} — DISJOINT grant from the serve cred, SHARED §13.1 revocation
|
|
6301
|
-
* family); an OPEN mesh uses a bare connection (no credential system to mint from - the broker
|
|
6302
|
-
* enforces nothing). The connection presents the CURRENT credential on every (re)connect (a
|
|
6303
|
-
* renewal is adopted without reconnecting the whole endpoint); the ActionContext bonds its
|
|
6304
|
-
* KV + JS + JSM to this one connection and space (SPEC 13.4), so a composition mixup cannot splice
|
|
6305
|
-
* goal state across brokers. */
|
|
6306
6589
|
/** #2073: dial the goal-writer's ONE standing connection and rebuild everything derived from it
|
|
6307
6590
|
* (the fence-resolving action context, the issuance-gate reader) — shared by the initial start
|
|
6308
6591
|
* and the closed-connection recovery ({@link onGoalWriterConnectionClosed}), so a redial never
|
|
@@ -6370,6 +6653,15 @@ export class Manager {
|
|
|
6370
6653
|
return;
|
|
6371
6654
|
console.error("! manager goal-writer could not be re-dialed; leaving the credential to the next renewal pass");
|
|
6372
6655
|
}
|
|
6656
|
+
/** P2 item 2 (spawn-as-action): stand up the standing self-mediated goal-writer connection +
|
|
6657
|
+
* ActionContext. Mode-dual, mirroring {@link registerManagerService}: an AUTH mesh uses the
|
|
6658
|
+
* scoped `goal-writer` credential already minted + family-STAGED inside registration's run block
|
|
6659
|
+
* ({@link mintAndStageGoalWriter} — DISJOINT grant from the serve cred, SHARED §13.1 revocation
|
|
6660
|
+
* family); an OPEN mesh uses a bare connection (no credential system to mint from - the broker
|
|
6661
|
+
* enforces nothing). The connection presents the CURRENT credential on every (re)connect (a
|
|
6662
|
+
* renewal is adopted without reconnecting the whole endpoint); the ActionContext bonds its
|
|
6663
|
+
* KV + JS + JSM to this one connection and space (SPEC 13.4), so a composition mixup cannot splice
|
|
6664
|
+
* goal state across brokers. */
|
|
6373
6665
|
async startGoalWriter() {
|
|
6374
6666
|
const identity = (this.auth || this.remoteAuthority) ? this.goalWriterIdentity : newIdentity();
|
|
6375
6667
|
// The mutable holder captured by the authenticator (mirrors the serve connection): a half-TTL
|
|
@@ -6431,18 +6723,6 @@ export class Manager {
|
|
|
6431
6723
|
});
|
|
6432
6724
|
return creds;
|
|
6433
6725
|
}
|
|
6434
|
-
/** P2 item 6: stand up the ONE §13.6 session plane on its own standing connection. Mode-dual,
|
|
6435
|
-
* mirroring {@link startGoalWriter}: an AUTH mesh presents the scoped `session-ledger` cred
|
|
6436
|
-
* already minted + family-staged inside registration's run block ({@link mintAndStageSessionLedger});
|
|
6437
|
-
* an OPEN mesh uses a bare connection (no credential system to mint from). The connection presents
|
|
6438
|
-
* the CURRENT credential on every (re)connect, so a half-TTL renewal is adopted without reconnecting.
|
|
6439
|
-
*
|
|
6440
|
-
* The offer SIGNER is a per-incarnation in-memory keypair: the static collapsed path mints AND
|
|
6441
|
-
* redeems the offer in one call ({@link ManagerSessionPlane.establishAttach}), so the manager
|
|
6442
|
-
* self-signs and self-verifies its own §13.6 grants and the keypair never leaves the process — a
|
|
6443
|
-
* holder never verifies the signature (it presents the grant back over the rail; the broker's
|
|
6444
|
-
* per-session caller cred is the holder's real fence). The plane's ledger lives in the DEDICATED
|
|
6445
|
-
* sessions bucket (createEndpointStreams provisioned it at registration). */
|
|
6446
6726
|
/** #2073: dial the session-ledger's ONE standing connection — shared by the initial start and the
|
|
6447
6727
|
* closed-connection recovery ({@link onSessionLedgerConnectionClosed}), the same shape {@link
|
|
6448
6728
|
* dialGoalWriter} uses for its rail. The KV/JetStream clients the session plane opened from the
|
|
@@ -6487,6 +6767,18 @@ export class Manager {
|
|
|
6487
6767
|
return;
|
|
6488
6768
|
console.error("! manager session-ledger could not be re-dialed; leaving the credential to the next renewal pass");
|
|
6489
6769
|
}
|
|
6770
|
+
/** P2 item 6: stand up the ONE §13.6 session plane on its own standing connection. Mode-dual,
|
|
6771
|
+
* mirroring {@link startGoalWriter}: an AUTH mesh presents the scoped `session-ledger` cred
|
|
6772
|
+
* already minted + family-staged inside registration's run block ({@link mintAndStageSessionLedger});
|
|
6773
|
+
* an OPEN mesh uses a bare connection (no credential system to mint from). The connection presents
|
|
6774
|
+
* the CURRENT credential on every (re)connect, so a half-TTL renewal is adopted without reconnecting.
|
|
6775
|
+
*
|
|
6776
|
+
* The offer SIGNER is a per-incarnation in-memory keypair: the static collapsed path mints AND
|
|
6777
|
+
* redeems the offer in one call ({@link ManagerSessionPlane.establishAttach}), so the manager
|
|
6778
|
+
* self-signs and self-verifies its own §13.6 grants and the keypair never leaves the process — a
|
|
6779
|
+
* holder never verifies the signature (it presents the grant back over the rail; the broker's
|
|
6780
|
+
* per-session caller cred is the holder's real fence). The plane's ledger lives in the DEDICATED
|
|
6781
|
+
* sessions bucket (createEndpointStreams provisioned it at registration). */
|
|
6490
6782
|
async startSessionPlane() {
|
|
6491
6783
|
const identity = (this.auth || this.remoteAuthority) ? this.sessionLedgerIdentity : newIdentity();
|
|
6492
6784
|
// The mutable holder captured by the authenticator (mirrors the goal-writer): a half-TTL renewal
|
|
@@ -8627,8 +8919,9 @@ export class Manager {
|
|
|
8627
8919
|
throw e;
|
|
8628
8920
|
}
|
|
8629
8921
|
}
|
|
8630
|
-
/** Managed agents cross-referenced with live presence (the manager sees the roster).
|
|
8631
|
-
|
|
8922
|
+
/** Managed agents cross-referenced with live presence (the manager sees the roster).
|
|
8923
|
+
*
|
|
8924
|
+
* `ownerFilter`: restrict to agents whose spawn-time stored `userOwner` equals it (the ps/status
|
|
8632
8925
|
* owner-domain bound); undefined = unbounded. {@link NO_OWNER_MATCHES} matches nothing. */
|
|
8633
8926
|
list(ownerFilter) {
|
|
8634
8927
|
const roster = new Map(this.ep.getRoster().map((p) => [p.card.name, p]));
|
|
@@ -8650,8 +8943,33 @@ export class Manager {
|
|
|
8650
8943
|
: undefined;
|
|
8651
8944
|
// The connector-reported provider from presence (#785): the live roster card's `meta.provider`,
|
|
8652
8945
|
// when it is a non-empty string. Absent when the connector reported none — never fabricated.
|
|
8653
|
-
const
|
|
8946
|
+
const seen = roster.get(a.name);
|
|
8947
|
+
const providerMeta = seen?.card.meta?.provider;
|
|
8654
8948
|
const provider = typeof providerMeta === "string" && providerMeta ? providerMeta : undefined;
|
|
8949
|
+
// The harness-reported condition beside the mesh status (#618): a turn that died upstream reads
|
|
8950
|
+
// `waiting` with `rate_limit` here, not a bare `waiting`. Absent when the connector reported none.
|
|
8951
|
+
// Projected field by field like `provider`: a malformed peer row must not fail this reply's schema.
|
|
8952
|
+
const reported = roster.get(a.name)?.condition;
|
|
8953
|
+
const condition = reported && typeof reported.code === "string" && reported.code
|
|
8954
|
+
? {
|
|
8955
|
+
code: reported.code,
|
|
8956
|
+
...(typeof reported.source === "string" ? { source: reported.source } : {}),
|
|
8957
|
+
...(typeof reported.message === "string" ? { message: reported.message } : {}),
|
|
8958
|
+
...(typeof reported.since === "number" && Number.isFinite(reported.since) ? { since: reported.since } : {}),
|
|
8959
|
+
}
|
|
8960
|
+
: undefined;
|
|
8961
|
+
// The seat's last harness-reported work progress (#618). `ts` is only its heartbeat, so a turn
|
|
8962
|
+
// that stopped advancing reads as an old `activeAt` beside a live process. Absent when none was reported.
|
|
8963
|
+
const activeAtReported = roster.get(a.name)?.activeAt;
|
|
8964
|
+
const activeAt = typeof activeAtReported === "number" && Number.isFinite(activeAtReported) ? activeAtReported : undefined;
|
|
8965
|
+
// How long an offline verdict has stood (#1208): the seat's last heartbeat before it went
|
|
8966
|
+
// offline. Only beside a verdict, since a stale or unpopulated view dates nothing, only from the
|
|
8967
|
+
// seat's own record, since `seen` is matched by name and a same-named foreign peer or an older
|
|
8968
|
+
// incarnation must not date this seat, and only when the seat's unvalidated `ts` fits the row
|
|
8969
|
+
// schema, so one odd record cannot fail the reply.
|
|
8970
|
+
const offlineSince = view.state === "current" && this.isOwnPresence(a, seen) && seen.status === "offline" && Number.isSafeInteger(seen.ts) && seen.ts >= 0
|
|
8971
|
+
? seen.ts
|
|
8972
|
+
: undefined;
|
|
8655
8973
|
return {
|
|
8656
8974
|
name: a.name,
|
|
8657
8975
|
// The spawned agent's id (nkey, or the user-mode principal) — lets an operator tool (e.g.
|
|
@@ -8664,10 +8982,13 @@ export class Manager {
|
|
|
8664
8982
|
mode: a.handle.kind,
|
|
8665
8983
|
status: a.handle.status(),
|
|
8666
8984
|
uptimeMs: Date.now() - a.startedAt,
|
|
8667
|
-
mesh:
|
|
8985
|
+
mesh: seen?.status ?? "absent",
|
|
8986
|
+
...(condition ? { condition } : {}),
|
|
8987
|
+
...(activeAt !== undefined ? { activeAt } : {}),
|
|
8668
8988
|
// `current` is the only state in which `mesh` is a verdict; the other two are the
|
|
8669
8989
|
// observer's own condition and travel on the row (older CLIs ignore the field).
|
|
8670
8990
|
meshView: view.state,
|
|
8991
|
+
...(offlineSince !== undefined ? { offlineSince } : {}),
|
|
8671
8992
|
// The incarnation coordinate (SPEC 13.1) — with `id`, exactly what a v0.4 caller needs to
|
|
8672
8993
|
// build a targeted (`despawn`/`attach`) request against THIS incarnation.
|
|
8673
8994
|
lifecycleUid: a.lifecycleUid,
|
|
@@ -8683,6 +9004,9 @@ export class Manager {
|
|
|
8683
9004
|
spawner: a.spawner,
|
|
8684
9005
|
instanceId: this.managerInstanceId,
|
|
8685
9006
|
host: hostname(),
|
|
9007
|
+
// #1500: the session a `--resume` seat forked, with its title and transcript hash once the
|
|
9008
|
+
// seat has recorded them.
|
|
9009
|
+
...(a.launch.forkSource ? { resume: this.forkProvenance(a) ?? { source: a.launch.forkSource } } : {}),
|
|
8686
9010
|
...(health && health.state !== "ok" ? { authHealth: health.state, authReason: health.reason } : {}),
|
|
8687
9011
|
};
|
|
8688
9012
|
});
|