@cotal-ai/manager 0.53.0 → 0.55.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands.js +29 -6
- package/dist/commands.js.map +1 -1
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -0
- package/dist/index.js.map +1 -1
- package/dist/manager-service-contract.d.ts +9 -1
- package/dist/manager-service-contract.d.ts.map +1 -1
- package/dist/manager-service-contract.js +22 -2
- package/dist/manager-service-contract.js.map +1 -1
- package/dist/manager.d.ts +101 -1
- package/dist/manager.d.ts.map +1 -1
- package/dist/manager.js +321 -15
- package/dist/manager.js.map +1 -1
- package/dist/remote-authority.d.ts +20 -1
- package/dist/remote-authority.d.ts.map +1 -1
- package/dist/remote-authority.js +109 -1
- package/dist/remote-authority.js.map +1 -1
- package/dist/remote-register.d.ts.map +1 -1
- package/dist/remote-register.js +17 -1
- package/dist/remote-register.js.map +1 -1
- package/dist/runtime/custodial-pty.d.ts +8 -0
- package/dist/runtime/custodial-pty.d.ts.map +1 -1
- package/dist/runtime/custodial-pty.js +31 -4
- package/dist/runtime/custodial-pty.js.map +1 -1
- package/package.json +10 -10
package/dist/manager.js
CHANGED
|
@@ -4,7 +4,7 @@ import { hostname } from "node:os";
|
|
|
4
4
|
import { credsAuthenticator } from "@nats-io/transport-node";
|
|
5
5
|
import { existsSync, lstatSync, readFileSync, realpathSync, rmSync, statSync } from "node:fs";
|
|
6
6
|
import { isAbsolute, join, dirname, resolve } from "node:path";
|
|
7
|
-
import { CotalEndpoint, DEFAULT_SERVER, DEV_OWNER, MANAGER_LEASE_RENEW_MS, STANDING_RENEWABLE_TTL_SEC, divergentSecretStoreNotice, parseSecretStoreIdentity, sameSecretStoreIdentity, agentFilePath, clearSpaceHistory, dialerFor, connectorServers, spawnEnvAllow, deprovisionAgent, firstFreeName, spawnNameError, idFromCreds, inspectCredHealth, composeWirePersona, loadAgentFile, listPersonaCatalog, personaCatalogDescription, personaCatalogReadable, loadCotalConfig, mintCreds, mintGeneration, mintAcceptedToken, openIssuedStore, acceptedBucket, issuedBucket, mintLifecycleUid, managedRetirementOpId, mkSecretDir, newIdentity, actionContext, parsePrincipalKey, parseShareSelection, principalKey, probeConnect, provisionAgent, provisionAgentDurables, registry, resolveAuthProvider, saveAgentFile, subjectMatches, AUTH_ENDPOINT, EP_CMD_RETIRE_LIFECYCLE, epRequestSubject, epCallerReplyFilter, parseEpSubject, controlServiceSubject, eventChannelPrincipal, } from "@cotal-ai/core";
|
|
7
|
+
import { CotalEndpoint, DEFAULT_SERVER, DEV_OWNER, MANAGER_LEASE_RENEW_MS, STANDING_RENEWABLE_TTL_SEC, assertLifecycleToken, divergentSecretStoreNotice, parseDaemonStoreAnswer, parseSecretStoreIdentity, sameSecretStoreIdentity, agentFilePath, clearSpaceHistory, dialerFor, connectorServers, spawnEnvAllow, deprovisionAgent, firstFreeName, spawnNameError, idFromCreds, inspectCredHealth, composeWirePersona, loadAgentFile, listPersonaCatalog, personaCatalogDescription, personaCatalogReadable, loadCotalConfig, mintCreds, mintGeneration, mintAcceptedToken, openIssuedStore, acceptedBucket, issuedBucket, mintLifecycleUid, managedRetirementOpId, mkSecretDir, newIdentity, actionContext, parsePrincipalKey, parseShareSelection, principalKey, probeConnect, provisionAgent, provisionAgentDurables, registry, resolveAuthProvider, saveAgentFile, subjectMatches, AUTH_ENDPOINT, EP_CMD_RETIRE_LIFECYCLE, epRequestSubject, epCallerReplyFilter, parseEpSubject, controlServiceSubject, eventChannelPrincipal, } from "@cotal-ai/core";
|
|
8
8
|
import { agentAuthState, agentCredsDir, agentLifecycleSecretFilePaths, agentSecretFilePaths, agentSecretKeyForFile, authDir, connectorInstallHint, DEFAULT_CONNECTOR, defaultAgentType, DELIVERY_CREDS_KIND, extensionConnectors, findCotalRoot, getSpaceAuth, hasUserAuthState, loadExtensionsManifest, loadManagerInstanceIdentity, loadMeshes, localTrustOfSpace, manifestExtensionNames, materializeFromManifest, materializeSecretToFile, MEMBERSHIP_RW_CREDS_KIND, mergeLaunchOptions, remintDaemonCreds, resolveOnPath, createManagerInstanceIdentity, spaceMaterialKey, SYSTEM_CREDS_FILES, userAuthStateDir, workspaceSecretStore, writeRenewalRecord } from "@cotal-ai/workspace";
|
|
9
9
|
import { createRuntime, isCustodialRuntime, requireRuntimeAdopt, requireRuntimeReap, RuntimeReapUnproven, } from "./runtime/index.js";
|
|
10
10
|
import { AttachEndpoint } from "./attach-endpoint.js";
|
|
@@ -505,6 +505,13 @@ export class Manager {
|
|
|
505
505
|
/** A renew is in flight. The tick must not start a second one: both would read the same cached
|
|
506
506
|
* revision, so whichever lands second CASes against a sequence the first already moved. */
|
|
507
507
|
leaseRenewInFlight = false;
|
|
508
|
+
/** The promise a currently in-flight `renewLease()` is running, so `stop()` can await it before
|
|
509
|
+
* releasing the key instead of racing it: a renew that lands after the cached revision was read
|
|
510
|
+
* for release would otherwise move the row out from under a same-cached-revision CAS delete. */
|
|
511
|
+
leaseRenewTask;
|
|
512
|
+
/** Set at the top of `stop()`. `renewLease` returns at once once this is set, and its `gone` arm
|
|
513
|
+
* does not re-acquire: a manager that is stopping never puts its own key back. */
|
|
514
|
+
leaseStopping = false;
|
|
508
515
|
/** The class-2 renewal owner's half-TTL schedule (D5 slice 5); armed only on auth meshes. */
|
|
509
516
|
credRenewTimer;
|
|
510
517
|
/** #1634: does THIS manager own the delivery daemon's credential renewal? True only while the
|
|
@@ -890,13 +897,27 @@ export class Manager {
|
|
|
890
897
|
// manager records each agent's read ACL at spawn (`commitAcl`, in provisionAgent) so the daemon can
|
|
891
898
|
// re-authorize it; that is the only Plane-3 state the manager touches, and it rides minting.
|
|
892
899
|
}
|
|
900
|
+
/** The delivery shard whose lease names the process that reloads the standing credentials.
|
|
901
|
+
* Delivery is single-shard today and the endpoint's own responder reads row 0; naming it here
|
|
902
|
+
* keeps the manager's binding check and that reply arguing about the same row. */
|
|
903
|
+
static DELIVERY_SHARD = 0;
|
|
893
904
|
/** Proof that this manager remints into the store the delivery daemon reloads from.
|
|
894
905
|
* Fingerprint-only `reloadCreds` is safe only after this. Two named, different stores
|
|
895
906
|
* refuse the pass. A daemon that is not bound yet is not a named store: start and
|
|
896
907
|
* remint still proceed (tests, delayed delivery), writing this manager's store. The
|
|
897
908
|
* challenge runs again before every remint, so a later daemon on a foreign store is
|
|
898
909
|
* refused then rather than certified by an earlier absence. A request timeout is not
|
|
899
|
-
* absence: a hung rail would skip the proof, so it fails closed.
|
|
910
|
+
* absence: a hung rail would skip the proof, so it fails closed.
|
|
911
|
+
*
|
|
912
|
+
* ABSENCE IS READ OFF THE LEASE ROW, never off the rail's no-responder shape alone (#1694).
|
|
913
|
+
* The rail is queue-grouped, so "no responder answered" is itself something the rail reports,
|
|
914
|
+
* and a responder that answers in a shape the client turns into that error makes the pass read
|
|
915
|
+
* "absent" while a holder is live on the row — certifying every later remint with no proof at
|
|
916
|
+
* all. So a no-responder outcome is decided by the row: a row that names a holder makes the
|
|
917
|
+
* pass refuse (undetermined — a live daemon went unanswered, which must not read as "no
|
|
918
|
+
* daemon"), and only a row that is absent or holderless settles "absent", the state start()
|
|
919
|
+
* treats as not-yet-bound. An unreadable row (a failed read, or a key that exists but does not
|
|
920
|
+
* parse) is a genuine unknown and refuses fail-closed, like a hung rail. */
|
|
900
921
|
async daemonStoreRelation() {
|
|
901
922
|
let reply;
|
|
902
923
|
try {
|
|
@@ -905,24 +926,77 @@ export class Manager {
|
|
|
905
926
|
catch (e) {
|
|
906
927
|
const msg = e.message;
|
|
907
928
|
if (isAbsentDeliveryAdmin(msg))
|
|
908
|
-
return
|
|
929
|
+
return this.absentByLeaseRow();
|
|
909
930
|
throw new Error(`could not challenge the delivery daemon's SecretStore: ${msg}`);
|
|
910
931
|
}
|
|
911
932
|
if (!reply.ok)
|
|
912
933
|
throw new Error(reply.error ?? "delivery daemon refused to name the SecretStore it reloads from");
|
|
913
|
-
let
|
|
934
|
+
let answer;
|
|
914
935
|
try {
|
|
915
|
-
|
|
936
|
+
answer = parseDaemonStoreAnswer(reply.data);
|
|
916
937
|
}
|
|
917
938
|
catch (e) {
|
|
918
|
-
throw new Error(`delivery daemon named an unreadable SecretStore: ${e.message}`
|
|
919
|
-
|
|
939
|
+
throw new Error(`delivery daemon named an unreadable SecretStore: ${e.message}. A daemon older ` +
|
|
940
|
+
`than #1694 replies with a bare store identity and no answerer binding; this manager ` +
|
|
941
|
+
`cannot establish which process answered, so nothing reminted - upgrade the delivery daemon`);
|
|
942
|
+
}
|
|
943
|
+
// The answerer's own claim first, so an honest non-holder is refused by its own admission and
|
|
944
|
+
// the operator-facing reason names that. This is the CHEAP half of the test and NOT the binding
|
|
945
|
+
// one: a responder that does not hold the lease can assert `true` here just as easily.
|
|
946
|
+
if (!answer.holdsDeliveryLease)
|
|
947
|
+
throw new Error("the delivery-admin rail was answered by a responder that does not hold this space's " +
|
|
948
|
+
"delivery lease, so its SecretStore is not the store the daemon reloads from - nothing reminted");
|
|
949
|
+
// THE BINDING TEST, measured by this manager rather than asserted by the answerer. The lease
|
|
950
|
+
// row's holder is read here under this manager's own credential; the answerer must BE that
|
|
951
|
+
// principal. A reply asserting the claim while some other process holds the shard fails here,
|
|
952
|
+
// which is the case a bare store identity could not see. An unreadable row is an unknown and
|
|
953
|
+
// refuses too: this test may never be satisfied by failing to run.
|
|
954
|
+
const holder = await this.ep.deliveryLeaseHolder(Manager.DELIVERY_SHARD);
|
|
955
|
+
if (holder === undefined)
|
|
956
|
+
throw new Error("this manager could not read the delivery lease row, so it cannot verify that the answering " +
|
|
957
|
+
"responder is the process that reloads the standing credentials - nothing reminted");
|
|
958
|
+
if (holder !== answer.responder)
|
|
959
|
+
throw new Error(`the delivery-admin rail was answered by ${answer.responder}, which is not the holder of ` +
|
|
960
|
+
`this space's delivery lease (${holder}), so its SecretStore is not the store the daemon ` +
|
|
961
|
+
`reloads from - nothing reminted`);
|
|
962
|
+
const daemon = answer.identity;
|
|
920
963
|
if (!sameSecretStoreIdentity(this.secretStoreIdentity, daemon)) {
|
|
921
964
|
console.error(`! ${divergentSecretStoreNotice(this.secretStoreIdentity, daemon)}`);
|
|
922
965
|
return "divergent";
|
|
923
966
|
}
|
|
924
967
|
return "shared";
|
|
925
968
|
}
|
|
969
|
+
/** Settle a no-responder challenge outcome from the lease row, never from the rail's own
|
|
970
|
+
* reporting shape (#1694). The three cases:
|
|
971
|
+
*
|
|
972
|
+
* 1. The row READS but names a holder: a live daemon holds the shard and the rail went
|
|
973
|
+
* unanswered. That is undetermined, not "no daemon" — treating it as absent would let this
|
|
974
|
+
* manager remint with no proof while a reloading daemon runs, which is exactly what the
|
|
975
|
+
* challenge exists to prevent. Refuse.
|
|
976
|
+
* 2. The row is unreadable — a read that fails (a denied or erroring bucket) OR a key that
|
|
977
|
+
* exists but does not parse as a lease record (`readDeliveryLeaseEntry` throws for both):
|
|
978
|
+
* undetermined, and it refuses the same fail-closed way a hung rail does. Never a silent
|
|
979
|
+
* "absent" over a row the manager could not read.
|
|
980
|
+
* 3. The row is absent or holderless (no key, a DEL/PURGE tombstone, or a row that names no
|
|
981
|
+
* holder): today's "absent", the state start() treats as a daemon that is not bound yet.
|
|
982
|
+
*
|
|
983
|
+
* `deliveryLeaseHolder` swallows its read failures into `undefined`, which would fold case 2
|
|
984
|
+
* into case 3 — so this reads the entry directly and lets the read's own failure refuse. */
|
|
985
|
+
async absentByLeaseRow() {
|
|
986
|
+
let entry;
|
|
987
|
+
try {
|
|
988
|
+
entry = await this.ep.readDeliveryLeaseEntry(Manager.DELIVERY_SHARD);
|
|
989
|
+
}
|
|
990
|
+
catch (e) {
|
|
991
|
+
throw new Error(`could not read the delivery lease row to settle whether the delivery daemon is absent ` +
|
|
992
|
+
`(${e.message}) - absence is undetermined, nothing reminted`);
|
|
993
|
+
}
|
|
994
|
+
const holder = entry?.info.holder;
|
|
995
|
+
if (holder !== undefined && holder !== "")
|
|
996
|
+
throw new Error(`the delivery-admin rail reported no responder while the delivery lease names a holder ` +
|
|
997
|
+
`(${holder}) - absence cannot be read off the rail alone, and nothing reminted`);
|
|
998
|
+
return "absent";
|
|
999
|
+
}
|
|
926
1000
|
/** Keep this manager's renewal lease alive between credential passes (#1634). The manager bucket
|
|
927
1001
|
* expires keys at MANAGER_LEASE_TTL_MS, far below the credential pass's cadence, so the lease
|
|
928
1002
|
* needs its own heartbeat like the per-instance liveness lease. A non-owner calls this too: the
|
|
@@ -955,7 +1029,23 @@ export class Manager {
|
|
|
955
1029
|
*
|
|
956
1030
|
* A manager failing either test serves the space normally and skips only the daemon remint. */
|
|
957
1031
|
async claimDaemonRenewalOwnership() {
|
|
958
|
-
|
|
1032
|
+
// EVERY refusal outcome drops a held lease, not only "divergent". The refusal paths THROW
|
|
1033
|
+
// (#1694: a non-holder answer, an unbindable answerer, an unreadable or holder-naming row on a
|
|
1034
|
+
// no-responder rail), and a thrown refusal caught by renewDaemonCreds would otherwise leave a
|
|
1035
|
+
// manager that once owned the renewal still holding the lease behind its own heartbeat — the
|
|
1036
|
+
// pass refuses, the log says so, and the space keeps a renewal owner that must not remint.
|
|
1037
|
+
// Deciding here, inside the claim, is what keeps the release on the refusal path itself.
|
|
1038
|
+
let relation;
|
|
1039
|
+
try {
|
|
1040
|
+
relation = await this.daemonStoreRelation();
|
|
1041
|
+
}
|
|
1042
|
+
catch (e) {
|
|
1043
|
+
if (this.daemonRenewalOwner)
|
|
1044
|
+
await this.ep.releaseDaemonRenewalLease().catch(() => { });
|
|
1045
|
+
this.daemonRenewalOwner = false;
|
|
1046
|
+
throw e;
|
|
1047
|
+
}
|
|
1048
|
+
if (relation === "divergent") {
|
|
959
1049
|
// Not our daemon's store: drop any lease we hold rather than sit on it, so the manager that
|
|
960
1050
|
// CAN remint is not blocked behind us.
|
|
961
1051
|
if (this.daemonRenewalOwner)
|
|
@@ -1634,6 +1724,7 @@ export class Manager {
|
|
|
1634
1724
|
throw new Error(`manager preservation shutdown incomplete: ${failures.join("; ")}`);
|
|
1635
1725
|
}
|
|
1636
1726
|
async stop(options = {}) {
|
|
1727
|
+
this.leaseStopping = true;
|
|
1637
1728
|
this.staticReconcileStopping = true;
|
|
1638
1729
|
const starting = this.startTask;
|
|
1639
1730
|
for (const item of this.staticReconcileItems.values()) {
|
|
@@ -1652,6 +1743,10 @@ export class Manager {
|
|
|
1652
1743
|
clearInterval(this.daemonRenewalLeaseTimer);
|
|
1653
1744
|
if (this.sessionKeyRenewTimer)
|
|
1654
1745
|
clearInterval(this.sessionKeyRenewTimer);
|
|
1746
|
+
// A renew already in flight when the timer was cleared above must be awaited, not raced: it can
|
|
1747
|
+
// still move the key's revision after we clear the timer, and releasing against a cached
|
|
1748
|
+
// revision it has since moved is a CAS the broker correctly refuses, silently, forever.
|
|
1749
|
+
await this.leaseRenewTask?.catch(() => { });
|
|
1655
1750
|
let releaseFailures = [];
|
|
1656
1751
|
if (this.maintenanceState === "active" && !this.resumeRequired) {
|
|
1657
1752
|
if (options.withAgents)
|
|
@@ -1663,7 +1758,7 @@ export class Manager {
|
|
|
1663
1758
|
// A signal after a partial preservation must never fall back into destructive teardown.
|
|
1664
1759
|
await this.stopRetainedAgentsOnExit();
|
|
1665
1760
|
}
|
|
1666
|
-
await this.
|
|
1761
|
+
await this.releaseOwnManagerLease();
|
|
1667
1762
|
// #1634: hand the renewal lease back on a clean stop so a sibling picks it up on its next pass
|
|
1668
1763
|
// rather than waiting out the bucket TTL. A non-owner holds no revision and this is a no-op.
|
|
1669
1764
|
await this.ep.releaseDaemonRenewalLease();
|
|
@@ -1686,6 +1781,30 @@ export class Manager {
|
|
|
1686
1781
|
if (releaseFailures.length)
|
|
1687
1782
|
throw new Error(`manager shutdown could not release every detachable seat: ${releaseFailures.join("; ")}`);
|
|
1688
1783
|
}
|
|
1784
|
+
/** Release this instance's liveness lease key on a clean stop, at the broker's own revision
|
|
1785
|
+
* rather than a cached one that a renew still in flight moments ago may have already moved past.
|
|
1786
|
+
* Re-reads the row and releases only what is PROVABLY this process's (absent, or another pid's,
|
|
1787
|
+
* is left alone — the same test {@link reconcileLease} uses). Nothing here throws: a stop must
|
|
1788
|
+
* complete either way, and a row that could not be released is left for the bucket TTL, with one
|
|
1789
|
+
* line of warning so the cost (a same-root restart waiting out the TTL) is not silent. */
|
|
1790
|
+
async releaseOwnManagerLease() {
|
|
1791
|
+
try {
|
|
1792
|
+
const own = await this.ep.readOwnManagerLease(this.managerInstanceId);
|
|
1793
|
+
if (own !== undefined && own.info.pid === process.pid)
|
|
1794
|
+
await this.ep.releaseManagerLease(this.managerInstanceId, own.revision);
|
|
1795
|
+
}
|
|
1796
|
+
catch {
|
|
1797
|
+
// Best-effort, like the release it wraps: a broker failure here is recovered by the bucket TTL.
|
|
1798
|
+
}
|
|
1799
|
+
try {
|
|
1800
|
+
const stillOurs = await this.ep.readOwnManagerLease(this.managerInstanceId);
|
|
1801
|
+
if (stillOurs !== undefined && stillOurs.info.pid === process.pid)
|
|
1802
|
+
console.error(`! manager instance ${this.managerInstanceId} could not release its liveness lease (space "${this.space}"); a same-root restart inside the bucket TTL will be refused`);
|
|
1803
|
+
}
|
|
1804
|
+
catch {
|
|
1805
|
+
// Could not confirm either way; the warning is best-effort, not a verdict.
|
|
1806
|
+
}
|
|
1807
|
+
}
|
|
1689
1808
|
/**
|
|
1690
1809
|
* REMOVE THIS INSTANCE'S SERVICE REGISTRATION on a clean stop (§13.5 deregistration).
|
|
1691
1810
|
*
|
|
@@ -1777,12 +1896,25 @@ export class Manager {
|
|
|
1777
1896
|
* on a live mesh turned a ten-second broker or KV outage into a manager outage that lasted until an
|
|
1778
1897
|
* operator noticed. */
|
|
1779
1898
|
async renewLease() {
|
|
1899
|
+
// A manager that is stopping never puts its own key back: a stop already fenced this loop, and
|
|
1900
|
+
// any tick or explicit call landing after that must be a no-op, not a fresh acquire attempt.
|
|
1901
|
+
if (this.leaseStopping)
|
|
1902
|
+
return;
|
|
1780
1903
|
// A renew that runs long must not be overlapped by the next tick: both would CAS against the same
|
|
1781
1904
|
// cached revision, so whichever lands second is refused over a sequence the first legitimately
|
|
1782
1905
|
// moved, a conflict this instance manufactured itself.
|
|
1783
1906
|
if (this.leaseRenewInFlight)
|
|
1784
1907
|
return;
|
|
1785
1908
|
this.leaseRenewInFlight = true;
|
|
1909
|
+
const task = this.renewLeaseAttempt();
|
|
1910
|
+
this.leaseRenewTask = task;
|
|
1911
|
+
return task;
|
|
1912
|
+
}
|
|
1913
|
+
/** The renew attempt itself, split out from {@link renewLease} only so `stop()` can hold the
|
|
1914
|
+
* promise a renew already in flight is running ({@link leaseRenewTask}) and await it before
|
|
1915
|
+
* releasing the key: releasing against a cached revision a live renew has since moved is a CAS
|
|
1916
|
+
* the broker correctly refuses, silently, forever. */
|
|
1917
|
+
async renewLeaseAttempt() {
|
|
1786
1918
|
try {
|
|
1787
1919
|
if (!this.leaseInfo || this.leaseRevision === undefined)
|
|
1788
1920
|
return;
|
|
@@ -1802,6 +1934,11 @@ export class Manager {
|
|
|
1802
1934
|
this.noteLease("held-unrenewed", `could not renew its liveness lease (${why}) but the key is still its own at revision ${verdict.revision}; serving, retrying`);
|
|
1803
1935
|
return;
|
|
1804
1936
|
case "gone": {
|
|
1937
|
+
// A stop that began while this renew was already in flight fences it here too: the
|
|
1938
|
+
// await above can cross the point where `stop()` set the flag, and a re-acquire past
|
|
1939
|
+
// that point would put a stopping process's key straight back.
|
|
1940
|
+
if (this.leaseStopping)
|
|
1941
|
+
return;
|
|
1805
1942
|
// The key being gone is a fact before the re-acquire is attempted, so a successful
|
|
1806
1943
|
// re-acquire is a change of state (gone to held) and prints, every time it happens.
|
|
1807
1944
|
const before = this.leaseState;
|
|
@@ -2232,10 +2369,25 @@ export class Manager {
|
|
|
2232
2369
|
const data = unwrap(await this.opModels(args(ctx)));
|
|
2233
2370
|
return { catalogs: Array.isArray(data) ? data : [data] };
|
|
2234
2371
|
}),
|
|
2372
|
+
goalResult: (ctx) => this.serveGated(ctx, async () => {
|
|
2373
|
+
const gw = this.goalWriter;
|
|
2374
|
+
if (!gw || gw.nc.isClosed())
|
|
2375
|
+
throw new EpEnvelopeError("unavailable", "the manager goal-reader connection is not standing; the accepted goal is unaffected");
|
|
2376
|
+
const goalId = args(ctx).goalId;
|
|
2377
|
+
// Only the broker-authenticated caller's full incarnation can name this read. The
|
|
2378
|
+
// trusted goal-writer keeps the raw EPF authority; serveEndpoint derives the reply rail.
|
|
2379
|
+
const result = await readGoalResult(gw.ctx, goalRefOf(ctx.subject, goalId));
|
|
2380
|
+
return { goalId, ...(result === undefined ? {} : { result }) };
|
|
2381
|
+
}),
|
|
2235
2382
|
resolveCwd: (ctx) => this.serveGated(ctx, () => this.resolveSpawnCwd(args(ctx).cwd)),
|
|
2236
2383
|
// P2 item 2: `spawn` is an ACTION - accept a goal + reply the acceptance floor payload, drive
|
|
2237
2384
|
// progress + terminal off-handler (no ~30s block). The blocking reply path is gone (pin 8).
|
|
2238
|
-
|
|
2385
|
+
// #373: arming the event plane (events: true, or the omission that arms it by default)
|
|
2386
|
+
// is operator reach on a user mesh — the spawn handler resolves the caller's admin tier
|
|
2387
|
+
// ONCE here (opLaunch takes the same signal as a parameter) and opStart's admission gate
|
|
2388
|
+
// decides on it BEFORE connector resolution or any grant mutation. Static meshes resolve
|
|
2389
|
+
// true and nothing changes.
|
|
2390
|
+
spawn: (ctx) => this.serveGated(ctx, async () => this.serveSpawnGoal(ctx, async (h) => this.opStart(args(ctx), callerOf(ctx), await this.epAdminReach(ctx.subject.caller), h, ctx.subject.route))),
|
|
2239
2391
|
despawn: (ctx) => this.serveGated(ctx, async () => {
|
|
2240
2392
|
const a = targetAgent(ctx);
|
|
2241
2393
|
const denied = await this.authorizeNamed(a, callerOf(ctx), await this.epAnyModeAdmin(ctx), ctx.subject.caller);
|
|
@@ -2542,6 +2694,8 @@ export class Manager {
|
|
|
2542
2694
|
* dead auth plane. Every failure is returned as the refusal sentence, with the grant + files
|
|
2543
2695
|
* rolled back. */
|
|
2544
2696
|
async provisionUserAgent(name, opts) {
|
|
2697
|
+
if (this.remoteAuthority?.enrollManagedAgent)
|
|
2698
|
+
return this.enrollUserAgent(name, opts, this.remoteAuthority.enrollManagedAgent);
|
|
2545
2699
|
const spawnerPr = opts.spawner ? parsePrincipalKey(opts.spawner) : null;
|
|
2546
2700
|
const owner = opts.specOwner ?? (spawnerPr && spawnerPr.owner.startsWith("u_") ? spawnerPr.owner : undefined);
|
|
2547
2701
|
if (!owner)
|
|
@@ -2620,7 +2774,7 @@ export class Manager {
|
|
|
2620
2774
|
"--health-file", healthPath,
|
|
2621
2775
|
];
|
|
2622
2776
|
await execBearerPreflight(bearerCmd);
|
|
2623
|
-
return { owner, files, launch: { owner, actor: name, sentinelCredsPath: sentinelPath, bearerCmd } };
|
|
2777
|
+
return { owner, lifecycleUid: opts.lifecycleUid, files, launch: { owner, actor: name, sentinelCredsPath: sentinelPath, bearerCmd } };
|
|
2624
2778
|
}
|
|
2625
2779
|
catch (e) {
|
|
2626
2780
|
// Roll back everything this attempt materialized — a refused spawn must leave no standing
|
|
@@ -2637,6 +2791,111 @@ export class Manager {
|
|
|
2637
2791
|
return { error: `agent auth preflight failed for "${name}": ${e.message}` };
|
|
2638
2792
|
}
|
|
2639
2793
|
}
|
|
2794
|
+
/**
|
|
2795
|
+
* HOSTED user-mode spawn provisioning (#1972): the participant arm of host-owned enrollment.
|
|
2796
|
+
*
|
|
2797
|
+
* A registered participant holds no ledger writer, no JetStream provisioner, and no signing seed,
|
|
2798
|
+
* so every step the local arm above performs itself is the HOST's here. What stays local is the
|
|
2799
|
+
* one thing that must: the standing `actorToken`. This method generates it, writes it at 0600
|
|
2800
|
+
* BEFORE the request, and sends only its SHA-256 digest — so the plaintext never crosses the
|
|
2801
|
+
* network and a host compromise cannot replay an agent's exchange secret.
|
|
2802
|
+
*
|
|
2803
|
+
* The HOST picks the lifecycle UID. It alone can see the retirement tombstones that make a UID
|
|
2804
|
+
* permanently unusable (a retired uid is refused forever by the retirement frontier), so a
|
|
2805
|
+
* participant-chosen uid could aim a fresh grant at a dead incarnation. The uid the host returns
|
|
2806
|
+
* is the one the spawn records and the one the child's durables are keyed by.
|
|
2807
|
+
*
|
|
2808
|
+
* The bearer argv selects `--exchange-url`: the local `--dir` arm reads a state directory this
|
|
2809
|
+
* machine does not have, and a signerless participant must never appear to hold one.
|
|
2810
|
+
*/
|
|
2811
|
+
async enrollUserAgent(name, opts, enroll) {
|
|
2812
|
+
let provider;
|
|
2813
|
+
try {
|
|
2814
|
+
provider = resolveAuthProvider();
|
|
2815
|
+
}
|
|
2816
|
+
catch (e) {
|
|
2817
|
+
return { error: e.message };
|
|
2818
|
+
}
|
|
2819
|
+
const secrets = this.secrets;
|
|
2820
|
+
// The token the agent will present to `POST /exchange` for the rest of its life. Generated
|
|
2821
|
+
// here, hashed here, and never sent: the request carries `tokenHash` alone.
|
|
2822
|
+
const actorToken = randomBytes(32).toString("base64url");
|
|
2823
|
+
const tokenHash = createHash("sha256").update(actorToken, "utf8").digest("hex");
|
|
2824
|
+
// Provisional paths under the participant's OWN uid: the token has to be on disk before the
|
|
2825
|
+
// request (a host that granted against a digest whose plaintext was lost would leave an agent
|
|
2826
|
+
// that can never exchange), and the host's uid is not known until the request answers. The
|
|
2827
|
+
// family is re-keyed to the host's uid below.
|
|
2828
|
+
const staged = agentLifecycleSecretFilePaths(this.workspaceRoot, this.space, name, opts.lifecycleUid);
|
|
2829
|
+
let files = staged;
|
|
2830
|
+
let material;
|
|
2831
|
+
try {
|
|
2832
|
+
await secrets.put(agentSecretKeyForFile(staged.actorToken, this.space), actorToken);
|
|
2833
|
+
await materializeSecretToFile(secrets, agentSecretKeyForFile(staged.actorToken, this.space), staged.actorToken);
|
|
2834
|
+
material = await enroll({
|
|
2835
|
+
target: {
|
|
2836
|
+
actor: name,
|
|
2837
|
+
tokenHash,
|
|
2838
|
+
...(opts.role !== undefined ? { role: opts.role } : {}),
|
|
2839
|
+
label: opts.label,
|
|
2840
|
+
...(opts.capabilities !== undefined ? { capabilities: opts.capabilities } : {}),
|
|
2841
|
+
...(opts.subscribe !== undefined ? { subscribe: opts.subscribe } : {}),
|
|
2842
|
+
allowSubscribe: opts.allowSubscribe,
|
|
2843
|
+
...(opts.allowPublish !== undefined ? { allowPublish: opts.allowPublish } : {}),
|
|
2844
|
+
},
|
|
2845
|
+
});
|
|
2846
|
+
if (material.actor !== name)
|
|
2847
|
+
throw new Error(`the host enrolled actor "${material.actor}", not the requested "${name}"`);
|
|
2848
|
+
const specOwner = opts.specOwner ?? (opts.spawner ? parsePrincipalKey(opts.spawner)?.owner : undefined);
|
|
2849
|
+
if (specOwner !== undefined && specOwner.startsWith("u_") && material.owner !== specOwner)
|
|
2850
|
+
throw new Error(`the host enrolled the agent under owner ${material.owner}, not the spawning owner ${specOwner}`);
|
|
2851
|
+
assertLifecycleToken(material.lifecycleUid, "host-enrolled managed agent lifecycleUid");
|
|
2852
|
+
// Re-key the family onto the HOST's uid (SPEC 13.1 name-disjointness on the FS): the child's
|
|
2853
|
+
// credential paths must embed the uid its broker footprint carries, never the provisional one.
|
|
2854
|
+
files = agentLifecycleSecretFilePaths(this.workspaceRoot, this.space, name, material.lifecycleUid);
|
|
2855
|
+
if (files.actorToken !== staged.actorToken) {
|
|
2856
|
+
await secrets.put(agentSecretKeyForFile(files.actorToken, this.space), actorToken);
|
|
2857
|
+
await materializeSecretToFile(secrets, agentSecretKeyForFile(files.actorToken, this.space), files.actorToken);
|
|
2858
|
+
await secrets.delete(agentSecretKeyForFile(staged.actorToken, this.space)).catch(() => { });
|
|
2859
|
+
rmSync(staged.actorToken, { force: true });
|
|
2860
|
+
}
|
|
2861
|
+
await secrets.put(agentSecretKeyForFile(files.sentinelCreds, this.space), material.sentinelCreds);
|
|
2862
|
+
await materializeSecretToFile(secrets, agentSecretKeyForFile(files.sentinelCreds, this.space), files.sentinelCreds);
|
|
2863
|
+
rmSync(files.health, { force: true }); // a fresh start opens a fresh health window
|
|
2864
|
+
const bearerCmd = [
|
|
2865
|
+
process.execPath,
|
|
2866
|
+
...process.execArgv,
|
|
2867
|
+
process.argv[1],
|
|
2868
|
+
provider.agentBearerCommand,
|
|
2869
|
+
// The signerless arm: the pinned public exchange, never a local state dir.
|
|
2870
|
+
"--exchange-url", material.agentBearerExchangeUrl,
|
|
2871
|
+
"--space", this.space,
|
|
2872
|
+
"--owner", material.owner,
|
|
2873
|
+
"--actor", name,
|
|
2874
|
+
"--token-file", files.actorToken,
|
|
2875
|
+
"--health-file", files.health,
|
|
2876
|
+
];
|
|
2877
|
+
await execBearerPreflight(bearerCmd);
|
|
2878
|
+
return {
|
|
2879
|
+
owner: material.owner,
|
|
2880
|
+
lifecycleUid: material.lifecycleUid,
|
|
2881
|
+
files,
|
|
2882
|
+
launch: { owner: material.owner, actor: name, sentinelCredsPath: files.sentinelCreds, bearerCmd },
|
|
2883
|
+
};
|
|
2884
|
+
}
|
|
2885
|
+
catch (e) {
|
|
2886
|
+
// Shred every local secret this attempt materialized. The HOST's rows are the host's to
|
|
2887
|
+
// reconcile: a participant holds no writer, so it must not pretend to roll back a grant or a
|
|
2888
|
+
// durable it never authored. A retry with the same requestId re-enters the same enrollment.
|
|
2889
|
+
for (const family of files.actorToken === staged.actorToken ? [staged] : [staged, files]) {
|
|
2890
|
+
await secrets.delete(agentSecretKeyForFile(family.actorToken, this.space)).catch(() => { });
|
|
2891
|
+
await secrets.delete(agentSecretKeyForFile(family.sentinelCreds, this.space)).catch(() => { });
|
|
2892
|
+
rmSync(family.actorToken, { force: true });
|
|
2893
|
+
rmSync(family.sentinelCreds, { force: true });
|
|
2894
|
+
rmSync(family.health, { force: true });
|
|
2895
|
+
}
|
|
2896
|
+
return { error: `agent auth preflight failed for "${name}": ${e.message}` };
|
|
2897
|
+
}
|
|
2898
|
+
}
|
|
2640
2899
|
/** Drop a live agent's slot. When `floor` is set and the agent died young (lived less than
|
|
2641
2900
|
* MIN_LIFETIME), push a cooling stamp so the freed slot still counts toward the ceiling until it
|
|
2642
2901
|
* expires — flooring the RECYCLE, not the call, so both free paths (despawn + exit/reap) are
|
|
@@ -3161,6 +3420,7 @@ export class Manager {
|
|
|
3161
3420
|
? { ...restart.opts, resume: undefined, prompt: undefined, continueSession }
|
|
3162
3421
|
: { ...restart.opts, resume: undefined, prompt: undefined };
|
|
3163
3422
|
const spec = connector.buildLaunch(opts);
|
|
3423
|
+
spec.env = { ...spec.env, COTAL_MANAGER_INSTANCE: this.managerInstanceId };
|
|
3164
3424
|
const wanted = this.managedPrincipal(a);
|
|
3165
3425
|
const joinedAfter = this.ep.getRoster()
|
|
3166
3426
|
.filter((p) => p.card.id === wanted && p.lifecycleUid === a.lifecycleUid)
|
|
@@ -3327,7 +3587,7 @@ export class Manager {
|
|
|
3327
3587
|
return false;
|
|
3328
3588
|
}
|
|
3329
3589
|
/** Parse an untyped control-plane `start` request into {@link StartAgentOpts}. */
|
|
3330
|
-
opStart(args, caller, hooks, route = "inst") {
|
|
3590
|
+
opStart(args, caller, admin, hooks, route = "inst") {
|
|
3331
3591
|
// `resume`, when present, must be a non-empty session id. An empty/whitespace value is a
|
|
3332
3592
|
// malformed request, not an implicit "spawn fresh" (no fallbacks). The CLI surfaces reject it,
|
|
3333
3593
|
// but a raw control message could otherwise slip an empty value through and silently start fresh.
|
|
@@ -3374,6 +3634,31 @@ export class Manager {
|
|
|
3374
3634
|
catch (e) {
|
|
3375
3635
|
return Promise.resolve({ ok: false, error: e.message });
|
|
3376
3636
|
}
|
|
3637
|
+
// #373: the event-plane admission gate. `startAgent` resolves `eventsRequired || opts.events
|
|
3638
|
+
// !== false`, so omission ARMS the plane — a gate on the explicit bit alone is bypassed by
|
|
3639
|
+
// omitting it. The gate is on the DECISION, and it runs before connector resolution and
|
|
3640
|
+
// before `startAgent` (which mints credentials), so a refusal provisions nothing.
|
|
3641
|
+
const requestedEvents = typeof args.events === "boolean" ? args.events : undefined;
|
|
3642
|
+
let eventsNotice;
|
|
3643
|
+
let events = requestedEvents;
|
|
3644
|
+
if (!admin) {
|
|
3645
|
+
if (requestedEvents === true)
|
|
3646
|
+
return Promise.resolve({
|
|
3647
|
+
ok: false,
|
|
3648
|
+
error: `spawn is operator reach; the caller's current ledger grant does not carry "admin" (SPEC 13.2): events: arming the event plane needs the admin tier`,
|
|
3649
|
+
});
|
|
3650
|
+
if (this.eventsRequired)
|
|
3651
|
+
return Promise.resolve({
|
|
3652
|
+
ok: false,
|
|
3653
|
+
error: `spawn is operator reach; the caller's current ledger grant does not carry "admin" (SPEC 13.2): space "${this.space}" requires the event plane by registration policy, which cannot be met without the admin tier`,
|
|
3654
|
+
});
|
|
3655
|
+
// Omission arms by default in startAgent; serve it DISARMED and say so (never a silent
|
|
3656
|
+
// downgrade). An explicit `false` is already the opt-out — served as-is, no notice.
|
|
3657
|
+
if (requestedEvents === undefined) {
|
|
3658
|
+
events = false;
|
|
3659
|
+
eventsNotice = `event plane not armed: arming it on spawn needs the admin tier; the spawn was served with events: false`;
|
|
3660
|
+
}
|
|
3661
|
+
}
|
|
3377
3662
|
return this.startAgent({
|
|
3378
3663
|
name: String(args.name ?? "").trim(),
|
|
3379
3664
|
agent: args.agent ? String(args.agent) : undefined,
|
|
@@ -3385,7 +3670,8 @@ export class Manager {
|
|
|
3385
3670
|
variant: args.variant ? String(args.variant) : undefined,
|
|
3386
3671
|
launchOptions: args.launchOptions,
|
|
3387
3672
|
resume: args.resume ? String(args.resume) : undefined,
|
|
3388
|
-
events
|
|
3673
|
+
events,
|
|
3674
|
+
eventsNotice,
|
|
3389
3675
|
cwd: args.cwd ? String(args.cwd) : undefined,
|
|
3390
3676
|
prompt: args.prompt ? String(args.prompt) : undefined,
|
|
3391
3677
|
subscribe,
|
|
@@ -3881,6 +4167,8 @@ export class Manager {
|
|
|
3881
4167
|
}
|
|
3882
4168
|
if (variant && !connector.supportsModelVariant)
|
|
3883
4169
|
return { ok: false, error: `${agent} connector does not support model variants (variant)` };
|
|
4170
|
+
if (prompt !== undefined && !connector.supportsPrompt)
|
|
4171
|
+
return { ok: false, error: `${agent} connector does not support an initial prompt (prompt)` };
|
|
3884
4172
|
// #4 A4 (panel): the roster the allocation consults must reflect the initial presence snapshot,
|
|
3885
4173
|
// or a spawn immediately after manager boot races an already-live unmanaged peer and re-opens the
|
|
3886
4174
|
// very collision black-hole this closes. Await the snapshot (bounded internally, fail-safe on an
|
|
@@ -3963,7 +4251,12 @@ export class Manager {
|
|
|
3963
4251
|
// The incarnation's lifecycle UID (SPEC 13.1), minted ONCE per spawn: every lifecycle-keyed
|
|
3964
4252
|
// broker resource (dm_/dlv_/chathist_ durables, ACL row, memberships) and the teardown
|
|
3965
4253
|
// credential carry it, so a same-name successor's footprint is name-disjoint by construction.
|
|
3966
|
-
|
|
4254
|
+
//
|
|
4255
|
+
// On the HOSTED enrollment arm (#1972) this value is PROVISIONAL. The host selects the
|
|
4256
|
+
// authoritative uid, because it alone reads the retirement tombstones that make a uid
|
|
4257
|
+
// permanently unusable, and the enrollment below rebinds this binding to what the host
|
|
4258
|
+
// returned before anything durable is keyed by it.
|
|
4259
|
+
let lifecycleUid = mintLifecycleUid();
|
|
3967
4260
|
// Reserve the seat's custody reference NOW, before any durable row and long before the
|
|
3968
4261
|
// launch. Minting it inside `runtime.spawn` (as it used to be) put the processes on disk
|
|
3969
4262
|
// before anything recorded how to address them, so a crash in that window left a live seat
|
|
@@ -4050,6 +4343,12 @@ export class Manager {
|
|
|
4050
4343
|
this.reserved.delete(name);
|
|
4051
4344
|
return { ok: false, error: prep.error };
|
|
4052
4345
|
}
|
|
4346
|
+
// THE HOST'S UID WINS (#1972). On the local arm this is the value passed in, unchanged. On
|
|
4347
|
+
// the hosted enrollment arm the host selected it, and from here every lifecycle-keyed thing
|
|
4348
|
+
// this spawn records — the slot, the launch, the managed row, the teardown credential — must
|
|
4349
|
+
// carry that value and not the provisional one, or the manager would address a footprint the
|
|
4350
|
+
// host never created.
|
|
4351
|
+
lifecycleUid = prep.lifecycleUid;
|
|
4053
4352
|
userLaunch = prep.launch;
|
|
4054
4353
|
userOwner = prep.owner;
|
|
4055
4354
|
provisioned = { id: principalKey(prep.owner, name).key, name, lifecycleUid, userOwner: prep.owner, secretPaths: prep.files, ...(custody ? { runtime: custody } : {}) };
|
|
@@ -4161,6 +4460,7 @@ export class Manager {
|
|
|
4161
4460
|
workspaceRoot: this.workspaceRoot,
|
|
4162
4461
|
};
|
|
4163
4462
|
const spec = connector.buildLaunch(launchOpts);
|
|
4463
|
+
spec.env = { ...spec.env, COTAL_MANAGER_INSTANCE: this.managerInstanceId };
|
|
4164
4464
|
const handle = await this.spawnCustodied(name, spec, cwd, custody);
|
|
4165
4465
|
hooks?.onLaunched?.(); // P2 item 2: the "launched" progress edge (process spawned, pre-presence)
|
|
4166
4466
|
const managed = {
|
|
@@ -4194,6 +4494,9 @@ export class Manager {
|
|
|
4194
4494
|
cwd,
|
|
4195
4495
|
model,
|
|
4196
4496
|
variant,
|
|
4497
|
+
// The kickoff prompt is part of the launch form: a stale-restart re-submits it, so the
|
|
4498
|
+
// recorded launch must carry it for the restart gate (and preserve) to see it.
|
|
4499
|
+
prompt,
|
|
4197
4500
|
subscribe,
|
|
4198
4501
|
allowSubscribe,
|
|
4199
4502
|
allowPublish,
|
|
@@ -4289,7 +4592,7 @@ export class Manager {
|
|
|
4289
4592
|
// canonicalJson (undefined never coerces to null, SPEC 13.6), so a role-less spawn would
|
|
4290
4593
|
// otherwise fail its succeeded terminal. The CLI/connector already render an absent role as
|
|
4291
4594
|
// "no role", so dropping the key preserves the reply (P2 item 2, surfaced by readiness:live).
|
|
4292
|
-
const okData = { name, agent, id: managed.id, mode: handle.kind, lifecycleUid, ...(role !== undefined ? { role } : {}) };
|
|
4595
|
+
const okData = { name, agent, id: managed.id, mode: handle.kind, lifecycleUid, ...(role !== undefined ? { role } : {}), ...(opts.eventsNotice !== undefined ? { eventsNotice: opts.eventsNotice } : {}) };
|
|
4293
4596
|
await hooks?.onOutcome?.({ kind: "succeeded", data: okData });
|
|
4294
4597
|
return { ok: true, data: okData };
|
|
4295
4598
|
}
|
|
@@ -4640,6 +4943,8 @@ export class Manager {
|
|
|
4640
4943
|
return { ok: false, error: `${connector.name} harness needs ${missing.join(", ")} on PATH - not found` };
|
|
4641
4944
|
if (entry.launch.variant && !connector.supportsModelVariant)
|
|
4642
4945
|
return { ok: false, error: `${connector.name} connector does not support model variants (variant)` };
|
|
4946
|
+
if (entry.launch.prompt !== undefined && !connector.supportsPrompt)
|
|
4947
|
+
return { ok: false, error: `${connector.name} connector does not support an initial prompt (prompt)` };
|
|
4643
4948
|
let retainedSession;
|
|
4644
4949
|
try {
|
|
4645
4950
|
retainedSession = this.retainedSessionId(entry, connector);
|
|
@@ -4720,6 +5025,7 @@ export class Manager {
|
|
|
4720
5025
|
workspaceRoot: this.workspaceRoot,
|
|
4721
5026
|
};
|
|
4722
5027
|
const spec = connector.buildLaunch(launchOpts);
|
|
5028
|
+
spec.env = { ...spec.env, COTAL_MANAGER_INSTANCE: this.managerInstanceId };
|
|
4723
5029
|
const value = { spec, launchOpts, ...authority };
|
|
4724
5030
|
prepared?.set(entry.name, value);
|
|
4725
5031
|
if (preflightOnly)
|