@cotal-ai/manager 0.62.0 → 0.64.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands.js +4 -4
- package/dist/commands.js.map +1 -1
- package/dist/endpoint-evict.d.ts +8 -9
- package/dist/endpoint-evict.d.ts.map +1 -1
- package/dist/endpoint-evict.js +35 -32
- package/dist/endpoint-evict.js.map +1 -1
- package/dist/holder-liveness.d.ts +2 -2
- package/dist/holder-liveness.js +2 -2
- package/dist/manager.d.ts +19 -5
- package/dist/manager.d.ts.map +1 -1
- package/dist/manager.js +144 -28
- package/dist/manager.js.map +1 -1
- package/dist/remote-register.d.ts.map +1 -1
- package/dist/remote-register.js +15 -5
- package/dist/remote-register.js.map +1 -1
- package/dist/run-hosting.d.ts +16 -3
- package/dist/run-hosting.d.ts.map +1 -1
- package/dist/run-hosting.js +39 -11
- package/dist/run-hosting.js.map +1 -1
- package/package.json +12 -12
package/dist/manager.js
CHANGED
|
@@ -4,11 +4,11 @@ import { hostname } from "node:os";
|
|
|
4
4
|
import { credsAuthenticator } from "@nats-io/transport-node";
|
|
5
5
|
import { existsSync, lstatSync, readFileSync, realpathSync, rmSync, statSync } from "node:fs";
|
|
6
6
|
import { isAbsolute, join, dirname, resolve } from "node:path";
|
|
7
|
-
import { CotalEndpoint, accountFromCreds, credsClaims, DEFAULT_SERVER, DEV_OWNER, MANAGER_LEASE_RENEW_MS, STANDING_RENEWABLE_TTL_SEC, assertLifecycleToken, discardLaunchArtifacts, divergentSecretStoreNotice, parseDaemonStoreAnswer, parseSecretStoreIdentity, sameSecretStoreIdentity, agentFilePath, clearSpaceHistory, dialerFor, connectorServers, spawnEnvAllow, deprovisionAgent, DeprovisionError, isConcreteChannel, deprovisionTargetPrincipal, firstFreeName, spawnNameError, idFromCreds, inspectCredHealth, credsRenewalDelayMs, composeWirePersona, loadAgentFile, listPersonaCatalog, personaCatalogDescription, personaCatalogReadable, loadCotalConfig, modelPolicyRefusal, mintCreds, mintGeneration, mintAcceptedToken, openIssuedStore, acceptedBucket, issuedBucket, mintLifecycleUid, managedRetirementOpId, mkSecretDir, newIdentity, actionContext, parsePrincipalKey, parseShareSelection, principalKey, probeConnect, provisionAgent, provisionAgentDurables, registry, resolveAuthProvider, saveAgentFile, subjectMatches, AUTH_ENDPOINT, EP_CMD_RETIRE_LIFECYCLE, controlServiceSubject, eventChannelPrincipal, resolveService, invokeCommand, } from "@cotal-ai/core";
|
|
7
|
+
import { CotalEndpoint, MANAGED_HANDOFF_KIND, accountFromCreds, credsClaims, DEFAULT_SERVER, DEV_OWNER, MANAGER_LEASE_RENEW_MS, STANDING_RENEWABLE_TTL_SEC, assertLifecycleToken, discardLaunchArtifacts, divergentSecretStoreNotice, parseDaemonStoreAnswer, parseSecretStoreIdentity, sameSecretStoreIdentity, agentFilePath, clearSpaceHistory, dialerFor, connectorServers, spawnEnvAllow, deprovisionAgent, DeprovisionError, isConcreteChannel, deprovisionTargetPrincipal, firstFreeName, spawnNameError, idFromCreds, inspectCredHealth, credsRenewalDelayMs, composeWirePersona, loadAgentFile, listPersonaCatalog, personaCatalogDescription, personaCatalogReadable, loadCotalConfig, modelPolicyRefusal, mintCreds, mintGeneration, mintAcceptedToken, openIssuedStore, acceptedBucket, issuedBucket, mintLifecycleUid, managedRetirementOpId, mkSecretDir, newIdentity, actionContext, parsePrincipalKey, parseShareSelection, principalKey, probeConnect, provisionAgent, provisionAgentDurables, registry, resolveAuthProvider, saveAgentFile, subjectMatches, AUTH_ENDPOINT, EP_CMD_RETIRE_LIFECYCLE, controlServiceSubject, eventChannelPrincipal, resolveService, invokeCommand, } from "@cotal-ai/core";
|
|
8
8
|
import { agentAuthState, agentCredsDir, agentLifecycleSecretFilePaths, agentSecretFilePaths, agentSecretKeyForFile, authDir, connectorInstallHint, DEFAULT_CONNECTOR, defaultAgentType, DELIVERY_CREDS_KIND, extensionConnectors, findCotalRoot, getSpaceAuth, hasUserAuthState, loadExtensionsManifest, loadManagerInstanceIdentity, loadMeshes, localTrustOfSpace, manifestExtensionNames, materializeFromManifest, materializeSecretToFile, MEMBERSHIP_RW_CREDS_KIND, mergeLaunchOptions, remintDaemonCreds, resolveOnPath, createManagerInstanceIdentity, spaceMaterialKey, SYSTEM_CREDS_FILES, userAuthStateDir, workspaceSecretStore, writeRenewalRecord } from "@cotal-ai/workspace";
|
|
9
9
|
import { createRuntime, isCustodialRuntime, requireRuntimeAdopt, requireRuntimeReap, RuntimeReapUnproven, } from "./runtime/index.js";
|
|
10
10
|
import { AttachEndpoint } from "./attach-endpoint.js";
|
|
11
|
-
import { DELIVERY_ADMIN_BOOT_WAIT_MS, isUnansweredDeliveryAdmin, makeManagerEndpointEvictionEvidence,
|
|
11
|
+
import { DELIVERY_ADMIN_BOOT_WAIT_MS, isUnansweredDeliveryAdmin, makeManagerEndpointEvictionEvidence, makeManagerEndpointHolderEvictor, untilDeliveryAdminAnswers } from "./endpoint-evict.js";
|
|
12
12
|
import { makeManagerHolderLivenessProbe } from "./holder-liveness.js";
|
|
13
13
|
import { GateReconcileRefused, reconcileEndpointGate } from "./reconcile-gate.js";
|
|
14
14
|
import { launchSpecForRun, materializePersona, launchAgentToStartOpts, parseLaunchSpec, persistLaunchSpec, readContinuityAssignment, writeContinuityAssignment } from "./launch.js";
|
|
@@ -302,6 +302,9 @@ export class Manager {
|
|
|
302
302
|
* a reader of that name could not tell the two apart (#382). Never pruned in-process: one string
|
|
303
303
|
* per numbered spawn. */
|
|
304
304
|
issuedNumberedNames = new Set();
|
|
305
|
+
/** Lifecycle UIDs already handed to `Runtime.spawnDelegated`: a lifecycle is handed off at most
|
|
306
|
+
* once, so a lost create acknowledgement can never become a second create. */
|
|
307
|
+
delegatedLaunched = new Set();
|
|
305
308
|
/** Expiry stamps (`startedAt + MIN_LIFETIME`) for slots that freed while still young — a
|
|
306
309
|
* count-only, lazily-pruned recycle floor (P4c). Pruned + summed into the ceiling gate. */
|
|
307
310
|
cooling = [];
|
|
@@ -519,6 +522,10 @@ export class Manager {
|
|
|
519
522
|
* daemon reloads from this manager's store AND this manager holds the per-space renewal lease.
|
|
520
523
|
* Starts false: a manager that has not proved both must not remint. */
|
|
521
524
|
daemonRenewalOwner = false;
|
|
525
|
+
/** #2561: may the lease heartbeat contend at all? Set by every claim, false while the daemon names
|
|
526
|
+
* another store or the challenge refused. Without it the heartbeat re-takes the lease the claim
|
|
527
|
+
* just released and holds it against every manager and `doctor auth --fix` that can remint. */
|
|
528
|
+
daemonRenewalEligible = false;
|
|
522
529
|
daemonRenewalLeaseTimer;
|
|
523
530
|
maintenanceState = "active";
|
|
524
531
|
lifecycleInFlight = 0;
|
|
@@ -861,8 +868,8 @@ export class Manager {
|
|
|
861
868
|
// second workspace root) has a distinct id ⇒ a distinct key ⇒ it coexists; the create THROWS only
|
|
862
869
|
// when the SAME instance id is already live (a same-root double-start, or a restart racing the
|
|
863
870
|
// crashed predecessor's not-yet-expired key), and we REFUSE loud. A crashed holder's key auto-expires
|
|
864
|
-
// (bucket TTL). Losing the key LATER
|
|
865
|
-
// retrying ({@link renewLease}).
|
|
871
|
+
// (bucket TTL). Losing the key LATER ends this process only when another process holds it; otherwise
|
|
872
|
+
// the renew loop puts it back or keeps retrying ({@link renewLease}).
|
|
866
873
|
this.leaseInfo = { holder: this.ep.ref().id, instanceId: this.managerInstanceId, runtime: this.runtime.kind, root: resolve(this.workspaceRoot), pid: process.pid };
|
|
867
874
|
try {
|
|
868
875
|
this.leaseRevision = await this.ep.acquireManagerLease(this.leaseInfo);
|
|
@@ -1150,7 +1157,7 @@ export class Manager {
|
|
|
1150
1157
|
* CAS is how a survivor picks up a dead owner's expired lease with no operator step. Never
|
|
1151
1158
|
* throws; a failed heartbeat leaves ownership to the next pass rather than ending the process. */
|
|
1152
1159
|
async keepDaemonRenewalLease() {
|
|
1153
|
-
if (!this.auth)
|
|
1160
|
+
if (!this.auth || !this.daemonRenewalEligible)
|
|
1154
1161
|
return;
|
|
1155
1162
|
try {
|
|
1156
1163
|
const held = await this.ep.holdDaemonRenewalLease(this.managerInstanceId);
|
|
@@ -1189,11 +1196,13 @@ export class Manager {
|
|
|
1189
1196
|
relation = await untilDeliveryAdminAnswers(waitMs, (requestMs) => this.daemonStoreRelation(requestMs(STORE_CHALLENGE_TIMEOUT_MS)), (reason, delayMs) => console.error(`! delivery-admin rail did not answer the store challenge (${reason}); retrying in ${delayMs / 1000}s`), (e) => e instanceof DeliveryAdminUnanswered);
|
|
1190
1197
|
}
|
|
1191
1198
|
catch (e) {
|
|
1199
|
+
this.daemonRenewalEligible = false;
|
|
1192
1200
|
if (this.daemonRenewalOwner)
|
|
1193
1201
|
await this.ep.releaseDaemonRenewalLease().catch(() => { });
|
|
1194
1202
|
this.daemonRenewalOwner = false;
|
|
1195
1203
|
throw e;
|
|
1196
1204
|
}
|
|
1205
|
+
this.daemonRenewalEligible = relation !== "divergent";
|
|
1197
1206
|
if (relation === "divergent") {
|
|
1198
1207
|
// Not our daemon's store: drop any lease we hold rather than sit on it, so the manager that
|
|
1199
1208
|
// CAN remint is not blocked behind us.
|
|
@@ -1666,6 +1675,11 @@ export class Manager {
|
|
|
1666
1675
|
error,
|
|
1667
1676
|
});
|
|
1668
1677
|
}
|
|
1678
|
+
// A later manager does not resume a delegated seat, and its close runs only inside its
|
|
1679
|
+
// retirement, so a cut that holds one is refused before any child stops.
|
|
1680
|
+
for (const a of this.agents.values())
|
|
1681
|
+
if (a.handedOff)
|
|
1682
|
+
failures.push({ name: a.name, id: a.id, error: "a delegated seat is not resumed by a later manager; stop it before preserving" });
|
|
1669
1683
|
const unverifiedStops = this.unverifiedStops.filter((stopped) => {
|
|
1670
1684
|
try {
|
|
1671
1685
|
if (!stopped.authoritative && stopped.handle.status() === "exited")
|
|
@@ -1974,7 +1988,7 @@ export class Manager {
|
|
|
1974
1988
|
}
|
|
1975
1989
|
}));
|
|
1976
1990
|
// Deprovision EVERY snapshot entry regardless of whether its stop failed (allSettled + a loud log).
|
|
1977
|
-
await Promise.allSettled(managed.filter((a) => !a.suppressCleanup).map((a) => this.deprovision(a).catch((e) => console.error(`deprovision ${a.name} (${a.id}) on shutdown: ${e.message}`))));
|
|
1991
|
+
await Promise.allSettled(managed.filter((a) => !a.suppressCleanup).map((a) => this.deprovision({ ...a, ...(a.handedOff ? { delegatedHandle: a.handle } : {}) }).catch((e) => console.error(`deprovision ${a.name} (${a.id}) on shutdown: ${e.message}`))));
|
|
1978
1992
|
if (failures.length)
|
|
1979
1993
|
throw new Error(`manager shutdown could not prove every seat exited: ${failures.join("; ")}`);
|
|
1980
1994
|
}
|
|
@@ -2003,7 +2017,11 @@ export class Manager {
|
|
|
2003
2017
|
const failures = [];
|
|
2004
2018
|
await Promise.all(managed.map(async (a) => {
|
|
2005
2019
|
try {
|
|
2006
|
-
a
|
|
2020
|
+
// No cut holds a delegated seat (runPreparation refuses one), so stopping it is its retirement.
|
|
2021
|
+
if (a.handedOff)
|
|
2022
|
+
await this.deprovision({ ...a, delegatedHandle: a.handle });
|
|
2023
|
+
else
|
|
2024
|
+
a.handle.stop({ graceful: false });
|
|
2007
2025
|
}
|
|
2008
2026
|
catch (e) {
|
|
2009
2027
|
failures.push(`${a.name}: stop failed: ${e.message}`);
|
|
@@ -2218,18 +2236,18 @@ export class Manager {
|
|
|
2218
2236
|
catch { /* best effort */ }
|
|
2219
2237
|
}
|
|
2220
2238
|
}
|
|
2221
|
-
/** Keep this instance's liveness key fresh, and
|
|
2239
|
+
/** Keep this instance's liveness key fresh, and end the process only when another process holds it.
|
|
2222
2240
|
*
|
|
2223
2241
|
* A renew that throws is a question, not a verdict: the request may have timed out with the write
|
|
2224
2242
|
* landed and only its acknowledgement lost, the key may have expired during a stall, or a same-id
|
|
2225
2243
|
* process may hold it. The verdict comes from RE-READING the key ({@link reconcileLease}), and each
|
|
2226
|
-
* answer has one response
|
|
2244
|
+
* answer has one response:
|
|
2227
2245
|
* - `held`: adopt the broker's revision and carry on.
|
|
2228
2246
|
* - `gone`: the key expired or was released while this process was still here, so put it back. The
|
|
2229
2247
|
* create is atomic, so a same-id process that got there first shows up as `taken` next tick.
|
|
2230
|
-
* - `taken`: a different process holds this instance's key
|
|
2231
|
-
*
|
|
2232
|
-
*
|
|
2248
|
+
* - `taken`: a different process holds this instance's key, so it took this instance over while
|
|
2249
|
+
* this one stalled. This process is the superseded one and stops serving by exiting (SPEC 13.1).
|
|
2250
|
+
* An open mesh has no barrier that would evict its serve connection, so nothing else stops it.
|
|
2233
2251
|
* - `unknown`: the broker could not be asked. Keep serving and ask again next tick, for as long as it
|
|
2234
2252
|
* takes. A manager that cannot reach its broker gains nothing by ending itself, and the seats it
|
|
2235
2253
|
* holds lose everything (a pty child dies with its parent).
|
|
@@ -2296,8 +2314,12 @@ export class Manager {
|
|
|
2296
2314
|
return;
|
|
2297
2315
|
}
|
|
2298
2316
|
case "taken":
|
|
2299
|
-
|
|
2300
|
-
|
|
2317
|
+
// No `stop()` on the way out: its deregistration deletes the instance's current
|
|
2318
|
+
// registration, which is the other process's now. That process took over on the premise
|
|
2319
|
+
// that this one was gone, so this exit leaves what a crash would, and seats a custodian or
|
|
2320
|
+
// multiplexer owns outlive it the same way.
|
|
2321
|
+
console.error(`! manager instance ${this.managerInstanceId} finds its liveness lease key held by ${verdict.by}, not by this process (renew: ${why}); that process serves this instance now, so this one stops serving and exits (space "${this.space}")`);
|
|
2322
|
+
process.exit(1);
|
|
2301
2323
|
case "unknown":
|
|
2302
2324
|
this.noteLease("unknown", `could not renew its liveness lease (${why}) or re-read it (${verdict.why}); serving, retrying until the broker answers`);
|
|
2303
2325
|
return;
|
|
@@ -2844,12 +2866,12 @@ export class Manager {
|
|
|
2844
2866
|
// The workflow-run family (SPEC 14.3): the manager hosts the driver. Reach is the broker's
|
|
2845
2867
|
// (`run` capability / privileged instrument rows); the serve gate is the maintenance fence.
|
|
2846
2868
|
runStart: (ctx) => this.serveGated(ctx, () => this.runHost().start(ctx, args(ctx))),
|
|
2847
|
-
runResume: (ctx) => this.serveGated(ctx, () => this.runHost().resume(args(ctx))),
|
|
2869
|
+
runResume: (ctx) => this.serveGated(ctx, () => this.runHost().resume(args(ctx), ctx)),
|
|
2848
2870
|
runAnswer: (ctx) => this.serveGated(ctx, () => {
|
|
2849
2871
|
const input = args(ctx);
|
|
2850
2872
|
if (input.amend === true)
|
|
2851
|
-
return this.runHost().amend(input, this.runAnswerer(ctx), (accepted) => this.authorizeRunAmend(ctx, accepted));
|
|
2852
|
-
return this.runHost().answer(input, this.runAnswerer(ctx), (open) => this.authorizeRunAnswer(ctx, input, open));
|
|
2873
|
+
return this.runHost().amend(input, this.runAnswerer(ctx), (accepted) => this.authorizeRunAmend(ctx, accepted), ctx);
|
|
2874
|
+
return this.runHost().answer(input, this.runAnswerer(ctx), (open) => this.authorizeRunAnswer(ctx, input, open), ctx);
|
|
2853
2875
|
}),
|
|
2854
2876
|
runStatus: (ctx) => this.serveGated(ctx, () => this.runHost().status(args(ctx))),
|
|
2855
2877
|
runPs: (ctx) => this.serveGated(ctx, () => this.runHost().list(args(ctx))),
|
|
@@ -3021,7 +3043,13 @@ export class Manager {
|
|
|
3021
3043
|
try {
|
|
3022
3044
|
if (graceful && process.platform === "win32" && a.control)
|
|
3023
3045
|
controlShutdown(a.control);
|
|
3024
|
-
|
|
3046
|
+
// A delegated seat's close runs only inside its retirement, after the host revoked its grant,
|
|
3047
|
+
// so stopping it starts that retirement. Every exit wait on a stop path then settles on that
|
|
3048
|
+
// close; the free path's own teardown joins the same flight.
|
|
3049
|
+
if (a.handedOff)
|
|
3050
|
+
this.trackDeprovision({ ...a, delegatedHandle: a.handle });
|
|
3051
|
+
else
|
|
3052
|
+
a.handle.stop({ graceful });
|
|
3025
3053
|
}
|
|
3026
3054
|
catch (e) {
|
|
3027
3055
|
console.error(`stop ${a.name} (${a.id}): ${e.message}`);
|
|
@@ -3282,6 +3310,7 @@ export class Manager {
|
|
|
3282
3310
|
lifecycleUid: material.lifecycleUid,
|
|
3283
3311
|
files,
|
|
3284
3312
|
launch: { owner: material.owner, actor: name, sentinelCredsPath: files.sentinelCreds, bearerCmd },
|
|
3313
|
+
enrolled: { material, actorToken },
|
|
3285
3314
|
};
|
|
3286
3315
|
}
|
|
3287
3316
|
catch (e) {
|
|
@@ -3309,6 +3338,43 @@ export class Manager {
|
|
|
3309
3338
|
return { error: `agent auth preflight failed for "${name}": ${rejectionText(e)}${leftover}` };
|
|
3310
3339
|
}
|
|
3311
3340
|
}
|
|
3341
|
+
/** This manager's registry record for its space, which pins the broker and IdP a delegated seat's
|
|
3342
|
+
* handoff carries. It must pin the exchange the host enrolls against, or the child would register
|
|
3343
|
+
* a mesh whose bearers come from somewhere else. */
|
|
3344
|
+
delegatedMeshRecord() {
|
|
3345
|
+
const record = loadMeshes().find((m) => m.space === this.space);
|
|
3346
|
+
if (!record?.userAuth)
|
|
3347
|
+
throw new Error(`a delegated seat needs this manager's user-mode registry record for space "${this.space}", and there is none`);
|
|
3348
|
+
if (record.userAuth.endpoints?.url !== this.remoteAuthority?.agentBearerExchangeUrl)
|
|
3349
|
+
throw new Error(`the registry record for space "${this.space}" pins a different exchange than the host's agentBearerExchangeUrl`);
|
|
3350
|
+
return record;
|
|
3351
|
+
}
|
|
3352
|
+
/** The one handoff a delegated seat receives, built from values this manager holds: the
|
|
3353
|
+
* enrollment result, the token it generated, and its own registry record. No path crosses. */
|
|
3354
|
+
composeManagedHandoff(enrolled, actorToken) {
|
|
3355
|
+
const record = this.delegatedMeshRecord();
|
|
3356
|
+
if (enrolled.agentBearerExchangeUrl !== record.userAuth.endpoints?.url)
|
|
3357
|
+
throw new Error(`the host enrolled "${enrolled.actor}" against a different exchange than this manager's registry record pins`);
|
|
3358
|
+
const { url, issuer, audience } = record.userAuth.idp;
|
|
3359
|
+
return {
|
|
3360
|
+
kind: MANAGED_HANDOFF_KIND,
|
|
3361
|
+
space: this.space,
|
|
3362
|
+
owner: enrolled.owner,
|
|
3363
|
+
actor: enrolled.actor,
|
|
3364
|
+
lifecycleUid: enrolled.lifecycleUid,
|
|
3365
|
+
server: record.server,
|
|
3366
|
+
tlsRequired: record.tlsRequired === true,
|
|
3367
|
+
authProvider: record.userAuth.provider,
|
|
3368
|
+
idp: { url, issuer, audience },
|
|
3369
|
+
exchangeUrl: enrolled.agentBearerExchangeUrl,
|
|
3370
|
+
sentinelCreds: enrolled.sentinelCreds,
|
|
3371
|
+
actorToken,
|
|
3372
|
+
subscribe: enrolled.subscribe,
|
|
3373
|
+
allowSubscribe: enrolled.allowSubscribe,
|
|
3374
|
+
allowPublish: enrolled.allowPublish,
|
|
3375
|
+
...(record.policy ? { policy: record.policy } : {}),
|
|
3376
|
+
};
|
|
3377
|
+
}
|
|
3312
3378
|
/**
|
|
3313
3379
|
* WHY A SEAT LEFT — one line, at the one place every free path passes through.
|
|
3314
3380
|
*
|
|
@@ -3414,7 +3480,7 @@ export class Manager {
|
|
|
3414
3480
|
if (this.userMode) {
|
|
3415
3481
|
const p = parsePrincipalKey(a.id);
|
|
3416
3482
|
if (p)
|
|
3417
|
-
this.retiring.set(a.name, { opId: retireOpId(a.lifecycleUid), lifecycleUid: a.lifecycleUid, owner: p.owner, actor: p.actor, agentId: a.id, userOwner: a.userOwner, delegated: a.delegated, secretPaths: a.secretPaths, launch: { allowSubscribe: a.launch.allowSubscribe }, startedAt: Date.now() });
|
|
3483
|
+
this.retiring.set(a.name, { opId: retireOpId(a.lifecycleUid), lifecycleUid: a.lifecycleUid, owner: p.owner, actor: p.actor, agentId: a.id, userOwner: a.userOwner, delegated: a.delegated, secretPaths: a.secretPaths, launch: { allowSubscribe: a.launch.allowSubscribe }, ...(a.handedOff ? { delegatedHandle: a.handle } : {}), startedAt: Date.now() });
|
|
3418
3484
|
}
|
|
3419
3485
|
else if (this.auth) {
|
|
3420
3486
|
// Unit B: a STATIC lifecycle now also holds its name pending its own terminal (the F1
|
|
@@ -3431,7 +3497,7 @@ export class Manager {
|
|
|
3431
3497
|
// is usually the seat ending, but a stop that could not prove the process gone leaves one the
|
|
3432
3498
|
// terminal must still address before it frees the alias.
|
|
3433
3499
|
if (!a.suppressCleanup && (this.maintenanceState === "active" || acceptedBeforeFence))
|
|
3434
|
-
this.trackDeprovision({ ...a, runtime: a.handle?.reference });
|
|
3500
|
+
this.trackDeprovision({ ...a, runtime: a.handle?.reference, ...(a.handedOff ? { delegatedHandle: a.handle } : {}) });
|
|
3435
3501
|
}
|
|
3436
3502
|
/** Tear down a departed agent's minted footprint (#159 B2, auth mode): its local-principal durables
|
|
3437
3503
|
* (`dm_local-<id>`, `dlv_local-<id>`), its read-ACL row, and its creds file — everything the spawn's
|
|
@@ -3483,6 +3549,13 @@ export class Manager {
|
|
|
3483
3549
|
target: { owner: target.owner, actor: target.actor, lifecycleUid: a.lifecycleUid },
|
|
3484
3550
|
opId: retireOpId(a.lifecycleUid),
|
|
3485
3551
|
});
|
|
3552
|
+
// A delegated seat's fenced close runs only now, after the grant is revoked so its child can
|
|
3553
|
+
// obtain no fresh bearer, and before the barrier, which must never run ahead of a create that
|
|
3554
|
+
// could still land. A close that cannot prove that yet throws here and keeps the alias held.
|
|
3555
|
+
if (a.delegatedHandle) {
|
|
3556
|
+
a.delegatedHandle.stop({ graceful: true });
|
|
3557
|
+
await this.awaitHandleExit(a.delegatedHandle);
|
|
3558
|
+
}
|
|
3486
3559
|
await this.requestRetirement(a);
|
|
3487
3560
|
return;
|
|
3488
3561
|
}
|
|
@@ -4648,6 +4721,28 @@ export class Manager {
|
|
|
4648
4721
|
if (this.userMode)
|
|
4649
4722
|
return { ok: false, error: "supervise is a restart policy this host cannot enforce: a user-mode seat has no static slot to keep the incarnation owned across a process death" };
|
|
4650
4723
|
}
|
|
4724
|
+
// A delegated seat starts outside this host from values alone, so a choice only this host can
|
|
4725
|
+
// honour is refused here, before enrollment, and named rather than dropped.
|
|
4726
|
+
if (this.runtime.spawnDelegated) {
|
|
4727
|
+
const delegatedBy = `runtime "${this.runtime.kind}" starts seats outside this host`;
|
|
4728
|
+
if (!this.remoteAuthority?.enrollManagedAgent)
|
|
4729
|
+
return { ok: false, error: `${delegatedBy}, which needs a host that enrolls managed agents (remoteAuthority.enrollManagedAgent); a delegated seat has no local grant path` };
|
|
4730
|
+
try {
|
|
4731
|
+
this.delegatedMeshRecord();
|
|
4732
|
+
}
|
|
4733
|
+
catch (e) {
|
|
4734
|
+
return { ok: false, error: e.message };
|
|
4735
|
+
}
|
|
4736
|
+
if (opts.resume)
|
|
4737
|
+
return { ok: false, error: `${delegatedBy}, so it cannot resume a session held on this host (resume)` };
|
|
4738
|
+
if (opts.resolved?.continuity === "exact")
|
|
4739
|
+
return { ok: false, error: `${delegatedBy}, so it cannot reopen a session held on this host (continuity: exact)` };
|
|
4740
|
+
if (typeof opts.cwd === "string" && opts.cwd !== "")
|
|
4741
|
+
return { ok: false, error: `${delegatedBy}, so it cannot run in a directory on this host's filesystem (cwd); the seat runs in its runtime resource's own directory` };
|
|
4742
|
+
const shared = Object.keys(connectorServers(loadCotalConfig(this.workspaceRoot), agent, parseShareSelection(opts.shareTools)));
|
|
4743
|
+
if (shared.length)
|
|
4744
|
+
return { ok: false, error: `${delegatedBy}, so it cannot share MCP servers that run on this host (${shared.join(", ")}); pass --share-tools none` };
|
|
4745
|
+
}
|
|
4651
4746
|
// A placed spawn's cwd is checked on this host BEFORE anything is minted or reserved: an absolute
|
|
4652
4747
|
// path this manager cannot realpath, or one that is not a directory, is refused here with the
|
|
4653
4748
|
// asked path, the reason and this host, so a misplaced spawn in a multi-manager space is
|
|
@@ -4784,7 +4879,7 @@ export class Manager {
|
|
|
4784
4879
|
return { ok: false, error: `the name "${identityName}" is still reconciling at manager startup; its prior lifecycle terminal owns this alias until it completes. Retry shortly.` };
|
|
4785
4880
|
const held = this.retiring.get(identityName);
|
|
4786
4881
|
if (held !== undefined) {
|
|
4787
|
-
void this.deprovision({ id: held.agentId, name: identityName, lifecycleUid: held.lifecycleUid, userOwner: held.userOwner, delegated: held.delegated, secretPaths: held.secretPaths, launch: held.launch }).catch(() => { });
|
|
4882
|
+
void this.deprovision({ id: held.agentId, name: identityName, lifecycleUid: held.lifecycleUid, userOwner: held.userOwner, delegated: held.delegated, secretPaths: held.secretPaths, delegatedHandle: held.delegatedHandle, launch: held.launch }).catch(() => { });
|
|
4788
4883
|
const err = lifecycleBlocked("failed-precondition", `the name "${identityName}" is reserved pending retirement: its previous agent's despawn started that lifecycle's teardown (footprint + standing-authority revoke + auth-side retirement), and the name frees only when all of it completes${held.lastError !== undefined ? ` (last attempt: ${held.lastError})` : ""}. NEXT: wait a moment and retry this spawn (retrying re-drives the whole teardown), or pick another name.`, { blockedOp: "retirement", headState: "retiring", opId: held.opId, remedy: "retry" });
|
|
4789
4884
|
return { ok: false, error: renderLifecycleBlocked(err.message, err), details: err.details };
|
|
4790
4885
|
}
|
|
@@ -4807,6 +4902,10 @@ export class Manager {
|
|
|
4807
4902
|
return { ok: false, error: e.message };
|
|
4808
4903
|
}
|
|
4809
4904
|
}
|
|
4905
|
+
// The child takes launch options as `--opt k=v`, so only string values can cross to it.
|
|
4906
|
+
const unportable = this.runtime.spawnDelegated ? Object.keys(launchOptions ?? {}).filter((k) => typeof launchOptions[k] !== "string") : [];
|
|
4907
|
+
if (unportable.length)
|
|
4908
|
+
return { ok: false, error: `runtime "${this.runtime.kind}" starts seats outside this host, so it cannot pass non-string launch options (${unportable.join(", ")})` };
|
|
4810
4909
|
// #4 A4 (panel): the roster the allocation consults must reflect the initial presence snapshot,
|
|
4811
4910
|
// or a spawn immediately after manager boot races an already-live unmanaged peer and re-opens the
|
|
4812
4911
|
// very collision black-hole this closes. Await the snapshot (bounded internally, fail-safe on an
|
|
@@ -4965,6 +5064,7 @@ export class Manager {
|
|
|
4965
5064
|
let issued;
|
|
4966
5065
|
let userLaunch;
|
|
4967
5066
|
let userOwner;
|
|
5067
|
+
let enrolled;
|
|
4968
5068
|
if (this.userMode) {
|
|
4969
5069
|
const prep = await this.provisionUserAgent(name, {
|
|
4970
5070
|
spawner,
|
|
@@ -4998,6 +5098,7 @@ export class Manager {
|
|
|
4998
5098
|
lifecycleUid = prep.lifecycleUid;
|
|
4999
5099
|
userLaunch = prep.launch;
|
|
5000
5100
|
userOwner = prep.owner;
|
|
5101
|
+
enrolled = prep.enrolled;
|
|
5001
5102
|
provisioned = { id: principalKey(prep.owner, name).key, name, lifecycleUid, userOwner: prep.owner, ...(opts.delegatedIntent ? { delegated: true } : {}), secretPaths: prep.files, ...(custody ? { runtime: custody } : {}) };
|
|
5002
5103
|
}
|
|
5003
5104
|
else if (this.auth) {
|
|
@@ -5110,9 +5211,22 @@ export class Manager {
|
|
|
5110
5211
|
// (possibly per-agent) launch cwd below. The cwd itself rides runtime.spawn, not the launch.
|
|
5111
5212
|
workspaceRoot: this.workspaceRoot,
|
|
5112
5213
|
};
|
|
5113
|
-
|
|
5114
|
-
|
|
5115
|
-
|
|
5214
|
+
// A delegated seat gets no local launch at all: no launch-material file, control token or
|
|
5215
|
+
// manager path is produced for it, only the handoff its runtime writes into the child.
|
|
5216
|
+
let spec;
|
|
5217
|
+
let handle;
|
|
5218
|
+
if (this.runtime.spawnDelegated) {
|
|
5219
|
+
const handoff = this.composeManagedHandoff(enrolled.material, enrolled.actorToken);
|
|
5220
|
+
if (this.delegatedLaunched.has(lifecycleUid))
|
|
5221
|
+
throw new Error(`lifecycle ${lifecycleUid} was already handed to runtime "${this.runtime.kind}"; a lifecycle is handed off at most once`);
|
|
5222
|
+
this.delegatedLaunched.add(lifecycleUid);
|
|
5223
|
+
handle = this.runtime.spawnDelegated({ agent, persona: readFileSync(configPath, "utf8"), role, model, variant, prompt, launchOptions: launchOptions, events }, handoff);
|
|
5224
|
+
}
|
|
5225
|
+
else {
|
|
5226
|
+
spec = connector.buildLaunch(launchOpts);
|
|
5227
|
+
spec.env = { ...spec.env, COTAL_MANAGER_INSTANCE: this.managerInstanceId };
|
|
5228
|
+
handle = await this.spawnCustodied(name, spec, cwd, custody);
|
|
5229
|
+
}
|
|
5116
5230
|
hooks?.onLaunched?.(); // P2 item 2: the "launched" progress edge (process spawned, pre-presence)
|
|
5117
5231
|
const managed = {
|
|
5118
5232
|
name,
|
|
@@ -5130,7 +5244,8 @@ export class Manager {
|
|
|
5130
5244
|
authorityParent: userLaunch && spawner && parsePrincipalKey(spawner) ? spawner : undefined,
|
|
5131
5245
|
startedAt: Date.now(),
|
|
5132
5246
|
handle,
|
|
5133
|
-
control: spec
|
|
5247
|
+
control: spec?.control,
|
|
5248
|
+
...(spec ? {} : { handedOff: true }),
|
|
5134
5249
|
launch: {
|
|
5135
5250
|
source: opts.resolved
|
|
5136
5251
|
? {
|
|
@@ -5156,7 +5271,7 @@ export class Manager {
|
|
|
5156
5271
|
events,
|
|
5157
5272
|
shareTools: opts.shareTools,
|
|
5158
5273
|
forkSource: opts.resume,
|
|
5159
|
-
...(opts.resume !== undefined && spec
|
|
5274
|
+
...(opts.resume !== undefined && spec?.resumeRecordPath ? { resumeRecordPath: spec.resumeRecordPath } : {}),
|
|
5160
5275
|
// Opaque values may contain secrets. Preserve only their keys and require the referenced
|
|
5161
5276
|
// persona/manifest to resolve the values again; imperative overrides have no safe payload.
|
|
5162
5277
|
unresolvedLaunchOptionKeys: opts.launchOptions && Object.keys(opts.launchOptions).length
|
|
@@ -5164,7 +5279,7 @@ export class Manager {
|
|
|
5164
5279
|
: undefined,
|
|
5165
5280
|
},
|
|
5166
5281
|
// A delegated agent is never restarted (SPEC 13.16), so its slot arms no recovery.
|
|
5167
|
-
...((connector.supportsSessionContinuation || opts.supervise !== undefined) && !opts.delegatedIntent
|
|
5282
|
+
...(spec && (connector.supportsSessionContinuation || opts.supervise !== undefined) && !opts.delegatedIntent
|
|
5168
5283
|
? {
|
|
5169
5284
|
restart: {
|
|
5170
5285
|
opts: launchOpts,
|
|
@@ -5331,6 +5446,7 @@ export class Manager {
|
|
|
5331
5446
|
userOwner: held.userOwner,
|
|
5332
5447
|
delegated: held.delegated,
|
|
5333
5448
|
secretPaths: held.secretPaths,
|
|
5449
|
+
delegatedHandle: held.delegatedHandle,
|
|
5334
5450
|
}).catch(() => { });
|
|
5335
5451
|
return {
|
|
5336
5452
|
ok: false,
|
|
@@ -6690,7 +6806,7 @@ export class Manager {
|
|
|
6690
6806
|
// error text — a crash-restart never silently skips eviction (no-fallbacks).
|
|
6691
6807
|
const barrier = endpointRegistrationBarrier(authKv, this.space, {
|
|
6692
6808
|
endpoint: MANAGER_ENDPOINT, instanceId: iid, opId: registrationOpId,
|
|
6693
|
-
...(auth ? { evict:
|
|
6809
|
+
...(auth ? { evict: makeManagerEndpointHolderEvictor({ space: this.space, servers: this.servers ?? DEFAULT_SERVER, auth, log: (line) => console.error(line), unreachableWaitMs: DELIVERY_ADMIN_BOOT_WAIT_MS }) } : {}),
|
|
6694
6810
|
});
|
|
6695
6811
|
const spec = { endpoint: MANAGER_ENDPOINT, owner: DEV_OWNER, clusterDigests: [artifacts.closureDigest], protocol: { v: 1 } };
|
|
6696
6812
|
const { registrationRevision } = await registerServiceInstance(recordsKv, {
|