@cotal-ai/manager 0.11.6 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +42 -0
- package/dist/commands.js +27 -4
- package/dist/commands.js.map +1 -1
- package/dist/index.d.ts +2 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -1
- package/dist/index.js.map +1 -1
- package/dist/launch.d.ts +6 -1
- package/dist/launch.d.ts.map +1 -1
- package/dist/launch.js +12 -2
- package/dist/launch.js.map +1 -1
- package/dist/manager.d.ts +210 -3
- package/dist/manager.d.ts.map +1 -1
- package/dist/manager.js +1137 -21
- package/dist/manager.js.map +1 -1
- package/dist/resume.d.ts +17 -0
- package/dist/resume.d.ts.map +1 -0
- package/dist/resume.js +138 -0
- package/dist/resume.js.map +1 -0
- package/dist/runtime/pty.d.ts.map +1 -1
- package/dist/runtime/pty.js +11 -0
- package/dist/runtime/pty.js.map +1 -1
- package/package.json +5 -5
package/dist/manager.js
CHANGED
|
@@ -1,13 +1,15 @@
|
|
|
1
1
|
import { execFile } from "node:child_process";
|
|
2
|
-
import {
|
|
2
|
+
import { createHash, randomBytes } from "node:crypto";
|
|
3
|
+
import { existsSync, lstatSync, readFileSync, rmSync } from "node:fs";
|
|
3
4
|
import { join, dirname, resolve } from "node:path";
|
|
4
|
-
import { CotalEndpoint, DEFAULT_SERVER, DEV_OWNER, MANAGER_LEASE_TTL_MS, STANDING_RENEWABLE_TTL_SEC, agentFilePath, clearSpaceHistory, connectorServers, deprovisionAgent, firstFreeName, loadAgentFile, loadCotalConfig, mintCreds, mkSecretDir, newIdentity, parsePrincipalKey, parseShareSelection, principalKey, provisionAgent, provisionAgentDurables, registry, resolveAuthProvider, saveAgentFile, writeSecretFile, subjectMatches, CONTROL_PRIVILEGED, CONTROL_SELF_SERVICE, CONTROL_ADMIN, } from "@cotal-ai/core";
|
|
5
|
-
import { agentAuthState, authDir, connectorInstallHint, DEFAULT_CONNECTOR, defaultAgentType, findCotalRoot, loadMeshes, loadSpaceAuth, manifestExtensionNames, materializeFromManifest, mergeLaunchOptions, remintDaemonCreds, resolveOnPath, userAuthStateDir, writeRenewalRecord } from "@cotal-ai/workspace";
|
|
5
|
+
import { CotalEndpoint, DEFAULT_SERVER, DEV_OWNER, MANAGER_LEASE_TTL_MS, STANDING_RENEWABLE_TTL_SEC, agentFilePath, clearSpaceHistory, connectorServers, deprovisionAgent, firstFreeName, idFromCreds, loadAgentFile, loadCotalConfig, mintCreds, mkSecretDir, newIdentity, parsePrincipalKey, parseShareSelection, principalKey, probeConnect, provisionAgent, provisionAgentDurables, registry, resolveAuthProvider, saveAgentFile, writeSecretFile, subjectMatches, CONTROL_PRIVILEGED, CONTROL_SELF_SERVICE, CONTROL_ADMIN, } from "@cotal-ai/core";
|
|
6
|
+
import { agentAuthState, authDir, connectorInstallHint, DEFAULT_CONNECTOR, defaultAgentType, findCotalRoot, loadMeshes, loadSpaceAuth, manifestExtensionNames, materializeFromManifest, mergeLaunchOptions, remintDaemonCreds, resolveOnPath, userAuthStateDir, workspaceSecretStore, writeRenewalRecord } from "@cotal-ai/workspace";
|
|
6
7
|
import { createRuntime, } from "./runtime/index.js";
|
|
7
8
|
import { AttachEndpoint } from "./attach-endpoint.js";
|
|
8
9
|
import { launchSpecForRun, materializePersona, launchAgentToStartOpts } from "./launch.js";
|
|
9
10
|
import { authorizeLaunch, authorizeNamedControl } from "./authorize.js";
|
|
10
11
|
import { controlShutdown } from "./control-shutdown.js";
|
|
12
|
+
import { parseResumeCommitArgs, parseResumeControlArgs, parseResumeFinalizeArgs } from "./resume.js";
|
|
11
13
|
/** Concurrency ceiling — the manager refuses to hold more than this many live + in-flight +
|
|
12
14
|
* cooling slots at once (P4a). Bounds a fork-bomb: spawn is a full agent process per call. */
|
|
13
15
|
const MAX_AGENTS = 50;
|
|
@@ -29,6 +31,9 @@ export const READINESS_TIMEOUT_MS = 30_000;
|
|
|
29
31
|
* `.catch`. Generous over the helper's 5s connect timeout to allow the two consumer-deletes + ACL purge
|
|
30
32
|
* + drain on a healthy-but-slow broker. */
|
|
31
33
|
const DEPROVISION_TIMEOUT_MS = 15_000;
|
|
34
|
+
/** A hard preservation stop should settle quickly. The manager still waits and reports a partial
|
|
35
|
+
* cut rather than pretending a child is gone. Held in ManagerOptions so fake runtimes can shorten it. */
|
|
36
|
+
const PRESERVE_STOP_TIMEOUT_MS = 10_000;
|
|
32
37
|
/** Sentinel owner-filter value that matches NO agent's `userOwner` (owner tokens never contain a
|
|
33
38
|
* dash) — what {@link Manager.psOwnerFilter} returns for an unparseable caller so a malformed
|
|
34
39
|
* principal fail-closes to an empty `ps` instead of an unbounded one. */
|
|
@@ -54,6 +59,9 @@ function withTimeout(p, ms, msg) {
|
|
|
54
59
|
});
|
|
55
60
|
return Promise.race([p.finally(() => clearTimeout(timer)), timeout]);
|
|
56
61
|
}
|
|
62
|
+
function sameStrings(a, b) {
|
|
63
|
+
return JSON.stringify([...(a ?? [])].sort()) === JSON.stringify([...(b ?? [])].sort());
|
|
64
|
+
}
|
|
57
65
|
/**
|
|
58
66
|
* The agent supervisor: a long-lived mesh node that owns agent process lifecycle.
|
|
59
67
|
* It serves control requests on the "manager" service and spawns/kills agents
|
|
@@ -68,6 +76,7 @@ export class Manager {
|
|
|
68
76
|
/** See {@link ManagerOptions.installedExtensions}. */
|
|
69
77
|
installedExtensions;
|
|
70
78
|
runtime;
|
|
79
|
+
preserveStopTimeoutMs;
|
|
71
80
|
agents = new Map();
|
|
72
81
|
/** Names whose spawn is in flight (reserved synchronously before the provision await) — counted
|
|
73
82
|
* toward the ceiling so two concurrent same-name spawns can't both pass the gate (P4a). */
|
|
@@ -92,6 +101,29 @@ export class Manager {
|
|
|
92
101
|
leaseTimer;
|
|
93
102
|
/** The class-2 renewal owner's half-TTL schedule (D5 slice 5); armed only on auth meshes. */
|
|
94
103
|
credRenewTimer;
|
|
104
|
+
maintenanceState = "active";
|
|
105
|
+
lifecycleInFlight = 0;
|
|
106
|
+
lifecycleDrainWaiters = [];
|
|
107
|
+
preservationTask;
|
|
108
|
+
preparationTask;
|
|
109
|
+
preservationGeneration = 0;
|
|
110
|
+
preservationAttemptId;
|
|
111
|
+
preservationStarted = false;
|
|
112
|
+
preservationFailures = [];
|
|
113
|
+
unverifiedStops = [];
|
|
114
|
+
preservationInventory;
|
|
115
|
+
resumeAttemptId;
|
|
116
|
+
resumeInventoryDigest;
|
|
117
|
+
resumeInventory;
|
|
118
|
+
resumeTask;
|
|
119
|
+
resumeResult;
|
|
120
|
+
resumeRequired = false;
|
|
121
|
+
resumeAwaitingCommit = false;
|
|
122
|
+
resumeCommitted = false;
|
|
123
|
+
resumeCommitTask;
|
|
124
|
+
resumeFinalized = false;
|
|
125
|
+
resumeDurableCommitToken;
|
|
126
|
+
resumedAgentNames = new Set();
|
|
95
127
|
constructor(opts) {
|
|
96
128
|
this.space = opts.space;
|
|
97
129
|
this.servers = opts.servers;
|
|
@@ -99,7 +131,17 @@ export class Manager {
|
|
|
99
131
|
this.workspaceRoot = opts.workspaceRoot ?? findCotalRoot();
|
|
100
132
|
this.installedExtensions = opts.installedExtensions ?? false;
|
|
101
133
|
this.runtime = createRuntime(opts.runtime ?? "auto", `cotal-${this.space}`);
|
|
102
|
-
this.
|
|
134
|
+
this.preserveStopTimeoutMs = opts.preserveStopTimeoutMs ?? PRESERVE_STOP_TIMEOUT_MS;
|
|
135
|
+
if (opts.resumeAttemptId && !/^[A-Za-z0-9_-]{1,128}$/.test(opts.resumeAttemptId))
|
|
136
|
+
throw new Error("resumeAttemptId must be a safe token (letters, digits, _, -; max 128)");
|
|
137
|
+
if (opts.resumeDurableCommitToken && !/^[a-f0-9]{64}$/.test(opts.resumeDurableCommitToken))
|
|
138
|
+
throw new Error("resumeDurableCommitToken must be a lowercase 32-byte token");
|
|
139
|
+
if (opts.resumeDurableCommitToken && !opts.resumeAttemptId)
|
|
140
|
+
throw new Error("resumeDurableCommitToken requires resumeAttemptId");
|
|
141
|
+
this.resumeAttemptId = opts.resumeAttemptId;
|
|
142
|
+
this.resumeRequired = opts.resumeAttemptId !== undefined;
|
|
143
|
+
this.resumeDurableCommitToken = opts.resumeDurableCommitToken;
|
|
144
|
+
this.attach = new AttachEndpoint((name) => this.maintenanceState === "active" && !this.resumeRequired ? this.agents.get(name)?.handle : undefined, () => this.list(),
|
|
103
145
|
// Initial /feed replay for a connecting console: the current peer roster.
|
|
104
146
|
() => [{ event: "roster", data: this.ep?.getRoster() ?? [] }], opts.consolePort ?? 0);
|
|
105
147
|
}
|
|
@@ -221,6 +263,9 @@ export class Manager {
|
|
|
221
263
|
* (no responder) is recorded honestly: the daemon's 75% source re-read remains the adoption backstop.
|
|
222
264
|
* Never throws — renewal failure must be LOUD (log + record), not fatal to the supervisor. */
|
|
223
265
|
async renewDaemonCreds() {
|
|
266
|
+
const release = this.beginLifecycle();
|
|
267
|
+
if (!release)
|
|
268
|
+
return;
|
|
224
269
|
try {
|
|
225
270
|
const results = await remintDaemonCreds(this.workspaceRoot);
|
|
226
271
|
const resigned = results.filter((r) => r.ok);
|
|
@@ -243,6 +288,362 @@ export class Manager {
|
|
|
243
288
|
catch (e) {
|
|
244
289
|
console.error(`! credential renewal pass failed: ${e.message}`);
|
|
245
290
|
}
|
|
291
|
+
finally {
|
|
292
|
+
release();
|
|
293
|
+
}
|
|
294
|
+
}
|
|
295
|
+
/** Admit one lifecycle/control operation while active. The synchronous increment is the fence:
|
|
296
|
+
* preserveState flips state before its first await, so work is either counted or rejected. */
|
|
297
|
+
beginLifecycle(resumeOperation = false) {
|
|
298
|
+
if (this.maintenanceState !== "active" || (this.resumeRequired && !resumeOperation))
|
|
299
|
+
return undefined;
|
|
300
|
+
this.lifecycleInFlight++;
|
|
301
|
+
let released = false;
|
|
302
|
+
return () => {
|
|
303
|
+
if (released)
|
|
304
|
+
return;
|
|
305
|
+
released = true;
|
|
306
|
+
this.releaseLifecycle();
|
|
307
|
+
};
|
|
308
|
+
}
|
|
309
|
+
releaseLifecycle() {
|
|
310
|
+
this.lifecycleInFlight--;
|
|
311
|
+
if (this.lifecycleInFlight !== 0)
|
|
312
|
+
return;
|
|
313
|
+
const waiters = this.lifecycleDrainWaiters;
|
|
314
|
+
this.lifecycleDrainWaiters = [];
|
|
315
|
+
for (const wake of waiters)
|
|
316
|
+
wake();
|
|
317
|
+
}
|
|
318
|
+
/** A cleanup spawned by accepted active-mode work is part of that work for maintenance draining,
|
|
319
|
+
* even where the ordinary control reply remains fire-and-forget. */
|
|
320
|
+
trackDeprovision(a, context = "") {
|
|
321
|
+
this.lifecycleInFlight++;
|
|
322
|
+
void this.deprovision(a)
|
|
323
|
+
.catch((e) => console.error(`deprovision${context ? ` ${context}` : ""} ${a.name} (${a.id}): ${e.message}`))
|
|
324
|
+
.finally(() => this.releaseLifecycle());
|
|
325
|
+
}
|
|
326
|
+
async awaitLifecycleDrain() {
|
|
327
|
+
if (this.lifecycleInFlight === 0)
|
|
328
|
+
return;
|
|
329
|
+
await new Promise((resolve) => this.lifecycleDrainWaiters.push(resolve));
|
|
330
|
+
}
|
|
331
|
+
maintenanceError() {
|
|
332
|
+
if (this.resumeRequired)
|
|
333
|
+
return `manager is waiting for resume attempt ${this.resumeAttemptId}; ordinary lifecycle/control work is fenced`;
|
|
334
|
+
return `manager is in ${this.maintenanceState} mode; new lifecycle/control work is fenced`;
|
|
335
|
+
}
|
|
336
|
+
/** Fence and build the inventory without stopping a child. The coordinator must durably persist
|
|
337
|
+
* this exact plan before calling commitPreservation with the same attempt id. */
|
|
338
|
+
preparePreservation(attemptId) {
|
|
339
|
+
if (!attemptId.trim())
|
|
340
|
+
return Promise.reject(new Error("preservation attemptId is required"));
|
|
341
|
+
if (this.preservationAttemptId && this.preservationAttemptId !== attemptId)
|
|
342
|
+
return Promise.reject(new Error(`manager is fenced for preservation attempt ${this.preservationAttemptId}; refusing different attempt ${attemptId}`));
|
|
343
|
+
if (this.maintenanceState === "preserved" && this.preservationInventory)
|
|
344
|
+
return Promise.resolve({ ok: true, attemptId, state: "preserved", inventory: this.preservationInventory, failures: [] });
|
|
345
|
+
if (this.preparationTask)
|
|
346
|
+
return this.preparationTask;
|
|
347
|
+
if (this.maintenanceState === "active") {
|
|
348
|
+
// The fence lands before any await. Accepted work has already incremented lifecycleInFlight.
|
|
349
|
+
this.maintenanceState = "preserving";
|
|
350
|
+
this.preservationAttemptId = attemptId;
|
|
351
|
+
this.preservationGeneration++;
|
|
352
|
+
if (this.credRenewTimer) {
|
|
353
|
+
clearInterval(this.credRenewTimer);
|
|
354
|
+
this.credRenewTimer = undefined;
|
|
355
|
+
}
|
|
356
|
+
}
|
|
357
|
+
const generation = this.preservationGeneration;
|
|
358
|
+
const task = this.runPreparation(attemptId, generation);
|
|
359
|
+
let wrapped;
|
|
360
|
+
wrapped = task.finally(() => {
|
|
361
|
+
if (this.preservationGeneration === generation && this.preparationTask === wrapped)
|
|
362
|
+
this.preparationTask = undefined;
|
|
363
|
+
});
|
|
364
|
+
this.preparationTask = wrapped;
|
|
365
|
+
return wrapped;
|
|
366
|
+
}
|
|
367
|
+
assertPreservationGeneration(attemptId, generation) {
|
|
368
|
+
if (this.preservationAttemptId !== attemptId || this.preservationGeneration !== generation)
|
|
369
|
+
throw new Error(`preservation attempt ${attemptId} was abandoned before preparation completed`);
|
|
370
|
+
}
|
|
371
|
+
async runPreparation(attemptId, generation) {
|
|
372
|
+
await this.awaitLifecycleDrain();
|
|
373
|
+
this.assertPreservationGeneration(attemptId, generation);
|
|
374
|
+
const inventory = this.preservationInventory ?? {
|
|
375
|
+
version: "cotal-manager-resume/v1",
|
|
376
|
+
space: this.space,
|
|
377
|
+
createdAt: new Date().toISOString(),
|
|
378
|
+
agents: [...this.agents.values()].map((a) => this.resumeEntry(a)),
|
|
379
|
+
};
|
|
380
|
+
const failures = [];
|
|
381
|
+
for (const entry of inventory.agents) {
|
|
382
|
+
const error = this.inventoryReferenceError(entry);
|
|
383
|
+
if (error)
|
|
384
|
+
failures.push({
|
|
385
|
+
name: entry.name,
|
|
386
|
+
id: entry.identity.mode === "user" ? principalKey(entry.identity.owner, entry.identity.actor).key : entry.identity.id,
|
|
387
|
+
error,
|
|
388
|
+
});
|
|
389
|
+
}
|
|
390
|
+
const unverifiedStops = this.unverifiedStops.filter((stopped) => {
|
|
391
|
+
try {
|
|
392
|
+
if (!stopped.authoritative && stopped.handle.status() === "exited")
|
|
393
|
+
return false;
|
|
394
|
+
}
|
|
395
|
+
catch { /* fail closed below */ }
|
|
396
|
+
failures.push({
|
|
397
|
+
name: stopped.name,
|
|
398
|
+
id: stopped.id,
|
|
399
|
+
error: stopped.error ?? `an earlier stop on runtime "${stopped.handle.kind}" cannot prove the child is gone`,
|
|
400
|
+
});
|
|
401
|
+
return true;
|
|
402
|
+
});
|
|
403
|
+
this.assertPreservationGeneration(attemptId, generation);
|
|
404
|
+
// The prepared inventory must round-trip through the EXACT resume control parser (schema and
|
|
405
|
+
// byte cap) NOW, before any child stops: a cut that cannot resume must fail at prepare time,
|
|
406
|
+
// never after listener exposure.
|
|
407
|
+
try {
|
|
408
|
+
parseResumeControlArgs({ attemptId, inventory });
|
|
409
|
+
}
|
|
410
|
+
catch (e) {
|
|
411
|
+
failures.push({ name: "<inventory>", id: attemptId, error: `prepared inventory would be rejected at resume: ${e.message}` });
|
|
412
|
+
}
|
|
413
|
+
this.preservationInventory = inventory;
|
|
414
|
+
this.preservationFailures = failures;
|
|
415
|
+
this.unverifiedStops = unverifiedStops;
|
|
416
|
+
return {
|
|
417
|
+
ok: failures.length === 0,
|
|
418
|
+
attemptId,
|
|
419
|
+
state: "prepared",
|
|
420
|
+
inventory,
|
|
421
|
+
failures: [...failures],
|
|
422
|
+
};
|
|
423
|
+
}
|
|
424
|
+
/** Stop children only after the coordinator has persisted the prepared inventory. Same-attempt
|
|
425
|
+
* retries are idempotent; a different attempt is refused. */
|
|
426
|
+
commitPreservation(attemptId) {
|
|
427
|
+
if (!this.preservationAttemptId || this.preservationAttemptId !== attemptId)
|
|
428
|
+
return Promise.reject(new Error(`preservation attempt ${attemptId} was not prepared by this manager`));
|
|
429
|
+
if (!this.preservationInventory)
|
|
430
|
+
return Promise.reject(new Error(`preservation attempt ${attemptId} has no prepared inventory`));
|
|
431
|
+
if (this.preservationFailures.length)
|
|
432
|
+
return Promise.resolve({
|
|
433
|
+
ok: false,
|
|
434
|
+
attemptId,
|
|
435
|
+
state: "preserving",
|
|
436
|
+
inventory: this.preservationInventory,
|
|
437
|
+
failures: [...this.preservationFailures],
|
|
438
|
+
});
|
|
439
|
+
if (this.maintenanceState === "preserved")
|
|
440
|
+
return Promise.resolve({ ok: true, attemptId, state: "preserved", inventory: this.preservationInventory, failures: [] });
|
|
441
|
+
if (this.preservationTask)
|
|
442
|
+
return this.preservationTask;
|
|
443
|
+
this.preservationStarted = true;
|
|
444
|
+
this.preservationTask = this.runPreservation(attemptId).finally(() => {
|
|
445
|
+
this.preservationTask = undefined;
|
|
446
|
+
});
|
|
447
|
+
return this.preservationTask;
|
|
448
|
+
}
|
|
449
|
+
/** Recover an abandoned prepare before any child stop. Once commit begins, preservation is
|
|
450
|
+
* irreversible and remains fenced until the coordinator records failure/recourse. */
|
|
451
|
+
abortPreservation(attemptId) {
|
|
452
|
+
if (this.preservationAttemptId !== attemptId)
|
|
453
|
+
throw new Error(`preservation attempt ${attemptId} is not the active manager attempt`);
|
|
454
|
+
if (this.preparationTask || this.lifecycleInFlight > 0)
|
|
455
|
+
throw new Error(`preservation attempt ${attemptId} is still preparing or draining accepted lifecycle work and cannot be aborted`);
|
|
456
|
+
if (this.preservationStarted || this.preservationTask || this.maintenanceState === "preserved")
|
|
457
|
+
throw new Error(`preservation attempt ${attemptId} has begun stopping children and cannot return to active mode`);
|
|
458
|
+
this.preservationGeneration++;
|
|
459
|
+
this.maintenanceState = "active";
|
|
460
|
+
this.preservationAttemptId = undefined;
|
|
461
|
+
this.preservationInventory = undefined;
|
|
462
|
+
this.preservationFailures = [];
|
|
463
|
+
if (this.auth && !this.credRenewTimer) {
|
|
464
|
+
this.credRenewTimer = setInterval(() => { void this.renewDaemonCreds(); }, (STANDING_RENEWABLE_TTL_SEC / 2) * 1000);
|
|
465
|
+
this.credRenewTimer.unref?.();
|
|
466
|
+
}
|
|
467
|
+
// Exit watchers were suppressed while the fence stood: reconcile every child that died during
|
|
468
|
+
// preparation now, or its slot/credential footprint would linger unreaped after the abort.
|
|
469
|
+
for (const agent of [...this.agents.values()]) {
|
|
470
|
+
try {
|
|
471
|
+
if (agent.handle.status() === "exited")
|
|
472
|
+
this.onAgentExit(agent);
|
|
473
|
+
}
|
|
474
|
+
catch { /* status unavailable - the exit watcher fires again on real exit */ }
|
|
475
|
+
}
|
|
476
|
+
}
|
|
477
|
+
/** In-process convenience that preserves the crash barrier by awaiting durable persistence between
|
|
478
|
+
* prepare and commit. Wire callers use the explicit two-phase admin operations. */
|
|
479
|
+
async preserveState(opts) {
|
|
480
|
+
const plan = await this.preparePreservation(opts.attemptId);
|
|
481
|
+
if (!plan.ok)
|
|
482
|
+
return { ok: false, attemptId: opts.attemptId, state: "preserving", inventory: plan.inventory, failures: plan.failures };
|
|
483
|
+
await opts.persistInventory(plan.inventory);
|
|
484
|
+
return this.commitPreservation(opts.attemptId);
|
|
485
|
+
}
|
|
486
|
+
async runPreservation(attemptId) {
|
|
487
|
+
const failures = [];
|
|
488
|
+
for (const a of [...this.agents.values()])
|
|
489
|
+
a.suppressCleanup = true;
|
|
490
|
+
await Promise.all([...this.agents.values()].map(async (a) => {
|
|
491
|
+
try {
|
|
492
|
+
// A preservation cut must not run the connector's logical leave/cleanup hooks.
|
|
493
|
+
a.handle.stop({ graceful: false });
|
|
494
|
+
}
|
|
495
|
+
catch (e) {
|
|
496
|
+
failures.push({ name: a.name, id: a.id, error: `stop failed: ${e.message}` });
|
|
497
|
+
return;
|
|
498
|
+
}
|
|
499
|
+
try {
|
|
500
|
+
await this.awaitHandleExit(a.handle);
|
|
501
|
+
if (this.agents.get(a.name) === a)
|
|
502
|
+
this.agents.delete(a.name);
|
|
503
|
+
}
|
|
504
|
+
catch (e) {
|
|
505
|
+
failures.push({ name: a.name, id: a.id, error: e.message });
|
|
506
|
+
}
|
|
507
|
+
}));
|
|
508
|
+
if (failures.length === 0)
|
|
509
|
+
this.maintenanceState = "preserved";
|
|
510
|
+
return {
|
|
511
|
+
ok: failures.length === 0,
|
|
512
|
+
attemptId,
|
|
513
|
+
state: this.maintenanceState === "preserved" ? "preserved" : "preserving",
|
|
514
|
+
inventory: this.preservationInventory,
|
|
515
|
+
failures,
|
|
516
|
+
};
|
|
517
|
+
}
|
|
518
|
+
async awaitHandleExit(handle) {
|
|
519
|
+
if (!handle.waitForExit)
|
|
520
|
+
throw new Error(`runtime "${handle.kind}" cannot prove child exit (AgentHandle.waitForExit is not implemented)`);
|
|
521
|
+
if (handle.status() === "exited")
|
|
522
|
+
return;
|
|
523
|
+
await withTimeout(handle.waitForExit(), this.preserveStopTimeoutMs, `child did not exit within ${this.preserveStopTimeoutMs}ms`);
|
|
524
|
+
if (handle.status() !== "exited")
|
|
525
|
+
throw new Error(`runtime "${handle.kind}" reported exit completion but status is still running`);
|
|
526
|
+
}
|
|
527
|
+
inventoryReferenceError(entry) {
|
|
528
|
+
if (entry.launch.source.kind === "manifest" && !entry.launch.source.runId)
|
|
529
|
+
return "resolved manifest launch has no retained runId";
|
|
530
|
+
if (entry.launch.unresolvedLaunchOptionKeys?.length)
|
|
531
|
+
return `imperative launch options have no non-secret durable source (${entry.launch.unresolvedLaunchOptionKeys.join(", ")})`;
|
|
532
|
+
if (!entry.dependencies.some((path) => resolve(path) === resolve(entry.launch.source.configPath)))
|
|
533
|
+
return `launch config is not declared as a retained dependency: ${entry.launch.source.configPath}`;
|
|
534
|
+
if (entry.launch.source.kind === "manifest" && entry.launch.source.runId) {
|
|
535
|
+
const specPath = join(this.workspaceRoot, ".cotal", "run", `${entry.launch.source.runId}.json`);
|
|
536
|
+
if (!entry.dependencies.some((path) => resolve(path) === resolve(specPath)))
|
|
537
|
+
return `manifest source is not declared as a retained dependency: ${specPath}`;
|
|
538
|
+
}
|
|
539
|
+
const required = [...entry.dependencies];
|
|
540
|
+
if (entry.identity.mode === "static")
|
|
541
|
+
required.push(entry.identity.credential.path);
|
|
542
|
+
if (entry.identity.mode === "user") {
|
|
543
|
+
required.push(entry.identity.actorToken.path, entry.identity.sentinelCredential.path);
|
|
544
|
+
}
|
|
545
|
+
for (const path of required) {
|
|
546
|
+
try {
|
|
547
|
+
const st = lstatSync(path);
|
|
548
|
+
if (!st.isFile() || st.isSymbolicLink())
|
|
549
|
+
return `retained reference is not a regular non-symlink file: ${path}`;
|
|
550
|
+
}
|
|
551
|
+
catch (e) {
|
|
552
|
+
return `retained reference unavailable: ${path} (${e.message})`;
|
|
553
|
+
}
|
|
554
|
+
}
|
|
555
|
+
if (process.platform !== "win32") {
|
|
556
|
+
const secrets = entry.identity.mode === "static"
|
|
557
|
+
? [entry.identity.credential.path]
|
|
558
|
+
: entry.identity.mode === "user"
|
|
559
|
+
? [entry.identity.actorToken.path, entry.identity.sentinelCredential.path]
|
|
560
|
+
: [];
|
|
561
|
+
for (const path of secrets)
|
|
562
|
+
if ((lstatSync(path).mode & 0o077) !== 0)
|
|
563
|
+
return `retained identity file is not private (expected 0600): ${path}`;
|
|
564
|
+
}
|
|
565
|
+
try {
|
|
566
|
+
if (this.fileDigest(entry.launch.source.configPath) !== entry.launch.source.configSha256)
|
|
567
|
+
return `launch config changed since it became effective: ${entry.launch.source.configPath}`;
|
|
568
|
+
if (entry.identity.mode === "static" && this.fileDigest(entry.identity.credential.path) !== entry.identity.credential.sha256)
|
|
569
|
+
return `retained credential changed after the cut: ${entry.identity.credential.path}`;
|
|
570
|
+
if (entry.identity.mode === "user" &&
|
|
571
|
+
(this.fileDigest(entry.identity.actorToken.path) !== entry.identity.actorToken.sha256 ||
|
|
572
|
+
this.fileDigest(entry.identity.sentinelCredential.path) !== entry.identity.sentinelCredential.sha256))
|
|
573
|
+
return `retained user identity files changed after the cut for ${entry.name}`;
|
|
574
|
+
if (entry.launch.source.kind === "manifest" && entry.launch.source.runId) {
|
|
575
|
+
const specPath = join(this.workspaceRoot, ".cotal", "run", `${entry.launch.source.runId}.json`);
|
|
576
|
+
if (!entry.launch.source.manifestSha256 || this.fileDigest(specPath) !== entry.launch.source.manifestSha256)
|
|
577
|
+
return `manifest source changed since it became effective: ${specPath}`;
|
|
578
|
+
}
|
|
579
|
+
}
|
|
580
|
+
catch (e) {
|
|
581
|
+
return `retained reference cannot be hashed: ${e.message}`;
|
|
582
|
+
}
|
|
583
|
+
return undefined;
|
|
584
|
+
}
|
|
585
|
+
fileDigest(path) {
|
|
586
|
+
return createHash("sha256").update(readFileSync(path)).digest("hex");
|
|
587
|
+
}
|
|
588
|
+
fileDigestOrEmpty(path) {
|
|
589
|
+
try {
|
|
590
|
+
return this.fileDigest(path);
|
|
591
|
+
}
|
|
592
|
+
catch {
|
|
593
|
+
return "";
|
|
594
|
+
}
|
|
595
|
+
}
|
|
596
|
+
resumeEntry(a) {
|
|
597
|
+
const principal = a.userOwner
|
|
598
|
+
? parsePrincipalKey(a.id)
|
|
599
|
+
: { owner: DEV_OWNER, actor: a.id };
|
|
600
|
+
if (!principal)
|
|
601
|
+
throw new Error(`managed agent ${a.name} has an invalid principal ${a.id}`);
|
|
602
|
+
const credsDir = join(authDir(this.workspaceRoot), "creds");
|
|
603
|
+
const staticCredsPath = join(credsDir, `${a.name}.creds`);
|
|
604
|
+
const actorTokenPath = join(credsDir, `${a.name}.actor-token`);
|
|
605
|
+
const sentinelPath = join(credsDir, `${a.name}.sentinel.creds`);
|
|
606
|
+
const identity = a.userOwner
|
|
607
|
+
? {
|
|
608
|
+
mode: "user",
|
|
609
|
+
owner: principal.owner,
|
|
610
|
+
actor: principal.actor,
|
|
611
|
+
actorToken: { kind: "file", path: actorTokenPath, sha256: this.fileDigestOrEmpty(actorTokenPath) },
|
|
612
|
+
sentinelCredential: { kind: "file", path: sentinelPath, sha256: this.fileDigestOrEmpty(sentinelPath) },
|
|
613
|
+
health: { kind: "file", path: join(credsDir, `${a.name}.auth-health.json`) },
|
|
614
|
+
}
|
|
615
|
+
: this.auth
|
|
616
|
+
? { mode: "static", id: principal.actor, credential: { kind: "file", path: staticCredsPath, sha256: this.fileDigestOrEmpty(staticCredsPath) } }
|
|
617
|
+
: { mode: "open", id: principal.actor };
|
|
618
|
+
const dependencies = [a.launch.source.configPath];
|
|
619
|
+
if (a.launch.source.kind === "manifest" && a.launch.source.runId)
|
|
620
|
+
dependencies.unshift(join(this.workspaceRoot, ".cotal", "run", `${a.launch.source.runId}.json`));
|
|
621
|
+
return {
|
|
622
|
+
space: this.space,
|
|
623
|
+
name: a.name,
|
|
624
|
+
role: a.role,
|
|
625
|
+
identity,
|
|
626
|
+
launch: {
|
|
627
|
+
connector: a.agent,
|
|
628
|
+
runtime: a.handle.kind,
|
|
629
|
+
cwd: a.launch.cwd,
|
|
630
|
+
source: a.launch.source,
|
|
631
|
+
model: a.launch.model,
|
|
632
|
+
variant: a.launch.variant,
|
|
633
|
+
subscribe: a.launch.subscribe,
|
|
634
|
+
allowSubscribe: a.launch.allowSubscribe,
|
|
635
|
+
allowPublish: a.launch.allowPublish,
|
|
636
|
+
capabilities: a.launch.capabilities,
|
|
637
|
+
transcript: a.launch.transcript,
|
|
638
|
+
shareTools: a.launch.shareTools,
|
|
639
|
+
forkSource: a.launch.forkSource,
|
|
640
|
+
unresolvedLaunchOptionKeys: a.launch.unresolvedLaunchOptionKeys,
|
|
641
|
+
},
|
|
642
|
+
dependencies,
|
|
643
|
+
spawner: a.spawner,
|
|
644
|
+
authorityParent: a.authorityParent,
|
|
645
|
+
startedAt: new Date(a.startedAt).toISOString(),
|
|
646
|
+
};
|
|
246
647
|
}
|
|
247
648
|
/** Tear down every managed agent's footprint — the shared teardown for EVERY manager-exit path (#159
|
|
248
649
|
* B2): graceful {@link stop} AND the fail-closed lease-loss exit ({@link renewLease}). A manager exit is
|
|
@@ -262,14 +663,44 @@ export class Manager {
|
|
|
262
663
|
this.stopHandle(a, false);
|
|
263
664
|
}
|
|
264
665
|
// Deprovision EVERY snapshot entry regardless of whether its stop failed (allSettled + a loud log).
|
|
265
|
-
await Promise.allSettled(managed.map((a) => this.deprovision(a).catch((e) => console.error(`deprovision ${a.name} (${a.id}) on shutdown: ${e.message}`))));
|
|
666
|
+
await Promise.allSettled(managed.filter((a) => !a.suppressCleanup).map((a) => this.deprovision(a).catch((e) => console.error(`deprovision ${a.name} (${a.id}) on shutdown: ${e.message}`))));
|
|
667
|
+
}
|
|
668
|
+
async stopRetainedAgentsOnExit() {
|
|
669
|
+
const managed = [...this.agents.values()];
|
|
670
|
+
for (const a of managed)
|
|
671
|
+
a.suppressCleanup = true;
|
|
672
|
+
const failures = [];
|
|
673
|
+
await Promise.all(managed.map(async (a) => {
|
|
674
|
+
try {
|
|
675
|
+
a.handle.stop({ graceful: false });
|
|
676
|
+
}
|
|
677
|
+
catch (e) {
|
|
678
|
+
failures.push(`${a.name}: stop failed: ${e.message}`);
|
|
679
|
+
}
|
|
680
|
+
try {
|
|
681
|
+
await this.awaitHandleExit(a.handle);
|
|
682
|
+
if (this.agents.get(a.name) === a)
|
|
683
|
+
this.agents.delete(a.name);
|
|
684
|
+
}
|
|
685
|
+
catch (e) {
|
|
686
|
+
failures.push(`${a.name}: ${e.message}`);
|
|
687
|
+
}
|
|
688
|
+
}));
|
|
689
|
+
if (failures.length)
|
|
690
|
+
throw new Error(`manager preservation shutdown incomplete: ${failures.join("; ")}`);
|
|
266
691
|
}
|
|
267
692
|
async stop() {
|
|
268
693
|
if (this.leaseTimer)
|
|
269
694
|
clearInterval(this.leaseTimer);
|
|
270
695
|
if (this.credRenewTimer)
|
|
271
696
|
clearInterval(this.credRenewTimer);
|
|
272
|
-
|
|
697
|
+
if (this.maintenanceState === "active" && !this.resumeRequired) {
|
|
698
|
+
await this.teardownManagedAgents(); // normal shutdown stays destructive (#159 B2)
|
|
699
|
+
}
|
|
700
|
+
else {
|
|
701
|
+
// A signal after a partial preservation must never fall back into destructive teardown.
|
|
702
|
+
await this.stopRetainedAgentsOnExit();
|
|
703
|
+
}
|
|
273
704
|
await this.ep.releaseManagerLease(this.leaseRevision);
|
|
274
705
|
await this.ep.stop();
|
|
275
706
|
await this.attach.stop();
|
|
@@ -279,9 +710,9 @@ export class Manager {
|
|
|
279
710
|
* with the new holder, and exit. We deliberately do NOT re-acquire (a replacement may already be live
|
|
280
711
|
* while we'd still be serving) and do NOT release the key — it now belongs to that replacement. */
|
|
281
712
|
async renewLease() {
|
|
282
|
-
if (!this.leaseInfo || this.leaseRevision === undefined)
|
|
283
|
-
return;
|
|
284
713
|
try {
|
|
714
|
+
if (!this.leaseInfo || this.leaseRevision === undefined)
|
|
715
|
+
return;
|
|
285
716
|
this.leaseRevision = await this.ep.renewManagerLease(this.leaseInfo, this.leaseRevision);
|
|
286
717
|
}
|
|
287
718
|
catch (e) {
|
|
@@ -291,7 +722,10 @@ export class Manager {
|
|
|
291
722
|
// Tear down our managed agents' footprints too (#159 B2) — this exit path leaks them otherwise. Do
|
|
292
723
|
// NOT release the lease key (it may belong to the replacement holder). Best-effort, like ep/attach.
|
|
293
724
|
try {
|
|
294
|
-
|
|
725
|
+
if (this.maintenanceState === "active" && !this.resumeRequired)
|
|
726
|
+
await this.teardownManagedAgents();
|
|
727
|
+
else
|
|
728
|
+
await this.stopRetainedAgentsOnExit();
|
|
295
729
|
}
|
|
296
730
|
catch { /* best effort */ }
|
|
297
731
|
try {
|
|
@@ -306,6 +740,187 @@ export class Manager {
|
|
|
306
740
|
}
|
|
307
741
|
}
|
|
308
742
|
async handle(req, tier) {
|
|
743
|
+
if (req.op === "finalizeResume") {
|
|
744
|
+
if (tier !== CONTROL_ADMIN)
|
|
745
|
+
return { ok: false, error: "finalizeResume is admin-only; not allowed on this control subject" };
|
|
746
|
+
let args;
|
|
747
|
+
try {
|
|
748
|
+
args = parseResumeFinalizeArgs(req.args);
|
|
749
|
+
}
|
|
750
|
+
catch (e) {
|
|
751
|
+
return { ok: false, error: e.message };
|
|
752
|
+
}
|
|
753
|
+
if (!this.resumeAttemptId || this.resumeAttemptId !== args.attemptId)
|
|
754
|
+
return { ok: false, error: `manager expects resume attempt ${this.resumeAttemptId ?? "<none>"}, not ${args.attemptId}` };
|
|
755
|
+
if (!this.resumeCommitted || !this.resumeDurableCommitToken)
|
|
756
|
+
return { ok: false, error: `resume attempt ${args.attemptId} has no successful commit to finalize` };
|
|
757
|
+
if (this.resumeDurableCommitToken !== args.durableCommitToken)
|
|
758
|
+
return { ok: false, error: `resume attempt ${args.attemptId} durable commit token does not match` };
|
|
759
|
+
if (this.resumeFinalized)
|
|
760
|
+
return { ok: true, data: { attemptId: args.attemptId, state: "active" } };
|
|
761
|
+
const inventory = this.resumeInventory;
|
|
762
|
+
if (!inventory)
|
|
763
|
+
return { ok: false, error: `resume attempt ${args.attemptId} has no bound inventory` };
|
|
764
|
+
let inactive;
|
|
765
|
+
try {
|
|
766
|
+
inactive = this.resumeLivenessErrors(inventory, this.ep.getRoster());
|
|
767
|
+
}
|
|
768
|
+
catch (e) {
|
|
769
|
+
return { ok: false, error: `resume attempt ${args.attemptId} cannot verify live principals at finalize: ${e.message}` };
|
|
770
|
+
}
|
|
771
|
+
if (inactive.length)
|
|
772
|
+
return { ok: false, error: `resume attempt ${args.attemptId} is not live at finalize: ${inactive.join("; ")}` };
|
|
773
|
+
for (const entry of this.resumeInventory?.agents ?? []) {
|
|
774
|
+
const managed = this.agents.get(entry.name);
|
|
775
|
+
if (managed)
|
|
776
|
+
managed.suppressCleanup = false;
|
|
777
|
+
}
|
|
778
|
+
this.resumeFinalized = true;
|
|
779
|
+
this.resumeRequired = false;
|
|
780
|
+
return { ok: true, data: { attemptId: args.attemptId, state: "active" } };
|
|
781
|
+
}
|
|
782
|
+
if (req.op === "commitResume") {
|
|
783
|
+
if (tier !== CONTROL_ADMIN)
|
|
784
|
+
return { ok: false, error: "commitResume is admin-only; not allowed on this control subject" };
|
|
785
|
+
let attemptId;
|
|
786
|
+
try {
|
|
787
|
+
attemptId = parseResumeCommitArgs(req.args).attemptId;
|
|
788
|
+
}
|
|
789
|
+
catch (e) {
|
|
790
|
+
return { ok: false, error: e.message };
|
|
791
|
+
}
|
|
792
|
+
if (!this.resumeAttemptId || this.resumeAttemptId !== attemptId)
|
|
793
|
+
return { ok: false, error: `manager expects resume attempt ${this.resumeAttemptId ?? "<none>"}, not ${attemptId}` };
|
|
794
|
+
if (this.resumeCommitted)
|
|
795
|
+
return {
|
|
796
|
+
ok: true,
|
|
797
|
+
data: {
|
|
798
|
+
attemptId,
|
|
799
|
+
state: this.resumeFinalized ? "active" : "awaitingFinalize",
|
|
800
|
+
durableCommitToken: this.resumeDurableCommitToken,
|
|
801
|
+
},
|
|
802
|
+
};
|
|
803
|
+
if (this.resumeCommitTask)
|
|
804
|
+
return this.resumeCommitTask;
|
|
805
|
+
const task = this.commitResumeActivation(attemptId);
|
|
806
|
+
this.resumeCommitTask = task;
|
|
807
|
+
try {
|
|
808
|
+
return await task;
|
|
809
|
+
}
|
|
810
|
+
finally {
|
|
811
|
+
if (this.resumeCommitTask === task)
|
|
812
|
+
this.resumeCommitTask = undefined;
|
|
813
|
+
}
|
|
814
|
+
}
|
|
815
|
+
if (req.op === "resumePreserved") {
|
|
816
|
+
if (tier !== CONTROL_ADMIN)
|
|
817
|
+
return { ok: false, error: "resumePreserved is admin-only; not allowed on this control subject" };
|
|
818
|
+
try {
|
|
819
|
+
const args = parseResumeControlArgs(req.args);
|
|
820
|
+
const inventoryDigest = createHash("sha256").update(JSON.stringify(args.inventory)).digest("hex");
|
|
821
|
+
if (!this.resumeAttemptId)
|
|
822
|
+
return { ok: false, error: "resumePreserved requires a manager started with --resume-attempt" };
|
|
823
|
+
if (this.resumeAttemptId !== args.attemptId)
|
|
824
|
+
return { ok: false, error: `manager expects resume attempt ${this.resumeAttemptId}, not ${args.attemptId}` };
|
|
825
|
+
if (this.resumeInventoryDigest && this.resumeInventoryDigest !== inventoryDigest)
|
|
826
|
+
return { ok: false, error: `resume attempt ${args.attemptId} is already bound to a different inventory` };
|
|
827
|
+
if (!this.resumeInventoryDigest) {
|
|
828
|
+
this.resumeInventoryDigest = inventoryDigest;
|
|
829
|
+
this.resumeInventory = args.inventory;
|
|
830
|
+
}
|
|
831
|
+
if (!this.resumeTask && !this.resumeResult) {
|
|
832
|
+
this.resumeTask = this.resumePreserved(args.inventory).then((result) => {
|
|
833
|
+
if (result.ok || this.resumedAgentNames.size > 0)
|
|
834
|
+
this.resumeResult = result;
|
|
835
|
+
return result;
|
|
836
|
+
}).finally(() => {
|
|
837
|
+
this.resumeTask = undefined;
|
|
838
|
+
});
|
|
839
|
+
}
|
|
840
|
+
const result = this.resumeResult ?? await this.resumeTask;
|
|
841
|
+
const data = { attemptId: args.attemptId, state: result.ok ? "awaitingCommit" : "degraded", ...result };
|
|
842
|
+
return result.ok
|
|
843
|
+
? { ok: true, data }
|
|
844
|
+
: { ok: false, data, error: result.error ?? "retained-agent resume failed" };
|
|
845
|
+
}
|
|
846
|
+
catch (e) {
|
|
847
|
+
return { ok: false, error: e.message };
|
|
848
|
+
}
|
|
849
|
+
}
|
|
850
|
+
if (req.op === "preparePreservation" || req.op === "commitPreservation" || req.op === "abortPreservation") {
|
|
851
|
+
if (tier !== CONTROL_ADMIN)
|
|
852
|
+
return { ok: false, error: `${req.op} is admin-only; not allowed on this control subject` };
|
|
853
|
+
if (this.resumeRequired)
|
|
854
|
+
return { ok: false, error: this.maintenanceError() };
|
|
855
|
+
const attemptId = String(req.args?.attemptId ?? "").trim();
|
|
856
|
+
if (!attemptId)
|
|
857
|
+
return { ok: false, error: `${req.op} requires attemptId` };
|
|
858
|
+
try {
|
|
859
|
+
if (req.op === "abortPreservation") {
|
|
860
|
+
this.abortPreservation(attemptId);
|
|
861
|
+
return { ok: true, data: { attemptId, state: "active" } };
|
|
862
|
+
}
|
|
863
|
+
const result = req.op === "preparePreservation"
|
|
864
|
+
? await this.preparePreservation(attemptId)
|
|
865
|
+
: await this.commitPreservation(attemptId);
|
|
866
|
+
return result.ok
|
|
867
|
+
? { ok: true, data: result }
|
|
868
|
+
: {
|
|
869
|
+
ok: false,
|
|
870
|
+
data: result,
|
|
871
|
+
error: `preservation incomplete: ${result.failures.map((f) => `${f.name}: ${f.error}`).join("; ")}`,
|
|
872
|
+
};
|
|
873
|
+
}
|
|
874
|
+
catch (e) {
|
|
875
|
+
return { ok: false, error: e.message };
|
|
876
|
+
}
|
|
877
|
+
}
|
|
878
|
+
const release = this.beginLifecycle();
|
|
879
|
+
if (!release)
|
|
880
|
+
return { ok: false, error: this.maintenanceError() };
|
|
881
|
+
try {
|
|
882
|
+
return await this.handleActive(req, tier);
|
|
883
|
+
}
|
|
884
|
+
finally {
|
|
885
|
+
release();
|
|
886
|
+
}
|
|
887
|
+
}
|
|
888
|
+
async commitResumeActivation(attemptId) {
|
|
889
|
+
if (!this.resumeAwaitingCommit || !this.resumeResult?.ok)
|
|
890
|
+
return { ok: false, error: `resume attempt ${attemptId} has no successful activation to commit` };
|
|
891
|
+
const inventory = this.resumeInventory;
|
|
892
|
+
if (!inventory)
|
|
893
|
+
return { ok: false, error: `resume attempt ${attemptId} has no bound inventory` };
|
|
894
|
+
const authority = await Promise.all(inventory.agents.map(async (entry) => {
|
|
895
|
+
try {
|
|
896
|
+
await this.validateRetainedAuthority(entry);
|
|
897
|
+
return undefined;
|
|
898
|
+
}
|
|
899
|
+
catch (e) {
|
|
900
|
+
return `${entry.name}: ${e.message}`;
|
|
901
|
+
}
|
|
902
|
+
}));
|
|
903
|
+
const drift = authority.filter((error) => error !== undefined);
|
|
904
|
+
if (drift.length)
|
|
905
|
+
return { ok: false, error: `resume attempt ${attemptId} retained authority changed before commit: ${drift.join("; ")}` };
|
|
906
|
+
let inactive;
|
|
907
|
+
try {
|
|
908
|
+
inactive = this.resumeLivenessErrors(inventory, this.ep.getRoster());
|
|
909
|
+
}
|
|
910
|
+
catch (e) {
|
|
911
|
+
return { ok: false, error: `resume attempt ${attemptId} cannot verify live principals: ${e.message}` };
|
|
912
|
+
}
|
|
913
|
+
if (inactive.length)
|
|
914
|
+
return { ok: false, error: `resume attempt ${attemptId} is not live at commit: ${inactive.join("; ")}` };
|
|
915
|
+
this.resumeAwaitingCommit = false;
|
|
916
|
+
this.resumeCommitted = true;
|
|
917
|
+
this.resumeDurableCommitToken ??= randomBytes(32).toString("hex");
|
|
918
|
+
return {
|
|
919
|
+
ok: true,
|
|
920
|
+
data: { attemptId, state: "awaitingFinalize", durableCommitToken: this.resumeDurableCommitToken },
|
|
921
|
+
};
|
|
922
|
+
}
|
|
923
|
+
async handleActive(req, tier) {
|
|
309
924
|
const args = req.args ?? {};
|
|
310
925
|
// `req.from.id` is non-forgeable in auth mode: serveControl rejects any request whose payload
|
|
311
926
|
// `from.id` doesn't match the subject sender (endpoint.ts). In open mode there are no creds, so
|
|
@@ -401,6 +1016,47 @@ export class Manager {
|
|
|
401
1016
|
managedPrincipal(a) {
|
|
402
1017
|
return a.userOwner ? a.id : principalKey(DEV_OWNER, a.id).key;
|
|
403
1018
|
}
|
|
1019
|
+
resumeLivenessErrors(inventory, roster) {
|
|
1020
|
+
const inactive = [];
|
|
1021
|
+
const expectedNames = new Set(inventory.agents.map((entry) => entry.name));
|
|
1022
|
+
for (const name of this.resumedAgentNames)
|
|
1023
|
+
if (!expectedNames.has(name))
|
|
1024
|
+
inactive.push(`${name} is not part of the bound inventory`);
|
|
1025
|
+
for (const entry of inventory.agents) {
|
|
1026
|
+
const managed = this.agents.get(entry.name);
|
|
1027
|
+
if (!managed) {
|
|
1028
|
+
inactive.push(`${entry.name} is no longer managed`);
|
|
1029
|
+
continue;
|
|
1030
|
+
}
|
|
1031
|
+
const expectedId = entry.identity.mode === "user"
|
|
1032
|
+
? principalKey(entry.identity.owner, entry.identity.actor).key
|
|
1033
|
+
: entry.identity.id;
|
|
1034
|
+
const expectedPrincipal = entry.identity.mode === "user"
|
|
1035
|
+
? expectedId
|
|
1036
|
+
: principalKey(DEV_OWNER, entry.identity.id).key;
|
|
1037
|
+
if (managed.id !== expectedId || this.managedPrincipal(managed) !== expectedPrincipal) {
|
|
1038
|
+
inactive.push(`${entry.name} no longer holds retained principal ${expectedPrincipal}`);
|
|
1039
|
+
continue;
|
|
1040
|
+
}
|
|
1041
|
+
if (managed.handle.name !== entry.name || managed.handle.kind !== entry.launch.runtime) {
|
|
1042
|
+
inactive.push(`${entry.name} is not attached to its exact retained ${entry.launch.runtime} handle`);
|
|
1043
|
+
continue;
|
|
1044
|
+
}
|
|
1045
|
+
try {
|
|
1046
|
+
if (managed.handle.status() !== "running") {
|
|
1047
|
+
inactive.push(`${entry.name} runtime is not running`);
|
|
1048
|
+
continue;
|
|
1049
|
+
}
|
|
1050
|
+
}
|
|
1051
|
+
catch (e) {
|
|
1052
|
+
inactive.push(`${entry.name} runtime status failed: ${e.message}`);
|
|
1053
|
+
continue;
|
|
1054
|
+
}
|
|
1055
|
+
if (!roster.some((presence) => presence.card.id === expectedPrincipal && presence.card.name === entry.name && presence.status !== "offline"))
|
|
1056
|
+
inactive.push(`${entry.name} principal ${expectedPrincipal} is not exactly present`);
|
|
1057
|
+
}
|
|
1058
|
+
return inactive;
|
|
1059
|
+
}
|
|
404
1060
|
/** Self-despawn (P2b): stop the managed agent whose id == the authenticated caller. The
|
|
405
1061
|
* no-name self-op can only ever resolve to the caller's OWN managed entry (ids are unique
|
|
406
1062
|
* per spawn + non-forgeable in auth mode), never a peer — so it's structurally incapable of
|
|
@@ -412,7 +1068,7 @@ export class Manager {
|
|
|
412
1068
|
return { ok: false, error: `self-stop: caller ${callerId} is not a managed agent` };
|
|
413
1069
|
const graceful = args.graceful !== false;
|
|
414
1070
|
this.stopHandle(target, graceful);
|
|
415
|
-
this.
|
|
1071
|
+
this.trackStoppedHandle(target, true);
|
|
416
1072
|
return { ok: true, data: { name: target.name, stopped: true, graceful } };
|
|
417
1073
|
}
|
|
418
1074
|
// Plane-3 durable join/leave/list ops moved OFF the manager onto the server-side delivery daemon's
|
|
@@ -441,6 +1097,49 @@ export class Manager {
|
|
|
441
1097
|
console.error(`stop ${a.name} (${a.id}): ${e.message}`);
|
|
442
1098
|
}
|
|
443
1099
|
}
|
|
1100
|
+
/** Keep an accepted stop inside the lifecycle drain until the runtime proves the child is gone,
|
|
1101
|
+
* so a maintenance prepare can never fence ahead of a child that is still dying.
|
|
1102
|
+
*
|
|
1103
|
+
* An operator-accepted stop frees its slot at once: `stop` replying ✓ means `ps` no longer lists
|
|
1104
|
+
* the agent. That cannot omit a still-live child from a cut, because runPreparation drains the
|
|
1105
|
+
* lifecycle BEFORE it reads the roster — the exit proof below is what closes the race, not the
|
|
1106
|
+
* slot lingering. A recursive reap (`requireAuthoritativeExit`) instead keeps the slot until the
|
|
1107
|
+
* wait proves exit: nobody asked for those children to be gone, so they stay managed until the
|
|
1108
|
+
* runtime says otherwise, and a runtime that cannot prove exit records an unverified stop. */
|
|
1109
|
+
trackStoppedHandle(a, floor, requireAuthoritativeExit = false) {
|
|
1110
|
+
if (!a.handle.waitForExit) {
|
|
1111
|
+
// Preserve ordinary external-runtime stop behavior, but retain enough evidence for a later
|
|
1112
|
+
// maintenance prepare to fail if that runtime still cannot prove the surface disappeared.
|
|
1113
|
+
this.unverifiedStops.push({
|
|
1114
|
+
name: a.name,
|
|
1115
|
+
id: a.id,
|
|
1116
|
+
handle: a.handle,
|
|
1117
|
+
authoritative: requireAuthoritativeExit,
|
|
1118
|
+
error: requireAuthoritativeExit
|
|
1119
|
+
? `recursive reap cannot prove exit on runtime "${a.handle.kind}" (AgentHandle.waitForExit is not implemented)`
|
|
1120
|
+
: undefined,
|
|
1121
|
+
});
|
|
1122
|
+
if (requireAuthoritativeExit)
|
|
1123
|
+
return;
|
|
1124
|
+
this.freeSlot(a, floor, true);
|
|
1125
|
+
return;
|
|
1126
|
+
}
|
|
1127
|
+
if (!requireAuthoritativeExit)
|
|
1128
|
+
this.freeSlot(a, floor, true);
|
|
1129
|
+
this.lifecycleInFlight++;
|
|
1130
|
+
void this.awaitHandleExit(a.handle)
|
|
1131
|
+
.then(() => this.freeSlot(a, floor, true)) // no-op once an accepted stop already freed it
|
|
1132
|
+
.catch((e) => {
|
|
1133
|
+
this.unverifiedStops.push({
|
|
1134
|
+
name: a.name,
|
|
1135
|
+
id: a.id,
|
|
1136
|
+
handle: a.handle,
|
|
1137
|
+
authoritative: true,
|
|
1138
|
+
error: `accepted stop could not prove exit: ${e.message}`,
|
|
1139
|
+
});
|
|
1140
|
+
})
|
|
1141
|
+
.finally(() => this.releaseLifecycle());
|
|
1142
|
+
}
|
|
444
1143
|
/** USER-MODE spawn provisioning (the gate-1 counterpart to the static mint block): resolve the
|
|
445
1144
|
* OWNER (ctl caller's principal, or the manifest's stamped owner — never a payload field),
|
|
446
1145
|
* pre-create the principal-keyed durables + ACL row on the ephemeral provisioner, author the
|
|
@@ -478,6 +1177,11 @@ export class Manager {
|
|
|
478
1177
|
// the spawner's own grant), so a refused delegation exits here having touched nothing beyond
|
|
479
1178
|
// the ledger: no durables, no broker footprint, nothing for a corrected respawn to race.
|
|
480
1179
|
const grant = await provider.grantAgent({
|
|
1180
|
+
// LOCAL composition, hardcoded: until the manager's entry is store-threaded (the same
|
|
1181
|
+
// later slice as its renewal-owner store, with/after the membership-rw reader migration),
|
|
1182
|
+
// a pure-KMS hosted manager CANNOT read the callout material this grant needs — hosted
|
|
1183
|
+
// user-mode spawn via the manager is UNAVAILABLE, not silently degraded, until then.
|
|
1184
|
+
store: workspaceSecretStore(this.workspaceRoot),
|
|
481
1185
|
dir,
|
|
482
1186
|
space: this.space,
|
|
483
1187
|
owner,
|
|
@@ -537,7 +1241,7 @@ export class Manager {
|
|
|
537
1241
|
* expires — flooring the RECYCLE, not the call, so both free paths (despawn + exit/reap) are
|
|
538
1242
|
* covered (P4c). Floor self + own-child despawn and natural exit; NEVER admin despawn (operator
|
|
539
1243
|
* emergency-kill stays unthrottled) and NEVER the reserved-rollback path (no cold-start paid). */
|
|
540
|
-
freeSlot(a, floor) {
|
|
1244
|
+
freeSlot(a, floor, acceptedBeforeFence = false) {
|
|
541
1245
|
if (this.agents.get(a.name) !== a)
|
|
542
1246
|
return; // already freed (exit raced despawn, etc.)
|
|
543
1247
|
this.agents.delete(a.name);
|
|
@@ -547,7 +1251,8 @@ export class Manager {
|
|
|
547
1251
|
// process is already gone, so this must never block the slot free or throw into the caller — it runs
|
|
548
1252
|
// detached, and a failure is logged loudly (never swallowed), not retried. The `agents` guard above
|
|
549
1253
|
// makes this fire exactly once per agent across every free path (despawn / self-stop / reap / exit).
|
|
550
|
-
|
|
1254
|
+
if (!a.suppressCleanup && (this.maintenanceState === "active" || acceptedBeforeFence))
|
|
1255
|
+
this.trackDeprovision(a);
|
|
551
1256
|
}
|
|
552
1257
|
/** Tear down a departed agent's minted footprint (#159 B2, auth mode): its local-principal durables
|
|
553
1258
|
* (`dm_local-<id>`, `dlv_local-<id>`), its read-ACL row, and its creds file — everything the spawn's
|
|
@@ -593,24 +1298,29 @@ export class Manager {
|
|
|
593
1298
|
// helper's own fail-fast connect). The durables/ACL row still fall to space teardown as a backstop.
|
|
594
1299
|
await withTimeout(deprovisionAgent({ servers: this.servers ?? DEFAULT_SERVER, space: this.space, targetId: a.id, creds }), DEPROVISION_TIMEOUT_MS, `deprovision ${a.name} (${a.id}): broker teardown timed out`);
|
|
595
1300
|
}
|
|
596
|
-
/** Reap a parent's children on its exit (P4b)
|
|
597
|
-
*
|
|
598
|
-
*
|
|
1301
|
+
/** Reap a parent's children on its exit (P4b). Every descendant remains managed until the runtime's
|
|
1302
|
+
* authoritative wait proves exit; the wait participates in the lifecycle drain, so preservation can
|
|
1303
|
+
* never omit a child that may still be alive. Recursive descendants are scheduled before their parent
|
|
1304
|
+
* slot can disappear. */
|
|
599
1305
|
reapChildrenOf(parentId) {
|
|
600
1306
|
for (const child of [...this.agents.values()]) {
|
|
601
1307
|
if (child.spawner !== parentId)
|
|
602
1308
|
continue;
|
|
1309
|
+
this.reapChildrenOf(this.managedPrincipal(child));
|
|
603
1310
|
this.stopHandle(child, false);
|
|
604
|
-
this.
|
|
605
|
-
this.reapChildrenOf(child.id);
|
|
1311
|
+
this.trackStoppedHandle(child, true, true);
|
|
606
1312
|
}
|
|
607
1313
|
}
|
|
608
1314
|
/** A managed agent's process exited on its own (crash, /exit, finished). Free its slot
|
|
609
1315
|
* (rate-floored — exit-driven churn counts) and reap any children it spawned. Idempotent via
|
|
610
1316
|
* freeSlot's identity guard, so a later graceful-stop SIGKILL firing exit again is a no-op. */
|
|
611
1317
|
onAgentExit(a) {
|
|
1318
|
+
// Preservation owns the child-stop snapshot. Exit watchers must neither delete that snapshot nor
|
|
1319
|
+
// trigger normal deprovision/reap while the cut is being formed.
|
|
1320
|
+
if (this.maintenanceState !== "active")
|
|
1321
|
+
return;
|
|
612
1322
|
this.freeSlot(a, true);
|
|
613
|
-
this.reapChildrenOf(a
|
|
1323
|
+
this.reapChildrenOf(this.managedPrincipal(a));
|
|
614
1324
|
}
|
|
615
1325
|
/** Agent names become `.cotal/agents/<name>.md` paths and mesh identities, so they must be bare
|
|
616
1326
|
* tokens, never a path — blocks traversal / arbitrary writes from a model-supplied name. */
|
|
@@ -835,7 +1545,7 @@ export class Manager {
|
|
|
835
1545
|
catch (e) {
|
|
836
1546
|
return { ok: false, error: e.message };
|
|
837
1547
|
}
|
|
838
|
-
const reply = await this.startAgent(launchAgentToStartOpts(la, configPath, spec.owner), caller);
|
|
1548
|
+
const reply = await this.startAgent(launchAgentToStartOpts(la, configPath, spec.owner, runId), caller);
|
|
839
1549
|
if (reply.ok)
|
|
840
1550
|
// `data.name` stays the spawned (numbered) identity — what creds are filed under and the ledger
|
|
841
1551
|
// keys on; `requested`/`runId`/`hash` give the CLI the manifest name + drift hash for the ledger.
|
|
@@ -849,6 +1559,17 @@ export class Manager {
|
|
|
849
1559
|
* defaulting to the manager's own id for roster/pre-spawn — recorded for the spawner
|
|
850
1560
|
* ledger (own-children despawn + reap-on-parent-exit). */
|
|
851
1561
|
async startAgent(opts, spawner) {
|
|
1562
|
+
const release = this.beginLifecycle();
|
|
1563
|
+
if (!release)
|
|
1564
|
+
return { ok: false, error: this.maintenanceError() };
|
|
1565
|
+
try {
|
|
1566
|
+
return await this.startAgentActive(opts, spawner);
|
|
1567
|
+
}
|
|
1568
|
+
finally {
|
|
1569
|
+
release();
|
|
1570
|
+
}
|
|
1571
|
+
}
|
|
1572
|
+
async startAgentActive(opts, spawner) {
|
|
852
1573
|
// The spawn argument is a persona REF — a filename in `.cotal/agents` (the unique spawn KEY), or
|
|
853
1574
|
// a path via `--config`. It is NOT the mesh identity: the identity comes from inside the file
|
|
854
1575
|
// (`name:`), so a persona can be filed descriptively (review-critic.md) yet present under a
|
|
@@ -1050,6 +1771,11 @@ export class Manager {
|
|
|
1050
1771
|
// arbitrary folders/repos. A relative path resolves against the workspace root; omitted → the
|
|
1051
1772
|
// agent shares the workspace root (the prior, unchanged behavior).
|
|
1052
1773
|
const cwd = opts.cwd ? resolve(this.workspaceRoot, opts.cwd) : this.workspaceRoot;
|
|
1774
|
+
const configSha256 = this.fileDigest(configPath);
|
|
1775
|
+
const manifestPath = opts.launchRef
|
|
1776
|
+
? join(this.workspaceRoot, ".cotal", "run", `${opts.launchRef.runId}.json`)
|
|
1777
|
+
: undefined;
|
|
1778
|
+
const manifestSha256 = manifestPath ? this.fileDigest(manifestPath) : undefined;
|
|
1053
1779
|
const spec = connector.buildLaunch({
|
|
1054
1780
|
space: this.space,
|
|
1055
1781
|
name,
|
|
@@ -1092,9 +1818,38 @@ export class Manager {
|
|
|
1092
1818
|
id: userLaunch ? principalKey(userLaunch.owner, name).key : identity.id,
|
|
1093
1819
|
...(userLaunch ? { userOwner } : { seed: identity.seed }),
|
|
1094
1820
|
spawner: spawner ?? this.ep.ref().id,
|
|
1821
|
+
authorityParent: userLaunch && spawner && parsePrincipalKey(spawner) ? spawner : undefined,
|
|
1095
1822
|
startedAt: Date.now(),
|
|
1096
1823
|
handle,
|
|
1097
1824
|
control: spec.control,
|
|
1825
|
+
launch: {
|
|
1826
|
+
source: opts.resolved
|
|
1827
|
+
? {
|
|
1828
|
+
kind: "manifest",
|
|
1829
|
+
runId: opts.launchRef?.runId,
|
|
1830
|
+
requested: opts.launchRef?.requested ?? opts.resolved.name,
|
|
1831
|
+
hash: opts.launchRef?.hash ?? opts.resolved.hash,
|
|
1832
|
+
configPath,
|
|
1833
|
+
configSha256,
|
|
1834
|
+
manifestSha256,
|
|
1835
|
+
}
|
|
1836
|
+
: { kind: "persona", ref, configPath, configSha256 },
|
|
1837
|
+
cwd,
|
|
1838
|
+
model,
|
|
1839
|
+
variant,
|
|
1840
|
+
subscribe,
|
|
1841
|
+
allowSubscribe,
|
|
1842
|
+
allowPublish,
|
|
1843
|
+
capabilities,
|
|
1844
|
+
transcript,
|
|
1845
|
+
shareTools: opts.shareTools,
|
|
1846
|
+
forkSource: opts.resume,
|
|
1847
|
+
// Opaque values may contain secrets. Preserve only their keys and require the referenced
|
|
1848
|
+
// persona/manifest to resolve the values again; imperative overrides have no safe payload.
|
|
1849
|
+
unresolvedLaunchOptionKeys: opts.launchOptions && Object.keys(opts.launchOptions).length
|
|
1850
|
+
? Object.keys(opts.launchOptions).sort()
|
|
1851
|
+
: undefined,
|
|
1852
|
+
},
|
|
1098
1853
|
};
|
|
1099
1854
|
this.agents.set(name, managed);
|
|
1100
1855
|
// The live slot now owns teardown — freeSlot deprovisions this identity on exit — so the
|
|
@@ -1128,9 +1883,370 @@ export class Manager {
|
|
|
1128
1883
|
// orphan down (detached, fail-loud) so a failed spawn leaves no creds/durables behind (#159 B).
|
|
1129
1884
|
if (provisioned) {
|
|
1130
1885
|
const orphan = provisioned;
|
|
1131
|
-
|
|
1886
|
+
this.trackDeprovision(orphan, "(orphaned spawn)");
|
|
1887
|
+
}
|
|
1888
|
+
}
|
|
1889
|
+
}
|
|
1890
|
+
/** Preflight the whole inventory before launching its first process, then adopt each exact retained
|
|
1891
|
+
* principal without provisioning. A later runtime launch failure is reported per-agent, but malformed
|
|
1892
|
+
* or missing inventory material can never produce a partially resumed set. */
|
|
1893
|
+
async resumePreserved(inventory) {
|
|
1894
|
+
const release = this.beginLifecycle(true);
|
|
1895
|
+
if (!release)
|
|
1896
|
+
return { ok: false, agents: [], error: this.maintenanceError() };
|
|
1897
|
+
const batchReservations = [];
|
|
1898
|
+
try {
|
|
1899
|
+
if (inventory.version !== "cotal-manager-resume/v1")
|
|
1900
|
+
return { ok: false, agents: [], error: `unsupported manager resume inventory version ${String(inventory.version)}` };
|
|
1901
|
+
if (inventory.space !== this.space)
|
|
1902
|
+
return { ok: false, agents: [], error: `resume inventory belongs to space "${inventory.space}", not "${this.space}"` };
|
|
1903
|
+
const seen = new Set();
|
|
1904
|
+
const principals = new Set();
|
|
1905
|
+
await this.ep.waitForPresenceSnapshot();
|
|
1906
|
+
const livePrincipals = new Set(this.ep.getRoster()
|
|
1907
|
+
.filter((presence) => presence.status !== "offline")
|
|
1908
|
+
.map((presence) => presence.card.id));
|
|
1909
|
+
if (this.agents.size + this.reserved.size + this.coolingCount() + inventory.agents.length > MAX_AGENTS)
|
|
1910
|
+
return { ok: false, agents: [], error: `resume inventory would exceed manager capacity (${MAX_AGENTS})` };
|
|
1911
|
+
for (const entry of inventory.agents) {
|
|
1912
|
+
if (seen.has(entry.name))
|
|
1913
|
+
return { ok: false, agents: [], error: `resume inventory contains duplicate agent name "${entry.name}"` };
|
|
1914
|
+
seen.add(entry.name);
|
|
1915
|
+
let principal;
|
|
1916
|
+
try {
|
|
1917
|
+
principal = entry.identity.mode === "user"
|
|
1918
|
+
? principalKey(entry.identity.owner, entry.identity.actor).key
|
|
1919
|
+
: principalKey(DEV_OWNER, entry.identity.id).key;
|
|
1920
|
+
}
|
|
1921
|
+
catch (e) {
|
|
1922
|
+
return { ok: false, agents: [], error: `invalid retained principal for ${entry.name}: ${e.message}` };
|
|
1923
|
+
}
|
|
1924
|
+
if (principals.has(principal))
|
|
1925
|
+
return { ok: false, agents: [], error: `resume inventory contains duplicate principal "${principal}"` };
|
|
1926
|
+
principals.add(principal);
|
|
1927
|
+
if (livePrincipals.has(principal))
|
|
1928
|
+
return { ok: false, agents: [], error: `retained principal "${principal}" is already live and this runtime cannot authoritatively adopt it` };
|
|
1929
|
+
if (this.agents.has(entry.name) || this.reserved.has(entry.name))
|
|
1930
|
+
return { ok: false, agents: [], error: `retained agent "${entry.name}" is already managed or reserved` };
|
|
1931
|
+
}
|
|
1932
|
+
for (const entry of inventory.agents) {
|
|
1933
|
+
this.reserved.add(entry.name);
|
|
1934
|
+
batchReservations.push(entry.name);
|
|
1935
|
+
}
|
|
1936
|
+
const prepared = new Map();
|
|
1937
|
+
const preflight = [];
|
|
1938
|
+
for (const entry of inventory.agents) {
|
|
1939
|
+
const reply = await this.resumePreservedAgent(entry, true, true, prepared);
|
|
1940
|
+
preflight.push({ name: entry.name, reply });
|
|
1941
|
+
}
|
|
1942
|
+
const preflightFailures = preflight.filter(({ reply }) => !reply.ok);
|
|
1943
|
+
if (preflightFailures.length)
|
|
1944
|
+
return {
|
|
1945
|
+
ok: false,
|
|
1946
|
+
agents: preflight,
|
|
1947
|
+
error: `${preflightFailures.length} retained agent${preflightFailures.length === 1 ? "" : "s"} failed preflight`,
|
|
1948
|
+
};
|
|
1949
|
+
const agents = [];
|
|
1950
|
+
for (let i = 0; i < inventory.agents.length; i++) {
|
|
1951
|
+
const entry = inventory.agents[i];
|
|
1952
|
+
const reply = await this.resumePreservedAgent(entry, false, true, prepared);
|
|
1953
|
+
agents.push({ name: entry.name, reply });
|
|
1954
|
+
if (!reply.ok) {
|
|
1955
|
+
for (const skipped of inventory.agents.slice(i + 1))
|
|
1956
|
+
agents.push({ name: skipped.name, reply: { ok: false, error: `not launched because ${entry.name} failed` } });
|
|
1957
|
+
return { ok: false, agents, error: reply.error };
|
|
1958
|
+
}
|
|
1959
|
+
}
|
|
1960
|
+
if (this.resumeAttemptId)
|
|
1961
|
+
this.resumeAwaitingCommit = true;
|
|
1962
|
+
return { ok: true, agents };
|
|
1963
|
+
}
|
|
1964
|
+
finally {
|
|
1965
|
+
for (const name of batchReservations)
|
|
1966
|
+
this.reserved.delete(name);
|
|
1967
|
+
release();
|
|
1968
|
+
}
|
|
1969
|
+
}
|
|
1970
|
+
/** Re-read every retained identity input and its current authority without provisioning. This runs
|
|
1971
|
+
* during whole-inventory preflight, immediately before each individual spawn, and at commit. */
|
|
1972
|
+
async validateRetainedAuthority(entry) {
|
|
1973
|
+
const referenceError = this.inventoryReferenceError(entry);
|
|
1974
|
+
if (referenceError)
|
|
1975
|
+
throw new Error(`retained agent ${entry.name}: ${referenceError}`);
|
|
1976
|
+
if (entry.identity.mode === "open") {
|
|
1977
|
+
if (this.auth || this.userMode)
|
|
1978
|
+
throw new Error(`retained agent ${entry.name} is open-mode but the current manager is authenticated`);
|
|
1979
|
+
return { id: entry.identity.id };
|
|
1980
|
+
}
|
|
1981
|
+
if (entry.identity.mode === "static") {
|
|
1982
|
+
if (!this.auth || this.userMode)
|
|
1983
|
+
throw new Error(`retained agent ${entry.name} is static-auth but the current manager is not`);
|
|
1984
|
+
const expected = resolve(authDir(this.workspaceRoot), "creds", `${entry.name}.creds`);
|
|
1985
|
+
if (resolve(entry.identity.credential.path) !== expected)
|
|
1986
|
+
throw new Error(`retained credential reference for ${entry.name} is not the manager-owned path ${expected}`);
|
|
1987
|
+
let credentialText;
|
|
1988
|
+
try {
|
|
1989
|
+
const st = lstatSync(expected);
|
|
1990
|
+
if (!st.isFile() || st.isSymbolicLink())
|
|
1991
|
+
throw new Error("not a regular non-symlink file");
|
|
1992
|
+
credentialText = readFileSync(expected, "utf8");
|
|
1993
|
+
const actual = idFromCreds(credentialText);
|
|
1994
|
+
if (actual !== entry.identity.id)
|
|
1995
|
+
throw new Error(`retained credential identity ${actual} does not match inventory principal ${entry.identity.id}`);
|
|
1996
|
+
}
|
|
1997
|
+
catch (e) {
|
|
1998
|
+
throw new Error(`retained credential for ${entry.name} is unusable: ${e.message}`);
|
|
1999
|
+
}
|
|
2000
|
+
const accepted = await this.probeStaticCredential(credentialText);
|
|
2001
|
+
if (!accepted.ok)
|
|
2002
|
+
throw new Error(`retained credential for ${entry.name} is not accepted by the current broker (${accepted.reason})`);
|
|
2003
|
+
return { id: entry.identity.id, creds: expected };
|
|
2004
|
+
}
|
|
2005
|
+
if (!this.userMode)
|
|
2006
|
+
throw new Error(`retained agent ${entry.name} is user-auth but the current manager is not`);
|
|
2007
|
+
try {
|
|
2008
|
+
const provider = resolveAuthProvider();
|
|
2009
|
+
const actorToken = readFileSync(entry.identity.actorToken.path, "utf8");
|
|
2010
|
+
const sentinelCreds = readFileSync(entry.identity.sentinelCredential.path, "utf8");
|
|
2011
|
+
const adopted = await provider.validateRetainedAgent({
|
|
2012
|
+
store: workspaceSecretStore(this.workspaceRoot),
|
|
2013
|
+
dir: userAuthStateDir(this.workspaceRoot, this.space),
|
|
2014
|
+
space: this.space,
|
|
2015
|
+
owner: entry.identity.owner,
|
|
2016
|
+
actor: entry.identity.actor,
|
|
2017
|
+
actorToken,
|
|
2018
|
+
sentinelCreds,
|
|
2019
|
+
});
|
|
2020
|
+
if (adopted.owner !== entry.identity.owner || adopted.actor !== entry.identity.actor)
|
|
2021
|
+
throw new Error(`auth provider returned a replacement principal; expected ${entry.identity.owner}.${entry.identity.actor}`);
|
|
2022
|
+
if (!sameStrings(adopted.allowSubscribe, entry.launch.allowSubscribe) ||
|
|
2023
|
+
!sameStrings(adopted.allowPublish, entry.launch.allowPublish) ||
|
|
2024
|
+
!sameStrings(adopted.scope, entry.launch.capabilities) ||
|
|
2025
|
+
adopted.role !== entry.role || adopted.parent !== entry.authorityParent)
|
|
2026
|
+
throw new Error(`retained user authority for ${entry.identity.owner}.${entry.identity.actor} no longer matches the inventory`);
|
|
2027
|
+
return {
|
|
2028
|
+
userAuth: {
|
|
2029
|
+
owner: entry.identity.owner,
|
|
2030
|
+
actor: entry.identity.actor,
|
|
2031
|
+
sentinelCredsPath: entry.identity.sentinelCredential.path,
|
|
2032
|
+
bearerCmd: [
|
|
2033
|
+
process.execPath,
|
|
2034
|
+
...process.execArgv,
|
|
2035
|
+
process.argv[1],
|
|
2036
|
+
provider.agentBearerCommand,
|
|
2037
|
+
"--dir", userAuthStateDir(this.workspaceRoot, this.space),
|
|
2038
|
+
"--space", this.space,
|
|
2039
|
+
"--owner", entry.identity.owner,
|
|
2040
|
+
"--actor", entry.identity.actor,
|
|
2041
|
+
"--token-file", entry.identity.actorToken.path,
|
|
2042
|
+
"--health-file", entry.identity.health.path,
|
|
2043
|
+
],
|
|
2044
|
+
},
|
|
2045
|
+
};
|
|
2046
|
+
}
|
|
2047
|
+
catch (e) {
|
|
2048
|
+
throw new Error(`retained user principal ${entry.identity.owner}.${entry.identity.actor} could not be reused: ${e.message}`);
|
|
2049
|
+
}
|
|
2050
|
+
}
|
|
2051
|
+
/** Validate/relaunch one retained inventory entry. Called only through resumePreserved so all
|
|
2052
|
+
* records pass the same preflight before the first child is exposed. */
|
|
2053
|
+
async resumePreservedAgent(entry, preflightOnly = false, batchReserved = false, prepared) {
|
|
2054
|
+
const release = this.beginLifecycle(batchReserved);
|
|
2055
|
+
if (!release)
|
|
2056
|
+
return { ok: false, error: this.maintenanceError() };
|
|
2057
|
+
try {
|
|
2058
|
+
if (entry.space !== this.space)
|
|
2059
|
+
return { ok: false, error: `retained agent ${entry.name} belongs to space "${entry.space}", not "${this.space}"` };
|
|
2060
|
+
if (entry.launch.runtime !== this.runtime.kind)
|
|
2061
|
+
return { ok: false, error: `retained agent ${entry.name} requires runtime "${entry.launch.runtime}", current manager uses "${this.runtime.kind}"` };
|
|
2062
|
+
const nameErr = this.nameError(entry.name);
|
|
2063
|
+
if (nameErr)
|
|
2064
|
+
return { ok: false, error: nameErr };
|
|
2065
|
+
if (this.agents.has(entry.name) || (!batchReserved && this.reserved.has(entry.name)))
|
|
2066
|
+
return { ok: false, error: `retained agent "${entry.name}" is already managed or reserved; same-principal resume never auto-numbers` };
|
|
2067
|
+
if (!batchReserved && this.agents.size + this.reserved.size + this.coolingCount() >= MAX_AGENTS)
|
|
2068
|
+
return { ok: false, error: `at capacity (${MAX_AGENTS} agents incl. in-flight + cooling); same-principal resume refused` };
|
|
2069
|
+
const cached = prepared?.get(entry.name);
|
|
2070
|
+
if (!preflightOnly && cached) {
|
|
2071
|
+
try {
|
|
2072
|
+
// Do not trust the earlier batch preflight across another agent's sequential readiness wait.
|
|
2073
|
+
await this.validateRetainedAuthority(entry);
|
|
2074
|
+
}
|
|
2075
|
+
catch (e) {
|
|
2076
|
+
return { ok: false, error: e.message };
|
|
2077
|
+
}
|
|
2078
|
+
return this.launchPreparedResume(entry, cached, batchReserved);
|
|
2079
|
+
}
|
|
2080
|
+
try {
|
|
2081
|
+
const cwd = lstatSync(entry.launch.cwd);
|
|
2082
|
+
if (!cwd.isDirectory() || cwd.isSymbolicLink())
|
|
2083
|
+
return { ok: false, error: `retained cwd is not a real directory: ${entry.launch.cwd}` };
|
|
2084
|
+
}
|
|
2085
|
+
catch (e) {
|
|
2086
|
+
return { ok: false, error: `retained cwd unavailable: ${entry.launch.cwd} (${e.message})` };
|
|
2087
|
+
}
|
|
2088
|
+
let connector;
|
|
2089
|
+
try {
|
|
2090
|
+
connector = registry.resolve("connector", entry.launch.connector);
|
|
2091
|
+
}
|
|
2092
|
+
catch (e) {
|
|
2093
|
+
return { ok: false, error: e.message };
|
|
2094
|
+
}
|
|
2095
|
+
const missing = (connector.requires ?? []).filter((bin) => !resolveOnPath(bin));
|
|
2096
|
+
if (missing.length)
|
|
2097
|
+
return { ok: false, error: `${connector.name} harness needs ${missing.join(", ")} on PATH - not found` };
|
|
2098
|
+
if (entry.launch.variant && !connector.supportsModelVariant)
|
|
2099
|
+
return { ok: false, error: `${connector.name} connector does not support model variants (variant)` };
|
|
2100
|
+
let launchOptions;
|
|
2101
|
+
if (entry.launch.source.kind === "manifest") {
|
|
2102
|
+
const launchSource = entry.launch.source;
|
|
2103
|
+
if (!launchSource.runId)
|
|
2104
|
+
return { ok: false, error: `retained manifest launch for ${entry.name} has no runId; refusing to guess a .cotal/run source` };
|
|
2105
|
+
let spec;
|
|
2106
|
+
try {
|
|
2107
|
+
const source = launchSpecForRun(this.workspaceRoot, launchSource.runId);
|
|
2108
|
+
if (source.space !== this.space)
|
|
2109
|
+
return { ok: false, error: `retained launch spec space "${source.space}" does not match manager space "${this.space}"` };
|
|
2110
|
+
spec = source.agents.find((a) => a.name === launchSource.requested);
|
|
2111
|
+
}
|
|
2112
|
+
catch (e) {
|
|
2113
|
+
return { ok: false, error: e.message };
|
|
2114
|
+
}
|
|
2115
|
+
if (!spec || spec.hash !== launchSource.hash)
|
|
2116
|
+
return { ok: false, error: `retained manifest agent ${launchSource.requested} is missing or its hash changed; refusing same-principal resume` };
|
|
2117
|
+
launchOptions = spec.launchOptions;
|
|
2118
|
+
}
|
|
2119
|
+
else {
|
|
2120
|
+
try {
|
|
2121
|
+
launchOptions = loadAgentFile(entry.launch.source.configPath).launchOptions;
|
|
2122
|
+
}
|
|
2123
|
+
catch (e) {
|
|
2124
|
+
return { ok: false, error: e.message };
|
|
2125
|
+
}
|
|
2126
|
+
}
|
|
2127
|
+
let authority;
|
|
2128
|
+
try {
|
|
2129
|
+
authority = await this.validateRetainedAuthority(entry);
|
|
2130
|
+
}
|
|
2131
|
+
catch (e) {
|
|
2132
|
+
return { ok: false, error: e.message };
|
|
2133
|
+
}
|
|
2134
|
+
try {
|
|
2135
|
+
const mcpServers = connectorServers(loadCotalConfig(this.workspaceRoot), entry.launch.connector, parseShareSelection(entry.launch.shareTools));
|
|
2136
|
+
const spec = connector.buildLaunch({
|
|
2137
|
+
space: this.space,
|
|
2138
|
+
name: entry.name,
|
|
2139
|
+
role: entry.role,
|
|
2140
|
+
id: authority.id,
|
|
2141
|
+
creds: authority.creds,
|
|
2142
|
+
userAuth: authority.userAuth,
|
|
2143
|
+
servers: this.servers,
|
|
2144
|
+
configPath: entry.launch.source.configPath,
|
|
2145
|
+
model: entry.launch.model,
|
|
2146
|
+
variant: entry.launch.variant,
|
|
2147
|
+
launchOptions,
|
|
2148
|
+
resume: entry.launch.forkSource,
|
|
2149
|
+
subscribe: entry.launch.subscribe,
|
|
2150
|
+
allowSubscribe: entry.launch.allowSubscribe,
|
|
2151
|
+
allowPublish: entry.launch.allowPublish,
|
|
2152
|
+
capabilities: entry.launch.capabilities,
|
|
2153
|
+
transcript: entry.launch.transcript,
|
|
2154
|
+
mcpServers,
|
|
2155
|
+
workspaceRoot: this.workspaceRoot,
|
|
2156
|
+
});
|
|
2157
|
+
const value = { spec, ...authority };
|
|
2158
|
+
prepared?.set(entry.name, value);
|
|
2159
|
+
if (preflightOnly)
|
|
2160
|
+
return { ok: true, data: { name: entry.name, preflight: true } };
|
|
2161
|
+
return this.launchPreparedResume(entry, value, batchReserved);
|
|
2162
|
+
}
|
|
2163
|
+
catch (e) {
|
|
2164
|
+
return { ok: false, error: e.message };
|
|
1132
2165
|
}
|
|
1133
2166
|
}
|
|
2167
|
+
finally {
|
|
2168
|
+
release();
|
|
2169
|
+
}
|
|
2170
|
+
}
|
|
2171
|
+
async launchPreparedResume(entry, prepared, batchReserved) {
|
|
2172
|
+
if (!batchReserved)
|
|
2173
|
+
this.reserved.add(entry.name);
|
|
2174
|
+
try {
|
|
2175
|
+
const handle = this.runtime.spawn(entry.name, prepared.spec, entry.launch.cwd);
|
|
2176
|
+
const managed = {
|
|
2177
|
+
name: entry.name,
|
|
2178
|
+
role: entry.role,
|
|
2179
|
+
agent: entry.launch.connector,
|
|
2180
|
+
id: entry.identity.mode === "user" ? principalKey(entry.identity.owner, entry.identity.actor).key : entry.identity.id,
|
|
2181
|
+
userOwner: entry.identity.mode === "user" ? entry.identity.owner : undefined,
|
|
2182
|
+
spawner: entry.spawner,
|
|
2183
|
+
authorityParent: entry.authorityParent,
|
|
2184
|
+
startedAt: Date.now(),
|
|
2185
|
+
handle,
|
|
2186
|
+
control: prepared.spec.control,
|
|
2187
|
+
launch: {
|
|
2188
|
+
source: entry.launch.source,
|
|
2189
|
+
cwd: entry.launch.cwd,
|
|
2190
|
+
model: entry.launch.model,
|
|
2191
|
+
variant: entry.launch.variant,
|
|
2192
|
+
subscribe: entry.launch.subscribe,
|
|
2193
|
+
allowSubscribe: entry.launch.allowSubscribe,
|
|
2194
|
+
allowPublish: entry.launch.allowPublish,
|
|
2195
|
+
capabilities: entry.launch.capabilities,
|
|
2196
|
+
transcript: entry.launch.transcript,
|
|
2197
|
+
shareTools: entry.launch.shareTools,
|
|
2198
|
+
forkSource: entry.launch.forkSource,
|
|
2199
|
+
},
|
|
2200
|
+
suppressCleanup: true,
|
|
2201
|
+
};
|
|
2202
|
+
this.agents.set(entry.name, managed);
|
|
2203
|
+
if (this.resumeAttemptId)
|
|
2204
|
+
this.resumedAgentNames.add(entry.name);
|
|
2205
|
+
const readiness = await this.awaitReadiness(managed);
|
|
2206
|
+
if (!readiness.ok && !readiness.uncertain)
|
|
2207
|
+
return { ok: false, error: readiness.detail };
|
|
2208
|
+
if (!readiness.ok) {
|
|
2209
|
+
this.watchExit(managed);
|
|
2210
|
+
this.watchResumeAdoption(managed);
|
|
2211
|
+
return { ok: false, error: readiness.detail };
|
|
2212
|
+
}
|
|
2213
|
+
if (!this.resumeAttemptId)
|
|
2214
|
+
managed.suppressCleanup = false;
|
|
2215
|
+
this.watchExit(managed);
|
|
2216
|
+
if (this.agents.get(managed.name) !== managed)
|
|
2217
|
+
return { ok: false, error: `${managed.name} exited immediately after same-principal readiness` };
|
|
2218
|
+
return {
|
|
2219
|
+
ok: true,
|
|
2220
|
+
data: { name: managed.name, role: managed.role, agent: managed.agent, id: managed.id, mode: handle.kind, resumed: true },
|
|
2221
|
+
};
|
|
2222
|
+
}
|
|
2223
|
+
catch (e) {
|
|
2224
|
+
return { ok: false, error: e.message };
|
|
2225
|
+
}
|
|
2226
|
+
finally {
|
|
2227
|
+
if (!batchReserved)
|
|
2228
|
+
this.reserved.delete(entry.name);
|
|
2229
|
+
}
|
|
2230
|
+
}
|
|
2231
|
+
probeStaticCredential(creds) {
|
|
2232
|
+
return probeConnect(this.servers ?? DEFAULT_SERVER, { creds, timeoutMs: 5_000 });
|
|
2233
|
+
}
|
|
2234
|
+
/** An uncertain resume remains non-destructive until exact-principal presence arrives later. */
|
|
2235
|
+
watchResumeAdoption(a) {
|
|
2236
|
+
const wanted = this.managedPrincipal(a);
|
|
2237
|
+
const onPresence = () => {
|
|
2238
|
+
if (this.agents.get(a.name) !== a) {
|
|
2239
|
+
this.ep.off("presence", onPresence);
|
|
2240
|
+
return;
|
|
2241
|
+
}
|
|
2242
|
+
if (!this.ep.getRoster().some((p) => p.card.id === wanted && p.status !== "offline"))
|
|
2243
|
+
return;
|
|
2244
|
+
if (!this.resumeRequired)
|
|
2245
|
+
a.suppressCleanup = false;
|
|
2246
|
+
this.ep.off("presence", onPresence);
|
|
2247
|
+
};
|
|
2248
|
+
this.ep.on("presence", onPresence);
|
|
2249
|
+
onPresence();
|
|
1134
2250
|
}
|
|
1135
2251
|
/** #159 B1: wait for a detached launch to reach a REAL outcome before replying — never a liveness-
|
|
1136
2252
|
* inferring timer. Races three:
|
|
@@ -1255,7 +2371,7 @@ export class Manager {
|
|
|
1255
2371
|
return { ok: false, error: denied };
|
|
1256
2372
|
const graceful = args.graceful !== false;
|
|
1257
2373
|
this.stopHandle(a, graceful);
|
|
1258
|
-
this.
|
|
2374
|
+
this.trackStoppedHandle(a, !admin);
|
|
1259
2375
|
return { ok: true, data: { name, stopped: true, graceful } };
|
|
1260
2376
|
}
|
|
1261
2377
|
/** Open a short-lived PROVISIONER connection, run the onboarding ops on it, and drain it (closure (ii),
|