@runuai/host 0.9.169 → 0.9.171
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/agent-cli.ts +56 -0
- package/lib/agent-errors.ts +22 -0
- package/lib/agents/codex.ts +222 -41
- package/lib/agents/machine-durable.ts +8 -2
- package/lib/agents/types.ts +16 -2
- package/lib/credentials-activation-authority.ts +158 -10
- package/lib/legacy-runtime-retirement.ts +43 -8
- package/lib/orchestrator.ts +72 -33
- package/package.json +1 -1
- package/src/main.ts +12 -0
- package/src/protocol.ts +93 -0
|
@@ -65,6 +65,14 @@
|
|
|
65
65
|
* transition the authority does. Should saying so fail, the
|
|
66
66
|
* runner is pending again, withdraws what it advertised, and says so later.
|
|
67
67
|
*
|
|
68
|
+
* What stops the switch is said to the cloud as well as logged (`status`,
|
|
69
|
+
* on every capability frame; `onStatusChange` when it changes): the latest
|
|
70
|
+
* refusal or blocker, who gave it (the cloud's answer, or this runner), each
|
|
71
|
+
* blocker apart and every Compose project the retirement could not prove
|
|
72
|
+
* Uai's. A transient one (the container runtime still preparing after a
|
|
73
|
+
* reconnect, an unanswered ask) never replaces a substantive one: an
|
|
74
|
+
* operator reading Connections sees what to fix, not what fixes itself.
|
|
75
|
+
*
|
|
68
76
|
* The record holds no secret: `{"schemaVersion":1,"step":"local",
|
|
69
77
|
* "stopped":[{"taskId":"…","runEpoch":3}]}`. One the runner cannot read
|
|
70
78
|
* counts as `local`, never as `legacy`: a damaged record may stand for a
|
|
@@ -88,6 +96,14 @@ import {
|
|
|
88
96
|
import { dirname } from "node:path";
|
|
89
97
|
|
|
90
98
|
import type { CredentialsActivationPhase } from "../src/cloud-credentials";
|
|
99
|
+
import {
|
|
100
|
+
MAX_CREDENTIALS_SWITCH_BLOCKER_CHARS,
|
|
101
|
+
MAX_CREDENTIALS_SWITCH_BLOCKERS,
|
|
102
|
+
MAX_CREDENTIALS_SWITCH_DETAIL_CHARS,
|
|
103
|
+
MAX_CREDENTIALS_SWITCH_PROJECTS,
|
|
104
|
+
type CredentialsSwitchCapability,
|
|
105
|
+
type CredentialsSwitchStuck,
|
|
106
|
+
} from "../src/protocol";
|
|
91
107
|
import type { CredentialsMode } from "./cloud-credential-host";
|
|
92
108
|
import type { RetirementPlan } from "./legacy-runtime-retirement";
|
|
93
109
|
import { isSafeHostTaskId } from "./task-identity";
|
|
@@ -167,6 +183,12 @@ export interface ActivationAuthorityDeps<Retirement = RetirementPlan> {
|
|
|
167
183
|
/** Ask the runner's supervisor for a restart. */
|
|
168
184
|
restart: (why: string) => void;
|
|
169
185
|
log: (line: string) => void;
|
|
186
|
+
/** `status()` changed: advertise it again. */
|
|
187
|
+
onStatusChange?: () => void;
|
|
188
|
+
/** What `status()` says of a blocker, made fit to leave the machine (the
|
|
189
|
+
* home directory as `~`). */
|
|
190
|
+
redact?: (text: string) => string;
|
|
191
|
+
now?: () => number;
|
|
170
192
|
}
|
|
171
193
|
|
|
172
194
|
export class CredentialsActivationAuthority<Retirement = RetirementPlan> {
|
|
@@ -191,6 +213,14 @@ export class CredentialsActivationAuthority<Retirement = RetirementPlan> {
|
|
|
191
213
|
#lastNote: string | null = null;
|
|
192
214
|
/** Leaving the runner's own credentials failed in this process. */
|
|
193
215
|
#stuck = false;
|
|
216
|
+
/** What stops the switch now, as `status()` says it. */
|
|
217
|
+
#blocker: CredentialsSwitchStuck | null = null;
|
|
218
|
+
/** Every Compose project the current blocker names, past the status's
|
|
219
|
+
* bound too: what the log lists, and what tells one blocker from another
|
|
220
|
+
* when only names past that bound changed. */
|
|
221
|
+
#blockerProjects = "";
|
|
222
|
+
/** `status()` as last told to `onStatusChange`'s caller. */
|
|
223
|
+
#published: string | null = null;
|
|
194
224
|
|
|
195
225
|
constructor(deps: ActivationAuthorityDeps<Retirement>) {
|
|
196
226
|
this.#deps = deps;
|
|
@@ -218,6 +248,13 @@ export class CredentialsActivationAuthority<Retirement = RetirementPlan> {
|
|
|
218
248
|
return this.#record?.step ?? null;
|
|
219
249
|
}
|
|
220
250
|
|
|
251
|
+
/** Where the switch stands, for the capability frame (see the header). */
|
|
252
|
+
status(): CredentialsSwitchCapability {
|
|
253
|
+
const state = this.mode();
|
|
254
|
+
const stuck = state === "active" ? null : this.#blocker;
|
|
255
|
+
return stuck === null ? { state } : { state, stuck: { ...stuck } };
|
|
256
|
+
}
|
|
257
|
+
|
|
221
258
|
mode(): CredentialsMode {
|
|
222
259
|
if (this.#record === null) {
|
|
223
260
|
return this.#initiallyPending ? "pending" : "legacy";
|
|
@@ -249,9 +286,12 @@ export class CredentialsActivationAuthority<Retirement = RetirementPlan> {
|
|
|
249
286
|
advance(): Promise<void> {
|
|
250
287
|
if (this.#advancing === null) {
|
|
251
288
|
this.#advancing = this.#advanceOnce()
|
|
252
|
-
.catch((error: unknown) =>
|
|
289
|
+
.catch((error: unknown) =>
|
|
290
|
+
this.#block("runner", error, `activation stopped: ${message(error)}; it tries again later`),
|
|
291
|
+
)
|
|
253
292
|
.finally(() => {
|
|
254
293
|
this.#advancing = null;
|
|
294
|
+
this.#publish();
|
|
255
295
|
});
|
|
256
296
|
}
|
|
257
297
|
return this.#advancing;
|
|
@@ -266,10 +306,13 @@ export class CredentialsActivationAuthority<Retirement = RetirementPlan> {
|
|
|
266
306
|
}
|
|
267
307
|
let plan: Retirement | null = null;
|
|
268
308
|
if (this.#record === null) {
|
|
269
|
-
if (!this.#deps.eligible())
|
|
270
|
-
|
|
309
|
+
if (!this.#deps.eligible()) {
|
|
310
|
+
this.#blocker = null;
|
|
311
|
+
return;
|
|
312
|
+
}
|
|
313
|
+
const prepared = await this.#ask("prepare");
|
|
271
314
|
if (!prepared.ok) {
|
|
272
|
-
this.#
|
|
315
|
+
this.#block("cloud", prepared.error, `the cloud did not agree to cloud credentials yet: ${prepared.error}`);
|
|
273
316
|
return;
|
|
274
317
|
}
|
|
275
318
|
const paused = this.#deps.pauseLegacyStarts();
|
|
@@ -287,7 +330,9 @@ export class CredentialsActivationAuthority<Retirement = RetirementPlan> {
|
|
|
287
330
|
this.#record ??= readActivationRecord(this.#deps.recordPath);
|
|
288
331
|
if (this.#record === null) {
|
|
289
332
|
paused.resume();
|
|
290
|
-
this.#
|
|
333
|
+
this.#block(
|
|
334
|
+
"runner",
|
|
335
|
+
error,
|
|
291
336
|
this.#initiallyPending
|
|
292
337
|
? `the v1-required task machine stays pending, with no v0 fallback: ${message(error)}`
|
|
293
338
|
: `the runner stays on its own credentials, nothing changed: ${message(error)}`,
|
|
@@ -295,6 +340,7 @@ export class CredentialsActivationAuthority<Retirement = RetirementPlan> {
|
|
|
295
340
|
return;
|
|
296
341
|
}
|
|
297
342
|
}
|
|
343
|
+
this.#blocker = null;
|
|
298
344
|
this.#deps.log("[credentials] switching to cloud credentials: leaving the runner's own");
|
|
299
345
|
}
|
|
300
346
|
if (!this.#synced()) return;
|
|
@@ -319,11 +365,16 @@ export class CredentialsActivationAuthority<Retirement = RetirementPlan> {
|
|
|
319
365
|
plan ??= await this.#deps.planRetirement();
|
|
320
366
|
await this.#retire(plan);
|
|
321
367
|
await this.#deps.claimFloor();
|
|
322
|
-
const committed = await this.#
|
|
368
|
+
const committed = await this.#ask("commit");
|
|
323
369
|
if (!committed.ok) {
|
|
324
|
-
this.#
|
|
370
|
+
this.#block(
|
|
371
|
+
"cloud",
|
|
372
|
+
committed.error,
|
|
373
|
+
`the cloud did not record this runner's switch yet: ${committed.error}; it runs no task meanwhile`,
|
|
374
|
+
);
|
|
325
375
|
return;
|
|
326
376
|
}
|
|
377
|
+
this.#blocker = null;
|
|
327
378
|
this.#write({ ...this.#record, step: "cloud" });
|
|
328
379
|
}
|
|
329
380
|
if (this.#record?.step === "cloud") {
|
|
@@ -350,7 +401,9 @@ export class CredentialsActivationAuthority<Retirement = RetirementPlan> {
|
|
|
350
401
|
this.#write(record);
|
|
351
402
|
}
|
|
352
403
|
} catch (error) {
|
|
353
|
-
this.#
|
|
404
|
+
this.#block(
|
|
405
|
+
"runner",
|
|
406
|
+
error,
|
|
354
407
|
`the activation record is not known to be on disk yet (${message(error)}): the switch waits, running no task, ` +
|
|
355
408
|
"until it is",
|
|
356
409
|
);
|
|
@@ -373,7 +426,9 @@ export class CredentialsActivationAuthority<Retirement = RetirementPlan> {
|
|
|
373
426
|
try {
|
|
374
427
|
await this.#deps.sweep();
|
|
375
428
|
} catch (error) {
|
|
376
|
-
this.#
|
|
429
|
+
this.#block(
|
|
430
|
+
"runner",
|
|
431
|
+
error,
|
|
377
432
|
`the recovery sweep failed (${message(error)}): the runner stays pending, running no task, and sweeps again later`,
|
|
378
433
|
);
|
|
379
434
|
return;
|
|
@@ -397,10 +452,15 @@ export class CredentialsActivationAuthority<Retirement = RetirementPlan> {
|
|
|
397
452
|
}
|
|
398
453
|
if (failure === null) {
|
|
399
454
|
this.#announced = true;
|
|
455
|
+
this.#blocker = null;
|
|
400
456
|
this.#deps.log("[credentials] tasks run on cloud credentials");
|
|
401
457
|
return;
|
|
402
458
|
}
|
|
403
|
-
this.#
|
|
459
|
+
this.#block(
|
|
460
|
+
"runner",
|
|
461
|
+
failure,
|
|
462
|
+
`saying the runner is active failed (${message(failure)}): it is pending again, and tries again later`,
|
|
463
|
+
);
|
|
404
464
|
try {
|
|
405
465
|
this.#deps.onActiveWithdrawn();
|
|
406
466
|
} catch (error) {
|
|
@@ -441,6 +501,67 @@ export class CredentialsActivationAuthority<Retirement = RetirementPlan> {
|
|
|
441
501
|
this.#durable = true;
|
|
442
502
|
}
|
|
443
503
|
|
|
504
|
+
/** One phase asked of the cloud. An ask that could not be made or was not
|
|
505
|
+
* answered says nothing about the switch: transient. */
|
|
506
|
+
async #ask(phase: CredentialsActivationPhase): Promise<ActivationAnswer> {
|
|
507
|
+
try {
|
|
508
|
+
return await this.#deps.ask(phase);
|
|
509
|
+
} catch (error) {
|
|
510
|
+
throw new TransientActivationError(message(error));
|
|
511
|
+
}
|
|
512
|
+
}
|
|
513
|
+
|
|
514
|
+
/** A refusal or blocker: logged, and what `status()` says unless it is
|
|
515
|
+
* transient and a substantive one is already said (see the header). */
|
|
516
|
+
#block(by: CredentialsSwitchStuck["by"], why: unknown, line: string): void {
|
|
517
|
+
this.#note(line);
|
|
518
|
+
const transient = isTransient(why);
|
|
519
|
+
if (transient && this.#blocker !== null && this.#blocker.transient !== true) return;
|
|
520
|
+
const redact = this.#deps.redact ?? ((text: string) => text);
|
|
521
|
+
const detail = bounded(redact(oneLine(typeof why === "string" ? why : message(why))), MAX_CREDENTIALS_SWITCH_DETAIL_CHARS);
|
|
522
|
+
const next: CredentialsSwitchStuck = { by, detail, since: (this.#deps.now ?? Date.now)() };
|
|
523
|
+
const structured = why as { blockers?: unknown; unprovenProjects?: unknown } | null;
|
|
524
|
+
const blockers = stringList(structured?.blockers);
|
|
525
|
+
if (blockers.length > 0) {
|
|
526
|
+
next.blockers = blockers
|
|
527
|
+
.slice(0, MAX_CREDENTIALS_SWITCH_BLOCKERS)
|
|
528
|
+
.map((blocker) => bounded(redact(oneLine(blocker)), MAX_CREDENTIALS_SWITCH_BLOCKER_CHARS));
|
|
529
|
+
}
|
|
530
|
+
const projects = stringList(structured?.unprovenProjects);
|
|
531
|
+
if (projects.length > 0) {
|
|
532
|
+
next.composeProjects = { names: projects.slice(0, MAX_CREDENTIALS_SWITCH_PROJECTS), total: projects.length };
|
|
533
|
+
}
|
|
534
|
+
if (transient) next.transient = true;
|
|
535
|
+
const previous = this.#blocker;
|
|
536
|
+
const allProjects = projects.join(" ");
|
|
537
|
+
// The same blocker again keeps when it was first seen. "The same" counts
|
|
538
|
+
// every project, not only the ones the status has room for.
|
|
539
|
+
if (previous !== null && sameBlocker(previous, next) && allProjects === this.#blockerProjects) return;
|
|
540
|
+
this.#blocker = next;
|
|
541
|
+
this.#blockerProjects = allProjects;
|
|
542
|
+
// The status names at most MAX_CREDENTIALS_SWITCH_PROJECTS, and the
|
|
543
|
+
// line above is cut short: the log names every one, once per blocker,
|
|
544
|
+
// for the operator Connections sends here for the rest.
|
|
545
|
+
if (projects.length > 0) {
|
|
546
|
+
this.#deps.log(
|
|
547
|
+
`[credentials] Compose projects nothing proves Uai's (${projects.length}), each to remove ` +
|
|
548
|
+
`(docker compose -p <name> down) or rename: ${allProjects}`,
|
|
549
|
+
);
|
|
550
|
+
}
|
|
551
|
+
}
|
|
552
|
+
|
|
553
|
+
/** `onStatusChange` when `status()` says something new. */
|
|
554
|
+
#publish(): void {
|
|
555
|
+
const now = JSON.stringify(this.status());
|
|
556
|
+
if (now === this.#published) return;
|
|
557
|
+
this.#published = now;
|
|
558
|
+
try {
|
|
559
|
+
this.#deps.onStatusChange?.();
|
|
560
|
+
} catch (error) {
|
|
561
|
+
this.#note(`advertising the switch's status failed: ${message(error)}`);
|
|
562
|
+
}
|
|
563
|
+
}
|
|
564
|
+
|
|
444
565
|
/** The same note twice in a row is logged once: a refusal is asked about
|
|
445
566
|
* again every few minutes. */
|
|
446
567
|
#note(line: string): void {
|
|
@@ -450,6 +571,33 @@ export class CredentialsActivationAuthority<Retirement = RetirementPlan> {
|
|
|
450
571
|
}
|
|
451
572
|
}
|
|
452
573
|
|
|
574
|
+
class TransientActivationError extends Error {
|
|
575
|
+
readonly transient = true;
|
|
576
|
+
}
|
|
577
|
+
|
|
578
|
+
function isTransient(why: unknown): boolean {
|
|
579
|
+
return why !== null && typeof why === "object" && (why as { transient?: unknown }).transient === true;
|
|
580
|
+
}
|
|
581
|
+
|
|
582
|
+
function sameBlocker(a: CredentialsSwitchStuck, b: CredentialsSwitchStuck): boolean {
|
|
583
|
+
const { since: _a, ...left } = a;
|
|
584
|
+
const { since: _b, ...right } = b;
|
|
585
|
+
return JSON.stringify(left) === JSON.stringify(right);
|
|
586
|
+
}
|
|
587
|
+
|
|
588
|
+
function stringList(value: unknown): string[] {
|
|
589
|
+
return Array.isArray(value) ? value.filter((item): item is string => typeof item === "string" && item !== "") : [];
|
|
590
|
+
}
|
|
591
|
+
|
|
592
|
+
function oneLine(text: string): string {
|
|
593
|
+
return text.replace(/\s*\n\s*(?:- )?/g, "; ").trim();
|
|
594
|
+
}
|
|
595
|
+
|
|
596
|
+
function bounded(text: string, max: number): string {
|
|
597
|
+
const nonEmpty = text === "" ? "unknown" : text;
|
|
598
|
+
return nonEmpty.length <= max ? nonEmpty : `${nonEmpty.slice(0, max - 1)}…`;
|
|
599
|
+
}
|
|
600
|
+
|
|
453
601
|
/** The record; null when there is none. One the runner cannot read is
|
|
454
602
|
* `local` with nothing owed (see the header). */
|
|
455
603
|
export function readActivationRecord(path: string): ActivationRecord | null {
|
|
@@ -22,7 +22,12 @@
|
|
|
22
22
|
* isolation), and which the operator removes or renames. A row with no
|
|
23
23
|
* runtime (a task that went down keeps its row, its runtime fields
|
|
24
24
|
* cleared) neither blocks nor vouches for anything. The plan records the
|
|
25
|
-
* engine it was made against.
|
|
25
|
+
* engine it was made against. Its rejection is a `RetirementBlockedError`:
|
|
26
|
+
* each blocker apart, and every unproven project named, so the cloud can
|
|
27
|
+
* show the operator all of them with the command that clears each
|
|
28
|
+
* (ADR-154 §13 build notes, 2026-10-27). Such a project is never removed
|
|
29
|
+
* here: a Compose label proves neither who made a stack nor that nothing
|
|
30
|
+
* uses it now (another runner may share this Docker and workspace root).
|
|
26
31
|
* - `retireLegacyTaskRuntimes` carries a plan out, only after the runner has
|
|
27
32
|
* committed to the switch. Each task's check, Stop and removal share its
|
|
28
33
|
* lifecycle slot: before its assignment closes and again before anything
|
|
@@ -117,6 +122,26 @@ export interface RetirementDeps extends RetirementPlanDeps {
|
|
|
117
122
|
log: (line: string) => void;
|
|
118
123
|
}
|
|
119
124
|
|
|
125
|
+
/**
|
|
126
|
+
* The plan's refusal: every blocker apart (`blockers`), and apart from them
|
|
127
|
+
* the `task-*` Compose projects nothing proves Uai's (all of them, for the
|
|
128
|
+
* operator to remove or rename). The message names both, as the log has.
|
|
129
|
+
* `transient`: nothing is wrong yet, the runner only cannot look now (its
|
|
130
|
+
* container runtime is still preparing); the next attempt looks again.
|
|
131
|
+
*/
|
|
132
|
+
export class RetirementBlockedError extends Error {
|
|
133
|
+
readonly blockers: string[];
|
|
134
|
+
readonly unprovenProjects: string[];
|
|
135
|
+
readonly transient: boolean;
|
|
136
|
+
constructor(message: string, options: { blockers?: string[]; unprovenProjects?: string[]; transient?: boolean } = {}) {
|
|
137
|
+
super(message);
|
|
138
|
+
this.name = "RetirementBlockedError";
|
|
139
|
+
this.blockers = options.blockers ?? [];
|
|
140
|
+
this.unprovenProjects = options.unprovenProjects ?? [];
|
|
141
|
+
this.transient = options.transient === true;
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
|
|
120
145
|
export interface RetirementReport {
|
|
121
146
|
/** Tasks whose runtime is gone (or was already). */
|
|
122
147
|
retired: string[];
|
|
@@ -136,8 +161,9 @@ export async function planLegacyTaskRuntimeRetirement(deps: RetirementPlanDeps):
|
|
|
136
161
|
const blockers: string[] = [];
|
|
137
162
|
const snapshot = deps.snapshot();
|
|
138
163
|
if (snapshot.runtimeProblem !== null || snapshot.machine === null) {
|
|
139
|
-
throw new
|
|
164
|
+
throw new RetirementBlockedError(
|
|
140
165
|
`task runtimes cannot be retired now: the container runtime is not ready (${snapshot.runtimeProblem ?? "no engine is pinned"})`,
|
|
166
|
+
{ transient: true },
|
|
141
167
|
);
|
|
142
168
|
}
|
|
143
169
|
if (!snapshot.fence.ok) blockers.push(snapshot.fence.detail ?? "the task rows do not match the pinned container runtime");
|
|
@@ -183,6 +209,7 @@ export async function planLegacyTaskRuntimeRetirement(deps: RetirementPlanDeps):
|
|
|
183
209
|
}
|
|
184
210
|
|
|
185
211
|
const appleOrphans: string[] = [];
|
|
212
|
+
const unprovenProjects: string[] = [];
|
|
186
213
|
try {
|
|
187
214
|
if (deps.apple) {
|
|
188
215
|
const named = new Set(rows.map((task) => `task-${task.taskId}-app`));
|
|
@@ -193,18 +220,26 @@ export async function planLegacyTaskRuntimeRetirement(deps: RetirementPlanDeps):
|
|
|
193
220
|
const named = new Set(rows.map((task) => `task-${task.taskId}`));
|
|
194
221
|
for (const project of await listTaskProjects(deps.docker)) {
|
|
195
222
|
if (named.has(project)) continue;
|
|
196
|
-
|
|
197
|
-
`Compose project ${project} has containers but no task on this runner: nothing proves it Uai's, so it is not removed. ` +
|
|
198
|
-
`If it was a task's, remove it (docker compose -p ${project} down); otherwise rename it`,
|
|
199
|
-
);
|
|
223
|
+
unprovenProjects.push(project);
|
|
200
224
|
}
|
|
201
225
|
}
|
|
202
226
|
} catch (error) {
|
|
203
227
|
blockers.push(`the task runtime inventory failed: ${message(error)}`);
|
|
204
228
|
}
|
|
205
229
|
|
|
206
|
-
if (blockers.length > 0) {
|
|
207
|
-
|
|
230
|
+
if (blockers.length > 0 || unprovenProjects.length > 0) {
|
|
231
|
+
const all = [
|
|
232
|
+
...blockers,
|
|
233
|
+
...unprovenProjects.map(
|
|
234
|
+
(project) =>
|
|
235
|
+
`Compose project ${project} has containers but no task on this runner: nothing proves it Uai's, so it is not removed. ` +
|
|
236
|
+
`If it was a task's, remove it (docker compose -p ${project} down); otherwise rename it`,
|
|
237
|
+
),
|
|
238
|
+
];
|
|
239
|
+
throw new RetirementBlockedError(`task runtimes cannot all be retired: ${all.join("; ").slice(0, 1500)}`, {
|
|
240
|
+
blockers,
|
|
241
|
+
unprovenProjects,
|
|
242
|
+
});
|
|
208
243
|
}
|
|
209
244
|
return { machine: snapshot.machine, removals, appleOrphans };
|
|
210
245
|
}
|
package/lib/orchestrator.ts
CHANGED
|
@@ -40,6 +40,7 @@ import { inArray } from "drizzle-orm";
|
|
|
40
40
|
|
|
41
41
|
import { getDb, schema } from "./db";
|
|
42
42
|
import { newId } from "./ulid";
|
|
43
|
+
import { agentErrorMessage, reportAgentError } from "./agent-errors";
|
|
43
44
|
import { mockAgentFactory } from "./agents/mock";
|
|
44
45
|
import { realAgentFactory } from "./agents/factory";
|
|
45
46
|
import { resolveAgentMode } from "./agents/mode";
|
|
@@ -50,6 +51,7 @@ import {
|
|
|
50
51
|
} from "./agents/session-state";
|
|
51
52
|
import {
|
|
52
53
|
type AgentEvent,
|
|
54
|
+
type AgentEventHandler,
|
|
53
55
|
type AgentSession,
|
|
54
56
|
type AgentSessionFactory,
|
|
55
57
|
type Roster,
|
|
@@ -709,7 +711,11 @@ const BROWSER_RETRY_COOLDOWN_MS = 60_000;
|
|
|
709
711
|
* poisons the runtime generation synchronously; normal detection/recovery owns
|
|
710
712
|
* the eventual ready edge and deferred-prompt replay.
|
|
711
713
|
*/
|
|
712
|
-
async function agentTerminationLostContainerRuntime(): Promise<boolean> {
|
|
714
|
+
async function agentTerminationLostContainerRuntime(channel: Channel): Promise<boolean> {
|
|
715
|
+
// Machine/self tasks have no Docker/Apple daemon. A failed Docker probe
|
|
716
|
+
// there would turn a real engine failure into silent runtime recovery.
|
|
717
|
+
if (isEnvironmentTransportProvider(channel.agentHandle?.descriptor.locator.provider) ||
|
|
718
|
+
isEnvironmentTransportProvider(getHostTask(channel.taskId)?.environmentProvider)) return false;
|
|
713
719
|
if (!backgroundContainerWorkAllowed()) return true;
|
|
714
720
|
let reachable = false;
|
|
715
721
|
try {
|
|
@@ -809,6 +815,7 @@ export interface OrchestratorTaskStopResult {
|
|
|
809
815
|
}
|
|
810
816
|
|
|
811
817
|
export class Orchestrator {
|
|
818
|
+
private readonly sessionEventHandlers = new WeakMap<AgentSession, AgentEventHandler>();
|
|
812
819
|
private readonly channels = new Map<string, Channel>();
|
|
813
820
|
private readonly channelSpecs = new Map<string, ChannelEnsureInput>();
|
|
814
821
|
/** Teardown tombstones prevent a cloud ensure from reopening a task between
|
|
@@ -2915,7 +2922,7 @@ export class Orchestrator {
|
|
|
2915
2922
|
// transport emission order across that await; otherwise the later boundary
|
|
2916
2923
|
// can consume/reorder prompt state while the earlier error still owns it.
|
|
2917
2924
|
let eventTail: Promise<void> = Promise.resolve();
|
|
2918
|
-
|
|
2925
|
+
const onEvent: AgentEventHandler = (event) => {
|
|
2919
2926
|
this.markChannelActivity(channel.taskId);
|
|
2920
2927
|
// Recycle/teardown deletes the old session before calling close().
|
|
2921
2928
|
// Adapters deliberately emit a final exit event; without this identity
|
|
@@ -2957,7 +2964,9 @@ export class Orchestrator {
|
|
|
2957
2964
|
() => channel.eventOperations.delete(operation),
|
|
2958
2965
|
() => channel.eventOperations.delete(operation),
|
|
2959
2966
|
);
|
|
2960
|
-
}
|
|
2967
|
+
};
|
|
2968
|
+
this.sessionEventHandlers.set(session, onEvent);
|
|
2969
|
+
session.onEvent(onEvent);
|
|
2961
2970
|
}
|
|
2962
2971
|
|
|
2963
2972
|
/**
|
|
@@ -4616,11 +4625,11 @@ export class Orchestrator {
|
|
|
4616
4625
|
}
|
|
4617
4626
|
}
|
|
4618
4627
|
|
|
4619
|
-
/** Consume
|
|
4620
|
-
private decrementActiveTurns(channel: Channel, agentId: string): number {
|
|
4628
|
+
/** Consume deliveries settled by one provider boundary. */
|
|
4629
|
+
private decrementActiveTurns(channel: Channel, agentId: string, settles = 1): number {
|
|
4621
4630
|
const remaining = Math.max(
|
|
4622
4631
|
0,
|
|
4623
|
-
(channel.activeTurns.get(agentId) ?? 1) -
|
|
4632
|
+
(channel.activeTurns.get(agentId) ?? 1) - settles,
|
|
4624
4633
|
);
|
|
4625
4634
|
if (remaining === 0) channel.activeTurns.delete(agentId);
|
|
4626
4635
|
else channel.activeTurns.set(agentId, remaining);
|
|
@@ -4759,26 +4768,29 @@ export class Orchestrator {
|
|
|
4759
4768
|
this.incrementActiveTurns(channel, agentId);
|
|
4760
4769
|
void session.send(text).catch((err: unknown) => {
|
|
4761
4770
|
if (turn) journal?.failed(sessionId, turn.turnId);
|
|
4771
|
+
const diagnosticMessage = `Could not send the message to ${agentId}: ${err instanceof Error ? err.message : String(err)}`;
|
|
4772
|
+
reportAgentError(diagnosticMessage, { taskId: channel.taskId, agentId, engine: session.kind },
|
|
4773
|
+
{ capture: !(err instanceof ContainerRuntimeUnavailableError) });
|
|
4762
4774
|
if (channel.sessions.get(agentId) === session) {
|
|
4763
|
-
this.removeAcceptedPrompt(channel, agentId, accepted);
|
|
4764
|
-
const remaining = this.decrementActiveTurns(channel, agentId);
|
|
4765
|
-
if (
|
|
4766
|
-
remaining === 0 &&
|
|
4767
|
-
channel.lastPrompt.get(agentId) === accepted
|
|
4768
|
-
) {
|
|
4769
|
-
channel.lastPrompt.delete(agentId);
|
|
4770
|
-
}
|
|
4771
4775
|
if (err instanceof ContainerRuntimeUnavailableError) {
|
|
4776
|
+
this.removeAcceptedPrompt(channel, agentId, accepted);
|
|
4777
|
+
const remaining = this.decrementActiveTurns(channel, agentId);
|
|
4778
|
+
if (remaining === 0 && channel.lastPrompt.get(agentId) === accepted) {
|
|
4779
|
+
channel.lastPrompt.delete(agentId);
|
|
4780
|
+
}
|
|
4772
4781
|
this.deferPromptUntilRuntimeReady(channel, agentId, accepted);
|
|
4773
4782
|
} else {
|
|
4774
|
-
|
|
4783
|
+
// Share the ordered terminal-event path with adapter failures. It
|
|
4784
|
+
// retires the active count and publishes the error/aborted boundary;
|
|
4785
|
+
// a concurrent adapter error or exit can consume it only once.
|
|
4786
|
+
this.sessionEventHandlers.get(session)?.({
|
|
4787
|
+
type: "error",
|
|
4788
|
+
message: agentErrorMessage(diagnosticMessage),
|
|
4789
|
+
diagnosticMessage,
|
|
4790
|
+
aborted: true,
|
|
4791
|
+
});
|
|
4775
4792
|
}
|
|
4776
4793
|
}
|
|
4777
|
-
console.warn(
|
|
4778
|
-
`[orchestrator] ${channel.taskId}/${agentId}: send failed: ${
|
|
4779
|
-
err instanceof Error ? err.message : String(err)
|
|
4780
|
-
}`,
|
|
4781
|
-
);
|
|
4782
4794
|
});
|
|
4783
4795
|
}
|
|
4784
4796
|
|
|
@@ -4794,13 +4806,14 @@ export class Orchestrator {
|
|
|
4794
4806
|
if (prompts.length === 0) channel.acceptedPrompts.delete(agentId);
|
|
4795
4807
|
}
|
|
4796
4808
|
|
|
4797
|
-
private
|
|
4809
|
+
private completeAcceptedPrompts(
|
|
4798
4810
|
channel: Channel,
|
|
4799
4811
|
agentId: string,
|
|
4812
|
+
settles = 1,
|
|
4800
4813
|
): void {
|
|
4801
4814
|
const prompts = channel.acceptedPrompts.get(agentId);
|
|
4802
4815
|
if (!prompts) return;
|
|
4803
|
-
prompts.
|
|
4816
|
+
prompts.splice(0, settles);
|
|
4804
4817
|
if (prompts.length === 0) channel.acceptedPrompts.delete(agentId);
|
|
4805
4818
|
}
|
|
4806
4819
|
|
|
@@ -5257,8 +5270,13 @@ export class Orchestrator {
|
|
|
5257
5270
|
});
|
|
5258
5271
|
break;
|
|
5259
5272
|
}
|
|
5273
|
+
case "notice": {
|
|
5274
|
+
this.emitSystemNote(channel.taskId, agentErrorMessage(`${agentId}: ${event.message}`));
|
|
5275
|
+
break;
|
|
5276
|
+
}
|
|
5260
5277
|
case "error": {
|
|
5261
|
-
|
|
5278
|
+
const diagnosticMessage = event.diagnosticMessage ?? event.message;
|
|
5279
|
+
if (await agentTerminationLostContainerRuntime(channel)) {
|
|
5262
5280
|
// `error` and `exit` can arrive back-to-back while the daemon probe
|
|
5263
5281
|
// is pending. Only the first terminal event still owns this session.
|
|
5264
5282
|
if (channel.sessions.get(agentId) !== session) break;
|
|
@@ -5272,7 +5290,7 @@ export class Orchestrator {
|
|
|
5272
5290
|
if (channel.sessions.get(agentId) !== session) break;
|
|
5273
5291
|
// The turn died with the session — drop its turn-state flags so a
|
|
5274
5292
|
// respawned session starts clean.
|
|
5275
|
-
|
|
5293
|
+
let abortBoundaryOnRosterChange =
|
|
5276
5294
|
this.needsAbortedTurnBoundary(channel, agentId);
|
|
5277
5295
|
this.retainRetiredTurnBoundary(
|
|
5278
5296
|
channel,
|
|
@@ -5280,6 +5298,14 @@ export class Orchestrator {
|
|
|
5280
5298
|
abortBoundaryOnRosterChange,
|
|
5281
5299
|
);
|
|
5282
5300
|
this.retireRunnerGeneration(channel, agentId);
|
|
5301
|
+
if (event.aborted) {
|
|
5302
|
+
this.emitRetiredTurnAbort(channel, agentId);
|
|
5303
|
+
const change = channel.rosterSessionChanges.get(agentId);
|
|
5304
|
+
if (change) change.abortPublished = true;
|
|
5305
|
+
// A recovery may replay the prompt, but the failed turn already
|
|
5306
|
+
// ended. Do not publish a second boundary for this generation.
|
|
5307
|
+
abortBoundaryOnRosterChange = false;
|
|
5308
|
+
}
|
|
5283
5309
|
// Claude under load (especially Docker Desktop macOS) occasionally
|
|
5284
5310
|
// unlinks ~/.claude.json mid-write during atomic config rewrites,
|
|
5285
5311
|
// and a concurrent claude spawn lands during the gap and exits 0
|
|
@@ -5325,7 +5351,7 @@ export class Orchestrator {
|
|
|
5325
5351
|
if (
|
|
5326
5352
|
agent &&
|
|
5327
5353
|
agent.kind !== "claude" &&
|
|
5328
|
-
isCredentialInvalidSignal(agent.kind,
|
|
5354
|
+
isCredentialInvalidSignal(agent.kind, diagnosticMessage)
|
|
5329
5355
|
) {
|
|
5330
5356
|
console.warn(
|
|
5331
5357
|
`[orchestrator] ${channel.taskId}/${agentId}: ${agent.kind} credential-invalid pattern matched: ${event.message.slice(0, 300)}`,
|
|
@@ -5407,7 +5433,7 @@ export class Orchestrator {
|
|
|
5407
5433
|
}
|
|
5408
5434
|
}
|
|
5409
5435
|
const isClaude = agent?.kind === "claude";
|
|
5410
|
-
if (isClaude && agent && isClaudeAuthRevoked(
|
|
5436
|
+
if (isClaude && agent && isClaudeAuthRevoked(diagnosticMessage)) {
|
|
5411
5437
|
// Token revoked/expired: the env-injected credential only changes on
|
|
5412
5438
|
// a fresh runner. Re-spawn on the re-resolved account (picks up a
|
|
5413
5439
|
// reconnected token); bounded so a still-bad token can't loop.
|
|
@@ -5424,7 +5450,7 @@ export class Orchestrator {
|
|
|
5424
5450
|
break;
|
|
5425
5451
|
}
|
|
5426
5452
|
const configMissing = CLAUDE_CONFIG_MISSING_PATTERNS.some((re) =>
|
|
5427
|
-
re.test(
|
|
5453
|
+
re.test(diagnosticMessage),
|
|
5428
5454
|
);
|
|
5429
5455
|
if (isClaude && configMissing) {
|
|
5430
5456
|
// This recovery path deliberately starts the next queued turn rather
|
|
@@ -5439,7 +5465,7 @@ export class Orchestrator {
|
|
|
5439
5465
|
break;
|
|
5440
5466
|
}
|
|
5441
5467
|
channel.lastPrompt.delete(agentId);
|
|
5442
|
-
|
|
5468
|
+
this.emitRetiredTurnAbort(channel, agentId);
|
|
5443
5469
|
this.emitHost({
|
|
5444
5470
|
kind: "agent.exit",
|
|
5445
5471
|
taskId: channel.taskId,
|
|
@@ -5469,9 +5495,11 @@ export class Orchestrator {
|
|
|
5469
5495
|
// turn's messages and wakes the mentioned peers with the full turn.
|
|
5470
5496
|
// An interrupted (ESC'd) turn goes out `aborted`: it's a half-turn, so
|
|
5471
5497
|
// the cloud discards the buffer instead of handing it to peers.
|
|
5472
|
-
const aborted = channel.interrupted.delete(agentId);
|
|
5473
|
-
|
|
5474
|
-
|
|
5498
|
+
const aborted = channel.interrupted.delete(agentId) || event.aborted === true;
|
|
5499
|
+
const settles = event.settles !== undefined && Number.isSafeInteger(event.settles) && event.settles > 0
|
|
5500
|
+
? event.settles : 1;
|
|
5501
|
+
this.completeAcceptedPrompts(channel, agentId, settles);
|
|
5502
|
+
const remainingTurns = this.decrementActiveTurns(channel, agentId, settles);
|
|
5475
5503
|
channel.openTurns.delete(agentId);
|
|
5476
5504
|
// ADR-076: a turn that completed means the current account is healthy —
|
|
5477
5505
|
// reset its rotation budget so a LATER rate-limit gets a fresh failover.
|
|
@@ -5522,7 +5550,15 @@ export class Orchestrator {
|
|
|
5522
5550
|
break;
|
|
5523
5551
|
}
|
|
5524
5552
|
case "exit":
|
|
5525
|
-
if (
|
|
5553
|
+
if (event.code === 0 && this.needsAbortedTurnBoundary(channel, agentId)) {
|
|
5554
|
+
await this.handleAgentEvent(channel, agentId, session, {
|
|
5555
|
+
type: "error",
|
|
5556
|
+
message: `${session.kind} stopped before replying to the message. Please try again.`,
|
|
5557
|
+
aborted: true,
|
|
5558
|
+
});
|
|
5559
|
+
break;
|
|
5560
|
+
}
|
|
5561
|
+
if (await agentTerminationLostContainerRuntime(channel)) {
|
|
5526
5562
|
if (channel.sessions.get(agentId) !== session) break;
|
|
5527
5563
|
await this.retireSessionForRuntimeLoss(
|
|
5528
5564
|
channel,
|
|
@@ -7319,7 +7355,10 @@ export function buildSystemPreamble(
|
|
|
7319
7355
|
`${taskBranch}\` from inside the project, then open a PR with \`gh pr`,
|
|
7320
7356
|
"create` (the container has gh authenticated). For multi-project",
|
|
7321
7357
|
"tasks, each project's PR is independent — open one per project whose",
|
|
7322
|
-
"worktree you actually changed.",
|
|
7358
|
+
"worktree you actually changed. For a separate branch, create it in",
|
|
7359
|
+
"this task's namespace — `git switch -c \"$(uai branch name fix-foo)\"`",
|
|
7360
|
+
"makes `uai/<taskId>/fix-foo`. Only task-branch and namespace PRs link to",
|
|
7361
|
+
"this task (reviews, `--pr`, Usage); a PR from any other branch does not.",
|
|
7323
7362
|
"",
|
|
7324
7363
|
"The `.uai/` directory under each task is uai's own scaffolding",
|
|
7325
7364
|
"(rendered Dockerfile, compose file, container scripts) — it is NOT",
|
package/package.json
CHANGED
package/src/main.ts
CHANGED
|
@@ -784,6 +784,9 @@ const credentialsActivation = new CredentialsActivationAuthority<CredentialsReti
|
|
|
784
784
|
requestManagedHostRestart();
|
|
785
785
|
},
|
|
786
786
|
log: (line) => console.warn(line),
|
|
787
|
+
// What stops the switch reaches the cloud too, not only this log.
|
|
788
|
+
onStatusChange: () => sendCapabilities(),
|
|
789
|
+
redact: redactHomeDirectory,
|
|
787
790
|
});
|
|
788
791
|
setCredentialsModeSource(() => credentialsActivation.mode());
|
|
789
792
|
/** ADR-154 §13: the steady/operator cadence. Active runners report their
|
|
@@ -1581,9 +1584,18 @@ function buildCapabilities(): HostCapabilities {
|
|
|
1581
1584
|
engineAccounts: engineAccountSummaries(),
|
|
1582
1585
|
engines: engineCatalogCapabilities(),
|
|
1583
1586
|
githubUsers: connectedUserIds(),
|
|
1587
|
+
// ADR-154 §13: where the switch to cloud credentials stands, and why.
|
|
1588
|
+
credentialsSwitch: credentialsActivation.status(),
|
|
1584
1589
|
};
|
|
1585
1590
|
}
|
|
1586
1591
|
|
|
1592
|
+
/** A path under the home directory, as `~/…`: a status line leaves the
|
|
1593
|
+
* machine, and the account name in a path is not the cloud's to keep. */
|
|
1594
|
+
function redactHomeDirectory(text: string): string {
|
|
1595
|
+
const home = homedir();
|
|
1596
|
+
return home.length > 1 ? text.split(home).join("~") : text;
|
|
1597
|
+
}
|
|
1598
|
+
|
|
1587
1599
|
// engineCliStatuses probes binaries asynchronously; capabilities build
|
|
1588
1600
|
// synchronously. The snapshot starts empty (cliInstalled: false) and each
|
|
1589
1601
|
// build kicks a refresh that re-advertises only when the answer changed.
|