@ours.network/fleet 1.0.5 → 1.1.0-nightly.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +122 -133
- package/dist/agent-recovery-gate.d.ts +17 -0
- package/dist/agent-recovery-gate.js +135 -0
- package/dist/application/capabilities.d.ts +0 -1
- package/dist/application/capabilities.js +3 -13
- package/dist/application/fleet-query-service.d.ts +0 -3
- package/dist/application/fleet-query-service.js +14 -25
- package/dist/application/role-creation-service.d.ts +5 -24
- package/dist/application/role-creation-service.js +27 -67
- package/dist/application/role-removal-service.js +6 -3
- package/dist/application/role-repository.d.ts +1 -2
- package/dist/application/role-repository.js +7 -7
- package/dist/application/session-control.d.ts +0 -12
- package/dist/application/session-control.js +0 -42
- package/dist/application/session-mutations.d.ts +5 -5
- package/dist/application/task-room-service.d.ts +1 -1
- package/dist/application/task-room-service.js +154 -16
- package/dist/application/types.d.ts +1 -8
- package/dist/briefing.js +12 -11
- package/dist/build-info.json +5 -5
- package/dist/cli.js +143 -63
- package/dist/config-yaml.d.ts +1 -1
- package/dist/config.d.ts +66 -6
- package/dist/config.js +398 -43
- package/dist/creation.d.ts +1 -1
- package/dist/creation.js +2 -2
- package/dist/daemon-recovery.d.ts +93 -0
- package/dist/daemon-recovery.js +328 -0
- package/dist/docs.d.ts +1 -1
- package/dist/docs.js +103 -127
- package/dist/doctor.js +18 -11
- package/dist/fleet-command-audit.d.ts +160 -0
- package/dist/fleet-command-audit.js +433 -0
- package/dist/fleet-proxy.d.ts +3 -1
- package/dist/fleet-proxy.js +30 -8
- package/dist/generated-agent-source.d.ts +9 -0
- package/dist/generated-agent-source.js +53 -0
- package/dist/harness/acp-session-transport.d.ts +8 -0
- package/dist/harness/acp-session-transport.js +3 -0
- package/dist/harness/agent-session.d.ts +51 -0
- package/dist/harness/agent-session.js +1 -0
- package/dist/harness/claude-code-session.d.ts +24 -0
- package/dist/harness/claude-code-session.js +36 -0
- package/dist/harness/claude-code.d.ts +2 -1
- package/dist/harness/claude-code.js +41 -81
- package/dist/harness/codex-session.d.ts +24 -0
- package/dist/harness/codex-session.js +36 -0
- package/dist/harness/codex.d.ts +2 -1
- package/dist/harness/codex.js +55 -45
- package/dist/harness/registry.js +4 -0
- package/dist/harness/types.d.ts +6 -38
- package/dist/index.d.ts +4 -6
- package/dist/index.js +2 -4
- package/dist/init-guidance.d.ts +1 -0
- package/dist/init-guidance.js +6 -0
- package/dist/isolation/resources.d.ts +3 -7
- package/dist/isolation/resources.js +3 -7
- package/dist/model-env.d.ts +4 -6
- package/dist/model-env.js +5 -8
- package/dist/monitor.d.ts +1 -24
- package/dist/monitor.js +5 -140
- package/dist/ops.js +4 -2
- package/dist/owner-channel/channel.d.ts +58 -2
- package/dist/owner-channel/channel.js +269 -29
- package/dist/owner-channel/commands.d.ts +2 -0
- package/dist/owner-channel/commands.js +31 -13
- package/dist/owner-channel/state.d.ts +2 -0
- package/dist/owner-channel/state.js +8 -2
- package/dist/preset-bootstrap.d.ts +10 -0
- package/dist/preset-bootstrap.js +85 -0
- package/dist/reports/artifact.d.ts +10 -0
- package/dist/reports/artifact.js +32 -0
- package/dist/reports/index.d.ts +5 -0
- package/dist/reports/index.js +5 -0
- package/dist/reports/render.d.ts +6 -0
- package/dist/reports/render.js +240 -0
- package/dist/reports/service.d.ts +42 -0
- package/dist/reports/service.js +57 -0
- package/dist/reports/tasks.d.ts +52 -0
- package/dist/reports/tasks.js +177 -0
- package/dist/reports/types.d.ts +315 -0
- package/dist/reports/types.js +6 -0
- package/dist/resolved-plan.d.ts +3 -2
- package/dist/resolved-plan.js +21 -5
- package/dist/rooms-tasks/cli.js +225 -40
- package/dist/rooms-tasks/config.js +24 -13
- package/dist/rooms-tasks/external-worker.d.ts +1 -0
- package/dist/rooms-tasks/external-worker.js +14 -9
- package/dist/rooms-tasks/provision.js +98 -36
- package/dist/rooms-tasks/templates.d.ts +0 -1
- package/dist/rooms-tasks/templates.js +9 -76
- package/dist/rooms-tasks/types.d.ts +10 -16
- package/dist/rooms-tasks/types.js +1 -4
- package/dist/runner.d.ts +13 -16
- package/dist/runner.js +110 -122
- package/dist/sensitive-config.d.ts +2 -0
- package/dist/sensitive-config.js +4 -0
- package/dist/session/acp.d.ts +7 -2
- package/dist/session/acp.js +29 -6
- package/dist/session/activity.d.ts +2 -3
- package/dist/session/activity.js +2 -3
- package/dist/session/arbiter.d.ts +4 -4
- package/dist/session/control.d.ts +17 -4
- package/dist/session/control.js +35 -0
- package/dist/session/types.d.ts +11 -5
- package/dist/spawn.d.ts +8 -33
- package/dist/spawn.js +53 -213
- package/dist/supervisor/none.d.ts +1 -1
- package/dist/supervisor/none.js +2 -2
- package/dist/temp-lifecycle.d.ts +1 -1
- package/dist/tmux.d.ts +1 -14
- package/dist/tmux.js +1 -51
- package/dist/watchdog/config.d.ts +10 -7
- package/dist/watchdog/config.js +30 -18
- package/dist/watchdog/run.js +12 -36
- package/dist/web/auth.d.ts +1 -1
- package/dist/web/fleet-config-service.d.ts +11 -8
- package/dist/web/fleet-config-service.js +384 -183
- package/dist/web/runtime.js +4 -15
- package/dist/web/server.d.ts +1 -3
- package/dist/web/server.js +3 -14
- package/dist/web/topology-promote.js +26 -19
- package/dist/web/topology.js +2 -2
- package/dist/web/yaml-document-edit.js +2 -0
- package/dist/web-app/assets/index-59GF-gsZ.css +1 -0
- package/dist/web-app/assets/index-BbTc5KtZ.js +10 -0
- package/dist/web-app/index.html +2 -2
- package/examples/fleet/agents/Alice.yaml +11 -0
- package/examples/fleet/agents/FleetCoordinator.yaml +7 -0
- package/examples/fleet/brains/claude-default.yaml +5 -0
- package/examples/fleet/roles/coordinator.yaml +5 -0
- package/examples/fleet/roles/developer.yaml +3 -0
- package/examples/fleet.yaml +74 -0
- package/package.json +5 -8
- package/presets/fleet/agents/Agent.yaml +3 -0
- package/presets/fleet/agents/Architect.yaml +3 -0
- package/presets/fleet/agents/Critic.yaml +3 -0
- package/presets/fleet/agents/Developer.yaml +3 -0
- package/presets/fleet/agents/Secretary.yaml +3 -0
- package/presets/fleet/agents/Tester.yaml +3 -0
- package/presets/fleet/brains/claude-default.yaml +5 -0
- package/presets/fleet/roles/Agent.yaml +6 -0
- package/presets/fleet/roles/Architect.yaml +6 -0
- package/presets/fleet/roles/Critic.yaml +6 -0
- package/presets/fleet/roles/Developer.yaml +6 -0
- package/presets/fleet/roles/Secretary.yaml +6 -0
- package/presets/fleet/roles/Tester.yaml +6 -0
- package/presets/fleet/room_templates/pair.yaml +11 -0
- package/presets/fleet/room_templates/single.yaml +8 -0
- package/presets/fleet/room_templates/team.yaml +12 -0
- package/presets/fleet.yaml +7 -0
- package/presets/manifest.json +5 -0
- package/dist/session/tmux.d.ts +0 -28
- package/dist/session/tmux.js +0 -80
- package/dist/web/terminal/bridge.d.ts +0 -27
- package/dist/web/terminal/bridge.js +0 -317
- package/dist/web-app/assets/TerminalView-C_G1ID2P.js +0 -9
- package/dist/web-app/assets/index-BCBK78hw.js +0 -10
- package/dist/web-app/assets/index-DuC-xnX4.css +0 -1
|
@@ -4,13 +4,9 @@ export interface ResourceArgs {
|
|
|
4
4
|
warnings: string[];
|
|
5
5
|
}
|
|
6
6
|
/**
|
|
7
|
-
* Build the `systemd-run --user --scope -p … --` prefix that caps the
|
|
8
|
-
* cgroup-v2 scope.
|
|
9
|
-
*
|
|
10
|
-
* only reliable per-agent limit is a transient scope at the pane itself. (Since
|
|
11
|
-
* #32 that server is per role rather than fleet-wide, which is what keeps one
|
|
12
|
-
* role's `stop` off every other role's pane — the limit still belongs on the
|
|
13
|
-
* pane.)
|
|
7
|
+
* Build the `systemd-run --user --scope -p … --` prefix that caps the agent
|
|
8
|
+
* process's cgroup-v2 scope. It composes outside the sandbox wrapper so both
|
|
9
|
+
* the adapter and its sandboxed child remain inside the transient scope.
|
|
14
10
|
*
|
|
15
11
|
* mem/pids are always enforced (their controllers are delegated to `--user` by
|
|
16
12
|
* default). cpu degrades to a warning when the cpu controller is not delegated.
|
|
@@ -1,12 +1,8 @@
|
|
|
1
1
|
import { readFileSync } from 'node:fs';
|
|
2
2
|
/**
|
|
3
|
-
* Build the `systemd-run --user --scope -p … --` prefix that caps the
|
|
4
|
-
* cgroup-v2 scope.
|
|
5
|
-
*
|
|
6
|
-
* only reliable per-agent limit is a transient scope at the pane itself. (Since
|
|
7
|
-
* #32 that server is per role rather than fleet-wide, which is what keeps one
|
|
8
|
-
* role's `stop` off every other role's pane — the limit still belongs on the
|
|
9
|
-
* pane.)
|
|
3
|
+
* Build the `systemd-run --user --scope -p … --` prefix that caps the agent
|
|
4
|
+
* process's cgroup-v2 scope. It composes outside the sandbox wrapper so both
|
|
5
|
+
* the adapter and its sandboxed child remain inside the transient scope.
|
|
10
6
|
*
|
|
11
7
|
* mem/pids are always enforced (their controllers are delegated to `--user` by
|
|
12
8
|
* default). cpu degrades to a warning when the cpu controller is not delegated.
|
package/dist/model-env.d.ts
CHANGED
|
@@ -3,22 +3,20 @@ import type { ResolvedRole } from './config.js';
|
|
|
3
3
|
* Which environment variable a harness reads to pin the model it RUNS.
|
|
4
4
|
*
|
|
5
5
|
* This is not a convenience: for `claude-code` it is the only channel that
|
|
6
|
-
* reaches the
|
|
7
|
-
*
|
|
8
|
-
* adapter resolves its model in this order — ANTHROPIC_MODEL, then
|
|
6
|
+
* reaches the Claude Code ACP adapter. The adapter resolves its model in this
|
|
7
|
+
* order — ANTHROPIC_MODEL, then
|
|
9
8
|
* `settings.model`, then a resumed session's live model, then its first
|
|
10
9
|
* catalogue entry. A role's declared model was therefore invisible to every
|
|
11
10
|
* ACP role, and a fleet-wide `defaults.env.ANTHROPIC_MODEL` silently outranked
|
|
12
11
|
* an explicitly requested one.
|
|
13
12
|
*/
|
|
14
|
-
export declare const MODEL_ENV_BY_HARNESS: Readonly<Record<string, string>>;
|
|
15
13
|
/** The model-pin variable for a harness, or undefined if it pins no model by env. */
|
|
16
14
|
export declare function modelEnvVar(harness: string | undefined): string | undefined;
|
|
17
15
|
export interface RoleModelEnvInput {
|
|
18
16
|
harness: string;
|
|
19
17
|
/** Already resolved by `resolveRoleModel` — may come from the fleet default. */
|
|
20
18
|
model: string | undefined;
|
|
21
|
-
/** True when the
|
|
19
|
+
/** True when the resolved Brain named a model, including `model: null`. */
|
|
22
20
|
modelWasExplicit: boolean;
|
|
23
21
|
defaultsEnv?: Record<string, string>;
|
|
24
22
|
roleEnv?: Record<string, string>;
|
|
@@ -37,7 +35,7 @@ export interface RoleModelEnv {
|
|
|
37
35
|
* never disagree.
|
|
38
36
|
*
|
|
39
37
|
* Precedence, highest first:
|
|
40
|
-
* 1. an explicit `model:`
|
|
38
|
+
* 1. an explicit `model:` on the resolved Brain
|
|
41
39
|
* 2. the role's own `env:` pin
|
|
42
40
|
* 3. the fleet `defaults.model`
|
|
43
41
|
* 4. the fleet `defaults.env` pin
|
package/dist/model-env.js
CHANGED
|
@@ -1,28 +1,25 @@
|
|
|
1
|
+
import { getAdapter } from './harness/registry.js';
|
|
1
2
|
/**
|
|
2
3
|
* Which environment variable a harness reads to pin the model it RUNS.
|
|
3
4
|
*
|
|
4
5
|
* This is not a convenience: for `claude-code` it is the only channel that
|
|
5
|
-
* reaches the
|
|
6
|
-
*
|
|
7
|
-
* adapter resolves its model in this order — ANTHROPIC_MODEL, then
|
|
6
|
+
* reaches the Claude Code ACP adapter. The adapter resolves its model in this
|
|
7
|
+
* order — ANTHROPIC_MODEL, then
|
|
8
8
|
* `settings.model`, then a resumed session's live model, then its first
|
|
9
9
|
* catalogue entry. A role's declared model was therefore invisible to every
|
|
10
10
|
* ACP role, and a fleet-wide `defaults.env.ANTHROPIC_MODEL` silently outranked
|
|
11
11
|
* an explicitly requested one.
|
|
12
12
|
*/
|
|
13
|
-
export const MODEL_ENV_BY_HARNESS = {
|
|
14
|
-
'claude-code': 'ANTHROPIC_MODEL',
|
|
15
|
-
};
|
|
16
13
|
/** The model-pin variable for a harness, or undefined if it pins no model by env. */
|
|
17
14
|
export function modelEnvVar(harness) {
|
|
18
|
-
return harness === undefined ? undefined :
|
|
15
|
+
return harness === undefined ? undefined : getAdapter(harness).agentSession.modelEnvironmentVariable();
|
|
19
16
|
}
|
|
20
17
|
/**
|
|
21
18
|
* Resolve a role's environment and its runtime model TOGETHER, so the two can
|
|
22
19
|
* never disagree.
|
|
23
20
|
*
|
|
24
21
|
* Precedence, highest first:
|
|
25
|
-
* 1. an explicit `model:`
|
|
22
|
+
* 1. an explicit `model:` on the resolved Brain
|
|
26
23
|
* 2. the role's own `env:` pin
|
|
27
24
|
* 3. the fleet `defaults.model`
|
|
28
25
|
* 4. the fleet `defaults.env` pin
|
package/dist/monitor.d.ts
CHANGED
|
@@ -26,15 +26,8 @@ export type FetchLike = (url: string, init?: {
|
|
|
26
26
|
headers?: Record<string, string>;
|
|
27
27
|
signal?: AbortSignal;
|
|
28
28
|
}) => Promise<FetchResponse>;
|
|
29
|
-
export interface MonitorTmux {
|
|
30
|
-
has(name: string): Promise<boolean>;
|
|
31
|
-
capture(name: string, lines?: number): Promise<string>;
|
|
32
|
-
sendText(name: string, text: string): Promise<void>;
|
|
33
|
-
sendKey(name: string, key: string): Promise<void>;
|
|
34
|
-
}
|
|
35
29
|
export interface MonitorDeps {
|
|
36
30
|
fetch: FetchLike;
|
|
37
|
-
tmux: MonitorTmux;
|
|
38
31
|
isAlive(pid: number): boolean;
|
|
39
32
|
sleep(ms: number): Promise<void>;
|
|
40
33
|
now(): number;
|
|
@@ -45,7 +38,7 @@ export interface MonitorDeps {
|
|
|
45
38
|
clear(t: ReturnType<typeof setTimeout>): void;
|
|
46
39
|
};
|
|
47
40
|
/**
|
|
48
|
-
* Structured prompt delivery used by
|
|
41
|
+
* Structured prompt delivery used by agent sessions.
|
|
49
42
|
* `succeeded` is the turn's TERMINAL result, not merely that the session took
|
|
50
43
|
* the prompt: a refused or cancelled wake was seen and not acted on, and must
|
|
51
44
|
* not commit the cursor.
|
|
@@ -208,14 +201,6 @@ export declare class Monitor {
|
|
|
208
201
|
/** Gather stragglers arriving within batch_ms so a burst lands as one line. */
|
|
209
202
|
private coalesce;
|
|
210
203
|
private deliver;
|
|
211
|
-
/**
|
|
212
|
-
* Watch the pane until the just-triggered turn settles, then fold its outcome
|
|
213
|
-
* into the API-error streak and republish `.monitor-status`. A completed turn
|
|
214
|
-
* (or an inconclusive give-up) resets/keeps the streak; an `API Error:` tail
|
|
215
|
-
* grows it. Once the streak reaches the threshold the status degrades; a later
|
|
216
|
-
* completed turn flips it back to armed. Detection only — no remediation (#19).
|
|
217
|
-
*/
|
|
218
|
-
private observeTurnOutcome;
|
|
219
204
|
/** Update the consecutive-API-error streak and derive `.monitor-status` from it. */
|
|
220
205
|
private recordTurn;
|
|
221
206
|
/**
|
|
@@ -226,14 +211,6 @@ export declare class Monitor {
|
|
|
226
211
|
* the composer is cleared by hand. Best-effort: a dead pane just makes the keys
|
|
227
212
|
* no-ops (delivery is still verified downstream).
|
|
228
213
|
*/
|
|
229
|
-
private clearComposer;
|
|
230
|
-
/**
|
|
231
|
-
* Block until the console can accept input; classify offline/stopped/ready, or
|
|
232
|
-
* `modal` when the pane still looks modal after `MODAL_GIVE_UP_MS`. The bound is
|
|
233
|
-
* what keeps a modal from wedging delivery silently: we still never `Enter` into
|
|
234
|
-
* the dialog, but the give-up is reported instead of retried forever.
|
|
235
|
-
*/
|
|
236
|
-
private awaitInjectable;
|
|
237
214
|
private doFetch;
|
|
238
215
|
private advance;
|
|
239
216
|
private persistCursor;
|
package/dist/monitor.js
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import { existsSync, readFileSync, renameSync, writeFileSync } from 'node:fs';
|
|
2
2
|
import { homedir } from 'node:os';
|
|
3
3
|
import { join } from 'node:path';
|
|
4
|
-
import { classifyFailureText } from './model-recovery.js';
|
|
5
4
|
// Code constants rather than user configuration.
|
|
6
5
|
const DEFAULT_PORT = 3050;
|
|
7
6
|
// The daemon normally holds for 25s, but that value is operator-configurable
|
|
@@ -522,7 +521,11 @@ export class Monitor {
|
|
|
522
521
|
}
|
|
523
522
|
async deliver(pid, batch) {
|
|
524
523
|
const line = formatNotificationLine(batch);
|
|
525
|
-
if (this.deps.delivery) {
|
|
524
|
+
if (!this.deps.delivery) {
|
|
525
|
+
this.degrade('delivery', 'structured agent-session delivery is unavailable');
|
|
526
|
+
return false;
|
|
527
|
+
}
|
|
528
|
+
{
|
|
526
529
|
const result = await this.deps.delivery.submit(line, { interrupt: this.cfg.interrupt });
|
|
527
530
|
if (!result.succeeded) {
|
|
528
531
|
// Name the reason: "refused" and "cancelled" are the agent's answer,
|
|
@@ -540,101 +543,6 @@ export class Monitor {
|
|
|
540
543
|
this.recordTurn('completed');
|
|
541
544
|
return true;
|
|
542
545
|
}
|
|
543
|
-
// Tmux exposes no authenticated tool lifecycle. `after_tool` therefore
|
|
544
|
-
// degrades to the existing non-cancelling injection path; never guess a
|
|
545
|
-
// boundary from pane text and never send C-c for this mode.
|
|
546
|
-
if (this.cfg.interrupt === true)
|
|
547
|
-
await this.deps.tmux.sendKey(this.name, 'C-c');
|
|
548
|
-
if (this.cfg.interrupt === 'after_tool')
|
|
549
|
-
this.degrade('safe-boundary', 'after_tool unsupported by tmux; using non-cancelling delivery');
|
|
550
|
-
const state = await this.awaitInjectable(pid);
|
|
551
|
-
if (state !== 'ready') {
|
|
552
|
-
if (state === 'offline')
|
|
553
|
-
this.degrade('offline', 'offline during delivery');
|
|
554
|
-
else if (state === 'modal')
|
|
555
|
-
this.degrade('modal', `modal wedge — pane held a dialog for ` +
|
|
556
|
-
`${MODAL_GIVE_UP_MS / 1000}s, wake not injected`);
|
|
557
|
-
return false;
|
|
558
|
-
}
|
|
559
|
-
await this.clearComposer(); // start from an empty composer
|
|
560
|
-
await this.deps.tmux.sendText(this.name, line); // send-keys -l + Enter
|
|
561
|
-
let delivered = false;
|
|
562
|
-
// Verify submission for THIS line even if stop() arrives mid-flight: the text
|
|
563
|
-
// is already in the composer and we want it submitted (at-least-once). A truly
|
|
564
|
-
// dead pane makes safeCapture return '' ⇒ not-in-composer ⇒ breaks, no wasted Enter.
|
|
565
|
-
for (let i = 0; i < MAX_ENTER_RETRIES;) {
|
|
566
|
-
await this.deps.sleep(POST_VERIFY_MS);
|
|
567
|
-
const capture = await safeCapture(this.deps.tmux, this.name);
|
|
568
|
-
if (!capture.ok) {
|
|
569
|
-
this.degrade('delivery', 'capture failed during injection verification');
|
|
570
|
-
return false;
|
|
571
|
-
}
|
|
572
|
-
// A dialog can appear after the initial send. Never let a verification
|
|
573
|
-
// retry confirm it. Wait under the same bounded modal policy as initial
|
|
574
|
-
// injection, then re-capture immediately before considering Enter.
|
|
575
|
-
if (looksModal(capture.pane)) {
|
|
576
|
-
const state = await this.awaitInjectable(pid);
|
|
577
|
-
if (state === 'ready')
|
|
578
|
-
continue;
|
|
579
|
-
if (state === 'offline')
|
|
580
|
-
this.degrade('offline', 'offline during injection verification');
|
|
581
|
-
else if (state === 'modal')
|
|
582
|
-
this.degrade('modal', `modal wedge during injection verification — ` +
|
|
583
|
-
`no Enter sent for ${MODAL_GIVE_UP_MS / 1000}s`);
|
|
584
|
-
return false;
|
|
585
|
-
}
|
|
586
|
-
if (!stillInComposer(capture.pane, line)) {
|
|
587
|
-
delivered = true;
|
|
588
|
-
break;
|
|
589
|
-
}
|
|
590
|
-
await this.deps.tmux.sendKey(this.name, 'Enter');
|
|
591
|
-
i++;
|
|
592
|
-
}
|
|
593
|
-
if (!delivered) {
|
|
594
|
-
this.degrade('delivery', 'injection unverified');
|
|
595
|
-
return false;
|
|
596
|
-
}
|
|
597
|
-
this.recover('delivery', 'modal');
|
|
598
|
-
// The wake landed and a turn started; observe how that turn terminates so a
|
|
599
|
-
// refusal-wedge (every turn dies with `API Error:` while delivery stays green)
|
|
600
|
-
// becomes visible in `.monitor-status` instead of masquerading as armed (#19).
|
|
601
|
-
const recoveryTriggered = await this.observeTurnOutcome(pid);
|
|
602
|
-
return !recoveryTriggered;
|
|
603
|
-
}
|
|
604
|
-
/**
|
|
605
|
-
* Watch the pane until the just-triggered turn settles, then fold its outcome
|
|
606
|
-
* into the API-error streak and republish `.monitor-status`. A completed turn
|
|
607
|
-
* (or an inconclusive give-up) resets/keeps the streak; an `API Error:` tail
|
|
608
|
-
* grows it. Once the streak reaches the threshold the status degrades; a later
|
|
609
|
-
* completed turn flips it back to armed. Detection only — no remediation (#19).
|
|
610
|
-
*/
|
|
611
|
-
async observeTurnOutcome(pid) {
|
|
612
|
-
for (let i = 0; i < TURN_OBSERVE_POLLS; i++) {
|
|
613
|
-
if (this.stopped)
|
|
614
|
-
return false; // shutting down — leave status
|
|
615
|
-
if (!this.deps.isAlive(pid) || !(await this.deps.tmux.has(this.name)))
|
|
616
|
-
return false; // loop marks offline
|
|
617
|
-
const capture = await safeCapture(this.deps.tmux, this.name);
|
|
618
|
-
if (!capture.ok) {
|
|
619
|
-
this.degrade('delivery', 'capture failed during turn observation');
|
|
620
|
-
return false;
|
|
621
|
-
}
|
|
622
|
-
if (looksApiError(capture.pane)) {
|
|
623
|
-
const evidence = classifyFailureText(capture.pane);
|
|
624
|
-
const recoveryTriggered = evidence
|
|
625
|
-
? this.deps.onFailureEvidence?.(evidence) === true
|
|
626
|
-
: false;
|
|
627
|
-
this.recordTurn('api-error');
|
|
628
|
-
return recoveryTriggered;
|
|
629
|
-
}
|
|
630
|
-
if (!looksRunning(capture.pane)) {
|
|
631
|
-
this.recordTurn('completed');
|
|
632
|
-
return false;
|
|
633
|
-
}
|
|
634
|
-
await this.deps.sleep(TURN_OBSERVE_INTERVAL_MS);
|
|
635
|
-
}
|
|
636
|
-
this.recordTurn('inconclusive'); // still running at give-up: hold the streak, don't re-arm
|
|
637
|
-
return false;
|
|
638
546
|
}
|
|
639
547
|
/** Update the consecutive-API-error streak and derive `.monitor-status` from it. */
|
|
640
548
|
recordTurn(outcome) {
|
|
@@ -658,41 +566,6 @@ export class Monitor {
|
|
|
658
566
|
* the composer is cleared by hand. Best-effort: a dead pane just makes the keys
|
|
659
567
|
* no-ops (delivery is still verified downstream).
|
|
660
568
|
*/
|
|
661
|
-
async clearComposer() {
|
|
662
|
-
for (const key of COMPOSER_CLEAR_KEYS)
|
|
663
|
-
await this.deps.tmux.sendKey(this.name, key);
|
|
664
|
-
}
|
|
665
|
-
/**
|
|
666
|
-
* Block until the console can accept input; classify offline/stopped/ready, or
|
|
667
|
-
* `modal` when the pane still looks modal after `MODAL_GIVE_UP_MS`. The bound is
|
|
668
|
-
* what keeps a modal from wedging delivery silently: we still never `Enter` into
|
|
669
|
-
* the dialog, but the give-up is reported instead of retried forever.
|
|
670
|
-
*/
|
|
671
|
-
async awaitInjectable(pid) {
|
|
672
|
-
let modalWaits = 0;
|
|
673
|
-
for (;;) {
|
|
674
|
-
if (this.stopped)
|
|
675
|
-
return 'stopped';
|
|
676
|
-
if (!this.deps.isAlive(pid) || !(await this.deps.tmux.has(this.name)))
|
|
677
|
-
return 'offline';
|
|
678
|
-
const now = this.deps.now();
|
|
679
|
-
if (now < this.bootDeadline) {
|
|
680
|
-
await this.deps.sleep(this.bootDeadline - now);
|
|
681
|
-
continue;
|
|
682
|
-
}
|
|
683
|
-
const capture = await safeCapture(this.deps.tmux, this.name);
|
|
684
|
-
if (!capture.ok) {
|
|
685
|
-
this.degrade('delivery', 'capture failed while checking session readiness');
|
|
686
|
-
await this.deps.sleep(MODAL_RETRY_MS);
|
|
687
|
-
continue;
|
|
688
|
-
}
|
|
689
|
-
if (!looksModal(capture.pane))
|
|
690
|
-
return 'ready';
|
|
691
|
-
if (modalWaits++ >= MAX_MODAL_WAITS)
|
|
692
|
-
return 'modal';
|
|
693
|
-
await this.deps.sleep(MODAL_RETRY_MS);
|
|
694
|
-
}
|
|
695
|
-
}
|
|
696
569
|
async doFetch(since, timeoutMs, timeoutKind) {
|
|
697
570
|
const ctrl = new AbortController();
|
|
698
571
|
this.currentAbort = ctrl;
|
|
@@ -820,12 +693,4 @@ export class Monitor {
|
|
|
820
693
|
export function createMonitor(o) {
|
|
821
694
|
return new Monitor(o);
|
|
822
695
|
}
|
|
823
|
-
async function safeCapture(tmux, name) {
|
|
824
|
-
try {
|
|
825
|
-
return { ok: true, pane: await tmux.capture(name) };
|
|
826
|
-
}
|
|
827
|
-
catch {
|
|
828
|
-
return { ok: false, pane: '' };
|
|
829
|
-
}
|
|
830
|
-
}
|
|
831
696
|
const msg = (e) => e?.message ?? String(e);
|
package/dist/ops.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { chmodSync, existsSync, mkdirSync, readFileSync, writeFileSync, rmSync, unlinkSync, } from 'node:fs';
|
|
2
2
|
import { join } from 'node:path';
|
|
3
3
|
import { randomUUID, createHash } from 'node:crypto';
|
|
4
|
-
import { agentDir,
|
|
4
|
+
import { agentDir, watchdogsRoot } from './paths.js';
|
|
5
5
|
import { findRole } from './config.js';
|
|
6
6
|
import { getAdapter } from './harness/registry.js';
|
|
7
7
|
import { generateBriefing } from './briefing.js';
|
|
@@ -9,6 +9,7 @@ import { resetRestartLedger } from './runner.js';
|
|
|
9
9
|
import { archiveTempState, stopTempSupervisor, tempSupervisorLiveness, } from './temp-lifecycle.js';
|
|
10
10
|
import { realExec } from './exec.js';
|
|
11
11
|
import { daemonIdentityProvisioner, reconcilePermanentRoleIdentities, } from './creation.js';
|
|
12
|
+
import { provesGeneratedAgentSource } from './generated-agent-source.js';
|
|
12
13
|
// Launch staggering now lives at the harness-launch point (the runner's start
|
|
13
14
|
// gate, driven by `start_stagger_ms`), so it covers systemd host-boot too — not
|
|
14
15
|
// just the `up`/`restart` command loop below. The old in-loop FLEET_START_STAGGER
|
|
@@ -264,9 +265,10 @@ export async function rmRole(cfg, name, deps) {
|
|
|
264
265
|
return;
|
|
265
266
|
}
|
|
266
267
|
const role = findRole(cfg, name);
|
|
268
|
+
const generatedSource = provesGeneratedAgentSource(agentDir(name), cfg.files[0], role.sourceFile);
|
|
267
269
|
await deps.backend.uninstall(name);
|
|
268
270
|
rmSync(agentDir(name), { recursive: true, force: true });
|
|
269
|
-
if (role.sourceFile
|
|
271
|
+
if (generatedSource && existsSync(role.sourceFile)) {
|
|
270
272
|
unlinkSync(role.sourceFile);
|
|
271
273
|
deps.log(`removed ${role.sourceFile}`);
|
|
272
274
|
}
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import { type OwnerChannelConfig } from '../config.js';
|
|
2
|
-
import { type
|
|
2
|
+
import { type AgentSession } from '../session/types.js';
|
|
3
3
|
import { type OwnerFleetOps } from './commands.js';
|
|
4
4
|
import type { ManagedFleetSpawnResult } from '../fleet-proxy.js';
|
|
5
|
+
import { type FleetAuditAttempt, type FleetAuditPresentation, type FleetCommandOutcomeClass } from '../fleet-command-audit.js';
|
|
5
6
|
import { type OursOps } from './ours-client.js';
|
|
6
7
|
import { type OwnerUpdatePhase } from './notices.js';
|
|
7
8
|
import { type OwnerEntry } from './state.js';
|
|
@@ -12,7 +13,7 @@ export interface OwnerChannelOptions {
|
|
|
12
13
|
/** Harness id of the role (e.g. 'claude-code', 'codex'); gates which slash commands may be forwarded. */
|
|
13
14
|
harness: string;
|
|
14
15
|
config: OwnerChannelConfig;
|
|
15
|
-
session:
|
|
16
|
+
session: AgentSession;
|
|
16
17
|
stateDir: string;
|
|
17
18
|
env?: Record<string, string>;
|
|
18
19
|
log(line: string): void;
|
|
@@ -27,14 +28,44 @@ export interface OwnerChannelOptions {
|
|
|
27
28
|
binderDeps?: OwnerBinderDeps;
|
|
28
29
|
/** Pre-acquired by the runner so the predecessor control socket remains reachable while waiting. */
|
|
29
30
|
binderLease?: OwnerBinderLease;
|
|
31
|
+
recoveryDeps?: {
|
|
32
|
+
now(): number;
|
|
33
|
+
setTimer(fn: () => void, ms: number): ReturnType<typeof setTimeout>;
|
|
34
|
+
clearTimer(timer: ReturnType<typeof setTimeout>): void;
|
|
35
|
+
deadlineMs?: number;
|
|
36
|
+
};
|
|
30
37
|
}
|
|
31
38
|
export interface OwnerChannelHandle {
|
|
32
39
|
start(): Promise<void>;
|
|
33
40
|
drain(): Promise<void>;
|
|
34
41
|
close(): Promise<void>;
|
|
42
|
+
/** Reattach this same supervisor-owned channel after a daemon generation change. */
|
|
43
|
+
recover?(epoch: string): Promise<void>;
|
|
44
|
+
/** False when shutdown skipped client disposal because quiescence was unproven. */
|
|
45
|
+
binderReleaseSafe?(): boolean;
|
|
35
46
|
manage(request: OwnerChannelManagementRequest): Promise<OwnerChannelManagementResult>;
|
|
36
47
|
/** Fleet-owned deterministic lifecycle notice; absent on legacy test doubles. */
|
|
37
48
|
notifyFleetSpawn?(event: ManagedFleetSpawnResult): Promise<void>;
|
|
49
|
+
beginFleetCommandAudit?(requestId: string, argv: string[]): Promise<FleetAuditAttempt>;
|
|
50
|
+
finishFleetCommandAudit?(input: {
|
|
51
|
+
correlationId: string;
|
|
52
|
+
class: FleetCommandOutcomeClass;
|
|
53
|
+
exitCode?: number;
|
|
54
|
+
effect: 'not_started' | 'completed' | 'unknown';
|
|
55
|
+
resourceIds?: Record<string, string>;
|
|
56
|
+
presentations?: FleetAuditPresentation[];
|
|
57
|
+
}): Promise<FleetAuditAttempt>;
|
|
58
|
+
}
|
|
59
|
+
export declare const OWNER_RECOVERY_QUEUE_CAPACITY = 32;
|
|
60
|
+
export declare const OWNER_RECOVERY_DEGRADED = "OWNER_RECOVERY_DEGRADED";
|
|
61
|
+
export declare class OwnerRecoveryDegradedError extends Error {
|
|
62
|
+
readonly code = "OWNER_RECOVERY_DEGRADED";
|
|
63
|
+
constructor();
|
|
64
|
+
}
|
|
65
|
+
export declare class OwnerRecoveryTimeoutError extends Error {
|
|
66
|
+
readonly stage: string;
|
|
67
|
+
readonly code = "OWNER_RECOVERY_DEGRADED";
|
|
68
|
+
constructor(stage: string);
|
|
38
69
|
}
|
|
39
70
|
export type OwnerChannelManagementRequest = {
|
|
40
71
|
action: 'contact_list';
|
|
@@ -162,17 +193,42 @@ export declare class OwnerChannel implements OwnerChannelHandle {
|
|
|
162
193
|
private readonly completionTasks;
|
|
163
194
|
private readonly activeRequests;
|
|
164
195
|
private managementTail;
|
|
196
|
+
private recoveryEpoch?;
|
|
197
|
+
private recoveryTask?;
|
|
198
|
+
private recoveryToken;
|
|
199
|
+
private recoveryQueued;
|
|
200
|
+
private watchGeneration;
|
|
201
|
+
/** A timed-out client operation that must settle before replacement is safe. */
|
|
202
|
+
private recoveryQuiescence?;
|
|
203
|
+
private releaseSafe;
|
|
165
204
|
private ready;
|
|
205
|
+
private startedOnce;
|
|
206
|
+
private shutdownDegraded;
|
|
166
207
|
private binder?;
|
|
167
208
|
private binderOwnedInternally;
|
|
168
209
|
private readonly fleetOps;
|
|
169
210
|
private readonly prepareRestart;
|
|
211
|
+
private readonly commandAudits;
|
|
170
212
|
constructor(options: OwnerChannelOptions);
|
|
171
213
|
start(): Promise<void>;
|
|
172
214
|
drain(): Promise<void>;
|
|
173
215
|
close(): Promise<void>;
|
|
216
|
+
binderReleaseSafe(): boolean;
|
|
174
217
|
manage(request: OwnerChannelManagementRequest): Promise<OwnerChannelManagementResult>;
|
|
175
218
|
notifyFleetSpawn(event: ManagedFleetSpawnResult): Promise<void>;
|
|
219
|
+
beginFleetCommandAudit(requestId: string, argv: string[]): Promise<FleetAuditAttempt>;
|
|
220
|
+
finishFleetCommandAudit(input: {
|
|
221
|
+
correlationId: string;
|
|
222
|
+
class: FleetCommandOutcomeClass;
|
|
223
|
+
exitCode?: number;
|
|
224
|
+
effect: 'not_started' | 'completed' | 'unknown';
|
|
225
|
+
resourceIds?: Record<string, string>;
|
|
226
|
+
presentations?: FleetAuditPresentation[];
|
|
227
|
+
}): Promise<FleetAuditAttempt>;
|
|
228
|
+
recover(epoch: string): Promise<void>;
|
|
229
|
+
private recoveryStage;
|
|
230
|
+
private writeShutdownState;
|
|
231
|
+
private queueManagement;
|
|
176
232
|
private manageNow;
|
|
177
233
|
/**
|
|
178
234
|
* The daemon reports established contacts and pending introductions as two
|