@ours.network/fleet 0.9.0 → 0.9.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/config.d.ts +7 -0
- package/dist/config.js +7 -1
- package/dist/doctor.js +31 -13
- package/dist/harness/claude-code.js +19 -3
- package/dist/monitor.d.ts +57 -3
- package/dist/monitor.js +153 -6
- package/dist/runner.js +6 -4
- package/package.json +1 -1
package/dist/config.d.ts
CHANGED
|
@@ -13,6 +13,13 @@ export interface MonitorConfig {
|
|
|
13
13
|
wake_sources: string[];
|
|
14
14
|
batch_ms: number;
|
|
15
15
|
inject: InjectMode;
|
|
16
|
+
/**
|
|
17
|
+
* Consecutive delivered wakes that must end in an `API Error:`-terminated turn
|
|
18
|
+
* (with no completed turn in between) before `.monitor-status` degrades to
|
|
19
|
+
* `turns failing (api error)` — the refusal-wedge detector (issue #19). Must be
|
|
20
|
+
* a positive integer; resolved default is 3. Optional so old snapshots resolve.
|
|
21
|
+
*/
|
|
22
|
+
turn_fail_threshold?: number;
|
|
16
23
|
}
|
|
17
24
|
/** Default wake sources when a role does not list its own (design §2). */
|
|
18
25
|
export declare const DEFAULT_WAKE_SOURCES: NotifyEventType[];
|
package/dist/config.js
CHANGED
|
@@ -10,9 +10,10 @@ export const NOTIFY_EVENT_TYPES = [
|
|
|
10
10
|
];
|
|
11
11
|
/** Default wake sources when a role does not list its own (design §2). */
|
|
12
12
|
export const DEFAULT_WAKE_SOURCES = ['message_received', 'file_received', 'local_contact_request', 'pending_message'];
|
|
13
|
-
const MONITOR_KEYS = ['enabled', 'wake_sources', 'batch_ms', 'inject'];
|
|
13
|
+
const MONITOR_KEYS = ['enabled', 'wake_sources', 'batch_ms', 'inject', 'turn_fail_threshold'];
|
|
14
14
|
const INJECT_MODES = ['notification', 'full'];
|
|
15
15
|
const MONITOR_DEFAULT_BATCH_MS = 2000;
|
|
16
|
+
const MONITOR_DEFAULT_TURN_FAIL_THRESHOLD = 3;
|
|
16
17
|
const isPlainObject = (v) => v !== null && typeof v === 'object' && !Array.isArray(v);
|
|
17
18
|
/** Validate a raw (role-level or merged) `monitor:` block; returns human-readable problems. */
|
|
18
19
|
export function validateMonitorConfig(raw) {
|
|
@@ -30,6 +31,10 @@ export function validateMonitorConfig(raw) {
|
|
|
30
31
|
problems.push('monitor.batch_ms: must be a non-negative number');
|
|
31
32
|
if (m.inject !== undefined && !INJECT_MODES.includes(m.inject))
|
|
32
33
|
problems.push(`monitor.inject: invalid value '${m.inject}'; allowed: ${INJECT_MODES.join(', ')}`);
|
|
34
|
+
if (m.turn_fail_threshold !== undefined
|
|
35
|
+
&& (typeof m.turn_fail_threshold !== 'number' || !Number.isInteger(m.turn_fail_threshold)
|
|
36
|
+
|| m.turn_fail_threshold < 1))
|
|
37
|
+
problems.push('monitor.turn_fail_threshold: must be a positive integer');
|
|
33
38
|
if (m.wake_sources !== undefined) {
|
|
34
39
|
if (!Array.isArray(m.wake_sources))
|
|
35
40
|
problems.push('monitor.wake_sources: must be a list');
|
|
@@ -168,6 +173,7 @@ export function resolveMonitorConfig(defMonitor, roleMonitor, labels = {}) {
|
|
|
168
173
|
wake_sources: merged.wake_sources ?? [...DEFAULT_WAKE_SOURCES],
|
|
169
174
|
batch_ms: merged.batch_ms ?? MONITOR_DEFAULT_BATCH_MS,
|
|
170
175
|
inject: merged.inject ?? 'notification',
|
|
176
|
+
turn_fail_threshold: merged.turn_fail_threshold ?? MONITOR_DEFAULT_TURN_FAIL_THRESHOLD,
|
|
171
177
|
};
|
|
172
178
|
}
|
|
173
179
|
export function findRole(cfg, name) {
|
package/dist/doctor.js
CHANGED
|
@@ -6,6 +6,7 @@ import { getAdapter } from './harness/registry.js';
|
|
|
6
6
|
import { agentDir, home, deriveXdgRuntimeDir } from './paths.js';
|
|
7
7
|
import { resolveIsolation } from './isolation/policy.js';
|
|
8
8
|
import { makeBubblewrapBackend } from './isolation/bubblewrap.js';
|
|
9
|
+
import { authResolutionHint, resolveEndpoint, } from './monitor.js';
|
|
9
10
|
/** Which cgroup-v2 controllers are delegated to this user manager (advisory). */
|
|
10
11
|
function cgroupDelegationDetail() {
|
|
11
12
|
try {
|
|
@@ -19,6 +20,21 @@ function cgroupDelegationDetail() {
|
|
|
19
20
|
return 'unknown (not cgroup-v2 or no delegation info)';
|
|
20
21
|
}
|
|
21
22
|
}
|
|
23
|
+
/** Resolve and deduplicate the effective daemon profiles used by monitored roles. */
|
|
24
|
+
function resolveMonitorProfiles(roles) {
|
|
25
|
+
const profiles = [];
|
|
26
|
+
for (const role of roles.filter(r => r.monitor?.enabled)) {
|
|
27
|
+
const endpoint = resolveEndpoint({ ...process.env, ...(role.env ?? {}) });
|
|
28
|
+
const token = endpoint.headers['x-ours-api-token'];
|
|
29
|
+
const existing = profiles.find(p => p.endpoint.origin === endpoint.origin
|
|
30
|
+
&& p.endpoint.headers['x-ours-api-token'] === token);
|
|
31
|
+
if (existing)
|
|
32
|
+
existing.roles.push(role.name);
|
|
33
|
+
else
|
|
34
|
+
profiles.push({ endpoint, roles: [role.name] });
|
|
35
|
+
}
|
|
36
|
+
return profiles;
|
|
37
|
+
}
|
|
22
38
|
/** Host-level + per-harness prerequisite report with actionable messages. */
|
|
23
39
|
export async function doctor(opts = {}, exec = realExec, platform = process.platform, fetchImpl = (u, i) => globalThis.fetch(u, i)) {
|
|
24
40
|
const checks = [];
|
|
@@ -110,34 +126,36 @@ export async function doctor(opts = {}, exec = realExec, platform = process.plat
|
|
|
110
126
|
// Monitor daemon-API reachability (design §5): only when a role is supervised.
|
|
111
127
|
// /state-dir is unauthenticated (liveness); /identities exercises the token so a
|
|
112
128
|
// shared-mode misconfig (401) surfaces here rather than as a silent deaf monitor.
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
const
|
|
116
|
-
const
|
|
129
|
+
const monitorProfiles = resolveMonitorProfiles(roles);
|
|
130
|
+
for (const profile of monitorProfiles) {
|
|
131
|
+
const { endpoint } = profile;
|
|
132
|
+
const checkName = monitorProfiles.length === 1
|
|
133
|
+
? 'monitor: daemon API'
|
|
134
|
+
: `monitor: daemon API (${profile.roles.join(', ')})`;
|
|
117
135
|
let ok = false, detail;
|
|
118
136
|
try {
|
|
119
|
-
const live = await fetchImpl(
|
|
137
|
+
const live = await fetchImpl(`${endpoint.origin}/state-dir`, {});
|
|
120
138
|
if (!live.ok) {
|
|
121
|
-
detail = `daemon on :${port} answered /state-dir with HTTP ${live.status} — not the ours daemon?`;
|
|
139
|
+
detail = `daemon on :${endpoint.port} answered /state-dir with HTTP ${live.status} — not the ours daemon?`;
|
|
122
140
|
}
|
|
123
141
|
else {
|
|
124
|
-
const auth = await fetchImpl(
|
|
142
|
+
const auth = await fetchImpl(`${endpoint.origin}/identities`, { headers: endpoint.headers });
|
|
125
143
|
if (auth.status === 401)
|
|
126
|
-
detail = `reachable on :${port} but the API token was rejected (401) —
|
|
127
|
-
|
|
144
|
+
detail = `reachable on :${endpoint.port} but the API token was rejected (401) — ` +
|
|
145
|
+
authResolutionHint(endpoint);
|
|
128
146
|
else if (!auth.ok)
|
|
129
|
-
detail = `reachable on :${port} but /identities returned HTTP ${auth.status}`;
|
|
147
|
+
detail = `reachable on :${endpoint.port} but /identities returned HTTP ${auth.status}`;
|
|
130
148
|
else {
|
|
131
149
|
ok = true;
|
|
132
|
-
detail = `reachable on :${port}, authorized — supervisor wake stream available`;
|
|
150
|
+
detail = `reachable on :${endpoint.port}, authorized — supervisor wake stream available`;
|
|
133
151
|
}
|
|
134
152
|
}
|
|
135
153
|
}
|
|
136
154
|
catch (e) {
|
|
137
|
-
detail = `unreachable on :${port} — monitored roles run degraded until it is up ` +
|
|
155
|
+
detail = `unreachable on :${endpoint.port} — monitored roles run degraded until it is up ` +
|
|
138
156
|
`(start it: ours-mcp start) [${e?.message ?? e}]`;
|
|
139
157
|
}
|
|
140
|
-
checks.push({ name:
|
|
158
|
+
checks.push({ name: checkName, ok, detail });
|
|
141
159
|
}
|
|
142
160
|
const harnesses = opts.harness
|
|
143
161
|
? [opts.harness]
|
|
@@ -38,6 +38,15 @@ export function pretrust(dir) {
|
|
|
38
38
|
e.projectOnboardingSeenCount = Math.max(e.projectOnboardingSeenCount ?? 0, 1);
|
|
39
39
|
writeFileSync(p, JSON.stringify(d, null, 2));
|
|
40
40
|
}
|
|
41
|
+
/**
|
|
42
|
+
* Shared Monitor-arming mandate (issue #16). Single-sourced so the briefing and
|
|
43
|
+
* the restart prompt can never diverge on how the agent must arm its monitor —
|
|
44
|
+
* it must be the Monitor TOOL, not a background Bash task (which never wakes the
|
|
45
|
+
* agent on output → an armed-looking but deaf monitor).
|
|
46
|
+
*/
|
|
47
|
+
const armMonitor = (id) => 'arm a **persistent Monitor** (the Monitor TOOL — NOT a background Bash command; a ' +
|
|
48
|
+
`background Bash task never wakes you on output) running \`ours-mcp watch "${id}"\` ` +
|
|
49
|
+
'so inbound ours mail wakes you';
|
|
41
50
|
export function makeClaudeCodeAdapter(exec = realExec) {
|
|
42
51
|
return {
|
|
43
52
|
id: 'claude-code',
|
|
@@ -108,7 +117,10 @@ export function makeClaudeCodeAdapter(exec = realExec) {
|
|
|
108
117
|
sendTool: 'send_message',
|
|
109
118
|
getMessagesTool: 'get_messages',
|
|
110
119
|
watchCommand: id => `ours-mcp watch "${id}"`,
|
|
111
|
-
monitorInstruction: id =>
|
|
120
|
+
monitorInstruction: id => {
|
|
121
|
+
const m = armMonitor(id);
|
|
122
|
+
return `${m.charAt(0).toUpperCase()}${m.slice(1)}.`;
|
|
123
|
+
},
|
|
112
124
|
supervisedWakeNote: () => 'Your mail wake-ups are delivered by the fleet supervisor directly into this console as ' +
|
|
113
125
|
'`[fleet-monitor]` lines — do NOT arm an in-session Monitor. When such a line appears, run ' +
|
|
114
126
|
'**get_messages** to drain the mail.',
|
|
@@ -116,8 +128,12 @@ export function makeClaudeCodeAdapter(exec = realExec) {
|
|
|
116
128
|
restartPrompt: (id, worklog, role) => `Session restarted. Re-bind your ours identity now (choose_identity name "${id}" force=true), ` +
|
|
117
129
|
(role?.monitor?.enabled
|
|
118
130
|
? 'then continue from '
|
|
119
|
-
: `
|
|
120
|
-
`${worklog}. Do not re-run whatever crashed you
|
|
131
|
+
: `then ${armMonitor(id)}, then continue from `) +
|
|
132
|
+
`${worklog}. Do not re-run whatever crashed you.` +
|
|
133
|
+
(role?.monitor?.enabled
|
|
134
|
+
? ' Your mail wakes arrive as `[fleet-monitor]` console lines from the supervisor — ' +
|
|
135
|
+
'do NOT arm an in-session Monitor.'
|
|
136
|
+
: ''),
|
|
121
137
|
},
|
|
122
138
|
exitPolicy: { cleanExitIsFresh: true, fastFailSecs: 20 },
|
|
123
139
|
};
|
package/dist/monitor.d.ts
CHANGED
|
@@ -39,11 +39,37 @@ export interface MonitorDeps {
|
|
|
39
39
|
clear(t: ReturnType<typeof setTimeout>): void;
|
|
40
40
|
};
|
|
41
41
|
}
|
|
42
|
-
/**
|
|
43
|
-
|
|
42
|
+
/** Best-effort daemon config (issue #17): the fields the MCP client reads. */
|
|
43
|
+
interface DaemonConfig {
|
|
44
|
+
apiToken?: string;
|
|
45
|
+
port?: number;
|
|
46
|
+
stateDir?: string;
|
|
47
|
+
}
|
|
48
|
+
/**
|
|
49
|
+
* Read the daemon config the way the MCP client does — best-effort. Any missing,
|
|
50
|
+
* malformed, or unreadable config yields `{}` so token resolution falls through
|
|
51
|
+
* (issue #17). Only the well-typed fields we consume are surfaced.
|
|
52
|
+
*/
|
|
53
|
+
export declare function readDaemonConfig(env: NodeJS.ProcessEnv): DaemonConfig;
|
|
54
|
+
/**
|
|
55
|
+
* Resolve the daemon API token exactly like the MCP client (issue #17), a 3-step
|
|
56
|
+
* chain: `OURS_API_TOKEN` (trimmed) → config `apiToken` (trimmed) → the 0600 owner
|
|
57
|
+
* token at `<stateDir>/daemon-token`. Never generates a token; a failed read of
|
|
58
|
+
* any source (missing/unreadable) silently falls through to the next.
|
|
59
|
+
*/
|
|
60
|
+
export declare function resolveApiToken(env: NodeJS.ProcessEnv, file?: DaemonConfig): string | undefined;
|
|
61
|
+
export interface DaemonEndpoint {
|
|
62
|
+
origin: string;
|
|
63
|
+
port: number;
|
|
64
|
+
configPath: string;
|
|
65
|
+
stateDir: string;
|
|
44
66
|
url(name: string): string;
|
|
45
67
|
headers: Record<string, string>;
|
|
46
|
-
}
|
|
68
|
+
}
|
|
69
|
+
/** Resolve the daemon endpoint + auth header from env → config → defaults. */
|
|
70
|
+
export declare function resolveEndpoint(env: NodeJS.ProcessEnv): DaemonEndpoint;
|
|
71
|
+
/** Actionable, secret-free description of every token source for this profile. */
|
|
72
|
+
export declare function authResolutionHint(ep: DaemonEndpoint): string;
|
|
47
73
|
/** Keep only the events whose type the role asked to wake on. */
|
|
48
74
|
export declare function filterEvents(events: NotifyEvent[], wakeSources: string[]): NotifyEvent[];
|
|
49
75
|
/**
|
|
@@ -59,6 +85,21 @@ export declare function formatNotificationLine(events: NotifyEvent[]): string;
|
|
|
59
85
|
* open question (a): refine empirically). A running turn is NOT modal.
|
|
60
86
|
*/
|
|
61
87
|
export declare function looksModal(pane: string): boolean;
|
|
88
|
+
/**
|
|
89
|
+
* Heuristic: did the turn shown in this pane TERMINATE in an API-level error?
|
|
90
|
+
* Claude Code renders a failed turn's tail as an `API Error:` line (a Usage-Policy
|
|
91
|
+
* refusal, a 4xx, etc.). We scan a generous tail window so the marker survives a
|
|
92
|
+
* trailing idle composer redrawn beneath it (design §3.2, refine empirically).
|
|
93
|
+
* The N-consecutive threshold in the Monitor debounces the odd false match.
|
|
94
|
+
*/
|
|
95
|
+
export declare function looksApiError(pane: string): boolean;
|
|
96
|
+
/**
|
|
97
|
+
* Heuristic: is a turn still RUNNING in this pane? Claude Code shows a live
|
|
98
|
+
* "esc to interrupt" footer (often with an elapsed-seconds meter) while a turn
|
|
99
|
+
* streams. Absence of any running marker — and no API error — means the turn has
|
|
100
|
+
* settled (completed). Kept a positive check so a quiet idle pane reads as done.
|
|
101
|
+
*/
|
|
102
|
+
export declare function looksRunning(pane: string): boolean;
|
|
62
103
|
export interface MonitorOpts {
|
|
63
104
|
name: string;
|
|
64
105
|
agentDir: string;
|
|
@@ -83,6 +124,8 @@ export declare class Monitor {
|
|
|
83
124
|
private stopped;
|
|
84
125
|
private bootDeadline;
|
|
85
126
|
private currentAbort;
|
|
127
|
+
private apiErrorStreak;
|
|
128
|
+
private readonly turnFailThreshold;
|
|
86
129
|
constructor(o: MonitorOpts);
|
|
87
130
|
/** Prime at the stream tip (or resume a persisted cursor if the daemon is down). */
|
|
88
131
|
prime(): Promise<void>;
|
|
@@ -92,6 +135,16 @@ export declare class Monitor {
|
|
|
92
135
|
/** Gather stragglers arriving within batch_ms so a burst lands as one line. */
|
|
93
136
|
private coalesce;
|
|
94
137
|
private deliver;
|
|
138
|
+
/**
|
|
139
|
+
* Watch the pane until the just-triggered turn settles, then fold its outcome
|
|
140
|
+
* into the API-error streak and republish `.monitor-status`. A completed turn
|
|
141
|
+
* (or an inconclusive give-up) resets/keeps the streak; an `API Error:` tail
|
|
142
|
+
* grows it. Once the streak reaches the threshold the status degrades; a later
|
|
143
|
+
* completed turn flips it back to armed. Detection only — no remediation (#19).
|
|
144
|
+
*/
|
|
145
|
+
private observeTurnOutcome;
|
|
146
|
+
/** Update the consecutive-API-error streak and derive `.monitor-status` from it. */
|
|
147
|
+
private recordTurn;
|
|
95
148
|
/** Block until the console can accept input; classify offline/stopped/ready. */
|
|
96
149
|
private awaitInjectable;
|
|
97
150
|
private doFetch;
|
|
@@ -101,3 +154,4 @@ export declare class Monitor {
|
|
|
101
154
|
private setStatus;
|
|
102
155
|
}
|
|
103
156
|
export declare function createMonitor(o: MonitorOpts): Monitor;
|
|
157
|
+
export {};
|
package/dist/monitor.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { existsSync, readFileSync, writeFileSync } from 'node:fs';
|
|
2
|
+
import { homedir } from 'node:os';
|
|
2
3
|
import { join } from 'node:path';
|
|
3
4
|
// Code constants (not config — YAGNI, design §2).
|
|
4
5
|
const DEFAULT_PORT = 3050;
|
|
@@ -12,17 +13,88 @@ const BACKOFF_STEP_MS = 1_000;
|
|
|
12
13
|
const BACKOFF_MAX_MS = 5_000;
|
|
13
14
|
const PREFIX = '[fleet-monitor]';
|
|
14
15
|
const MAX_LINE = 260;
|
|
16
|
+
// Turn-outcome observation (issue #19): after a delivered wake, watch the pane
|
|
17
|
+
// until the triggered turn settles, then classify it. Code constants (not config):
|
|
18
|
+
const TURN_OBSERVE_POLLS = 20; // give up after ~POLLS × INTERVAL of a still-running turn
|
|
19
|
+
const TURN_OBSERVE_INTERVAL_MS = 1_500;
|
|
20
|
+
const DEFAULT_TURN_FAIL_THRESHOLD = 3; // fallback when the resolved config omits it
|
|
15
21
|
class AuthError extends Error {
|
|
16
22
|
}
|
|
17
|
-
/**
|
|
23
|
+
/** Path to the daemon config the MCP client uses: OURS_CONFIG ?? real ~/.ours/config.json. */
|
|
24
|
+
const daemonConfigPath = (env) => env.OURS_CONFIG ?? join(homedir(), '.ours', 'config.json');
|
|
25
|
+
/** Match ours-mcp's env integer semantics: parseInt, invalid → absent. */
|
|
26
|
+
function envInt(env, name) {
|
|
27
|
+
const raw = env[name];
|
|
28
|
+
if (raw === undefined)
|
|
29
|
+
return undefined;
|
|
30
|
+
const n = parseInt(raw, 10);
|
|
31
|
+
return Number.isNaN(n) ? undefined : n;
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
34
|
+
* Read the daemon config the way the MCP client does — best-effort. Any missing,
|
|
35
|
+
* malformed, or unreadable config yields `{}` so token resolution falls through
|
|
36
|
+
* (issue #17). Only the well-typed fields we consume are surfaced.
|
|
37
|
+
*/
|
|
38
|
+
export function readDaemonConfig(env) {
|
|
39
|
+
try {
|
|
40
|
+
const p = JSON.parse(readFileSync(daemonConfigPath(env), 'utf8'));
|
|
41
|
+
const o = {};
|
|
42
|
+
if (typeof p.apiToken === 'string' && p.apiToken.trim())
|
|
43
|
+
o.apiToken = p.apiToken.trim();
|
|
44
|
+
if (typeof p.port === 'number' && Number.isFinite(p.port))
|
|
45
|
+
o.port = p.port;
|
|
46
|
+
if (typeof p.stateDir === 'string')
|
|
47
|
+
o.stateDir = p.stateDir;
|
|
48
|
+
return o;
|
|
49
|
+
}
|
|
50
|
+
catch {
|
|
51
|
+
return {};
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
/**
|
|
55
|
+
* Resolve the daemon API token exactly like the MCP client (issue #17), a 3-step
|
|
56
|
+
* chain: `OURS_API_TOKEN` (trimmed) → config `apiToken` (trimmed) → the 0600 owner
|
|
57
|
+
* token at `<stateDir>/daemon-token`. Never generates a token; a failed read of
|
|
58
|
+
* any source (missing/unreadable) silently falls through to the next.
|
|
59
|
+
*/
|
|
60
|
+
export function resolveApiToken(env, file = readDaemonConfig(env)) {
|
|
61
|
+
const e = env.OURS_API_TOKEN?.trim();
|
|
62
|
+
if (e)
|
|
63
|
+
return e;
|
|
64
|
+
if (file.apiToken)
|
|
65
|
+
return file.apiToken;
|
|
66
|
+
const sd = env.OURS_STATE_DIR ?? file.stateDir ?? join(homedir(), '.ours');
|
|
67
|
+
try {
|
|
68
|
+
const t = readFileSync(join(sd, 'daemon-token'), 'utf8').trim();
|
|
69
|
+
if (t)
|
|
70
|
+
return t;
|
|
71
|
+
}
|
|
72
|
+
catch { /* missing/unreadable (e.g. cross-user 0600) → fall through */ }
|
|
73
|
+
return undefined;
|
|
74
|
+
}
|
|
75
|
+
/** Resolve the daemon endpoint + auth header from env → config → defaults. */
|
|
18
76
|
export function resolveEndpoint(env) {
|
|
19
|
-
const
|
|
20
|
-
const
|
|
77
|
+
const file = readDaemonConfig(env);
|
|
78
|
+
const port = envInt(env, 'OURS_PORT') ?? file.port ?? DEFAULT_PORT;
|
|
79
|
+
const configPath = daemonConfigPath(env);
|
|
80
|
+
const stateDir = env.OURS_STATE_DIR ?? file.stateDir ?? join(homedir(), '.ours');
|
|
81
|
+
const token = resolveApiToken(env, file);
|
|
82
|
+
const origin = `http://127.0.0.1:${port}`;
|
|
21
83
|
return {
|
|
22
|
-
|
|
84
|
+
origin,
|
|
85
|
+
port,
|
|
86
|
+
configPath,
|
|
87
|
+
stateDir,
|
|
88
|
+
url: (name) => `${origin}/identities/${encodeURIComponent(name)}/notifications`,
|
|
23
89
|
headers: token ? { 'x-ours-api-token': token } : {},
|
|
24
90
|
};
|
|
25
91
|
}
|
|
92
|
+
/** Actionable, secret-free description of every token source for this profile. */
|
|
93
|
+
export function authResolutionHint(ep) {
|
|
94
|
+
const tokenPath = join(ep.stateDir, 'daemon-token');
|
|
95
|
+
return `set OURS_API_TOKEN, set apiToken in ${JSON.stringify(ep.configPath)}, or ensure ` +
|
|
96
|
+
`${JSON.stringify(tokenPath)} is readable by the fleet supervisor`;
|
|
97
|
+
}
|
|
26
98
|
/** Keep only the events whose type the role asked to wake on. */
|
|
27
99
|
export function filterEvents(events, wakeSources) {
|
|
28
100
|
const set = new Set(wakeSources);
|
|
@@ -84,6 +156,31 @@ export function looksModal(pane) {
|
|
|
84
156
|
const hasNumbered = /(^|\n)\s*[❯>]?\s*\d+[.)]\s+\S/.test(pane);
|
|
85
157
|
return hasPointer && hasNumbered;
|
|
86
158
|
}
|
|
159
|
+
/**
|
|
160
|
+
* Heuristic: did the turn shown in this pane TERMINATE in an API-level error?
|
|
161
|
+
* Claude Code renders a failed turn's tail as an `API Error:` line (a Usage-Policy
|
|
162
|
+
* refusal, a 4xx, etc.). We scan a generous tail window so the marker survives a
|
|
163
|
+
* trailing idle composer redrawn beneath it (design §3.2, refine empirically).
|
|
164
|
+
* The N-consecutive threshold in the Monitor debounces the odd false match.
|
|
165
|
+
*/
|
|
166
|
+
export function looksApiError(pane) {
|
|
167
|
+
const tail = pane.split('\n').slice(-15).join('\n');
|
|
168
|
+
return /\bAPI Error\b/i.test(tail);
|
|
169
|
+
}
|
|
170
|
+
/**
|
|
171
|
+
* Heuristic: is a turn still RUNNING in this pane? Claude Code shows a live
|
|
172
|
+
* "esc to interrupt" footer (often with an elapsed-seconds meter) while a turn
|
|
173
|
+
* streams. Absence of any running marker — and no API error — means the turn has
|
|
174
|
+
* settled (completed). Kept a positive check so a quiet idle pane reads as done.
|
|
175
|
+
*/
|
|
176
|
+
export function looksRunning(pane) {
|
|
177
|
+
const tail = pane.split('\n').slice(-6).join('\n');
|
|
178
|
+
if (/esc to interrupt/i.test(tail))
|
|
179
|
+
return true; // Claude Code's running footer
|
|
180
|
+
if (/\(\s*\d+s\b/.test(tail))
|
|
181
|
+
return true; // "(12s · … tokens)" elapsed meter
|
|
182
|
+
return false;
|
|
183
|
+
}
|
|
87
184
|
/** Is the injected line still sitting unsubmitted in the composer (bottom of pane)? */
|
|
88
185
|
function stillInComposer(pane, line) {
|
|
89
186
|
const frag = line.slice(0, 48);
|
|
@@ -102,6 +199,10 @@ export class Monitor {
|
|
|
102
199
|
stopped = false;
|
|
103
200
|
bootDeadline = 0;
|
|
104
201
|
currentAbort = null;
|
|
202
|
+
// Refusal-wedge detector (issue #19): consecutive delivered wakes whose turn
|
|
203
|
+
// ended in an API error with no completed turn in between.
|
|
204
|
+
apiErrorStreak = 0;
|
|
205
|
+
turnFailThreshold;
|
|
105
206
|
constructor(o) {
|
|
106
207
|
this.name = o.name;
|
|
107
208
|
this.cfg = o.cfg;
|
|
@@ -109,6 +210,8 @@ export class Monitor {
|
|
|
109
210
|
this.ep = resolveEndpoint(o.deps.env);
|
|
110
211
|
this.statusPath = join(o.agentDir, '.monitor-status');
|
|
111
212
|
this.cursorPath = join(o.agentDir, '.notify-cursor');
|
|
213
|
+
const n = o.cfg.turn_fail_threshold;
|
|
214
|
+
this.turnFailThreshold = typeof n === 'number' && n >= 1 ? n : DEFAULT_TURN_FAIL_THRESHOLD;
|
|
112
215
|
}
|
|
113
216
|
/** Prime at the stream tip (or resume a persisted cursor if the daemon is down). */
|
|
114
217
|
async prime() {
|
|
@@ -207,7 +310,51 @@ export class Monitor {
|
|
|
207
310
|
}
|
|
208
311
|
await this.deps.tmux.sendKey(this.name, 'Enter');
|
|
209
312
|
}
|
|
210
|
-
|
|
313
|
+
if (!delivered) {
|
|
314
|
+
this.setStatus('degraded: injection unverified');
|
|
315
|
+
return;
|
|
316
|
+
}
|
|
317
|
+
// The wake landed and a turn started; observe how that turn terminates so a
|
|
318
|
+
// refusal-wedge (every turn dies with `API Error:` while delivery stays green)
|
|
319
|
+
// becomes visible in `.monitor-status` instead of masquerading as armed (#19).
|
|
320
|
+
await this.observeTurnOutcome(pid);
|
|
321
|
+
}
|
|
322
|
+
/**
|
|
323
|
+
* Watch the pane until the just-triggered turn settles, then fold its outcome
|
|
324
|
+
* into the API-error streak and republish `.monitor-status`. A completed turn
|
|
325
|
+
* (or an inconclusive give-up) resets/keeps the streak; an `API Error:` tail
|
|
326
|
+
* grows it. Once the streak reaches the threshold the status degrades; a later
|
|
327
|
+
* completed turn flips it back to armed. Detection only — no remediation (#19).
|
|
328
|
+
*/
|
|
329
|
+
async observeTurnOutcome(pid) {
|
|
330
|
+
for (let i = 0; i < TURN_OBSERVE_POLLS; i++) {
|
|
331
|
+
if (this.stopped)
|
|
332
|
+
return; // shutting down — leave status
|
|
333
|
+
if (!this.deps.isAlive(pid) || !(await this.deps.tmux.has(this.name)))
|
|
334
|
+
return; // loop marks offline
|
|
335
|
+
const pane = await safeCapture(this.deps.tmux, this.name);
|
|
336
|
+
if (looksApiError(pane)) {
|
|
337
|
+
this.recordTurn('api-error');
|
|
338
|
+
return;
|
|
339
|
+
}
|
|
340
|
+
if (!looksRunning(pane)) {
|
|
341
|
+
this.recordTurn('completed');
|
|
342
|
+
return;
|
|
343
|
+
}
|
|
344
|
+
await this.deps.sleep(TURN_OBSERVE_INTERVAL_MS);
|
|
345
|
+
}
|
|
346
|
+
this.recordTurn('inconclusive'); // still running at give-up: hold the streak, don't re-arm
|
|
347
|
+
}
|
|
348
|
+
/** Update the consecutive-API-error streak and derive `.monitor-status` from it. */
|
|
349
|
+
recordTurn(outcome) {
|
|
350
|
+
if (outcome === 'api-error')
|
|
351
|
+
this.apiErrorStreak++;
|
|
352
|
+
else if (outcome === 'completed')
|
|
353
|
+
this.apiErrorStreak = 0;
|
|
354
|
+
// 'inconclusive' leaves the streak (and therefore the status) unchanged.
|
|
355
|
+
this.setStatus(this.apiErrorStreak >= this.turnFailThreshold
|
|
356
|
+
? 'degraded: turns failing (api error)'
|
|
357
|
+
: 'armed');
|
|
211
358
|
}
|
|
212
359
|
/** Block until the console can accept input; classify offline/stopped/ready. */
|
|
213
360
|
async awaitInjectable(pid) {
|
|
@@ -242,7 +389,7 @@ export class Monitor {
|
|
|
242
389
|
this.currentAbort = null;
|
|
243
390
|
}
|
|
244
391
|
if (resp.status === 401)
|
|
245
|
-
throw new AuthError(
|
|
392
|
+
throw new AuthError(`daemon rejected the API token (401) — ${authResolutionHint(this.ep)}`);
|
|
246
393
|
if (!resp.ok)
|
|
247
394
|
throw new Error(`daemon returned HTTP ${resp.status}`);
|
|
248
395
|
return resp.json();
|
package/dist/runner.js
CHANGED
|
@@ -41,8 +41,8 @@ export function buildPaneCommand(launch, roleEnv, exitStatusPath, paneArgv = lau
|
|
|
41
41
|
const cmd = paneArgv.map(shq).join(' ');
|
|
42
42
|
return `${envPfx} ${cmd}; echo $? > ${shq(exitStatusPath)}`;
|
|
43
43
|
}
|
|
44
|
-
/** Adapt the
|
|
45
|
-
function monitorDeps(deps) {
|
|
44
|
+
/** Adapt runner deps and the role's daemon-profile overrides for the monitor. */
|
|
45
|
+
function monitorDeps(deps, roleEnv) {
|
|
46
46
|
return {
|
|
47
47
|
fetch: deps.fetch,
|
|
48
48
|
tmux: deps.tmux,
|
|
@@ -50,7 +50,9 @@ function monitorDeps(deps) {
|
|
|
50
50
|
sleep: deps.sleep,
|
|
51
51
|
now: deps.now,
|
|
52
52
|
log: deps.log,
|
|
53
|
-
env
|
|
53
|
+
// The agent pane receives role.env too. The supervisor monitor must select
|
|
54
|
+
// the same ours daemon profile, with per-role values winning over service env.
|
|
55
|
+
env: { ...process.env, ...(roleEnv ?? {}) },
|
|
54
56
|
timers: { set: (fn, ms) => setTimeout(fn, ms), clear: t => clearTimeout(t) },
|
|
55
57
|
};
|
|
56
58
|
}
|
|
@@ -232,7 +234,7 @@ export async function runOnce(name, opts = {}, partialDeps = {}) {
|
|
|
232
234
|
// as disabled (monitor may be undefined on an old role.yaml).
|
|
233
235
|
const monitor = role.monitor?.enabled ? deps.createMonitor({
|
|
234
236
|
name, agentDir: dir, cfg: role.monitor,
|
|
235
|
-
deps: monitorDeps(deps),
|
|
237
|
+
deps: monitorDeps(deps, role.env),
|
|
236
238
|
}) : null;
|
|
237
239
|
if (monitor)
|
|
238
240
|
await monitor.prime();
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@ours.network/fleet",
|
|
3
|
-
"version": "0.9.
|
|
3
|
+
"version": "0.9.2",
|
|
4
4
|
"description": "Harness-agnostic fleet of persistent, identity-bound AI agents. Declarative fleet.yaml, tmux consoles, systemd/launchd supervision, ours.network messaging.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "FSL-1.1-Apache-2.0",
|