@ours.network/fleet 0.8.0 → 0.9.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +19 -0
- package/dist/cli.js +0 -1
- package/dist/config.d.ts +2 -0
- package/dist/config.js +10 -1
- package/dist/doctor.js +31 -13
- package/dist/harness/claude-code.js +19 -3
- package/dist/monitor.d.ts +30 -3
- package/dist/monitor.js +72 -5
- package/dist/ops.d.ts +0 -1
- package/dist/ops.js +5 -9
- package/dist/runner.d.ts +11 -0
- package/dist/runner.js +101 -6
- package/dist/spawn.js +6 -0
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -180,6 +180,7 @@ the default `~/fleet.yaml` on its very first crash-restart and fail to resolve.
|
|
|
180
180
|
|
|
181
181
|
```yaml
|
|
182
182
|
vars: { work_root: /home/me/work } # ${var} substitution anywhere below
|
|
183
|
+
start_stagger_ms: 0 # delay between agent LAUNCHES (host-wide, ms); 0 = no stagger
|
|
183
184
|
defaults:
|
|
184
185
|
harness: claude-code # for roles that don't set one
|
|
185
186
|
model: claude-fable-5 # default model for roles that don't set one (per-role model / --model wins)
|
|
@@ -240,6 +241,24 @@ role's `harness_options`, so a fleet can set common Codex permission/profile def
|
|
|
240
241
|
and override individual keys per role. `monitor` merges the same way — a role block
|
|
241
242
|
overrides `defaults.monitor` key-by-key.
|
|
242
243
|
|
|
244
|
+
### Start staggering
|
|
245
|
+
|
|
246
|
+
`start_stagger_ms` (top-level, host-wide, default `0`) spaces out agent **launches**
|
|
247
|
+
so a burst of boots doesn't hit the harness/API rate limit (429) all at once. When
|
|
248
|
+
set, each launch is held until at least `start_stagger_ms` after the previous one
|
|
249
|
+
across the whole host. It is enforced at the harness-launch point inside the runner,
|
|
250
|
+
so — unlike a delay in the `up`/`restart` command loop — it also covers **systemd
|
|
251
|
+
host boot**, where every agent's unit starts concurrently. The gate is time-based:
|
|
252
|
+
a lone start or a solo crash-restart waits **zero**; only genuinely concurrent
|
|
253
|
+
launches are spread out. Example: `start_stagger_ms: 4000` on a 7-agent fleet spaces
|
|
254
|
+
their boots ~4 s apart instead of firing all seven at once.
|
|
255
|
+
|
|
256
|
+
> **Migration (v0.9+):** this replaces the old `FLEET_START_STAGGER` environment
|
|
257
|
+
> variable, which only staggered the `ours-fleet up`/`restart` command loop (not
|
|
258
|
+
> host boot) and defaulted to 5 s. `FLEET_START_STAGGER` is **retired** — set
|
|
259
|
+
> `start_stagger_ms` in `fleet.yaml` instead (note: milliseconds, and the default
|
|
260
|
+
> is now `0`, so add it explicitly if you relied on the old implicit 5 s spacing).
|
|
261
|
+
|
|
243
262
|
### Mail monitor
|
|
244
263
|
|
|
245
264
|
With `monitor.enabled` (the default), the **supervisor** delivers a role's mail
|
package/dist/cli.js
CHANGED
|
@@ -26,7 +26,6 @@ catch {
|
|
|
26
26
|
const deps = () => ({
|
|
27
27
|
backend: pickBackend(),
|
|
28
28
|
binPath,
|
|
29
|
-
sleep: ms => new Promise(r => setTimeout(r, ms)),
|
|
30
29
|
log: l => console.log(l),
|
|
31
30
|
});
|
|
32
31
|
const die = (e) => { console.error(String(e instanceof Error ? e.message : e)); process.exit(1); };
|
package/dist/config.d.ts
CHANGED
|
@@ -48,6 +48,8 @@ export interface FleetConfig {
|
|
|
48
48
|
vars: Record<string, string>;
|
|
49
49
|
defaults: Record<string, unknown>;
|
|
50
50
|
files: string[];
|
|
51
|
+
/** Fleet-wide delay (ms) enforced between agent launches to avoid boot bursts (0 = none). */
|
|
52
|
+
startStaggerMs: number;
|
|
51
53
|
}
|
|
52
54
|
export declare class ConfigError extends Error {
|
|
53
55
|
}
|
package/dist/config.js
CHANGED
|
@@ -86,6 +86,7 @@ export function loadConfig(configPath) {
|
|
|
86
86
|
const baseDoc = docs.length && docs[0].file === base ? docs[0].doc : {};
|
|
87
87
|
const vars = (baseDoc.vars ?? {});
|
|
88
88
|
const defaults = (baseDoc.defaults ?? {});
|
|
89
|
+
const startStaggerMs = resolveStartStaggerMs(baseDoc.start_stagger_ms, base);
|
|
89
90
|
const seen = new Map();
|
|
90
91
|
const roles = [];
|
|
91
92
|
for (const { file, doc } of docs) {
|
|
@@ -132,7 +133,15 @@ export function loadConfig(configPath) {
|
|
|
132
133
|
});
|
|
133
134
|
}
|
|
134
135
|
}
|
|
135
|
-
return { roles, vars, defaults, files };
|
|
136
|
+
return { roles, vars, defaults, files, startStaggerMs };
|
|
137
|
+
}
|
|
138
|
+
/** Validate the top-level `start_stagger_ms` (supervisor launch spacing); default 0. */
|
|
139
|
+
function resolveStartStaggerMs(raw, base) {
|
|
140
|
+
if (raw === undefined || raw === null)
|
|
141
|
+
return 0;
|
|
142
|
+
if (typeof raw !== 'number' || !Number.isFinite(raw) || raw < 0)
|
|
143
|
+
throw new ConfigError(`${base}: start_stagger_ms must be a non-negative number of milliseconds (got ${JSON.stringify(raw)})`);
|
|
144
|
+
return raw;
|
|
136
145
|
}
|
|
137
146
|
/**
|
|
138
147
|
* Merge `defaults.monitor` under the role's own `monitor:` key-by-key, validate the
|
package/dist/doctor.js
CHANGED
|
@@ -6,6 +6,7 @@ import { getAdapter } from './harness/registry.js';
|
|
|
6
6
|
import { agentDir, home, deriveXdgRuntimeDir } from './paths.js';
|
|
7
7
|
import { resolveIsolation } from './isolation/policy.js';
|
|
8
8
|
import { makeBubblewrapBackend } from './isolation/bubblewrap.js';
|
|
9
|
+
import { authResolutionHint, resolveEndpoint, } from './monitor.js';
|
|
9
10
|
/** Which cgroup-v2 controllers are delegated to this user manager (advisory). */
|
|
10
11
|
function cgroupDelegationDetail() {
|
|
11
12
|
try {
|
|
@@ -19,6 +20,21 @@ function cgroupDelegationDetail() {
|
|
|
19
20
|
return 'unknown (not cgroup-v2 or no delegation info)';
|
|
20
21
|
}
|
|
21
22
|
}
|
|
23
|
+
/** Resolve and deduplicate the effective daemon profiles used by monitored roles. */
|
|
24
|
+
function resolveMonitorProfiles(roles) {
|
|
25
|
+
const profiles = [];
|
|
26
|
+
for (const role of roles.filter(r => r.monitor?.enabled)) {
|
|
27
|
+
const endpoint = resolveEndpoint({ ...process.env, ...(role.env ?? {}) });
|
|
28
|
+
const token = endpoint.headers['x-ours-api-token'];
|
|
29
|
+
const existing = profiles.find(p => p.endpoint.origin === endpoint.origin
|
|
30
|
+
&& p.endpoint.headers['x-ours-api-token'] === token);
|
|
31
|
+
if (existing)
|
|
32
|
+
existing.roles.push(role.name);
|
|
33
|
+
else
|
|
34
|
+
profiles.push({ endpoint, roles: [role.name] });
|
|
35
|
+
}
|
|
36
|
+
return profiles;
|
|
37
|
+
}
|
|
22
38
|
/** Host-level + per-harness prerequisite report with actionable messages. */
|
|
23
39
|
export async function doctor(opts = {}, exec = realExec, platform = process.platform, fetchImpl = (u, i) => globalThis.fetch(u, i)) {
|
|
24
40
|
const checks = [];
|
|
@@ -110,34 +126,36 @@ export async function doctor(opts = {}, exec = realExec, platform = process.plat
|
|
|
110
126
|
// Monitor daemon-API reachability (design §5): only when a role is supervised.
|
|
111
127
|
// /state-dir is unauthenticated (liveness); /identities exercises the token so a
|
|
112
128
|
// shared-mode misconfig (401) surfaces here rather than as a silent deaf monitor.
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
const
|
|
116
|
-
const
|
|
129
|
+
const monitorProfiles = resolveMonitorProfiles(roles);
|
|
130
|
+
for (const profile of monitorProfiles) {
|
|
131
|
+
const { endpoint } = profile;
|
|
132
|
+
const checkName = monitorProfiles.length === 1
|
|
133
|
+
? 'monitor: daemon API'
|
|
134
|
+
: `monitor: daemon API (${profile.roles.join(', ')})`;
|
|
117
135
|
let ok = false, detail;
|
|
118
136
|
try {
|
|
119
|
-
const live = await fetchImpl(
|
|
137
|
+
const live = await fetchImpl(`${endpoint.origin}/state-dir`, {});
|
|
120
138
|
if (!live.ok) {
|
|
121
|
-
detail = `daemon on :${port} answered /state-dir with HTTP ${live.status} — not the ours daemon?`;
|
|
139
|
+
detail = `daemon on :${endpoint.port} answered /state-dir with HTTP ${live.status} — not the ours daemon?`;
|
|
122
140
|
}
|
|
123
141
|
else {
|
|
124
|
-
const auth = await fetchImpl(
|
|
142
|
+
const auth = await fetchImpl(`${endpoint.origin}/identities`, { headers: endpoint.headers });
|
|
125
143
|
if (auth.status === 401)
|
|
126
|
-
detail = `reachable on :${port} but the API token was rejected (401) —
|
|
127
|
-
|
|
144
|
+
detail = `reachable on :${endpoint.port} but the API token was rejected (401) — ` +
|
|
145
|
+
authResolutionHint(endpoint);
|
|
128
146
|
else if (!auth.ok)
|
|
129
|
-
detail = `reachable on :${port} but /identities returned HTTP ${auth.status}`;
|
|
147
|
+
detail = `reachable on :${endpoint.port} but /identities returned HTTP ${auth.status}`;
|
|
130
148
|
else {
|
|
131
149
|
ok = true;
|
|
132
|
-
detail = `reachable on :${port}, authorized — supervisor wake stream available`;
|
|
150
|
+
detail = `reachable on :${endpoint.port}, authorized — supervisor wake stream available`;
|
|
133
151
|
}
|
|
134
152
|
}
|
|
135
153
|
}
|
|
136
154
|
catch (e) {
|
|
137
|
-
detail = `unreachable on :${port} — monitored roles run degraded until it is up ` +
|
|
155
|
+
detail = `unreachable on :${endpoint.port} — monitored roles run degraded until it is up ` +
|
|
138
156
|
`(start it: ours-mcp start) [${e?.message ?? e}]`;
|
|
139
157
|
}
|
|
140
|
-
checks.push({ name:
|
|
158
|
+
checks.push({ name: checkName, ok, detail });
|
|
141
159
|
}
|
|
142
160
|
const harnesses = opts.harness
|
|
143
161
|
? [opts.harness]
|
|
@@ -38,6 +38,15 @@ export function pretrust(dir) {
|
|
|
38
38
|
e.projectOnboardingSeenCount = Math.max(e.projectOnboardingSeenCount ?? 0, 1);
|
|
39
39
|
writeFileSync(p, JSON.stringify(d, null, 2));
|
|
40
40
|
}
|
|
41
|
+
/**
|
|
42
|
+
* Shared Monitor-arming mandate (issue #16). Single-sourced so the briefing and
|
|
43
|
+
* the restart prompt can never diverge on how the agent must arm its monitor —
|
|
44
|
+
* it must be the Monitor TOOL, not a background Bash task (which never wakes the
|
|
45
|
+
* agent on output → an armed-looking but deaf monitor).
|
|
46
|
+
*/
|
|
47
|
+
const armMonitor = (id) => 'arm a **persistent Monitor** (the Monitor TOOL — NOT a background Bash command; a ' +
|
|
48
|
+
`background Bash task never wakes you on output) running \`ours-mcp watch "${id}"\` ` +
|
|
49
|
+
'so inbound ours mail wakes you';
|
|
41
50
|
export function makeClaudeCodeAdapter(exec = realExec) {
|
|
42
51
|
return {
|
|
43
52
|
id: 'claude-code',
|
|
@@ -108,7 +117,10 @@ export function makeClaudeCodeAdapter(exec = realExec) {
|
|
|
108
117
|
sendTool: 'send_message',
|
|
109
118
|
getMessagesTool: 'get_messages',
|
|
110
119
|
watchCommand: id => `ours-mcp watch "${id}"`,
|
|
111
|
-
monitorInstruction: id =>
|
|
120
|
+
monitorInstruction: id => {
|
|
121
|
+
const m = armMonitor(id);
|
|
122
|
+
return `${m.charAt(0).toUpperCase()}${m.slice(1)}.`;
|
|
123
|
+
},
|
|
112
124
|
supervisedWakeNote: () => 'Your mail wake-ups are delivered by the fleet supervisor directly into this console as ' +
|
|
113
125
|
'`[fleet-monitor]` lines — do NOT arm an in-session Monitor. When such a line appears, run ' +
|
|
114
126
|
'**get_messages** to drain the mail.',
|
|
@@ -116,8 +128,12 @@ export function makeClaudeCodeAdapter(exec = realExec) {
|
|
|
116
128
|
restartPrompt: (id, worklog, role) => `Session restarted. Re-bind your ours identity now (choose_identity name "${id}" force=true), ` +
|
|
117
129
|
(role?.monitor?.enabled
|
|
118
130
|
? 'then continue from '
|
|
119
|
-
: `
|
|
120
|
-
`${worklog}. Do not re-run whatever crashed you
|
|
131
|
+
: `then ${armMonitor(id)}, then continue from `) +
|
|
132
|
+
`${worklog}. Do not re-run whatever crashed you.` +
|
|
133
|
+
(role?.monitor?.enabled
|
|
134
|
+
? ' Your mail wakes arrive as `[fleet-monitor]` console lines from the supervisor — ' +
|
|
135
|
+
'do NOT arm an in-session Monitor.'
|
|
136
|
+
: ''),
|
|
121
137
|
},
|
|
122
138
|
exitPolicy: { cleanExitIsFresh: true, fastFailSecs: 20 },
|
|
123
139
|
};
|
package/dist/monitor.d.ts
CHANGED
|
@@ -39,11 +39,37 @@ export interface MonitorDeps {
|
|
|
39
39
|
clear(t: ReturnType<typeof setTimeout>): void;
|
|
40
40
|
};
|
|
41
41
|
}
|
|
42
|
-
/**
|
|
43
|
-
|
|
42
|
+
/** Best-effort daemon config (issue #17): the fields the MCP client reads. */
|
|
43
|
+
interface DaemonConfig {
|
|
44
|
+
apiToken?: string;
|
|
45
|
+
port?: number;
|
|
46
|
+
stateDir?: string;
|
|
47
|
+
}
|
|
48
|
+
/**
|
|
49
|
+
* Read the daemon config the way the MCP client does — best-effort. Any missing,
|
|
50
|
+
* malformed, or unreadable config yields `{}` so token resolution falls through
|
|
51
|
+
* (issue #17). Only the well-typed fields we consume are surfaced.
|
|
52
|
+
*/
|
|
53
|
+
export declare function readDaemonConfig(env: NodeJS.ProcessEnv): DaemonConfig;
|
|
54
|
+
/**
|
|
55
|
+
* Resolve the daemon API token exactly like the MCP client (issue #17), a 3-step
|
|
56
|
+
* chain: `OURS_API_TOKEN` (trimmed) → config `apiToken` (trimmed) → the 0600 owner
|
|
57
|
+
* token at `<stateDir>/daemon-token`. Never generates a token; a failed read of
|
|
58
|
+
* any source (missing/unreadable) silently falls through to the next.
|
|
59
|
+
*/
|
|
60
|
+
export declare function resolveApiToken(env: NodeJS.ProcessEnv, file?: DaemonConfig): string | undefined;
|
|
61
|
+
export interface DaemonEndpoint {
|
|
62
|
+
origin: string;
|
|
63
|
+
port: number;
|
|
64
|
+
configPath: string;
|
|
65
|
+
stateDir: string;
|
|
44
66
|
url(name: string): string;
|
|
45
67
|
headers: Record<string, string>;
|
|
46
|
-
}
|
|
68
|
+
}
|
|
69
|
+
/** Resolve the daemon endpoint + auth header from env → config → defaults. */
|
|
70
|
+
export declare function resolveEndpoint(env: NodeJS.ProcessEnv): DaemonEndpoint;
|
|
71
|
+
/** Actionable, secret-free description of every token source for this profile. */
|
|
72
|
+
export declare function authResolutionHint(ep: DaemonEndpoint): string;
|
|
47
73
|
/** Keep only the events whose type the role asked to wake on. */
|
|
48
74
|
export declare function filterEvents(events: NotifyEvent[], wakeSources: string[]): NotifyEvent[];
|
|
49
75
|
/**
|
|
@@ -101,3 +127,4 @@ export declare class Monitor {
|
|
|
101
127
|
private setStatus;
|
|
102
128
|
}
|
|
103
129
|
export declare function createMonitor(o: MonitorOpts): Monitor;
|
|
130
|
+
export {};
|
package/dist/monitor.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { existsSync, readFileSync, writeFileSync } from 'node:fs';
|
|
2
|
+
import { homedir } from 'node:os';
|
|
2
3
|
import { join } from 'node:path';
|
|
3
4
|
// Code constants (not config — YAGNI, design §2).
|
|
4
5
|
const DEFAULT_PORT = 3050;
|
|
@@ -14,15 +15,81 @@ const PREFIX = '[fleet-monitor]';
|
|
|
14
15
|
const MAX_LINE = 260;
|
|
15
16
|
class AuthError extends Error {
|
|
16
17
|
}
|
|
17
|
-
/**
|
|
18
|
+
/** Path to the daemon config the MCP client uses: OURS_CONFIG ?? real ~/.ours/config.json. */
|
|
19
|
+
const daemonConfigPath = (env) => env.OURS_CONFIG ?? join(homedir(), '.ours', 'config.json');
|
|
20
|
+
/** Match ours-mcp's env integer semantics: parseInt, invalid → absent. */
|
|
21
|
+
function envInt(env, name) {
|
|
22
|
+
const raw = env[name];
|
|
23
|
+
if (raw === undefined)
|
|
24
|
+
return undefined;
|
|
25
|
+
const n = parseInt(raw, 10);
|
|
26
|
+
return Number.isNaN(n) ? undefined : n;
|
|
27
|
+
}
|
|
28
|
+
/**
|
|
29
|
+
* Read the daemon config the way the MCP client does — best-effort. Any missing,
|
|
30
|
+
* malformed, or unreadable config yields `{}` so token resolution falls through
|
|
31
|
+
* (issue #17). Only the well-typed fields we consume are surfaced.
|
|
32
|
+
*/
|
|
33
|
+
export function readDaemonConfig(env) {
|
|
34
|
+
try {
|
|
35
|
+
const p = JSON.parse(readFileSync(daemonConfigPath(env), 'utf8'));
|
|
36
|
+
const o = {};
|
|
37
|
+
if (typeof p.apiToken === 'string' && p.apiToken.trim())
|
|
38
|
+
o.apiToken = p.apiToken.trim();
|
|
39
|
+
if (typeof p.port === 'number' && Number.isFinite(p.port))
|
|
40
|
+
o.port = p.port;
|
|
41
|
+
if (typeof p.stateDir === 'string')
|
|
42
|
+
o.stateDir = p.stateDir;
|
|
43
|
+
return o;
|
|
44
|
+
}
|
|
45
|
+
catch {
|
|
46
|
+
return {};
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* Resolve the daemon API token exactly like the MCP client (issue #17), a 3-step
|
|
51
|
+
* chain: `OURS_API_TOKEN` (trimmed) → config `apiToken` (trimmed) → the 0600 owner
|
|
52
|
+
* token at `<stateDir>/daemon-token`. Never generates a token; a failed read of
|
|
53
|
+
* any source (missing/unreadable) silently falls through to the next.
|
|
54
|
+
*/
|
|
55
|
+
export function resolveApiToken(env, file = readDaemonConfig(env)) {
|
|
56
|
+
const e = env.OURS_API_TOKEN?.trim();
|
|
57
|
+
if (e)
|
|
58
|
+
return e;
|
|
59
|
+
if (file.apiToken)
|
|
60
|
+
return file.apiToken;
|
|
61
|
+
const sd = env.OURS_STATE_DIR ?? file.stateDir ?? join(homedir(), '.ours');
|
|
62
|
+
try {
|
|
63
|
+
const t = readFileSync(join(sd, 'daemon-token'), 'utf8').trim();
|
|
64
|
+
if (t)
|
|
65
|
+
return t;
|
|
66
|
+
}
|
|
67
|
+
catch { /* missing/unreadable (e.g. cross-user 0600) → fall through */ }
|
|
68
|
+
return undefined;
|
|
69
|
+
}
|
|
70
|
+
/** Resolve the daemon endpoint + auth header from env → config → defaults. */
|
|
18
71
|
export function resolveEndpoint(env) {
|
|
19
|
-
const
|
|
20
|
-
const
|
|
72
|
+
const file = readDaemonConfig(env);
|
|
73
|
+
const port = envInt(env, 'OURS_PORT') ?? file.port ?? DEFAULT_PORT;
|
|
74
|
+
const configPath = daemonConfigPath(env);
|
|
75
|
+
const stateDir = env.OURS_STATE_DIR ?? file.stateDir ?? join(homedir(), '.ours');
|
|
76
|
+
const token = resolveApiToken(env, file);
|
|
77
|
+
const origin = `http://127.0.0.1:${port}`;
|
|
21
78
|
return {
|
|
22
|
-
|
|
79
|
+
origin,
|
|
80
|
+
port,
|
|
81
|
+
configPath,
|
|
82
|
+
stateDir,
|
|
83
|
+
url: (name) => `${origin}/identities/${encodeURIComponent(name)}/notifications`,
|
|
23
84
|
headers: token ? { 'x-ours-api-token': token } : {},
|
|
24
85
|
};
|
|
25
86
|
}
|
|
87
|
+
/** Actionable, secret-free description of every token source for this profile. */
|
|
88
|
+
export function authResolutionHint(ep) {
|
|
89
|
+
const tokenPath = join(ep.stateDir, 'daemon-token');
|
|
90
|
+
return `set OURS_API_TOKEN, set apiToken in ${JSON.stringify(ep.configPath)}, or ensure ` +
|
|
91
|
+
`${JSON.stringify(tokenPath)} is readable by the fleet supervisor`;
|
|
92
|
+
}
|
|
26
93
|
/** Keep only the events whose type the role asked to wake on. */
|
|
27
94
|
export function filterEvents(events, wakeSources) {
|
|
28
95
|
const set = new Set(wakeSources);
|
|
@@ -242,7 +309,7 @@ export class Monitor {
|
|
|
242
309
|
this.currentAbort = null;
|
|
243
310
|
}
|
|
244
311
|
if (resp.status === 401)
|
|
245
|
-
throw new AuthError(
|
|
312
|
+
throw new AuthError(`daemon rejected the API token (401) — ${authResolutionHint(this.ep)}`);
|
|
246
313
|
if (!resp.ok)
|
|
247
314
|
throw new Error(`daemon returned HTTP ${resp.status}`);
|
|
248
315
|
return resp.json();
|
package/dist/ops.d.ts
CHANGED
|
@@ -3,7 +3,6 @@ import type { SupervisorBackend } from './supervisor/types.js';
|
|
|
3
3
|
export interface OpsDeps {
|
|
4
4
|
backend: SupervisorBackend;
|
|
5
5
|
binPath: string;
|
|
6
|
-
sleep(ms: number): Promise<void>;
|
|
7
6
|
log(line: string): void;
|
|
8
7
|
}
|
|
9
8
|
/** Materialize a role's state dir from config: briefing + markers. Returns the dir. */
|
package/dist/ops.js
CHANGED
|
@@ -5,7 +5,11 @@ import { agentDir, fleetDDir } from './paths.js';
|
|
|
5
5
|
import { findRole } from './config.js';
|
|
6
6
|
import { getAdapter } from './harness/registry.js';
|
|
7
7
|
import { generateBriefing } from './briefing.js';
|
|
8
|
-
|
|
8
|
+
// Launch staggering now lives at the harness-launch point (the runner's start
|
|
9
|
+
// gate, driven by `start_stagger_ms`), so it covers systemd host-boot too — not
|
|
10
|
+
// just the `up`/`restart` command loop below. The old in-loop FLEET_START_STAGGER
|
|
11
|
+
// sleep is retired; `up`/`restart` fire installs promptly and the gate spaces the
|
|
12
|
+
// resulting launches.
|
|
9
13
|
/** Materialize a role's state dir from config: briefing + markers. Returns the dir. */
|
|
10
14
|
export function applyRole(role, opts = {}) {
|
|
11
15
|
const adapter = getAdapter(role.harness);
|
|
@@ -43,11 +47,7 @@ function selectRoles(cfg, names) {
|
|
|
43
47
|
}
|
|
44
48
|
/** Create/start roles declaratively. Idempotent; active roles keep their context. */
|
|
45
49
|
export async function up(cfg, names, deps, configPath) {
|
|
46
|
-
let first = true;
|
|
47
50
|
for (const role of selectRoles(cfg, names)) {
|
|
48
|
-
if (!first)
|
|
49
|
-
await deps.sleep(STAGGER_MS());
|
|
50
|
-
first = false;
|
|
51
51
|
const dir = applyRole(role, { configPath });
|
|
52
52
|
// If the role isn't running, boot fresh so it reads the briefing we just wrote.
|
|
53
53
|
const status = await deps.backend.status(role.name).catch(() => '');
|
|
@@ -70,11 +70,7 @@ export async function down(cfg, names, deps) {
|
|
|
70
70
|
}
|
|
71
71
|
/** Re-sync from config + bounce. mode 'keep' resumes context; 'fresh' wipes it. */
|
|
72
72
|
export async function restartRoles(cfg, names, deps, mode, configPath) {
|
|
73
|
-
let first = true;
|
|
74
73
|
for (const role of selectRoles(cfg, names)) {
|
|
75
|
-
if (!first)
|
|
76
|
-
await deps.sleep(STAGGER_MS());
|
|
77
|
-
first = false;
|
|
78
74
|
applyRole(role, { fresh: mode === 'fresh', configPath });
|
|
79
75
|
await deps.backend.restart(role.name);
|
|
80
76
|
deps.log(mode === 'fresh'
|
package/dist/runner.d.ts
CHANGED
|
@@ -24,6 +24,17 @@ export interface RunnerDeps {
|
|
|
24
24
|
* still sees the real exit code.
|
|
25
25
|
*/
|
|
26
26
|
export declare function buildPaneCommand(launch: Launch, roleEnv: Record<string, string> | undefined, exitStatusPath: string, paneArgv?: string[]): string;
|
|
27
|
+
/** Filename spawnTemp writes into a temp agent dir to carry the fleet start-stagger. */
|
|
28
|
+
export declare const START_STAGGER_FILE = ".start-stagger-ms";
|
|
29
|
+
/**
|
|
30
|
+
* Reserve this process's launch slot on the host-wide start gate and return the
|
|
31
|
+
* wall-clock time it may launch at. A tiny atomic mutex (mkdir is atomic across
|
|
32
|
+
* processes) guards a single `.last-launch` timestamp: each launcher takes the
|
|
33
|
+
* next slot = max(now, last + staggerMs), so concurrent boots serialize and spread
|
|
34
|
+
* out by staggerMs while a lone/idle start returns `now` (zero wait). A crashed
|
|
35
|
+
* launcher's stale lock is broken so the gate can never deadlock the fleet.
|
|
36
|
+
*/
|
|
37
|
+
export declare function reserveLaunchSlot(root: string, staggerMs: number, deps: Pick<RunnerDeps, 'now' | 'sleep' | 'log'>): Promise<number>;
|
|
27
38
|
/** Read a temp role's config snapshot written by spawnTemp. */
|
|
28
39
|
export declare function loadTempRole(name: string): ResolvedRole;
|
|
29
40
|
/** One supervised session lifecycle. The supervisor re-invokes us after we return. */
|
package/dist/runner.js
CHANGED
|
@@ -2,7 +2,7 @@ import { existsSync, readFileSync, writeFileSync, rmSync, mkdirSync } from 'node
|
|
|
2
2
|
import { join } from 'node:path';
|
|
3
3
|
import { randomUUID } from 'node:crypto';
|
|
4
4
|
import { parse } from 'yaml';
|
|
5
|
-
import { agentDir, home } from './paths.js';
|
|
5
|
+
import { agentDir, home, stateRoot } from './paths.js';
|
|
6
6
|
import { loadConfig, findRole } from './config.js';
|
|
7
7
|
import { getAdapter } from './harness/registry.js';
|
|
8
8
|
import { Tmux } from './tmux.js';
|
|
@@ -41,8 +41,8 @@ export function buildPaneCommand(launch, roleEnv, exitStatusPath, paneArgv = lau
|
|
|
41
41
|
const cmd = paneArgv.map(shq).join(' ');
|
|
42
42
|
return `${envPfx} ${cmd}; echo $? > ${shq(exitStatusPath)}`;
|
|
43
43
|
}
|
|
44
|
-
/** Adapt the
|
|
45
|
-
function monitorDeps(deps) {
|
|
44
|
+
/** Adapt runner deps and the role's daemon-profile overrides for the monitor. */
|
|
45
|
+
function monitorDeps(deps, roleEnv) {
|
|
46
46
|
return {
|
|
47
47
|
fetch: deps.fetch,
|
|
48
48
|
tmux: deps.tmux,
|
|
@@ -50,10 +50,78 @@ function monitorDeps(deps) {
|
|
|
50
50
|
sleep: deps.sleep,
|
|
51
51
|
now: deps.now,
|
|
52
52
|
log: deps.log,
|
|
53
|
-
env
|
|
53
|
+
// The agent pane receives role.env too. The supervisor monitor must select
|
|
54
|
+
// the same ours daemon profile, with per-role values winning over service env.
|
|
55
|
+
env: { ...process.env, ...(roleEnv ?? {}) },
|
|
54
56
|
timers: { set: (fn, ms) => setTimeout(fn, ms), clear: t => clearTimeout(t) },
|
|
55
57
|
};
|
|
56
58
|
}
|
|
59
|
+
/** Filename spawnTemp writes into a temp agent dir to carry the fleet start-stagger. */
|
|
60
|
+
export const START_STAGGER_FILE = '.start-stagger-ms';
|
|
61
|
+
/** Read the start-stagger a temp agent was spawned with (0 if none / unreadable). */
|
|
62
|
+
function readStartStagger(dir) {
|
|
63
|
+
try {
|
|
64
|
+
const n = parseInt(readFileSync(join(dir, START_STAGGER_FILE), 'utf8').trim(), 10);
|
|
65
|
+
return Number.isFinite(n) && n > 0 ? n : 0;
|
|
66
|
+
}
|
|
67
|
+
catch {
|
|
68
|
+
return 0;
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
/**
|
|
72
|
+
* Reserve this process's launch slot on the host-wide start gate and return the
|
|
73
|
+
* wall-clock time it may launch at. A tiny atomic mutex (mkdir is atomic across
|
|
74
|
+
* processes) guards a single `.last-launch` timestamp: each launcher takes the
|
|
75
|
+
* next slot = max(now, last + staggerMs), so concurrent boots serialize and spread
|
|
76
|
+
* out by staggerMs while a lone/idle start returns `now` (zero wait). A crashed
|
|
77
|
+
* launcher's stale lock is broken so the gate can never deadlock the fleet.
|
|
78
|
+
*/
|
|
79
|
+
export async function reserveLaunchSlot(root, staggerMs, deps) {
|
|
80
|
+
mkdirSync(root, { recursive: true });
|
|
81
|
+
const lockDir = join(root, '.launch-gate.lock');
|
|
82
|
+
const lockTsFile = join(lockDir, 'ts');
|
|
83
|
+
const tsFile = join(root, '.last-launch');
|
|
84
|
+
const staleMs = Math.max(staggerMs * 4, 10_000);
|
|
85
|
+
const POLL_MS = 50;
|
|
86
|
+
const readTs = (p) => {
|
|
87
|
+
try {
|
|
88
|
+
const n = parseInt(readFileSync(p, 'utf8').trim(), 10);
|
|
89
|
+
return Number.isFinite(n) ? n : null;
|
|
90
|
+
}
|
|
91
|
+
catch {
|
|
92
|
+
return null;
|
|
93
|
+
}
|
|
94
|
+
};
|
|
95
|
+
for (let waited = 0;;) {
|
|
96
|
+
try {
|
|
97
|
+
mkdirSync(lockDir);
|
|
98
|
+
writeFileSync(lockTsFile, String(deps.now()));
|
|
99
|
+
break;
|
|
100
|
+
}
|
|
101
|
+
catch (e) {
|
|
102
|
+
if (e.code !== 'EEXIST')
|
|
103
|
+
throw e;
|
|
104
|
+
const lockTs = readTs(lockTsFile);
|
|
105
|
+
const stale = lockTs !== null && deps.now() - lockTs > staleMs;
|
|
106
|
+
if (stale || waited > staleMs * 2) { // break a crashed launcher's lock; never deadlock
|
|
107
|
+
rmSync(lockDir, { recursive: true, force: true });
|
|
108
|
+
continue;
|
|
109
|
+
}
|
|
110
|
+
await deps.sleep(POLL_MS);
|
|
111
|
+
waited += POLL_MS;
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
try {
|
|
115
|
+
const now = deps.now();
|
|
116
|
+
const last = readTs(tsFile);
|
|
117
|
+
const target = last === null ? now : Math.max(now, last + staggerMs);
|
|
118
|
+
writeFileSync(tsFile, String(target));
|
|
119
|
+
return target;
|
|
120
|
+
}
|
|
121
|
+
finally {
|
|
122
|
+
rmSync(lockDir, { recursive: true, force: true });
|
|
123
|
+
}
|
|
124
|
+
}
|
|
57
125
|
/** Read a temp role's config snapshot written by spawnTemp. */
|
|
58
126
|
export function loadTempRole(name) {
|
|
59
127
|
const p = join(agentDir(name, true), 'role.yaml');
|
|
@@ -86,7 +154,20 @@ export async function runOnce(name, opts = {}, partialDeps = {}) {
|
|
|
86
154
|
const temp = opts.temp === true;
|
|
87
155
|
const dir = agentDir(name, temp);
|
|
88
156
|
const configPath = temp ? opts.configPath : resolveConfigPath(dir, opts.configPath);
|
|
89
|
-
|
|
157
|
+
// Resolve the role and the fleet-wide start-stagger. Permanent roles read the
|
|
158
|
+
// live config; temp/detached agents read the value spawnTemp snapshotted into
|
|
159
|
+
// their dir (they have no config path threaded through the detached supervisor).
|
|
160
|
+
let role;
|
|
161
|
+
let staggerMs;
|
|
162
|
+
if (temp) {
|
|
163
|
+
role = loadTempRole(name);
|
|
164
|
+
staggerMs = readStartStagger(dir);
|
|
165
|
+
}
|
|
166
|
+
else {
|
|
167
|
+
const cfg = loadConfig(configPath);
|
|
168
|
+
role = findRole(cfg, name);
|
|
169
|
+
staggerMs = cfg.startStaggerMs;
|
|
170
|
+
}
|
|
90
171
|
const adapter = getAdapter(role.harness);
|
|
91
172
|
mkdirSync(dir, { recursive: true });
|
|
92
173
|
const sidFile = join(dir, '.session-id');
|
|
@@ -132,6 +213,20 @@ export async function runOnce(name, opts = {}, partialDeps = {}) {
|
|
|
132
213
|
if (rprefix.length)
|
|
133
214
|
paneArgv = [...rprefix, ...paneArgv];
|
|
134
215
|
}
|
|
216
|
+
// Start-stagger: space this launch at least start_stagger_ms after the previous
|
|
217
|
+
// agent launch across the whole host, so a burst of boots (systemd starts every
|
|
218
|
+
// user unit concurrently on boot; `ours-fleet up`/restart-all bulk-start) does not
|
|
219
|
+
// hit the harness/API rate limit at once. Time-based via a shared launch gate, so
|
|
220
|
+
// a lone start or a solo crash-restart waits zero. Applied right before the harness
|
|
221
|
+
// launch (tmux.newSession); the cheap monitor prime still runs immediately after.
|
|
222
|
+
if (staggerMs > 0) {
|
|
223
|
+
const slot = await reserveLaunchSlot(stateRoot(), staggerMs, deps);
|
|
224
|
+
const wait = slot - deps.now();
|
|
225
|
+
if (wait > 0) {
|
|
226
|
+
deps.log(`[${name}] start-stagger: holding ${wait}ms before launch`);
|
|
227
|
+
await deps.sleep(wait);
|
|
228
|
+
}
|
|
229
|
+
}
|
|
135
230
|
// Supervisor mail monitor (design §1): prime the notification cursor at the
|
|
136
231
|
// stream tip BEFORE the session launches so no arrival is missed during boot
|
|
137
232
|
// (backlog before the tip is the SessionStart hook's job). Disabled roles keep
|
|
@@ -139,7 +234,7 @@ export async function runOnce(name, opts = {}, partialDeps = {}) {
|
|
|
139
234
|
// as disabled (monitor may be undefined on an old role.yaml).
|
|
140
235
|
const monitor = role.monitor?.enabled ? deps.createMonitor({
|
|
141
236
|
name, agentDir: dir, cfg: role.monitor,
|
|
142
|
-
deps: monitorDeps(deps),
|
|
237
|
+
deps: monitorDeps(deps, role.env),
|
|
143
238
|
}) : null;
|
|
144
239
|
if (monitor)
|
|
145
240
|
await monitor.prime();
|
package/dist/spawn.js
CHANGED
|
@@ -5,6 +5,7 @@ import { stringify } from 'yaml';
|
|
|
5
5
|
import { agentDir, fleetDDir } from './paths.js';
|
|
6
6
|
import { loadConfig, resolveMonitorConfig } from './config.js';
|
|
7
7
|
import { applyRole, up } from './ops.js';
|
|
8
|
+
import { START_STAGGER_FILE } from './runner.js';
|
|
8
9
|
function roleFromOpts(o, defaultHarness) {
|
|
9
10
|
const r = {};
|
|
10
11
|
if (o.harness)
|
|
@@ -96,6 +97,11 @@ export async function spawnTemp(o, binPath, launch = detachedSupervisor) {
|
|
|
96
97
|
};
|
|
97
98
|
const dir = applyRole(role, { temp: true });
|
|
98
99
|
writeFileSync(join(dir, 'role.yaml'), stringify(role));
|
|
100
|
+
// Snapshot the fleet start-stagger so the detached temp supervisor (no config path
|
|
101
|
+
// threaded through it) honors the same launch gate — a burst of temp spawns spaces
|
|
102
|
+
// out; a lone temp spawn still waits zero (time-based gate).
|
|
103
|
+
if (cfg.startStaggerMs > 0)
|
|
104
|
+
writeFileSync(join(dir, START_STAGGER_FILE), String(cfg.startStaggerMs));
|
|
99
105
|
// Run the supervisor DETACHED — NOT inside a tmux session named <name>.
|
|
100
106
|
// `_run-temp` -> runOnce() creates AND kills the tmux session <name> for the
|
|
101
107
|
// agent itself; a supervisor sharing that session name would SIGHUP its own
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@ours.network/fleet",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.9.1",
|
|
4
4
|
"description": "Harness-agnostic fleet of persistent, identity-bound AI agents. Declarative fleet.yaml, tmux consoles, systemd/launchd supervision, ours.network messaging.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "FSL-1.1-Apache-2.0",
|