@ours.network/fleet 0.8.0 → 0.9.0

package/README.md

@@ -180,6 +180,7 @@ the default `~/fleet.yaml` on its very first crash-restart and fail to resolve.
 
 ```yaml
 vars: { work_root: /home/me/work }      # ${var} substitution anywhere below
+start_stagger_ms: 0                     # delay between agent LAUNCHES (host-wide, ms); 0 = no stagger
 defaults:
   harness: claude-code                  # for roles that don't set one
   model: claude-fable-5                 # default model for roles that don't set one (per-role model / --model wins)
@@ -240,6 +241,24 @@ role's `harness_options`, so a fleet can set common Codex permission/profile def
 and override individual keys per role. `monitor` merges the same way — a role block
 overrides `defaults.monitor` key-by-key.
 
+### Start staggering
+
+`start_stagger_ms` (top-level, host-wide, default `0`) spaces out agent **launches**
+so a burst of boots doesn't hit the harness/API rate limit (429) all at once. When
+set, each launch is held until at least `start_stagger_ms` after the previous one
+across the whole host. It is enforced at the harness-launch point inside the runner,
+so — unlike a delay in the `up`/`restart` command loop — it also covers **systemd
+host boot**, where every agent's unit starts concurrently. The gate is time-based:
+a lone start or a solo crash-restart waits **zero**; only genuinely concurrent
+launches are spread out. Example: `start_stagger_ms: 4000` on a 7-agent fleet spaces
+their boots ~4 s apart instead of firing all seven at once.
+
+> **Migration (v0.9+):** this replaces the old `FLEET_START_STAGGER` environment
+> variable, which only staggered the `ours-fleet up`/`restart` command loop (not
+> host boot) and defaulted to 5 s. `FLEET_START_STAGGER` is **retired** — set
+> `start_stagger_ms` in `fleet.yaml` instead (note: milliseconds, and the default
+> is now `0`, so add it explicitly if you relied on the old implicit 5 s spacing).
+
 ### Mail monitor
 
 With `monitor.enabled` (the default), the **supervisor** delivers a role's mail

package/dist/cli.js

@@ -26,7 +26,6 @@ catch {
 const deps = () => ({
     backend: pickBackend(),
     binPath,
-    sleep: ms => new Promise(r => setTimeout(r, ms)),
     log: l => console.log(l),
 });
 const die = (e) => { console.error(String(e instanceof Error ? e.message : e)); process.exit(1); };

package/dist/config.d.ts

@@ -48,6 +48,8 @@ export interface FleetConfig {
     vars: Record<string, string>;
     defaults: Record<string, unknown>;
     files: string[];
+    /** Fleet-wide delay (ms) enforced between agent launches to avoid boot bursts (0 = none). */
+    startStaggerMs: number;
 }
 export declare class ConfigError extends Error {
 }

package/dist/config.js

@@ -86,6 +86,7 @@ export function loadConfig(configPath) {
     const baseDoc = docs.length && docs[0].file === base ? docs[0].doc : {};
     const vars = (baseDoc.vars ?? {});
     const defaults = (baseDoc.defaults ?? {});
+    const startStaggerMs = resolveStartStaggerMs(baseDoc.start_stagger_ms, base);
     const seen = new Map();
     const roles = [];
     for (const { file, doc } of docs) {
@@ -132,7 +133,15 @@ export function loadConfig(configPath) {
             });
         }
     }
-    return { roles, vars, defaults, files };
+    return { roles, vars, defaults, files, startStaggerMs };
+}
+/** Validate the top-level `start_stagger_ms` (supervisor launch spacing); default 0. */
+function resolveStartStaggerMs(raw, base) {
+    if (raw === undefined || raw === null)
+        return 0;
+    if (typeof raw !== 'number' || !Number.isFinite(raw) || raw < 0)
+        throw new ConfigError(`${base}: start_stagger_ms must be a non-negative number of milliseconds (got ${JSON.stringify(raw)})`);
+    return raw;
 }
 /**
  * Merge `defaults.monitor` under the role's own `monitor:` key-by-key, validate the

package/dist/ops.d.ts

@@ -3,7 +3,6 @@ import type { SupervisorBackend } from './supervisor/types.js';
 export interface OpsDeps {
     backend: SupervisorBackend;
     binPath: string;
-    sleep(ms: number): Promise<void>;
     log(line: string): void;
 }
 /** Materialize a role's state dir from config: briefing + markers. Returns the dir. */

package/dist/ops.js

@@ -5,7 +5,11 @@ import { agentDir, fleetDDir } from './paths.js';
 import { findRole } from './config.js';
 import { getAdapter } from './harness/registry.js';
 import { generateBriefing } from './briefing.js';
-const STAGGER_MS = () => 1000 * Number(process.env.FLEET_START_STAGGER ?? 5);
+// Launch staggering now lives at the harness-launch point (the runner's start
+// gate, driven by `start_stagger_ms`), so it covers systemd host-boot too — not
+// just the `up`/`restart` command loop below. The old in-loop FLEET_START_STAGGER
+// sleep is retired; `up`/`restart` fire installs promptly and the gate spaces the
+// resulting launches.
 /** Materialize a role's state dir from config: briefing + markers. Returns the dir. */
 export function applyRole(role, opts = {}) {
     const adapter = getAdapter(role.harness);
@@ -43,11 +47,7 @@ function selectRoles(cfg, names) {
 }
 /** Create/start roles declaratively. Idempotent; active roles keep their context. */
 export async function up(cfg, names, deps, configPath) {
-    let first = true;
     for (const role of selectRoles(cfg, names)) {
-        if (!first)
-            await deps.sleep(STAGGER_MS());
-        first = false;
         const dir = applyRole(role, { configPath });
         // If the role isn't running, boot fresh so it reads the briefing we just wrote.
         const status = await deps.backend.status(role.name).catch(() => '');
@@ -70,11 +70,7 @@ export async function down(cfg, names, deps) {
 }
 /** Re-sync from config + bounce. mode 'keep' resumes context; 'fresh' wipes it. */
 export async function restartRoles(cfg, names, deps, mode, configPath) {
-    let first = true;
     for (const role of selectRoles(cfg, names)) {
-        if (!first)
-            await deps.sleep(STAGGER_MS());
-        first = false;
         applyRole(role, { fresh: mode === 'fresh', configPath });
         await deps.backend.restart(role.name);
         deps.log(mode === 'fresh'

package/dist/runner.d.ts

@@ -24,6 +24,17 @@ export interface RunnerDeps {
  * still sees the real exit code.
  */
 export declare function buildPaneCommand(launch: Launch, roleEnv: Record<string, string> | undefined, exitStatusPath: string, paneArgv?: string[]): string;
+/** Filename spawnTemp writes into a temp agent dir to carry the fleet start-stagger. */
+export declare const START_STAGGER_FILE = ".start-stagger-ms";
+/**
+ * Reserve this process's launch slot on the host-wide start gate and return the
+ * wall-clock time it may launch at. A tiny atomic mutex (mkdir is atomic across
+ * processes) guards a single `.last-launch` timestamp: each launcher takes the
+ * next slot = max(now, last + staggerMs), so concurrent boots serialize and spread
+ * out by staggerMs while a lone/idle start returns `now` (zero wait). A crashed
+ * launcher's stale lock is broken so the gate can never deadlock the fleet.
+ */
+export declare function reserveLaunchSlot(root: string, staggerMs: number, deps: Pick<RunnerDeps, 'now' | 'sleep' | 'log'>): Promise<number>;
 /** Read a temp role's config snapshot written by spawnTemp. */
 export declare function loadTempRole(name: string): ResolvedRole;
 /** One supervised session lifecycle. The supervisor re-invokes us after we return. */

package/dist/runner.js

@@ -2,7 +2,7 @@ import { existsSync, readFileSync, writeFileSync, rmSync, mkdirSync } from 'node
 import { join } from 'node:path';
 import { randomUUID } from 'node:crypto';
 import { parse } from 'yaml';
-import { agentDir, home } from './paths.js';
+import { agentDir, home, stateRoot } from './paths.js';
 import { loadConfig, findRole } from './config.js';
 import { getAdapter } from './harness/registry.js';
 import { Tmux } from './tmux.js';
@@ -54,6 +54,72 @@ function monitorDeps(deps) {
         timers: { set: (fn, ms) => setTimeout(fn, ms), clear: t => clearTimeout(t) },
     };
 }
+/** Filename spawnTemp writes into a temp agent dir to carry the fleet start-stagger. */
+export const START_STAGGER_FILE = '.start-stagger-ms';
+/** Read the start-stagger a temp agent was spawned with (0 if none / unreadable). */
+function readStartStagger(dir) {
+    try {
+        const n = parseInt(readFileSync(join(dir, START_STAGGER_FILE), 'utf8').trim(), 10);
+        return Number.isFinite(n) && n > 0 ? n : 0;
+    }
+    catch {
+        return 0;
+    }
+}
+/**
+ * Reserve this process's launch slot on the host-wide start gate and return the
+ * wall-clock time it may launch at. A tiny atomic mutex (mkdir is atomic across
+ * processes) guards a single `.last-launch` timestamp: each launcher takes the
+ * next slot = max(now, last + staggerMs), so concurrent boots serialize and spread
+ * out by staggerMs while a lone/idle start returns `now` (zero wait). A crashed
+ * launcher's stale lock is broken so the gate can never deadlock the fleet.
+ */
+export async function reserveLaunchSlot(root, staggerMs, deps) {
+    mkdirSync(root, { recursive: true });
+    const lockDir = join(root, '.launch-gate.lock');
+    const lockTsFile = join(lockDir, 'ts');
+    const tsFile = join(root, '.last-launch');
+    const staleMs = Math.max(staggerMs * 4, 10_000);
+    const POLL_MS = 50;
+    const readTs = (p) => {
+        try {
+            const n = parseInt(readFileSync(p, 'utf8').trim(), 10);
+            return Number.isFinite(n) ? n : null;
+        }
+        catch {
+            return null;
+        }
+    };
+    for (let waited = 0;;) {
+        try {
+            mkdirSync(lockDir);
+            writeFileSync(lockTsFile, String(deps.now()));
+            break;
+        }
+        catch (e) {
+            if (e.code !== 'EEXIST')
+                throw e;
+            const lockTs = readTs(lockTsFile);
+            const stale = lockTs !== null && deps.now() - lockTs > staleMs;
+            if (stale || waited > staleMs * 2) { // break a crashed launcher's lock; never deadlock
+                rmSync(lockDir, { recursive: true, force: true });
+                continue;
+            }
+            await deps.sleep(POLL_MS);
+            waited += POLL_MS;
+        }
+    }
+    try {
+        const now = deps.now();
+        const last = readTs(tsFile);
+        const target = last === null ? now : Math.max(now, last + staggerMs);
+        writeFileSync(tsFile, String(target));
+        return target;
+    }
+    finally {
+        rmSync(lockDir, { recursive: true, force: true });
+    }
+}
 /** Read a temp role's config snapshot written by spawnTemp. */
 export function loadTempRole(name) {
     const p = join(agentDir(name, true), 'role.yaml');
@@ -86,7 +152,20 @@ export async function runOnce(name, opts = {}, partialDeps = {}) {
     const temp = opts.temp === true;
     const dir = agentDir(name, temp);
     const configPath = temp ? opts.configPath : resolveConfigPath(dir, opts.configPath);
-    const role = temp ? loadTempRole(name) : findRole(loadConfig(configPath), name);
+    // Resolve the role and the fleet-wide start-stagger. Permanent roles read the
+    // live config; temp/detached agents read the value spawnTemp snapshotted into
+    // their dir (they have no config path threaded through the detached supervisor).
+    let role;
+    let staggerMs;
+    if (temp) {
+        role = loadTempRole(name);
+        staggerMs = readStartStagger(dir);
+    }
+    else {
+        const cfg = loadConfig(configPath);
+        role = findRole(cfg, name);
+        staggerMs = cfg.startStaggerMs;
+    }
     const adapter = getAdapter(role.harness);
     mkdirSync(dir, { recursive: true });
     const sidFile = join(dir, '.session-id');
@@ -132,6 +211,20 @@ export async function runOnce(name, opts = {}, partialDeps = {}) {
         if (rprefix.length)
             paneArgv = [...rprefix, ...paneArgv];
     }
+    // Start-stagger: space this launch at least start_stagger_ms after the previous
+    // agent launch across the whole host, so a burst of boots (systemd starts every
+    // user unit concurrently on boot; `ours-fleet up`/restart-all bulk-start) does not
+    // hit the harness/API rate limit at once. Time-based via a shared launch gate, so
+    // a lone start or a solo crash-restart waits zero. Applied right before the harness
+    // launch (tmux.newSession); the cheap monitor prime still runs immediately after.
+    if (staggerMs > 0) {
+        const slot = await reserveLaunchSlot(stateRoot(), staggerMs, deps);
+        const wait = slot - deps.now();
+        if (wait > 0) {
+            deps.log(`[${name}] start-stagger: holding ${wait}ms before launch`);
+            await deps.sleep(wait);
+        }
+    }
     // Supervisor mail monitor (design §1): prime the notification cursor at the
     // stream tip BEFORE the session launches so no arrival is missed during boot
     // (backlog before the tip is the SessionStart hook's job). Disabled roles keep

package/dist/spawn.js

@@ -5,6 +5,7 @@ import { stringify } from 'yaml';
 import { agentDir, fleetDDir } from './paths.js';
 import { loadConfig, resolveMonitorConfig } from './config.js';
 import { applyRole, up } from './ops.js';
+import { START_STAGGER_FILE } from './runner.js';
 function roleFromOpts(o, defaultHarness) {
     const r = {};
     if (o.harness)
@@ -96,6 +97,11 @@ export async function spawnTemp(o, binPath, launch = detachedSupervisor) {
     };
     const dir = applyRole(role, { temp: true });
     writeFileSync(join(dir, 'role.yaml'), stringify(role));
+    // Snapshot the fleet start-stagger so the detached temp supervisor (no config path
+    // threaded through it) honors the same launch gate — a burst of temp spawns spaces
+    // out; a lone temp spawn still waits zero (time-based gate).
+    if (cfg.startStaggerMs > 0)
+        writeFileSync(join(dir, START_STAGGER_FILE), String(cfg.startStaggerMs));
     // Run the supervisor DETACHED — NOT inside a tmux session named <name>.
     // `_run-temp` -> runOnce() creates AND kills the tmux session <name> for the
     // agent itself; a supervisor sharing that session name would SIGHUP its own

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ours.network/fleet",
-  "version": "0.8.0",
+  "version": "0.9.0",
   "description": "Harness-agnostic fleet of persistent, identity-bound AI agents. Declarative fleet.yaml, tmux consoles, systemd/launchd supervision, ours.network messaging.",
   "type": "module",
   "license": "FSL-1.1-Apache-2.0",