@ours.network/fleet 0.17.0 → 0.17.2

package/dist/harness/codex.js

@@ -1,10 +1,11 @@
-import { mkdirSync } from 'node:fs';
-import { join } from 'node:path';
+import { chmodSync, existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
+import { dirname, join, resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
 import { agentDir, home } from '../paths.js';
 import { realExec } from '../exec.js';
 import { registerAdapter } from './registry.js';
 import { harnessRuntimeDir } from '../isolation/policy.js';
-import { bundledAcpAgent } from './acp-agent.js';
+import { bundledAcpAgent, resolveBundledAcpAgent } from './acp-agent.js';
 const OPTION_KEYS = [
     'launcher', 'sandbox', 'approval', 'permission_mode', 'search', 'profile', 'config', 'add_dirs',
     'monitor',
@@ -14,6 +15,12 @@ const LAUNCHERS = ['auto', 'ours-codex', 'codex'];
 const SANDBOX_MODES = ['read-only', 'workspace-write', 'danger-full-access'];
 /** Codex CLI's accepted `--ask-for-approval` values. */
 const APPROVAL_POLICIES = ['untrusted', 'on-request', 'never'];
+const BUNDLED_CODEX_ACP_VERSION = '1.1.7';
+const CODEX_ACP_PACKAGE = '@agentclientprotocol/codex-acp';
+const CODEX_PROXY_APPROVAL_ENV = 'OURS_FLEET_CODEX_APPROVAL';
+const CODEX_PROXY_SANDBOX_ENV = 'OURS_FLEET_CODEX_SANDBOX';
+const CODEX_PROXY_REAL_PATH_ENV = 'OURS_FLEET_REAL_CODEX_PATH';
+const CODEX_PROXY_MANIFEST_ENV = 'OURS_FLEET_CODEX_ACP_MANIFEST';
 /**
  * What an unattended role can actually do under Codex's native settings.
  * `on-request` and `untrusted` stop to ask, and with no console attached that
@@ -55,6 +62,22 @@ function acpAgentMode(role) {
         return 'agent-full-access';
     return undefined;
 }
+function acpModePermissions(mode) {
+    if (mode === 'read-only')
+        return { approval: 'on-request', sandbox: 'read-only' };
+    if (mode === 'agent-full-access')
+        return { approval: 'never', sandbox: 'danger-full-access' };
+    return { approval: 'on-request', sandbox: 'workspace-write' };
+}
+function fleetModeForApproval(nativeMode) {
+    if (nativeMode === 'never')
+        return 'allow';
+    if (nativeMode === 'on-request')
+        return 'auto';
+    if (nativeMode === 'untrusted')
+        return 'ask';
+    throw new Error(`unsupported Codex approval policy '${nativeMode}'`);
+}
 /** Resolve & validate the per-role approval policy, throwing on an unknown value. */
 function approvalPolicy(role) {
     const o = role.harness_options;
@@ -76,6 +99,59 @@ function approvalPolicy(role) {
 function launcherMode(role) {
     return role.harness_options?.launcher ?? 'auto';
 }
+function bundledCodexAcp() {
+    return resolveBundledAcpAgent(CODEX_ACP_PACKAGE, 'codex-acp', 'codex-acp');
+}
+function canOverrideBundledAcpApproval() {
+    const resolution = bundledCodexAcp();
+    return resolution.bundled && resolution.version === BUNDLED_CODEX_ACP_VERSION
+        && resolution.manifestPath !== undefined && compiledProxyModule() !== undefined;
+}
+function compiledProxyModule() {
+    const adjacent = fileURLToPath(new URL('./codex-app-server-proxy.js', import.meta.url));
+    if (existsSync(adjacent))
+        return adjacent;
+    // Vitest imports src/ directly; globalSetup builds the executable module in dist/.
+    const fromSource = resolve(dirname(fileURLToPath(import.meta.url)), '../../dist/harness', 'codex-app-server-proxy.js');
+    if (existsSync(fromSource))
+        return fromSource;
+    return undefined;
+}
+const shellQuote = (value) => `'${value.replaceAll("'", "'\\''")}'`;
+/**
+ * Materialize the tiny CODEX_PATH executable inside the role's own state dir.
+ * Keeping it there makes the proxy available under both ordinary and isolated
+ * launches without adding another host path to the filesystem boundary.
+ */
+function codexAcpEnvironment(role, dirs) {
+    if (role.session !== 'acp' || role.session_options?.acp?.command != null)
+        return {};
+    const resolution = bundledCodexAcp();
+    if (!resolution.bundled || resolution.version !== BUNDLED_CODEX_ACP_VERSION
+        || !resolution.manifestPath)
+        return {};
+    const runtimeDir = harnessRuntimeDir(dirs.stateDir, 'codex');
+    mkdirSync(runtimeDir, { recursive: true });
+    const proxyModule = join(runtimeDir, 'app-server-proxy.mjs');
+    const source = compiledProxyModule();
+    if (!source)
+        throw new Error('Codex app-server proxy is missing; rebuild ours-fleet');
+    writeFileSync(proxyModule, readFileSync(source, 'utf8'), { mode: 0o600 });
+    const windows = process.platform === 'win32';
+    const command = join(runtimeDir, windows ? 'codex-app-server-proxy.cmd' : 'codex-app-server-proxy');
+    const script = windows
+        ? `@echo off\r\n"${process.execPath.replaceAll('"', '""')}" "${proxyModule.replaceAll('"', '""')}" %*\r\n`
+        : `#!/bin/sh\nexec ${shellQuote(process.execPath)} ${shellQuote(proxyModule)} "$@"\n`;
+    writeFileSync(command, script, { mode: 0o700 });
+    chmodSync(command, 0o700);
+    return {
+        CODEX_PATH: command,
+        [CODEX_PROXY_APPROVAL_ENV]: approvalPolicy(role) ?? 'on-request',
+        [CODEX_PROXY_SANDBOX_ENV]: sandboxMode(role) ?? 'workspace-write',
+        [CODEX_PROXY_MANIFEST_ENV]: resolution.manifestPath,
+        ...(process.env.CODEX_PATH ? { [CODEX_PROXY_REAL_PATH_ENV]: process.env.CODEX_PATH } : {}),
+    };
+}
 function encodeTomlValue(value) {
     if (typeof value === 'string')
         return JSON.stringify(value);
@@ -209,7 +285,7 @@ export function makeCodexAdapter(exec = realExec) {
             if (requested === 'ours-codex' && !hasOursCodex)
                 throw new Error('harness_options.launcher is ours-codex, but ours-codex is not on PATH; install @ours.network/codex or use launcher: auto');
             const command = requested === 'codex' ? 'codex' : hasOursCodex ? 'ours-codex' : 'codex';
-            return { argv: [], env: {}, command };
+            return { argv: [], env: codexAcpEnvironment(role, dirs), command };
         },
         buildLaunch(role, mode, _s, prep) {
             const stateDir = roleStateDir(role);
@@ -227,7 +303,7 @@ export function makeCodexAdapter(exec = realExec) {
                 ? [...configured]
                 : typeof configured === 'string'
                     ? ['sh', '-c', configured]
-                    : bundledAcpAgent('@agentclientprotocol/codex-acp', 'codex-acp', 'codex-acp');
+                    : bundledAcpAgent(CODEX_ACP_PACKAGE, 'codex-acp', 'codex-acp');
             const initialMode = acpAgentMode(role);
             return {
                 argv,
@@ -282,6 +358,27 @@ export function makeCodexAdapter(exec = realExec) {
                 return translated;
             const approval = approvalPolicy(role) ?? 'on-request';
             const sandbox = sandboxMode(role) ?? 'workspace-write';
+            if (role.session === 'acp') {
+                const mode = acpAgentMode(role) ?? 'agent';
+                const configured = role.session_options?.acp?.command;
+                const overrideAvailable = configured == null && canOverrideBundledAcpApproval();
+                const actual = overrideAvailable ? { approval, sandbox } : acpModePermissions(mode);
+                const exact = actual.approval === approval && actual.sandbox === sandbox;
+                return {
+                    ...translated,
+                    native: { mode, ...actual },
+                    exact,
+                    warnings: exact ? [] : [configured != null
+                            ? `custom ACP command cannot be verified against approval=${approval} sandbox=${sandbox}; `
+                                + `its '${mode}' mode is conservatively treated as approval=${actual.approval} `
+                                + `sandbox=${actual.sandbox}`
+                            : `codex-acp mode '${mode}' actually uses approval=${actual.approval} `
+                                + `sandbox=${actual.sandbox}, and the bundled ${BUNDLED_CODEX_ACP_VERSION} `
+                                + `app-server override is unavailable; this does not exactly represent `
+                                + `approval=${approval} sandbox=${sandbox}`],
+                    capabilities: codexCapabilities(actual.approval, actual.sandbox),
+                };
+            }
             return {
                 ...translated,
                 native: { approval, sandbox },
@@ -289,13 +386,21 @@ export function makeCodexAdapter(exec = realExec) {
             };
         },
         effectivePermissionMode(role) {
+            if (role.session === 'acp') {
+                const nativeMode = acpAgentMode(role) ?? 'agent';
+                if (role.session_options?.acp?.command == null && canOverrideBundledAcpApproval()) {
+                    const approval = approvalPolicy(role) ?? 'on-request';
+                    return { fleetMode: fleetModeForApproval(approval), nativeMode };
+                }
+                return {
+                    fleetMode: fleetModeForApproval(acpModePermissions(nativeMode).approval), nativeMode,
+                };
+            }
             const nativeMode = approvalPolicy(role) ?? 'untrusted';
-            const fleetMode = nativeMode === 'never' ? 'allow'
-                : nativeMode === 'on-request' ? 'auto'
-                    : nativeMode === 'untrusted' ? 'ask' : undefined;
-            if (!fleetMode)
-                throw new Error(`unsupported Codex approval policy '${nativeMode}'`);
-            return { fleetMode, nativeMode };
+            return { fleetMode: fleetModeForApproval(nativeMode), nativeMode };
+        },
+        inheritedPermissionMode(role) {
+            return fleetModeForApproval(approvalPolicy(role) ?? 'untrusted');
         },
         vocabulary: {
             bindTool: 'choose_identity',

package/dist/harness/types.d.ts

@@ -111,6 +111,8 @@ export interface HarnessAdapter {
         fleetMode: FleetPermissionMode;
         nativeMode: string;
     };
+    /** Configured portable intent to inherit when a live runtime preset is narrower. */
+    inheritedPermissionMode?(role: ResolvedRole): FleetPermissionMode;
     /**
      * REQUIRED. Every adapter must either translate neutral permissions or
      * explicitly declare that it cannot. Enforced at registration.

package/dist/index.d.ts

@@ -1,7 +1,8 @@
 export { loadConfig, findRole, resolvePermissions, ConfigError } from './config.js';
 export type { FleetConfig, ResolvedRole, RoleConfig, OverseeEntry, SessionBackendId, CommonPermissions, FleetPermissionMode, ApprovalMode, SessionOptions, OwnerChannelConfig, } from './config.js';
 export type { HarnessAdapter, BriefingVocab, ExitPolicy, PrereqReport, PrereqCheck, SessionPrep, SessionState, Launch, AcpLaunch, PermissionTranslation, RoleDirs, ValidationError, } from './harness/types.js';
-export type { SessionHandle, SessionSnapshot, SessionEvent, TurnResult, } from './session/types.js';
+export type { SessionHandle, SessionSnapshot, SessionEvent, TurnResult, InterruptOutcome, InterruptResult, TurnCancellationSource, QueuedPrompt, ExitRecord, } from './session/types.js';
+export { interruptOutcome } from './session/types.js';
 export { AcpSession } from './session/acp.js';
 export { OwnerChannel } from './owner-channel/channel.js';
 export { TmuxSession } from './session/tmux.js';

package/dist/index.js

@@ -1,4 +1,5 @@
 export { loadConfig, findRole, resolvePermissions, ConfigError } from './config.js';
+export { interruptOutcome } from './session/types.js';
 export { AcpSession } from './session/acp.js';
 export { OwnerChannel } from './owner-channel/channel.js';
 export { TmuxSession } from './session/tmux.js';

package/dist/loops/manager.d.ts

@@ -1,12 +1,22 @@
 import type { ResolvedRoleLoop } from './config.js';
-import { type ScheduledLoopsFile } from './state.js';
+import { type ScheduledLoopsFile, type StateWriter } from './state.js';
 import { RoleTurnArbiter } from '../session/arbiter.js';
 export interface LoopManagerDeps {
     now(): number;
     setTimer(callback: () => void, ms: number): unknown;
     clearTimer(timer: unknown): void;
     log(line: string): void;
+    /** Persistence seam; defaults to the atomic replace. Tests inject write faults here. */
+    writeState?: StateWriter;
+    /** Test seam for the post-cancellation abandon bound; production uses the default. */
+    cancelAbandonMs?: number;
 }
+/**
+ * Recovery delay for the nth consecutive failure: 1s doubling to the poll
+ * ceiling. Bounded at both ends — never a busy retry, never longer than the
+ * normal cadence, so recovery is noticed within a minute of the fault clearing.
+ */
+export declare function backoffMs(consecutiveFailures: number): number;
 export interface LoopActionResult {
     state: 'started' | 'skipped_busy' | 'disabled' | 'unavailable';
     runId?: string;
@@ -29,6 +39,8 @@ export declare class ScheduledLoopManager implements ScheduledLoopManagerHandle
     private timer?;
     private readonly runTimeouts;
     private stopping;
+    /** Consecutive unforeseen poll failures; drives the recovery backoff. */
+    private pollFailures;
     constructor(role: string, definitions: ResolvedRoleLoop[], stateDir: string, arbiter: RoleTurnArbiter, deps: LoopManagerDeps);
     start(): void;
     stop(): Promise<void>;
@@ -40,10 +52,39 @@ export declare class ScheduledLoopManager implements ScheduledLoopManagerHandle
     /** Public fake-clock seam; timer callbacks call the same transition. */
     poll(): Promise<void>;
     private attempt;
+    /**
+     * Last resort for a run whose cancellation never settles. Without it a single
+     * unsettled turn keeps `activeRunId` set for the life of the process, and
+     * every later tick reports `skipped_busy` forever — the loop looks scheduled
+     * while nothing has run since. Releasing the slot is honest, and it is safe
+     * because `finish` still refuses to double-report a run it no longer owns.
+     *
+     * Releasing the loop's own slot is only half of it. The cancellation that
+     * never settled is still holding the arbiter's admission boundary, so the
+     * generation it belongs to has to be retired in the same step — otherwise the
+     * loop believes it is free while every later producer, scheduled or owner,
+     * queues behind a promise that will never resolve.
+     */
+    private armAbandon;
     private finish;
     private advance;
     private skipMissed;
     private skipRestartMisses;
+    /**
+     * A run the store could not record is dropped, not retried: the cursor has
+     * already moved, so this can never become a busy loop, and the outage is
+     * visible as a skip with a failed health rather than as silence.
+     */
+    private skipUnpersisted;
     private schedule;
+    private arm;
+    /**
+     * The residual safety net. Persistence failures are absorbed by the store, so
+     * reaching here means something unforeseen threw — and whatever it was, the
+     * manager must stay armed. The previous behaviour left no timer at all, which
+     * turned one transient ENOSPC into every loop on the role being silently gone
+     * until the process was restarted.
+     */
+    private recover;
     private envelope;
 }

package/dist/loops/manager.js

@@ -1,5 +1,23 @@
 import { randomUUID } from 'node:crypto';
 import { ScheduledLoopStateStore, deterministicJitter, increment, } from './state.js';
+/** The manager never sleeps longer than this, so one poll per minute is the floor rate. */
+const POLL_CEILING_MS = 60_000;
+/**
+ * Recovery delay for the nth consecutive failure: 1s doubling to the poll
+ * ceiling. Bounded at both ends — never a busy retry, never longer than the
+ * normal cadence, so recovery is noticed within a minute of the fault clearing.
+ */
+export function backoffMs(consecutiveFailures) {
+    const exponent = Math.min(Math.max(consecutiveFailures, 1) - 1, 6);
+    return Math.min(POLL_CEILING_MS, 1_000 * 2 ** exponent);
+}
+/**
+ * How long a cancelled run may stay `active` before the loop stops believing it
+ * will ever settle. Deliberately longer than the session layer's own escalation
+ * (cancel grace + SIGTERM->SIGKILL grace), so this fires only when that recovery
+ * did not, and never races a run that is about to report honestly.
+ */
+const LOOP_CANCEL_ABANDON_MS = 60_000;
 export class ScheduledLoopManager {
     role;
     arbiter;
@@ -9,13 +27,15 @@ export class ScheduledLoopManager {
     timer;
     runTimeouts = new Set();
     stopping = false;
+    /** Consecutive unforeseen poll failures; drives the recovery backoff. */
+    pollFailures = 0;
     constructor(role, definitions, stateDir, arbiter, deps) {
         this.role = role;
         this.arbiter = arbiter;
         this.deps = deps;
         for (const definition of definitions)
             this.definitions.set(definition.name, definition);
-        this.store = new ScheduledLoopStateStore(stateDir, role, definitions, deps.now(), deps.log);
+        this.store = new ScheduledLoopStateStore(stateDir, role, definitions, deps.now(), deps.log, deps.writeState);
     }
     start() {
         if (!this.store.fresh)
@@ -102,10 +122,21 @@ export class ScheduledLoopManager {
             }
             const scheduledAt = Date.parse(state.nextScheduledAt);
             this.advance(definition, state);
-            this.store.persist();
+            // The claim has to be durable before the turn is submitted: a run that is
+            // not on disk is a run a restart cannot see, and the occurrence would be
+            // submitted a second time. At-most-once outranks running this occurrence.
+            if (!this.store.persist()) {
+                this.skipUnpersisted(definition, state, now);
+                continue;
+            }
             await this.attempt(definition, state, scheduledAt);
         }
         this.store.state.clock.lastWallMs = now;
+        if (this.pollFailures) {
+            this.store.clearAnomaly('manager_task_failed');
+            this.deps.log(`[${this.role}] scheduled loop manager recovered after ${this.pollFailures} failed poll(s)`);
+            this.pollFailures = 0;
+        }
         this.store.persist();
         this.schedule();
     }
@@ -153,9 +184,16 @@ export class ScheduledLoopManager {
             if (state.activeRunId !== runId || this.stopping)
                 return;
             this.deps.log(`[${this.role}] loop ${definition.name} timed out run=${runId.slice(0, 11)} after ${timeoutMs}ms; cancelling`);
-            void this.arbiter.interrupt('scheduled-loop').catch(error => {
+            void this.arbiter.interrupt('scheduled-loop').then(outcome => {
+                // Forced recovery IS a successful cancellation; record how it ended
+                // rather than reporting the interrupt itself as a failure.
+                if (outcome?.state === 'forced')
+                    this.deps.log(`[${this.role}] loop ${definition.name} cancellation enforced `
+                        + `run=${runId.slice(0, 11)} reason=${outcome.reasonCode ?? 'forced'}`);
+            }).catch(error => {
                 this.deps.log(`[${this.role}] loop ${definition.name} timeout cancellation failed: ${error?.name ?? 'Error'}`);
             });
+            this.armAbandon(definition, state, runId);
         }, timeoutMs);
         this.runTimeouts.add(timeout);
         void result.queued.completion.then(turn => {
@@ -165,6 +203,38 @@ export class ScheduledLoopManager {
         });
         return { state: 'started', runId };
     }
+    /**
+     * Last resort for a run whose cancellation never settles. Without it a single
+     * unsettled turn keeps `activeRunId` set for the life of the process, and
+     * every later tick reports `skipped_busy` forever — the loop looks scheduled
+     * while nothing has run since. Releasing the slot is honest, and it is safe
+     * because `finish` still refuses to double-report a run it no longer owns.
+     *
+     * Releasing the loop's own slot is only half of it. The cancellation that
+     * never settled is still holding the arbiter's admission boundary, so the
+     * generation it belongs to has to be retired in the same step — otherwise the
+     * loop believes it is free while every later producer, scheduled or owner,
+     * queues behind a promise that will never resolve.
+     */
+    armAbandon(definition, state, runId) {
+        const abandon = this.deps.setTimer(() => {
+            this.runTimeouts.delete(abandon);
+            if (state.activeRunId !== runId || this.stopping)
+                return;
+            state.activeRunId = null;
+            state.lastFinishedAt = new Date(this.deps.now()).toISOString();
+            state.counts.failed = increment(state.counts.failed);
+            state.lastOutcome = 'abandoned_unsettled';
+            state.lastError = { kind: 'abandoned_unsettled', at: state.lastFinishedAt };
+            this.store.state.health = 'degraded';
+            this.store.state.anomaly = 'abandoned_unsettled';
+            this.store.persist();
+            this.arbiter.retireStalledAdmission();
+            this.deps.log(`[${this.role}] loop ${definition.name} abandoned run=${runId.slice(0, 11)}: `
+                + 'cancellation never settled; admission released');
+        }, this.deps.cancelAbandonMs ?? LOOP_CANCEL_ABANDON_MS);
+        this.runTimeouts.add(abandon);
+    }
     finish(definition, state, runId, result) {
         if (state.activeRunId !== runId)
             return;
@@ -216,6 +286,19 @@ export class ScheduledLoopManager {
                 this.skipMissed(definition, state, now);
         }
     }
+    /**
+     * A run the store could not record is dropped, not retried: the cursor has
+     * already moved, so this can never become a busy loop, and the outage is
+     * visible as a skip with a failed health rather than as silence.
+     */
+    skipUnpersisted(definition, state, now) {
+        state.counts.skipped = increment(state.counts.skipped);
+        state.lastOutcome = 'skipped_unpersisted';
+        state.lastFinishedAt = new Date(now).toISOString();
+        state.lastError = { kind: 'persist_failed', at: state.lastFinishedAt };
+        this.deps.log(`[${this.role}] loop ${definition.name} skipped_unpersisted `
+            + `(${this.store.lastPersistError ?? 'state write failed'})`);
+    }
     schedule() {
         if (this.timer !== undefined)
             this.deps.clearTimer(this.timer);
@@ -226,20 +309,36 @@ export class ScheduledLoopManager {
             const state = this.store.state.loops[definition.name];
             return definition.enabled && !state.operatorDisabled;
         }).map(definition => Date.parse(this.store.state.loops[definition.name].nextDueAt));
-        if (!due.length)
+        // While writes are failing the manager keeps a bounded probe armed even with
+        // nothing due, so health stops lying the moment the disk comes back.
+        const probe = this.store.persistFailures ? backoffMs(this.store.persistFailures) : undefined;
+        if (!due.length && probe === undefined)
             return;
-        const delay = Math.max(0, Math.min(...due) - this.deps.now());
-        this.timer = this.deps.setTimer(() => {
-            void this.poll().catch(error => {
-                this.store.state.health = 'failed';
-                this.store.state.anomaly = 'manager_task_failed';
-                try {
-                    this.store.persist();
-                }
-                catch { }
-                this.deps.log(`[${this.role}] scheduled loop manager failed: ${error?.name ?? 'Error'}`);
-            });
-        }, Math.min(delay, 60_000));
+        const delay = due.length ? Math.max(0, Math.min(...due) - this.deps.now()) : POLL_CEILING_MS;
+        this.arm(Math.min(delay, POLL_CEILING_MS, probe ?? POLL_CEILING_MS));
+    }
+    arm(delayMs) {
+        this.timer = this.deps.setTimer(() => { void this.poll().catch(error => this.recover(error)); }, delayMs);
+    }
+    /**
+     * The residual safety net. Persistence failures are absorbed by the store, so
+     * reaching here means something unforeseen threw — and whatever it was, the
+     * manager must stay armed. The previous behaviour left no timer at all, which
+     * turned one transient ENOSPC into every loop on the role being silently gone
+     * until the process was restarted.
+     */
+    recover(error) {
+        this.pollFailures = increment(this.pollFailures);
+        this.store.state.health = 'failed';
+        this.store.state.anomaly = 'manager_task_failed';
+        this.store.persist();
+        this.deps.log(`[${this.role}] scheduled loop manager failed: ${error?.name ?? 'Error'}`
+            + ` (attempt ${this.pollFailures}, retrying in ${backoffMs(this.pollFailures)}ms)`);
+        if (this.stopping)
+            return;
+        if (this.timer !== undefined)
+            this.deps.clearTimer(this.timer);
+        this.arm(backoffMs(this.pollFailures));
     }
     envelope(definition, runId, scheduledAt) {
         return [

package/dist/loops/state.d.ts

@@ -42,13 +42,57 @@ export interface ScheduledLoopsFile {
 export declare const increment: (value: number, amount?: number) => number;
 export declare function deterministicJitter(role: string, loop: string, nominalMs: number, maximumMs: number): number;
 export declare function scheduledLoopsPath(stateDir: string): string;
+/**
+ * A live manager rewrites the checkpoint at least once per poll ceiling (60s).
+ * Past this bound the recorded `health` describes a manager that is no longer
+ * updating it — a dead scheduler, or one whose writes are failing, which is
+ * exactly the case where the field still reads `healthy` because the flip to
+ * `failed` could not be written. Readers must not repeat a stale field as fact.
+ */
+export declare const STORED_STATE_STALE_MS: number;
+export interface StoredLoopVerdict {
+    health: ScheduledLoopsFile['health'] | 'stale';
+    recorded: ScheduledLoopsFile['health'];
+    stale: boolean;
+    ageMs: number;
+    /** Whether any loop still owes a run — nothing is expected of the file if not. */
+    scheduled: boolean;
+}
+/** Truthful health for a reader that only has the stored file to go on. */
+export declare function storedLoopHealth(file: ScheduledLoopsFile, now: number): StoredLoopVerdict;
 export declare function readScheduledLoops(stateDir: string): ScheduledLoopsFile | undefined;
+/** Persistence seam. Production uses the atomic replace; faults are injected here. */
+export type StateWriter = (path: string, contents: string, mode: number) => void;
 export declare class ScheduledLoopStateStore {
     private readonly log;
+    private readonly write;
     readonly path: string;
     readonly fresh: boolean;
     state: ScheduledLoopsFile;
-    constructor(stateDir: string, role: string, definitions: ResolvedRoleLoop[], now: number, log: (line: string) => void);
+    /** Consecutive failed checkpoints; 0 whenever the stored file is current. */
+    persistFailures: number;
+    lastPersistError: string | null;
+    /** Health displaced by the persist_failed marker, restored when a write lands. */
+    private suppressed;
+    constructor(stateDir: string, role: string, definitions: ResolvedRoleLoop[], now: number, log: (line: string) => void, write?: StateWriter);
     reconcile(definitions: ResolvedRoleLoop[], now: number, recoverActive?: boolean): void;
-    persist(): void;
+    /**
+     * Drop a transient anomaly once its cause is over — including one currently
+     * displaced by `persist_failed`, which would otherwise come back the moment a
+     * write finally lands.
+     */
+    clearAnomaly(kind: string): void;
+    /**
+     * Checkpoint the state. A write failure — ENOSPC is the one seen in the field —
+     * must never propagate: every caller sits under a timer callback, and an
+     * exception there kills the scheduling chain for the life of the process while
+     * the last-written file goes on claiming the loops are healthy. So the failure
+     * is recorded in memory instead, where `status()` and the live control socket
+     * report it immediately, and the caller decides what to do with `false`.
+     *
+     * The `failed` marker is applied optimistically-in-reverse: it is rolled back
+     * just before each attempt, so that the write which finally lands records the
+     * real health rather than the outage that is now over.
+     */
+    persist(): boolean;
 }

package/dist/loops/state.js

@@ -17,6 +17,25 @@ export function deterministicJitter(role, loop, nominalMs, maximumMs) {
 export function scheduledLoopsPath(stateDir) {
     return join(stateDir, '.scheduled-loops.json');
 }
+/**
+ * A live manager rewrites the checkpoint at least once per poll ceiling (60s).
+ * Past this bound the recorded `health` describes a manager that is no longer
+ * updating it — a dead scheduler, or one whose writes are failing, which is
+ * exactly the case where the field still reads `healthy` because the flip to
+ * `failed` could not be written. Readers must not repeat a stale field as fact.
+ */
+export const STORED_STATE_STALE_MS = 5 * 60_000;
+/** Truthful health for a reader that only has the stored file to go on. */
+export function storedLoopHealth(file, now) {
+    const ageMs = Math.max(0, now - file.clock.lastWallMs);
+    // A role with nothing left to schedule stops polling on purpose, so its
+    // checkpoint stops advancing on purpose too. Only a role that still owes
+    // someone a run can be stale; calling a deliberately idle one stale would
+    // teach operators to ignore the word.
+    const scheduled = Object.values(file.loops).some(loop => loop.enabled && !loop.operatorDisabled);
+    const stale = scheduled && ageMs > STORED_STATE_STALE_MS;
+    return { health: stale ? 'stale' : file.health, recorded: file.health, stale, ageMs, scheduled };
+}
 export function readScheduledLoops(stateDir) {
     try {
         const path = scheduledLoopsPath(stateDir);
@@ -31,11 +50,18 @@ export function readScheduledLoops(stateDir) {
 }
 export class ScheduledLoopStateStore {
     log;
+    write;
     path;
     fresh;
     state;
-    constructor(stateDir, role, definitions, now, log) {
+    /** Consecutive failed checkpoints; 0 whenever the stored file is current. */
+    persistFailures = 0;
+    lastPersistError = null;
+    /** Health displaced by the persist_failed marker, restored when a write lands. */
+    suppressed = null;
+    constructor(stateDir, role, definitions, now, log, write = replaceFileAtomically) {
         this.log = log;
+        this.write = write;
         this.path = scheduledLoopsPath(stateDir);
         let restored;
         let corrupt = false;
@@ -113,9 +139,61 @@ export class ScheduledLoopStateStore {
         this.state.clock.lastWallMs = now;
         this.persist();
     }
+    /**
+     * Drop a transient anomaly once its cause is over — including one currently
+     * displaced by `persist_failed`, which would otherwise come back the moment a
+     * write finally lands.
+     */
+    clearAnomaly(kind) {
+        if (this.suppressed?.anomaly === kind)
+            this.suppressed = { health: 'healthy', anomaly: null };
+        if (this.state.anomaly === kind) {
+            this.state.health = 'healthy';
+            this.state.anomaly = null;
+        }
+    }
+    /**
+     * Checkpoint the state. A write failure — ENOSPC is the one seen in the field —
+     * must never propagate: every caller sits under a timer callback, and an
+     * exception there kills the scheduling chain for the life of the process while
+     * the last-written file goes on claiming the loops are healthy. So the failure
+     * is recorded in memory instead, where `status()` and the live control socket
+     * report it immediately, and the caller decides what to do with `false`.
+     *
+     * The `failed` marker is applied optimistically-in-reverse: it is rolled back
+     * just before each attempt, so that the write which finally lands records the
+     * real health rather than the outage that is now over.
+     */
     persist() {
-        replaceFileAtomically(this.path, JSON.stringify(this.state, null, 2) + '\n', 0o600);
-        chmodSync(this.path, 0o600);
+        if (this.suppressed && this.state.anomaly === 'persist_failed') {
+            this.state.health = this.suppressed.health;
+            this.state.anomaly = this.suppressed.anomaly;
+        }
+        try {
+            this.write(this.path, JSON.stringify(this.state, null, 2) + '\n', 0o600);
+            chmodSync(this.path, 0o600);
+        }
+        catch (error) {
+            const code = error?.code ?? error?.name ?? 'Error';
+            if (this.state.anomaly !== 'persist_failed')
+                this.suppressed = { health: this.state.health, anomaly: this.state.anomaly };
+            this.state.health = 'failed';
+            this.state.anomaly = 'persist_failed';
+            this.lastPersistError = code;
+            this.persistFailures = increment(this.persistFailures);
+            if (this.persistFailures === 1)
+                this.log(`[${this.state.role}] scheduled loop state write failing (${code}); `
+                    + 'cadence held and health reported failed until it lands');
+            return false;
+        }
+        if (this.persistFailures) {
+            this.log(`[${this.state.role}] scheduled loop persistence recovered after `
+                + `${this.persistFailures} failed write(s) (${this.lastPersistError})`);
+            this.persistFailures = 0;
+            this.lastPersistError = null;
+        }
+        this.suppressed = null;
+        return true;
     }
 }
 function validFile(value) {