@ours.network/fleet 0.11.1 → 0.13.0

package/dist/owner-channel/tasks.js

@@ -0,0 +1,246 @@
+import { createHash, randomBytes } from 'node:crypto';
+import { chmodSync, existsSync, readFileSync } from 'node:fs';
+import { replaceFileAtomically } from '../atomic-file.js';
+export const OWNER_TASK_TTL_MS = 7 * 24 * 60 * 60 * 1_000;
+export const OWNER_TASK_MAX_OPEN = 32;
+export const OWNER_TASK_MAX_PER_OWNER = 8;
+export const OWNER_TASK_MAX_REPORTS = 20;
+export const OWNER_TASK_REPORT_MIN_INTERVAL_MS = 5_000;
+const MAX_TOMBSTONES = 256;
+const TOMBSTONE_TTL_MS = 30 * 24 * 60 * 60 * 1_000;
+const HEX_64 = /^[a-f0-9]{64}$/;
+const CID = /^[A-Fa-f0-9]{64}$/;
+/**
+ * Durable proactive-task routing. It deliberately stores no report body: only
+ * the authenticated route, bounded hashes/counters, and delivery state.
+ */
+export class OwnerTaskState {
+    path;
+    tasks = [];
+    tombstones = [];
+    corruptReason;
+    constructor(path) {
+        this.path = path;
+        if (!existsSync(path))
+            return;
+        try {
+            const raw = JSON.parse(readFileSync(path, 'utf8'));
+            if (raw.version !== 1 || !Array.isArray(raw.tasks) || !Array.isArray(raw.tombstones)
+                || raw.tasks.length > OWNER_TASK_MAX_OPEN || raw.tombstones.length > MAX_TOMBSTONES
+                || !raw.tasks.every(task => this.validTask(task))
+                || !raw.tombstones.every(tombstone => this.validTombstone(tombstone)))
+                throw new Error('invalid or unbounded task state');
+            const ids = [...raw.tasks.map(task => task.id), ...raw.tombstones.map(item => item.id)];
+            if (new Set(ids).size !== ids.length)
+                throw new Error('duplicate task ID');
+            this.tasks = raw.tasks.map(task => ({ ...task, digests: [...task.digests] }));
+            this.tombstones = raw.tombstones.map(item => ({ ...item }));
+            // A crash or lost response after the durable pre-send marker makes the
+            // result unknowable. Never resend it automatically and risk duplication.
+            let recovered = false;
+            for (const task of this.tasks) {
+                if (task.status === 'sending') {
+                    task.status = 'uncertain';
+                    recovered = true;
+                }
+            }
+            chmodSync(path, 0o600);
+            if (recovered)
+                this.persist();
+        }
+        catch {
+            this.tasks = [];
+            this.tombstones = [];
+            this.corruptReason = 'invalid persisted owner task state';
+            try {
+                chmodSync(path, 0o600);
+            }
+            catch { /* remain fail-closed */ }
+        }
+    }
+    integrity() {
+        return this.corruptReason ? { ok: false, error: this.corruptReason } : { ok: true };
+    }
+    open(route, now = Date.now()) {
+        this.assertHealthy();
+        if (!HEX_64.test(route.requestId) || !CID.test(route.contact)
+            || typeof route.wireId !== 'string' || route.wireId.length < 1 || route.wireId.length > 1_024)
+            throw new Error('owner task route is invalid or exceeds its bounds');
+        this.cleanup(now);
+        if (this.tasks.length >= OWNER_TASK_MAX_OPEN)
+            throw new Error(`owner channel is limited to ${OWNER_TASK_MAX_OPEN} open tasks`);
+        if (this.tasks.filter(task => task.contact === route.contact).length >= OWNER_TASK_MAX_PER_OWNER)
+            throw new Error(`an owner is limited to ${OWNER_TASK_MAX_PER_OWNER} open tasks per role`);
+        let id;
+        do {
+            id = randomBytes(32).toString('hex');
+        } while (this.tasks.some(task => task.id === id) || this.tombstones.some(item => item.id === id));
+        const task = {
+            id, ...route, createdAt: now, expiresAt: now + OWNER_TASK_TTL_MS,
+            status: 'open', sequence: 0, reportCount: 0, digests: [],
+        };
+        this.mutate(() => { this.tasks.push(task); });
+        return { ...task, digests: [] };
+    }
+    route(taskId, now = Date.now()) {
+        this.assertHealthy();
+        this.cleanup(now);
+        const task = this.findOpen(taskId);
+        return {
+            ...task, digests: [...task.digests],
+            ...(task.pending ? { pending: { ...task.pending } } : {}),
+        };
+    }
+    beginReport(taskId, phase, digest, chars, bytes, now = Date.now()) {
+        this.assertHealthy();
+        this.cleanup(now);
+        const task = this.findOpen(taskId);
+        if (task.status === 'uncertain')
+            throw new Error('task report delivery outcome is uncertain; refusing to resend or reorder reports');
+        if (task.status !== 'open')
+            throw new Error('task report is already being delivered');
+        if (task.digests.includes(digest))
+            throw new Error('duplicate task report refused');
+        if (task.reportCount >= OWNER_TASK_MAX_REPORTS)
+            throw new Error(`owner task is limited to ${OWNER_TASK_MAX_REPORTS} reports`);
+        if (task.lastReportAt !== undefined && now - task.lastReportAt < OWNER_TASK_REPORT_MIN_INTERVAL_MS)
+            throw new Error(`owner task reports are rate-limited to one every ${OWNER_TASK_REPORT_MIN_INTERVAL_MS}ms`);
+        this.mutate(() => {
+            task.status = 'sending';
+            task.pending = { digest, phase, chars, bytes };
+        });
+        return { ...task, digests: [...task.digests], pending: { ...task.pending } };
+    }
+    delivered(taskId, digest, terminal, now = Date.now()) {
+        this.assertHealthy();
+        const task = this.tasks.find(item => item.id === taskId);
+        if (!task || task.status !== 'sending' || task.pending?.digest !== digest)
+            throw new Error('task report delivery state changed unexpectedly');
+        const sequence = task.sequence + 1;
+        this.mutate(() => {
+            task.sequence = sequence;
+            task.reportCount++;
+            task.lastReportAt = now;
+            task.digests.push(digest);
+            task.pending = undefined;
+            if (terminal)
+                this.remove(task, 'closed', now);
+            else
+                task.status = 'open';
+        });
+        return sequence;
+    }
+    uncertain(taskId, digest) {
+        this.assertHealthy();
+        const task = this.tasks.find(item => item.id === taskId);
+        if (!task || task.pending?.digest !== digest)
+            return;
+        this.mutate(() => { task.status = 'uncertain'; });
+    }
+    revoke(contact, now = Date.now()) {
+        this.assertHealthy();
+        const revoked = this.tasks.filter(task => task.contact === contact);
+        if (!revoked.length)
+            return 0;
+        this.mutate(() => { for (const task of revoked)
+            this.remove(task, 'revoked', now); });
+        return revoked.length;
+    }
+    cleanup(now = Date.now(), effectiveOwners) {
+        this.assertHealthy();
+        const expired = this.tasks.filter(task => task.expiresAt <= now);
+        const revoked = effectiveOwners
+            ? this.tasks.filter(task => !effectiveOwners.has(task.contact) && !expired.includes(task)) : [];
+        const oldTombstones = this.tombstones.filter(item => now - item.at > TOMBSTONE_TTL_MS);
+        if (!expired.length && !revoked.length && !oldTombstones.length)
+            return 0;
+        this.mutate(() => {
+            for (const task of expired)
+                this.remove(task, 'expired', now);
+            for (const task of revoked)
+                this.remove(task, 'revoked', now);
+            this.tombstones = this.tombstones
+                .filter(item => now - item.at <= TOMBSTONE_TTL_MS).slice(-MAX_TOMBSTONES);
+        });
+        return expired.length + revoked.length;
+    }
+    findOpen(taskId) {
+        if (!HEX_64.test(taskId))
+            throw new Error('owner task ID must be exactly 64 lowercase hexadecimal characters');
+        const task = this.tasks.find(item => item.id === taskId);
+        if (task)
+            return task;
+        const tombstone = this.tombstones.find(item => item.id === taskId);
+        if (tombstone)
+            throw new Error(`owner task is ${tombstone.state}`);
+        throw new Error('unknown owner task ID');
+    }
+    remove(task, state, now) {
+        this.tasks = this.tasks.filter(item => item !== task);
+        this.tombstones.push({ id: task.id, state, at: now });
+        this.tombstones = this.tombstones.slice(-MAX_TOMBSTONES);
+    }
+    mutate(change) {
+        const snapshot = JSON.stringify({ tasks: this.tasks, tombstones: this.tombstones });
+        change();
+        try {
+            this.persist();
+        }
+        catch (error) {
+            const old = JSON.parse(snapshot);
+            this.tasks = old.tasks;
+            this.tombstones = old.tombstones;
+            throw error;
+        }
+    }
+    persist() {
+        replaceFileAtomically(this.path, JSON.stringify({
+            version: 1, tasks: this.tasks, tombstones: this.tombstones,
+        }) + '\n', 0o600);
+        chmodSync(this.path, 0o600);
+    }
+    assertHealthy() {
+        if (this.corruptReason)
+            throw new Error(`owner task state is corrupt; refusing operation: ${this.corruptReason}`);
+    }
+    validTask(value) {
+        if (!value || typeof value !== 'object')
+            return false;
+        const task = value;
+        return HEX_64.test(task.id) && HEX_64.test(task.requestId) && CID.test(task.contact)
+            && typeof task.wireId === 'string' && task.wireId.length > 0 && task.wireId.length <= 1_024
+            && Number.isSafeInteger(task.createdAt) && task.createdAt >= 0
+            && Number.isSafeInteger(task.expiresAt) && task.expiresAt - task.createdAt === OWNER_TASK_TTL_MS
+            && ['open', 'sending', 'uncertain'].includes(task.status)
+            && Number.isSafeInteger(task.sequence) && task.sequence >= 0
+            && Number.isSafeInteger(task.reportCount) && task.reportCount >= 0
+            && task.reportCount <= OWNER_TASK_MAX_REPORTS
+            && Array.isArray(task.digests) && task.digests.length <= OWNER_TASK_MAX_REPORTS
+            && task.digests.every(digest => HEX_64.test(digest))
+            && new Set(task.digests).size === task.digests.length
+            && task.sequence === task.reportCount && task.reportCount === task.digests.length
+            && (task.reportCount === 0
+                ? task.lastReportAt === undefined
+                : Number.isSafeInteger(task.lastReportAt) && task.lastReportAt >= task.createdAt)
+            && (task.pending === undefined || this.validPending(task.pending))
+            && (task.pending === undefined || !task.digests.includes(task.pending.digest))
+            && (task.status === 'open' ? task.pending === undefined : task.pending !== undefined);
+    }
+    validPending(value) {
+        if (!value || typeof value !== 'object')
+            return false;
+        const pending = value;
+        return HEX_64.test(pending.digest) && ['progress', 'done', 'blocked'].includes(pending.phase)
+            && Number.isSafeInteger(pending.chars) && pending.chars >= 1 && pending.chars <= 280
+            && Number.isSafeInteger(pending.bytes) && pending.bytes >= 1 && pending.bytes <= 1_024;
+    }
+    validTombstone(value) {
+        if (!value || typeof value !== 'object')
+            return false;
+        const item = value;
+        return HEX_64.test(item.id) && ['closed', 'expired', 'revoked'].includes(item.state)
+            && Number.isSafeInteger(item.at);
+    }
+}
+export const ownerTaskDigest = (phase, message) => createHash('sha256').update(`${phase}\0${message}`).digest('hex');
+export const ownerTaskAuditId = (value) => createHash('sha256').update(value).digest('hex').slice(0, 12);

package/dist/resolved-plan.js

@@ -20,6 +20,12 @@ export function resolvedPlan(cfg) {
         startStaggerMs: cfg.startStaggerMs,
         diagnostics: cfg.diagnostics.map(diagnostic => ({ ...diagnostic })),
         roles: cfg.roles.map(resolvedRolePlan),
+        loops: cfg.loops.map(loop => sortedObject({
+            name: loop.name, selectors: [...loop.selectors], roles: [...loop.roleNames],
+            intervalMs: loop.intervalMs, initialDelayMs: loop.initialDelayMs, jitterMs: loop.jitterMs,
+            enabled: loop.enabled, sourceFile: loop.sourceFile,
+            prompt: { bytes: loop.promptBytes, sha256: loop.promptHash },
+        })),
         watchdogs: cfg.watchdogs.map(w => sortedObject({
             name: w.name, sourceFile: w.sourceFile, enabled: w.enabled,
             intervalMs: w.intervalMs, coordinator: w.coordinator, watch: [...w.watch],
@@ -60,6 +66,12 @@ export function resolvedRolePlan(role) {
             ],
         },
         monitor: role.monitor,
+        ownerChannel: role.owner_channel ?? null,
+        loops: (role.loops ?? []).map(loop => ({
+            name: loop.name, enabled: loop.enabled, intervalMs: loop.intervalMs,
+            initialDelayMs: loop.initialDelayMs, jitterMs: loop.jitterMs,
+            definitionHash: loop.definitionHash, prompt: { bytes: loop.promptBytes, sha256: loop.promptHash },
+        })),
         isolation: role.isolation ?? null,
         worklog: role.worklog ?? null,
         authProxy: role.auth_proxy ?? null,

package/dist/runner.d.ts

@@ -4,6 +4,7 @@ import { Tmux } from './tmux.js';
 import { type MonitorHandle, type MonitorOpts, type FetchLike } from './monitor.js';
 import { type Exec } from './exec.js';
 import type { ExitRecord } from './session/types.js';
+import { type OwnerChannelHandle, type OwnerChannelOptions } from './owner-channel/channel.js';
 export interface RunnerDeps {
     tmux: Tmux;
     exec: Exec;
@@ -16,6 +17,8 @@ export interface RunnerDeps {
     fetch: FetchLike;
     /** Construct the supervisor mail monitor (injectable so tests stub it out). */
     createMonitor(opts: MonitorOpts): MonitorHandle;
+    /** Construct trusted owner ingress (injectable for lifecycle tests). */
+    createOwnerChannel(opts: OwnerChannelOptions): OwnerChannelHandle;
     /** Lets a test (or a shutdown path) end the supervised restart loop. */
     shouldStop?(): boolean;
 }

package/dist/runner.js

@@ -18,6 +18,9 @@ import { TmuxSession } from './session/tmux.js';
 import { classifyShellStatus } from './session/types.js';
 import { effectiveModelForRole, modelRecoveryHeld, reconcileModelRecovery, recordModelFailure, classifyFailureText, } from './model-recovery.js';
 import { rotateWorklog } from './worklog.js';
+import { OwnerChannel } from './owner-channel/channel.js';
+import { RoleTurnArbiter } from './session/arbiter.js';
+import { ScheduledLoopManager, } from './loops/manager.js';
 const defaultDeps = () => ({
     tmux: new Tmux(),
     exec: realExec,
@@ -34,6 +37,7 @@ const defaultDeps = () => ({
     log: line => process.stderr.write(line + '\n'),
     fetch: (url, init) => globalThis.fetch(url, init),
     createMonitor: opts => createMonitor(opts),
+    createOwnerChannel: opts => new OwnerChannel(opts),
 });
 const MONITOR_OWNER_FILE = '.monitor-owner';
 /**
@@ -415,6 +419,13 @@ export async function runOnce(name, opts = {}, partialDeps = {}) {
     let unsubscribeRecovery;
     let monitorLoop;
     let acpStartupComplete = false;
+    let ownerChannel;
+    let loopManager;
+    let arbiter;
+    let reloadLoopConfig;
+    let loopGeneration = JSON.stringify((role.loops ?? []).map(loop => [
+        loop.name, loop.definitionHash, loop.promptHash,
+    ]));
     if (sessionBackend === 'acp') {
         const perms = role.permissions ?? resolvePermissions(undefined, undefined);
         // Say once, at startup, that this role will decide permission requests by
@@ -434,7 +445,8 @@ export async function runOnce(name, opts = {}, partialDeps = {}) {
             log: deps.log,
         });
         pid = acpSession.pid;
-        sessionHandle = acpSession;
+        arbiter = new RoleTurnArbiter(acpSession);
+        sessionHandle = arbiter;
         unsubscribeRecovery = acpSession.subscribe(event => {
             if (event.kind !== 'error' || !event.text)
                 return;
@@ -442,7 +454,7 @@ export async function runOnce(name, opts = {}, partialDeps = {}) {
             if (evidence)
                 resolvedMonitorDeps.onFailureEvidence?.(evidence);
         });
-        control = new RoleControlServer(dir, acpSession, deps.log);
+        control = new RoleControlServer(dir, arbiter, deps.log);
         await control.start();
         resolvedMonitorDeps.delivery = {
             // A wake is only delivered when its turn TERMINATES successfully. A
@@ -454,7 +466,11 @@ export async function runOnce(name, opts = {}, partialDeps = {}) {
                 // steer into the live turn instead; after it completes, honor the
                 // configured interrupt policy normally.
                 const interrupt = options?.interrupt === true && acpStartupComplete;
-                const result = await acpSession.submitPrompt(text, { ...options, interrupt, steer: true });
+                const result = await arbiter.submitPrompt(text, {
+                    ...options, interrupt, steer: true,
+                    ...(interrupt ? { interruptSource: 'fleet-monitor' } : {}),
+                    origin: { kind: 'fleet-monitor' },
+                });
                 const steered = result.accepted
                     && (result.detail === 'injected' || result.detail === 'startedNewTurn');
                 return {
@@ -470,7 +486,7 @@ export async function runOnce(name, opts = {}, partialDeps = {}) {
         // Wait for the first turn's TERMINAL result. An agent that accepts the
         // startup prompt and then refuses it has not started; logging the role as
         // up would hide a role that never read its briefing.
-        const starting = acpSession.submitPrompt(firstPrompt);
+        const starting = arbiter.submitPrompt(firstPrompt, { origin: { kind: 'startup' } });
         // Monitoring starts immediately. The delivery adapter above downgrades
         // interruption to steering until this startup turn reaches a terminal
         // success, so there is neither a deaf gap nor a boot-cancellation loop.
@@ -500,6 +516,71 @@ export async function runOnce(name, opts = {}, partialDeps = {}) {
                 `${started.detail ? `: ${started.detail}` : ''}`);
         }
         acpStartupComplete = true;
+        if (role.owner_channel) {
+            ownerChannel = deps.createOwnerChannel({
+                role: name,
+                config: role.owner_channel,
+                session: arbiter,
+                stateDir: dir,
+                env: role.env,
+                log: deps.log,
+            });
+            try {
+                await ownerChannel.start();
+            }
+            catch (error) {
+                monitor?.stop();
+                if (monitorLoop)
+                    await monitorLoop;
+                await control.close();
+                await acpSession.close();
+                unsubscribeRecovery?.();
+                throw new Error(`[${name}] owner channel failed to start: `
+                    + `${error?.message ?? String(error)}`);
+            }
+            control.setOwnerChannel(ownerChannel);
+        }
+        reloadLoopConfig = async () => {
+            const nextRole = findRole(loadConfig(configPath), name);
+            const definitions = nextRole.loops ?? [];
+            const generation = JSON.stringify(definitions.map(loop => [
+                loop.name, loop.definitionHash, loop.promptHash,
+            ]));
+            if (generation === loopGeneration && (loopManager || !definitions.length))
+                return { changed: false, loops: definitions.length };
+            if (!loopManager && definitions.length) {
+                loopManager = new ScheduledLoopManager(name, definitions, dir, arbiter, {
+                    now: deps.now,
+                    setTimer: (callback, ms) => setTimeout(callback, ms),
+                    clearTimer: timer => clearTimeout(timer),
+                    log: deps.log,
+                });
+                control.setLoopManager(loopManager);
+                loopManager.start();
+            }
+            else {
+                loopManager?.reconcile(definitions);
+            }
+            loopGeneration = generation;
+            deps.log(`[${name}] scheduled loops reloaded (${definitions.length} definitions)`);
+            return { changed: true, loops: definitions.length };
+        };
+        control.setConfigReloader(reloadLoopConfig);
+        if (role.loops?.length) {
+            try {
+                loopManager = new ScheduledLoopManager(name, role.loops, dir, arbiter, {
+                    now: deps.now,
+                    setTimer: (callback, ms) => setTimeout(callback, ms),
+                    clearTimer: timer => clearTimeout(timer),
+                    log: deps.log,
+                });
+                control.setLoopManager(loopManager);
+                loopManager.start();
+            }
+            catch (error) {
+                deps.log(`[${name}] scheduled loop manager unavailable: ${error?.name ?? 'Error'}`);
+            }
+        }
     }
     else {
         await deps.tmux.kill(name);
@@ -520,8 +601,34 @@ export async function runOnce(name, opts = {}, partialDeps = {}) {
     // pane pid is known and is stopped when that pid dies (task dies with runner).
     monitorLoop ??= monitor?.run(pid);
     const start = deps.now();
-    while (sessionHandle.isAlive())
+    let nextLoopReloadAt = deps.now() + 30_000;
+    let lastReloadError = '';
+    while (sessionHandle.isAlive()) {
         await deps.sleep(2000);
+        const now = deps.now();
+        if (reloadLoopConfig && now >= nextLoopReloadAt) {
+            nextLoopReloadAt = now + 30_000;
+            try {
+                await reloadLoopConfig();
+                lastReloadError = '';
+            }
+            catch (error) {
+                const message = error?.message ?? String(error);
+                if (message !== lastReloadError)
+                    deps.log(`[${name}] scheduled loop config reload rejected: ${message}`);
+                lastReloadError = message;
+            }
+        }
+    }
+    if (loopManager) {
+        control?.setLoopManager(undefined);
+        await loopManager.stop();
+    }
+    control?.setConfigReloader(undefined);
+    if (ownerChannel) {
+        control?.setOwnerChannel(undefined);
+        await ownerChannel.close();
+    }
     if (monitor) {
         monitor.stop();
         await monitorLoop;

package/dist/session/acp.d.ts

@@ -1,5 +1,5 @@
 import type { CommonPermissions } from '../config.js';
-import type { ExitRecord, QueuedPrompt, SessionEvent, SessionHandle, SessionSnapshot, SubmitPromptOptions, TurnOutcome, TurnResult } from './types.js';
+import type { ExitRecord, QueuedPrompt, SessionEvent, SessionHandle, SessionSnapshot, SubmitPromptOptions, TurnCancellationSource, TurnOutcome, TurnResult } from './types.js';
 export interface AcpSessionOptions {
     name: string;
     argv: string[];
@@ -38,6 +38,7 @@ export declare class AcpSession implements SessionHandle {
     private steeringSupported;
     private capabilities?;
     private controllerCount;
+    private activeTurn?;
     private constructor();
     static start(options: AcpSessionOptions): Promise<AcpSession>;
     isAlive(): boolean;
@@ -49,7 +50,8 @@ export declare class AcpSession implements SessionHandle {
      */
     queuePrompt(text: string, options?: SubmitPromptOptions): Promise<QueuedPrompt>;
     submitPrompt(text: string, options?: SubmitPromptOptions): Promise<TurnResult>;
-    interrupt(): Promise<void>;
+    interrupt(source?: TurnCancellationSource): Promise<void>;
+    private cancelActive;
     respondPermission(permissionId: string, optionId: string): boolean;
     eventsSince(seq: number): SessionEvent[];
     subscribe(listener: (event: SessionEvent) => void): () => void;