@ours.network/fleet 1.1.0-nightly.5 → 1.1.0-nightly.7

package/dist/owner-channel/channel.js

@@ -3,6 +3,7 @@ import { existsSync, readFileSync } from 'node:fs';
 import { mkdir, readFile, readdir, rm } from 'node:fs/promises';
 import { join } from 'node:path';
 import { canonicalCid } from '../config.js';
+import { DAEMON_RECOVERY_DEADLINE_MS } from '../daemon-recovery.js';
 import { replaceFileAtomically } from '../atomic-file.js';
 import { TaskRoomApplicationService } from '../application/task-room-service.js';
 import { RoleLifecycleService } from '../application/role-command-service.js';
@@ -14,6 +15,7 @@ import { ACP_CANCEL_DEADLINE_EXCEEDED, SessionControlError, } from '../session/t
 import { VERSION } from '../version.js';
 import { renderMarkdownFailure, renderMarkdownResult, roomStatus, taskStatus, } from '../rooms-tasks/markdown.js';
 import { dispatchOwnerCommand, fleetCliOps, isOwnerCommandText, } from './commands.js';
+import { FleetCommandAuditStore, renderFleetAuditInvocation, renderFleetAuditOutcome, } from '../fleet-command-audit.js';
 import { OURS_BOUND_ELSEWHERE, OursSdkClient, oursErrorCode, } from './ours-client.js';
 import { ownerNotices, } from './notices.js';
 import { DuplicateSendError, OwnerAuthorizationState, OwnerChannelState, OwnerConversationState, } from './state.js';
@@ -21,6 +23,24 @@ import { OwnerTaskState, ownerTaskAuditId, ownerTaskDigest, } from './tasks.js';
 import { AttachmentRecoveryState, admitAttachments, cleanupAttachmentRoot, parseIncomingAttachments, parseRetrievedAttachments, prepareAttachmentDirectory, recoveredAttachment, removeRequestDirectory, safeField, validateAttachmentSelection, validateAttachmentRelaySelection, writeRecoveredAttachment, } from './attachments.js';
 import { acquireOwnerBinderLease, OWNER_BIND_HANDOFF_TIMEOUT_MS, } from './binder.js';
 import { MessageRecoveryState } from './message-recovery.js';
+export const OWNER_RECOVERY_QUEUE_CAPACITY = 32;
+export const OWNER_RECOVERY_DEGRADED = 'OWNER_RECOVERY_DEGRADED';
+export class OwnerRecoveryDegradedError extends Error {
+    code = OWNER_RECOVERY_DEGRADED;
+    constructor() {
+        super('owner channel recovery is degraded');
+        this.name = 'OwnerRecoveryDegradedError';
+    }
+}
+export class OwnerRecoveryTimeoutError extends Error {
+    stage;
+    code = OWNER_RECOVERY_DEGRADED;
+    constructor(stage) {
+        super(`owner channel recovery timed out at ${stage}`);
+        this.stage = stage;
+        this.name = 'OwnerRecoveryTimeoutError';
+    }
+}
 const OWNER_UPDATE_MIN_INTERVAL_MS = 5_000;
 const OWNER_UPDATE_MAX_COUNT = 20;
 const OWNER_UPDATE_MAX_CHARS = 280;
@@ -76,11 +96,22 @@ export class OwnerChannel {
     completionTasks = new Set();
     activeRequests = new Map();
     managementTail = Promise.resolve();
+    recoveryEpoch;
+    recoveryTask;
+    recoveryToken = 0;
+    recoveryQueued = 0;
+    watchGeneration = 0;
+    /** A timed-out client operation that must settle before replacement is safe. */
+    recoveryQuiescence;
+    releaseSafe = true;
     ready = false;
+    startedOnce = false;
+    shutdownDegraded = false;
     binder;
     binderOwnedInternally = false;
     fleetOps;
     prepareRestart;
+    commandAudits;
     constructor(options) {
         this.options = options;
         this.client = options.client ?? new OursSdkClient(options.env, line => options.log(`[${options.role}] owner channel ${line}`));
@@ -96,6 +127,7 @@ export class OwnerChannel {
             await lifecycle.prepareRestart({ roleIds: [role], mode });
         });
         this.state = new OwnerChannelState(join(options.stateDir, '.owner-channel-state.json'));
+        this.commandAudits = new FleetCommandAuditStore(join(options.stateDir, '.owner-channel-command-audit.json'));
         this.authorizations = new OwnerAuthorizationState(join(options.stateDir, '.owner-channel-owners.json'), options.config.owners);
         this.conversations = new OwnerConversationState(join(options.stateDir, '.owner-channel-conversations.json'));
         this.tasks = new OwnerTaskState(join(options.stateDir, '.owner-channel-tasks.json'));
@@ -124,6 +156,7 @@ export class OwnerChannel {
     }
     async start() {
         this.stopping = false;
+        this.shutdownDegraded = false;
         this.binderOwnedInternally = !this.options.binderLease;
         this.binder = this.options.binderLease ?? await acquireOwnerBinderLease(this.options.stateDir, this.options.role, this.options.config.identity, this.options.binderDeps);
         try {
@@ -164,11 +197,12 @@ export class OwnerChannel {
             this.attachmentRecovery.cleanup(Date.now(), this.attachmentConfig.retention_ms);
             void cleanupAttachmentRoot(this.attachmentRoot, Date.now(), this.attachmentConfig.retention_ms).catch(error => this.logError('attachment crash cleanup failed', error));
         }
+        this.startedOnce = true;
         this.ready = true;
         // Do not make role startup wait for an old owner request to finish a turn.
         // watchLoop itself drains before every establishment, including this first
         // one, so there is no drain-to-tip race.
-        this.watchTask = this.watchLoop();
+        this.watchTask = this.watchLoop(++this.watchGeneration);
     }
     drain() {
         this.drainRequested = true;
@@ -183,28 +217,50 @@ export class OwnerChannel {
         return this.drainTask;
     }
     async close() {
+        if (this.recoveryTask)
+            this.shutdownDegraded = true;
         this.stopping = true;
         this.ready = false;
-        this.watchAbort?.abort();
-        await this.watchTask?.catch(error => this.logError('watch shutdown failed', error));
+        this.recoveryToken++;
+        this.watchGeneration++;
+        const staleWatch = this.watchTask;
         this.watchTask = undefined;
-        await this.managementTail;
+        this.watchAbort?.abort();
+        const now = this.options.recoveryDeps?.now ?? (() => Date.now());
+        const deadlineAt = now()
+            + (this.options.recoveryDeps?.deadlineMs ?? DAEMON_RECOVERY_DEADLINE_MS);
+        let disposed = false;
         try {
-            await this.client.close();
+            await this.recoveryStage('shutdown_quiescence', Promise.all([
+                staleWatch?.catch(error => this.logError('watch shutdown failed', error)),
+                this.drainTask?.catch(error => this.logError('drain shutdown failed', error)),
+                this.managementTail.then(() => undefined),
+                this.recoveryQuiescence,
+            ]), deadlineAt);
+            await this.recoveryStage('shutdown_close', this.client.close(), deadlineAt);
+            disposed = true;
+            this.writeShutdownState('closed', 'OWNER_SHUTDOWN_CLOSED', now());
+        }
+        catch (error) {
+            this.shutdownDegraded = true;
+            this.releaseSafe = false;
+            const stage = error instanceof OwnerRecoveryTimeoutError ? error.stage : 'shutdown_unknown';
+            this.writeShutdownState('degraded', `OWNER_${stage.toUpperCase()}_TIMEOUT`, now());
+            this.options.log(`[${this.options.role}] owner shutdown degraded stage=${stage}`);
         }
         finally {
-            if (this.binderOwnedInternally)
+            if (disposed && this.binderOwnedInternally)
                 this.binder?.release();
-            this.binder = undefined;
+            if (disposed)
+                this.binder = undefined;
         }
     }
+    binderReleaseSafe() { return this.releaseSafe; }
     manage(request) {
-        const run = this.managementTail.then(() => this.manageNow(request));
-        this.managementTail = run.then(() => undefined, () => undefined);
-        return run;
+        return this.queueManagement(() => this.manageNow(request));
     }
     notifyFleetSpawn(event) {
-        const run = this.managementTail.then(async () => {
+        return this.queueManagement(async () => {
             if (!this.ready || this.stopping)
                 throw new Error('owner-channel MCP client is unavailable');
             const model = event.model ? `, model ${event.model}` : '';
@@ -220,9 +276,187 @@ export class OwnerChannel {
             await this.sendProactiveMessage(`🧑‍💻 ${event.caller} spawned ${event.lifetime} agent ${event.role} `
                 + `(${event.harness}/${event.session}${model}; ${monitor}${permission}).${inherited}`, `fleet-spawn\0${event.creationActionId}`, 0);
         });
+    }
+    beginFleetCommandAudit(requestId, argv) {
+        const run = this.managementTail.then(async () => {
+            let attempt = this.commandAudits.begin(requestId, this.options.role, argv);
+            if (attempt.invocation === 'delivered')
+                return attempt;
+            if (attempt.invocation === 'uncertain')
+                throw new Error('fleet command invocation delivery is uncertain; execution is denied');
+            if (!this.ready || this.stopping) {
+                this.commandAudits.invocation(attempt.correlationId, this.options.role, 'uncertain');
+                this.options.log(`[${this.options.role}] fleet command invocation ${attempt.correlationId} `
+                    + 'delivery unavailable; execution denied');
+                throw new Error('owner-channel MCP client is unavailable; invocation recorded uncertain');
+            }
+            try {
+                await this.sendProactiveMessage(renderFleetAuditInvocation(attempt), `fleet-command-invocation\0${attempt.correlationId}`, 0);
+            }
+            catch (error) {
+                this.commandAudits.invocation(attempt.correlationId, this.options.role, 'uncertain');
+                throw error;
+            }
+            attempt = this.commandAudits.invocation(attempt.correlationId, this.options.role, 'delivered');
+            return attempt;
+        });
+        this.managementTail = run.then(() => undefined, () => undefined);
+        return run;
+    }
+    finishFleetCommandAudit(input) {
+        const run = this.managementTail.then(async () => {
+            let attempt = this.commandAudits.finish(input.correlationId, this.options.role, {
+                class: input.class, effect: input.effect,
+                ...(input.exitCode === undefined ? {} : { exitCode: input.exitCode }),
+                ...(input.resourceIds ? { resourceIds: input.resourceIds } : {}),
+                ...(input.presentation ? { presentation: input.presentation } : {}),
+            });
+            if (attempt.outcome?.delivery === 'delivered')
+                return attempt;
+            if (attempt.outcome?.delivery === 'uncertain')
+                throw new Error(`fleet command outcome delivery is uncertain after execution; effect status=${attempt.outcome.effect}`);
+            if (!this.ready || this.stopping) {
+                this.commandAudits.outcome(attempt.correlationId, this.options.role, 'uncertain');
+                this.options.log(`[${this.options.role}] fleet command outcome ${attempt.correlationId} `
+                    + `delivery unavailable after execution; effect status=${attempt.outcome?.effect}`);
+                throw new Error(`owner-channel unavailable after execution; effect status=${attempt.outcome?.effect}`);
+            }
+            try {
+                await this.sendProactiveMessage(renderFleetAuditOutcome(attempt), `fleet-command-outcome\0${attempt.correlationId}`, 0);
+            }
+            catch (error) {
+                this.commandAudits.outcome(attempt.correlationId, this.options.role, 'uncertain');
+                throw new Error(`fleet command outcome delivery is uncertain after execution; effect status=${attempt.outcome?.effect}`, { cause: error });
+            }
+            attempt = this.commandAudits.outcome(attempt.correlationId, this.options.role, 'delivered');
+            return attempt;
+        });
         this.managementTail = run.then(() => undefined, () => undefined);
         return run;
     }
+    recover(epoch) {
+        if (this.stopping)
+            return Promise.reject(new Error('owner channel is stopping'));
+        if (this.recoveryEpoch === epoch && this.recoveryTask)
+            return this.recoveryTask;
+        this.recoveryEpoch = epoch;
+        this.ready = false;
+        const token = ++this.recoveryToken;
+        const watchGeneration = ++this.watchGeneration;
+        const staleWatch = this.watchTask;
+        this.watchTask = undefined;
+        this.watchAbort?.abort();
+        const superseded = () => this.stopping || token !== this.recoveryToken;
+        const now = this.options.recoveryDeps?.now ?? (() => Date.now());
+        const deadlineAt = now()
+            + (this.options.recoveryDeps?.deadlineMs ?? DAEMON_RECOVERY_DEADLINE_MS);
+        const run = this.managementTail.then(async () => {
+            if (superseded())
+                throw new Error('owner recovery epoch superseded');
+            if (this.recoveryQuiescence) {
+                const debt = this.recoveryQuiescence;
+                await this.recoveryStage('prior_quiescence', debt, deadlineAt);
+                if (this.recoveryQuiescence === debt)
+                    this.recoveryQuiescence = undefined;
+            }
+            // Quiesce every stale-client reader before replacing its transport. A
+            // watch can be inside the shared drain task when its abort lands.
+            await this.recoveryStage('stale_quiescence', Promise.all([
+                staleWatch?.catch(error => this.logError('stale watch shutdown failed', error)),
+                this.drainTask?.catch(error => this.logError('stale drain shutdown failed', error)),
+            ]), deadlineAt);
+            if (superseded())
+                throw new Error('owner recovery epoch superseded');
+            await this.recoveryStage('close', this.client.close(), deadlineAt);
+            if (superseded())
+                throw new Error('owner recovery epoch superseded');
+            try {
+                await this.recoveryStage('start', this.client.start(), deadlineAt);
+                if (superseded())
+                    throw new Error('owner recovery epoch superseded');
+                await this.recoveryStage('bind', this.client.bindIdentity(this.options.config.identity), deadlineAt);
+                if (superseded())
+                    throw new Error('owner recovery epoch superseded');
+                // Journal-aware drain is the only operation allowed to mark owner mail
+                // read. The final listing is a typed, read-only channel probe.
+                await this.recoveryStage('drain', this.drain(), deadlineAt);
+                await this.recoveryStage('list', this.client.listIncomingMessages(), deadlineAt);
+                if (superseded())
+                    throw new Error('owner recovery epoch superseded');
+                this.ready = true;
+                this.watchTask = this.watchLoop(watchGeneration);
+            }
+            catch (error) {
+                this.ready = false;
+                if (token === this.recoveryToken)
+                    this.recoveryToken++;
+                // A timed-out operation may still be using this client. Never overlap
+                // its disposal; the recorded quiescence debt gates the next retry.
+                if (!this.recoveryQuiescence) {
+                    try {
+                        await this.recoveryStage('close_after_failure', this.client.close(), deadlineAt);
+                    }
+                    catch (closeError) {
+                        this.logError('recovery client close failed', closeError);
+                    }
+                }
+                throw error;
+            }
+        });
+        this.managementTail = run.then(() => undefined, () => undefined);
+        const task = run.finally(() => {
+            if (this.recoveryTask === task)
+                this.recoveryTask = undefined;
+        });
+        this.recoveryTask = task;
+        return task;
+    }
+    recoveryStage(stage, operation, deadlineAt) {
+        const now = this.options.recoveryDeps?.now ?? (() => Date.now());
+        const remaining = Math.max(0, deadlineAt - now());
+        const setTimer = this.options.recoveryDeps?.setTimer ?? ((fn, ms) => setTimeout(fn, ms));
+        const clearTimer = this.options.recoveryDeps?.clearTimer ?? (timer => clearTimeout(timer));
+        let timer;
+        const timeout = new Promise((_resolve, reject) => {
+            timer = setTimer(() => reject(new OwnerRecoveryTimeoutError(stage)), remaining);
+        });
+        return Promise.race([operation, timeout]).catch(error => {
+            if (error instanceof OwnerRecoveryTimeoutError) {
+                const debt = operation.then(() => undefined, () => undefined);
+                this.recoveryQuiescence = debt;
+            }
+            throw error;
+        }).finally(() => clearTimer(timer));
+    }
+    writeShutdownState(state, reason, at) {
+        try {
+            replaceFileAtomically(join(this.options.stateDir, '.owner-channel-shutdown.json'), JSON.stringify({ version: 1, state, reason, at: new Date(at).toISOString() }, null, 2) + '\n');
+        }
+        catch (error) {
+            this.logError('shutdown status write failed', error);
+        }
+    }
+    queueManagement(operation) {
+        const deferred = this.recoveryTask !== undefined && !this.ready;
+        if (!this.ready && !deferred)
+            return Promise.reject(!this.startedOnce || (this.stopping && !this.shutdownDegraded)
+                ? new Error('owner-channel MCP client is unavailable')
+                : new OwnerRecoveryDegradedError());
+        if (deferred && this.recoveryQueued >= OWNER_RECOVERY_QUEUE_CAPACITY)
+            return Promise.reject(new Error('owner recovery queue is full'));
+        if (deferred)
+            this.recoveryQueued++;
+        const run = this.managementTail.then(() => {
+            if (!this.ready || this.stopping)
+                throw new OwnerRecoveryDegradedError();
+            return operation();
+        });
+        this.managementTail = run.then(() => undefined, () => undefined);
+        return run.finally(() => {
+            if (deferred)
+                this.recoveryQueued = Math.max(0, this.recoveryQueued - 1);
+        });
+    }
     async manageNow(request) {
         if (!this.ready || this.stopping)
             throw new Error('owner-channel MCP client is unavailable');
@@ -1876,7 +2110,7 @@ export class OwnerChannel {
         }
         return undefined;
     }
-    async watchLoop() {
+    async watchLoop(generation) {
         const sleep = this.options.binderDeps?.sleep
             ?? (ms => new Promise(resolve => setTimeout(resolve, ms)));
         const restored = this.readWatchState();
@@ -1885,7 +2119,7 @@ export class OwnerChannel {
             state = this.writeWatchState(state, 'OWNER_WATCH_STATE_RECOVERED');
         let delayMs = 1_000;
         let attempts = 0;
-        while (!this.stopping) {
+        while (!this.stopping && generation === this.watchGeneration) {
             const ctrl = new AbortController();
             this.watchAbort = ctrl;
             try {
@@ -1895,12 +2129,12 @@ export class OwnerChannel {
                 // fall into a gap. Persistent history plus fleet's body-free claim
                 // journal is authoritative; durable wire-ID dedupe makes hints harmless.
                 await this.drain();
-                if (this.stopping)
+                if (this.stopping || generation !== this.watchGeneration)
                     return;
                 state = this.writeWatchState(state, 'OWNER_WATCH_CONNECTING', { reconnected: attempts > 0 });
                 attempts++;
                 for await (const _event of this.client.watchNotifications(this.options.config.identity, { since: 0, signal: ctrl.signal })) {
-                    if (this.stopping)
+                    if (this.stopping || generation !== this.watchGeneration)
                         return;
                     state = this.writeWatchState(state, 'OWNER_WATCH_CONNECTED', { resetFailures: true });
                     delayMs = 1_000;
@@ -1909,11 +2143,11 @@ export class OwnerChannel {
                     // recorded before a managed request is dispatched.
                     await this.drain();
                 }
-                if (!this.stopping)
+                if (!this.stopping && generation === this.watchGeneration)
                     throw new Error('ours SDK notification stream ended');
             }
             catch (error) {
-                if (this.stopping)
+                if (this.stopping || generation !== this.watchGeneration)
                     return;
                 // SDK 2 deliberately hides transport status behind its typed stream.
                 // Do not parse error prose to rediscover it: every failure follows the