@ours.network/fleet 1.2.0-nightly.7 → 1.2.0-nightly.71

package/dist/notifications/fleet-producer.js

@@ -0,0 +1,237 @@
+import { observeFleetResources } from '../application/resource-observer.js';
+import { roomsDir } from '../rooms-tasks/room-state.js';
+import { cleanupDir, drainNotificationTargetCleanup, notificationAgentId } from './target-cleanup.js';
+import { closeSync, existsSync, lstatSync, openSync, readSync, readFileSync, readdirSync, statSync } from 'node:fs';
+import { join } from 'node:path';
+import { createHash } from 'node:crypto';
+import { NotificationOutbox } from './outbox.js';
+/** Unlike a UI inventory, unread classification must not treat unreadable records as absence. */
+function taskRoleAssociations() {
+    const dir = roomsDir(), result = new Map();
+    let files;
+    try {
+        files = readdirSync(dir);
+    }
+    catch (error) {
+        if (error.code === 'ENOENT')
+            return result;
+        throw error;
+    }
+    for (const name of files.filter(name => name.endsWith('.json'))) {
+        const file = join(dir, name), stat = lstatSync(file);
+        if (!stat.isFile() || stat.isSymbolicLink() || stat.size > 1024 * 1024)
+            throw Error('Unreadable room association');
+        const room = JSON.parse(readFileSync(file, 'utf8'));
+        if (!room || typeof room.room_id !== 'string' || !Array.isArray(room.member_seats))
+            throw Error('Invalid room association');
+        if (room.task_id === undefined)
+            continue;
+        if (typeof room.task_id !== 'string' || !/^[A-Za-z0-9_-]{1,120}$/.test(room.task_id))
+            throw Error('Invalid task association');
+        for (const seat of room.member_seats) {
+            if (typeof seat.role_name !== 'string')
+                throw Error('Invalid member association');
+            if (result.has(seat.role_name) && result.get(seat.role_name) !== room.task_id)
+                throw Error('Ambiguous task association');
+            result.set(seat.role_name, room.task_id);
+        }
+    }
+    return result;
+}
+/** Read durable supervisor ledgers without restarting agents or attaching a
+ * conversation controller. Each tick reads at most 256 KiB per role, from its
+ * checkpoint; it never loads a complete transcript or requests model history. */
+export class FleetNotificationProducer {
+    roots;
+    stateDir;
+    config;
+    warn;
+    queues = new Map();
+    observation = new AbortController();
+    continuation;
+    stopped = false;
+    existing = new Set();
+    cleanup;
+    constructor(roots, stateDir, config, warn = () => { }) {
+        this.roots = roots;
+        this.stateDir = stateDir;
+        this.config = config;
+        this.warn = warn;
+        observeFleetResources({ retryWatchFailures: true, roots, paths: [...roots, roomsDir(), cleanupDir()] }, state => {
+            if (state.health === 'healthy')
+                this.poll();
+            else
+                this.warn('Notification discovery observation unavailable; checkpoint retained');
+        }, this.observation.signal);
+        for (const root of roots)
+            for (const name of this.names(root))
+                this.existing.add(join(root, name));
+        this.poll();
+    }
+    names(root) {
+        try {
+            return readdirSync(root, { withFileTypes: true }).filter(d => d.isDirectory() && /^[a-zA-Z0-9_-]+$/.test(d.name)).map(d => d.name);
+        }
+        catch {
+            return [];
+        }
+    }
+    poll() {
+        if (this.stopped)
+            return;
+        this.cleanup ??= drainNotificationTargetCleanup(this.config, undefined, undefined, { beforeDelete: async (url) => {
+                const name = notificationAgentId(url);
+                if (!name || !/^[A-Za-z0-9_-]+$/.test(name))
+                    return false;
+                // A recreated or retained role still has a valid target. Filesystem errors cannot prove removal.
+                for (const root of this.roots) {
+                    try {
+                        lstatSync(join(root, name));
+                        return false;
+                    }
+                    catch (error) {
+                        if (error.code !== 'ENOENT')
+                            throw error;
+                    }
+                }
+                for (const root of this.roots) {
+                    const roleDir = join(root, name), queue = this.queues.get(roleDir);
+                    if (queue) {
+                        await queue.retire();
+                        this.queues.delete(roleDir);
+                    }
+                    else {
+                        const file = join(this.stateDir, createHash('sha256').update(roleDir).digest('hex') + '.json');
+                        if (existsSync(file)) {
+                            const queue = new NotificationOutbox(file, this.config, async () => { throw Error('retired target'); });
+                            try {
+                                await queue.retire();
+                            }
+                            finally {
+                                await queue.close();
+                            }
+                        }
+                    }
+                }
+                return true;
+            } }).catch(() => { }).finally(() => {
+            this.cleanup = undefined;
+            try {
+                if (readdirSync(cleanupDir()).some(name => /^[a-f0-9]{64}\.json$/.test(name)))
+                    this.resume(3000);
+            }
+            catch { }
+        });
+        let taskRoles;
+        try {
+            taskRoles = taskRoleAssociations();
+        }
+        catch {
+            this.warn('Notification task association unavailable; delivering unclassified events');
+        }
+        for (const root of this.roots)
+            for (const roleId of this.names(root)) {
+                const roleDir = join(root, roleId), dir = join(roleDir, '.conversation');
+                try {
+                    if (!existsSync(dir) || lstatSync(dir).isSymbolicLink())
+                        continue;
+                    const segments = readdirSync(dir).filter(n => /^events-\d{6}\.jsonl$/.test(n) && !lstatSync(join(dir, n)).isSymbolicLink()).sort();
+                    if (!segments.length)
+                        continue;
+                    let outbox = this.queues.get(roleDir);
+                    if (!outbox) {
+                        const key = createHash('sha256').update(roleDir).digest('hex');
+                        outbox = new NotificationOutbox(join(this.stateDir, key + '.json'), this.config, async (value, eventId) => ({ eventId, title: `${roleId}: agent completed`, body: 'Your agent finished responding. Open the chat to view the result.',
+                            url: `/fleet/chats/agents/${encodeURIComponent(roleId)}`, ...value }), this.warn);
+                        if (!outbox.checkpoint) {
+                            const segment = this.existing.has(roleDir) ? segments.at(-1) : segments[0];
+                            outbox.saveCheckpoint({ segment, offset: this.existing.has(roleDir) ? statSync(join(dir, segment)).size : 0, prompts: [] });
+                        }
+                        this.queues.set(roleDir, outbox);
+                    }
+                    const saved = outbox.checkpoint;
+                    const current = { ...saved, prompts: [...saved.prompts], messages: { ...saved.messages } };
+                    let index = segments.indexOf(current.segment);
+                    if (index < 0) {
+                        this.warn('Notification ledger segment unavailable; checkpoint retained');
+                        continue;
+                    }
+                    const file = join(dir, current.segment), size = statSync(file).size;
+                    if (size < current.offset) {
+                        this.warn('Notification ledger truncated; checkpoint retained');
+                        continue;
+                    }
+                    if (size === current.offset && index < segments.length - 1) {
+                        current.segment = segments[++index];
+                        current.offset = 0;
+                    }
+                    const fd = openSync(join(dir, current.segment), 'r'), buffer = Buffer.alloc(256 * 1024);
+                    let count;
+                    try {
+                        count = readSync(fd, buffer, 0, buffer.length, current.offset);
+                    }
+                    finally {
+                        closeSync(fd);
+                    }
+                    if (current.offset + count < statSync(join(dir, current.segment)).size || index < segments.length - 1)
+                        this.resume();
+                    let position = 0;
+                    while (position < count) {
+                        const end = buffer.indexOf(10, position);
+                        if (end < 0 || end >= count) {
+                            // Oversized model/media events are irrelevant to prompt/completion
+                            // correlation. Skip in bounded chunks; retain partial small lines.
+                            if (current.discarding || (position === 0 && count === buffer.length)) {
+                                current.offset += count - position;
+                                current.discarding = true;
+                                this.warn('Notification observer skipping oversized ledger record');
+                            }
+                            break;
+                        }
+                        if (current.discarding) {
+                            current.offset += end - position + 1;
+                            position = end + 1;
+                            current.discarding = false;
+                            continue;
+                        }
+                        const event = JSON.parse(buffer.subarray(position, end).toString('utf8'));
+                        if (event.schemaVersion !== 1 || event.roleId !== roleId)
+                            throw new Error('invalid ledger event');
+                        const prompt = `${event.sessionGeneration}:${event.promptId}`;
+                        if (event.kind === 'prompt.admitted' && event.promptId && ['browser', 'owner_admin_console', 'owner_channel'].includes(event.source ?? '')) {
+                            if (!current.prompts.includes(prompt))
+                                current.prompts.push(prompt);
+                        }
+                        if (['message.chunk', 'message.replace'].includes(event.kind) && event.promptId && event.messageId && current.prompts.includes(prompt))
+                            current.messages[prompt] = event.messageId;
+                        if (event.kind === 'turn.completed' && event.promptId && current.prompts.includes(prompt)) {
+                            if (!outbox.enqueue(`${roleId}:${event.sessionGeneration}:${event.seq}`, { ...(taskRoles ? { taskId: taskRoles.get(roleId) ?? null } : {}), url: `/fleet/chats/agents/${encodeURIComponent(roleId)}#fleet-message-${encodeURIComponent(current.messages[prompt] ?? event.promptId)}` })) {
+                                this.resume(1000);
+                                break;
+                            }
+                            current.prompts = current.prompts.filter(p => p !== prompt);
+                            delete current.messages[prompt];
+                        }
+                        current.offset += end - position + 1;
+                        position = end + 1;
+                    }
+                    if (current.segment !== saved.segment || current.offset !== saved.offset)
+                        outbox.saveCheckpoint(current);
+                }
+                catch {
+                    this.warn('Notification ledger admission pending; checkpoint will replay');
+                    // Retained, not-yet-admitted ledger work needs its own next-due wake;
+                    // the outbox sender can retry only events that were already admitted.
+                    this.resume(1000);
+                }
+            }
+    }
+    resume(delay = 0) {
+        if (this.continuation || this.stopped)
+            return;
+        this.continuation = setTimeout(() => { this.continuation = undefined; this.poll(); }, delay);
+        this.continuation.unref();
+    }
+    async drain() { await Promise.all([...this.queues.values()].map(q => q.drain())); }
+    async close() { this.stopped = true; this.observation.abort(); clearTimeout(this.continuation); await this.cleanup; await Promise.all([...this.queues.values()].map(q => q.close())); }
+}

package/dist/notifications/outbox.d.ts

@@ -0,0 +1,46 @@
+export interface NotificationInput {
+    eventId: string;
+    title: string;
+    body: string;
+    url: string;
+    taskId?: string | null;
+}
+export interface ProducerConfig {
+    /** Service base URL. With a gateway credential it is the server gateway's notifications route. */
+    origin: string;
+    token: string;
+    /** Daemon API credential that authenticates this client to the server gateway. */
+    gatewayCredential?: string;
+}
+type ProfileLocation = {
+    serverUrl: string;
+    expectedInstanceId: string;
+    credentialPath: string;
+};
+/** The installer delivers the producer credential beside the profile credential, bound to that server installation. */
+export declare const PRODUCER_CREDENTIAL_FILE = "notifications-producer.json";
+export declare function producerConfig(env?: NodeJS.ProcessEnv, profile?: () => ProfileLocation | undefined, warn?: (message: string) => void): ProducerConfig | undefined;
+/** Durable producer queue. Service dedupe makes crash-after-acceptance replay safe. */
+export declare class NotificationOutbox {
+    private readonly file;
+    private readonly config;
+    private readonly project;
+    private readonly warn;
+    private entries;
+    cursor?: number;
+    checkpoint?: unknown;
+    private running?;
+    private retired;
+    private timer;
+    constructor(file: string, config: ProducerConfig, project: (value: unknown, id: string) => Promise<NotificationInput>, warn?: (message: string) => void);
+    private persist;
+    advance(cursor: number): void;
+    saveCheckpoint(checkpoint: unknown): void;
+    enqueue(id: string, value: unknown, cursor?: number | undefined): boolean;
+    drain(): Promise<void>;
+    private drainOnce;
+    /** Wait for the current send before clearing queued sends and retiring its target. */
+    retire(): Promise<void>;
+    close(): Promise<void>;
+}
+export {};

package/dist/notifications/outbox.js

@@ -0,0 +1,158 @@
+import { closeSync, existsSync, fsyncSync, mkdirSync, openSync, readFileSync, renameSync, writeFileSync } from 'node:fs';
+import { dirname, join } from 'node:path';
+import { readPrivateFile } from '@ours.network/sdk/connector';
+import { readClientProfile } from '../client-profile.js';
+function defaultProfile() {
+    try {
+        return readClientProfile();
+    }
+    catch {
+        return undefined;
+    }
+}
+/** The installer delivers the producer credential beside the profile credential, bound to that server installation. */
+export const PRODUCER_CREDENTIAL_FILE = 'notifications-producer.json';
+const loopback = (url) => ['127.0.0.1', '[::1]', 'localhost'].includes(url.hostname);
+export function producerConfig(env = process.env, profile = defaultProfile, warn = () => { }) {
+    const origin = env.OURS_NOTIFICATIONS_ORIGIN, token = env.OURS_NOTIFICATIONS_PRODUCER_TOKEN;
+    if (origin || token) {
+        if (!origin || !token || token.length < 32)
+            throw new Error('notifications producer requires origin and scoped token');
+        const url = new URL(origin);
+        if (!['http:', 'https:'].includes(url.protocol) || url.origin !== origin || url.username || url.password)
+            throw new Error('invalid notification service origin');
+        if (url.protocol === 'http:' && !loopback(url))
+            throw new Error('notification credentials require HTTPS or loopback');
+        return { origin, token };
+    }
+    const selected = profile();
+    if (!selected)
+        return;
+    const file = join(dirname(selected.credentialPath), PRODUCER_CREDENTIAL_FILE);
+    if (!existsSync(file))
+        return;
+    // Reasons are fixed text: parser and file errors can quote the secret-bearing contents.
+    let reason = 'it is not a private owner-only file';
+    try {
+        const text = readPrivateFile(file, 4096).toString('utf8');
+        reason = 'it is not valid producer JSON';
+        const bound = JSON.parse(text);
+        reason = 'it belongs to another server installation';
+        if (!bound || bound.schema !== 1 || bound.serverUrl !== selected.serverUrl || bound.expectedInstanceId !== selected.expectedInstanceId)
+            throw new Error();
+        reason = 'its credential is invalid';
+        if (typeof bound.token !== 'string' || bound.token.length < 32)
+            throw new Error();
+        reason = 'the server requires HTTPS or loopback';
+        const server = new URL(selected.serverUrl);
+        if (server.protocol !== 'https:' && !(server.protocol === 'http:' && loopback(server)))
+            throw new Error();
+        reason = 'the client credential is unreadable';
+        const gatewayCredential = readPrivateFile(selected.credentialPath, 4096).toString('utf8').trim();
+        return { origin: selected.serverUrl.replace(/\/$/, '') + '/notifications', token: bound.token, gatewayCredential };
+    }
+    catch {
+        warn(`Notifications are not produced: ${file} is unusable (${reason}). Re-run the installer to repair it.`);
+        return;
+    }
+}
+/** Durable producer queue. Service dedupe makes crash-after-acceptance replay safe. */
+export class NotificationOutbox {
+    file;
+    config;
+    project;
+    warn;
+    entries;
+    cursor;
+    checkpoint;
+    running;
+    retired = false;
+    timer;
+    constructor(file, config, project, warn = () => { }) {
+        this.file = file;
+        this.config = config;
+        this.project = project;
+        this.warn = warn;
+        mkdirSync(dirname(file), { recursive: true, mode: 0o700 });
+        const saved = existsSync(file) ? JSON.parse(readFileSync(file, 'utf8')) : { entries: [] };
+        this.entries = Array.isArray(saved) ? saved : saved.entries;
+        this.cursor = saved.cursor;
+        this.checkpoint = saved.checkpoint;
+        if (!Array.isArray(this.entries))
+            throw new Error('invalid notification outbox');
+        this.timer = setInterval(() => { void this.drain(); }, 3000);
+        this.timer.unref();
+    }
+    persist(entries, cursor = this.cursor, checkpoint = this.checkpoint) {
+        const tmp = this.file + '.tmp', fd = openSync(tmp, 'w', 0o600);
+        try {
+            writeFileSync(fd, JSON.stringify({ entries, cursor, checkpoint }));
+            fsyncSync(fd);
+        }
+        finally {
+            closeSync(fd);
+        }
+        renameSync(tmp, this.file);
+        const dir = openSync(dirname(this.file), 'r');
+        try {
+            fsyncSync(dir);
+        }
+        finally {
+            closeSync(dir);
+        }
+        this.entries = entries;
+        this.cursor = cursor;
+        this.checkpoint = checkpoint;
+    }
+    advance(cursor) { this.persist(this.entries, cursor); }
+    saveCheckpoint(checkpoint) { this.persist(this.entries, this.cursor, checkpoint); }
+    enqueue(id, value, cursor = this.cursor) {
+        if (this.retired)
+            return false;
+        if (this.entries.some(e => e.id === id)) {
+            if (cursor !== this.cursor)
+                this.advance(cursor);
+            return true;
+        }
+        if (this.entries.length >= 2000)
+            return false;
+        this.persist([...this.entries, { id, value }], cursor);
+        return true;
+    }
+    drain() {
+        if (this.retired)
+            return Promise.resolve();
+        return this.running ??= this.drainOnce().finally(() => { this.running = undefined; });
+    }
+    async drainOnce() {
+        for (const entry of [...this.entries].slice(0, 32)) {
+            if (this.retired)
+                break;
+            try {
+                const value = entry.payload ?? await this.project(entry.value, entry.id);
+                // Freeze the canonical projection before any network attempt. A crash
+                // after service acceptance must replay the identical dedupe payload.
+                if (!entry.payload)
+                    this.persist(this.entries.map(e => e.id === entry.id ? { ...e, payload: value } : e));
+                const response = await fetch(this.config.origin + '/api/v1/send', {
+                    method: 'POST', redirect: 'error', signal: AbortSignal.timeout(10_000),
+                    headers: { ...(this.config.gatewayCredential
+                            ? { 'X-Ours-Api-Token': this.config.gatewayCredential, 'X-Ours-Notifications-Producer': this.config.token }
+                            : { Authorization: `Bearer ${this.config.token}` }), 'Content-Type': 'application/json' },
+                    body: JSON.stringify(value),
+                });
+                await response.body?.cancel();
+                if (!response.ok)
+                    throw new Error('notification service rejected event');
+                this.persist(this.entries.filter(e => e.id !== entry.id));
+            }
+            catch {
+                this.warn('Notification producer delivery pending; retrying from durable outbox');
+                break;
+            }
+        }
+    }
+    /** Wait for the current send before clearing queued sends and retiring its target. */
+    async retire() { this.retired = true; clearInterval(this.timer); await this.running; this.persist([]); }
+    async close() { clearInterval(this.timer); await this.running; }
+}

package/dist/notifications/target-cleanup.d.ts

@@ -0,0 +1,11 @@
+import { type ProducerConfig } from './outbox.js';
+export declare const cleanupDir: () => string;
+/** Only the canonical agent namespace can authorize role-target cleanup. */
+export declare function notificationAgentId(path: string): string | null;
+/** Queue only after the owning lifecycle has established deletion, never on a failed inventory read. */
+export declare function retireNotificationTarget(url: string, config?: ProducerConfig | undefined, dir?: string, request?: typeof fetch): Promise<void>;
+/** Persisted retries also run from the ordinary web notification producer after restart. */
+export declare function drainNotificationTargetCleanup(config: ProducerConfig, dir?: string, request?: typeof fetch, options?: {
+    retain?: boolean;
+    beforeDelete?: (url: string) => Promise<boolean>;
+}): Promise<void>;

package/dist/notifications/target-cleanup.js

@@ -0,0 +1,63 @@
+import { createHash } from 'node:crypto';
+import { existsSync, mkdirSync, readdirSync, readFileSync, rmSync } from 'node:fs';
+import { join } from 'node:path';
+import { replaceFileAtomically } from '../atomic-file.js';
+import { stateRoot } from '../paths.js';
+import { producerConfig } from './outbox.js';
+export const cleanupDir = () => join(stateRoot(), 'notifications', 'removed-targets');
+/** Only the canonical agent namespace can authorize role-target cleanup. */
+export function notificationAgentId(path) {
+    try {
+        if (typeof path !== 'string' || /[\\\x00-\x20]|%(?:2f|5c|00|0[ad])/i.test(path))
+            return null;
+        const match = /^\/fleet\/chats\/agents\/([^/?#]+)(?:[?#]|$)/.exec(path);
+        if (!match)
+            return null;
+        const url = new URL(path, 'https://ours.invalid');
+        if (['source', 'chat', 'task', 'detail'].some(key => url.searchParams.has(key)))
+            return null;
+        const id = decodeURIComponent(match[1]);
+        return /^[A-Za-z0-9_-]+$/.test(id) ? id : null;
+    }
+    catch {
+        return null;
+    }
+}
+/** Queue only after the owning lifecycle has established deletion, never on a failed inventory read. */
+export async function retireNotificationTarget(url, config = producerConfig(), dir = cleanupDir(), request = fetch) {
+    if (!config)
+        return;
+    mkdirSync(dir, { recursive: true, mode: 0o700 });
+    replaceFileAtomically(join(dir, createHash('sha256').update(url).digest('hex') + '.json'), JSON.stringify({ url }) + '\n', 0o600);
+    await drainNotificationTargetCleanup(config, dir, request, { retain: true });
+}
+/** Persisted retries also run from the ordinary web notification producer after restart. */
+export async function drainNotificationTargetCleanup(config, dir = cleanupDir(), request = fetch, options = {}) {
+    if (!existsSync(dir))
+        return;
+    let attempts = 0;
+    for (const file of readdirSync(dir).filter(file => /^[a-f0-9]{64}\.json$/.test(file))) {
+        if (attempts >= 32)
+            break;
+        try {
+            const path = join(dir, file), value = JSON.parse(readFileSync(path, 'utf8'));
+            if (!notificationAgentId(value.url))
+                continue;
+            attempts++;
+            if (options.beforeDelete && !(await options.beforeDelete(value.url))) {
+                rmSync(path, { force: true });
+                continue;
+            }
+            const response = await request(config.origin + '/api/v1/delete-target', { method: 'POST', redirect: 'error', signal: AbortSignal.timeout(10000),
+                headers: { ...(config.gatewayCredential ? { 'X-Ours-Api-Token': config.gatewayCredential, 'X-Ours-Notifications-Producer': config.token } : { Authorization: `Bearer ${config.token}` }), 'Content-Type': 'application/json' }, body: JSON.stringify(value) });
+            await response.body?.cancel();
+            if (!response.ok)
+                break;
+            if (!options.retain)
+                rmSync(path, { force: true });
+        }
+        catch {
+            break; /* No credentials or response bodies are included in logs. */
+        }
+    }
+}

package/dist/notifications/task-producer.d.ts

@@ -0,0 +1,20 @@
+import type { ProducerConfig } from './outbox.js';
+/** The Fleet manager owns this sender even when the web console is stopped.
+ * Events are already durable in their owning task commit, so no second outbox
+ * admission/crash window is needed. A failed task does not block other tasks. */
+export declare class TaskNotificationProducer {
+    private readonly config;
+    private readonly warn;
+    private readonly request;
+    private timer?;
+    private dirty;
+    private running?;
+    private offset;
+    private stopped;
+    private readonly shutdown;
+    constructor(config: ProducerConfig, warn?: (line: string) => void, request?: typeof fetch);
+    drain(): Promise<void>;
+    private retry;
+    private drainOnce;
+    close(): Promise<void>;
+}

package/dist/notifications/task-producer.js

@@ -0,0 +1,100 @@
+import { observeFleetResources } from '../application/resource-observer.js';
+import { tasksDir } from '../rooms-tasks/task-state.js';
+import { acknowledgeTaskNotification, pendingTaskNotifications } from '../rooms-tasks/task-state.js';
+/** The Fleet manager owns this sender even when the web console is stopped.
+ * Events are already durable in their owning task commit, so no second outbox
+ * admission/crash window is needed. A failed task does not block other tasks. */
+export class TaskNotificationProducer {
+    config;
+    warn;
+    request;
+    timer;
+    dirty = false;
+    running;
+    offset = 0;
+    stopped = false;
+    shutdown = new AbortController();
+    constructor(config, warn = () => { }, request = fetch) {
+        this.config = config;
+        this.warn = warn;
+        this.request = request;
+        // Subscribe before startup replay: a commit while fetch is pending needs a second pass.
+        observeFleetResources({ retryWatchFailures: true, paths: [tasksDir()], roots: [] }, state => {
+            if (state.health !== 'healthy')
+                this.warn('Task notification observation unavailable; committed events retained');
+            else {
+                this.dirty = true;
+                void this.drain();
+            }
+        }, this.shutdown.signal);
+        void this.drain();
+    }
+    drain() {
+        if (this.stopped)
+            return Promise.resolve();
+        if (this.running)
+            return this.running;
+        this.dirty = false;
+        clearTimeout(this.timer);
+        this.timer = undefined;
+        this.running = this.drainOnce().catch(() => {
+            this.warn('Task notification snapshot unavailable; committed events retained');
+            this.retry();
+        }).finally(() => {
+            this.running = undefined;
+            if (this.dirty && !this.stopped) {
+                this.dirty = false;
+                void this.drain();
+            }
+        });
+        return this.running;
+    }
+    retry() {
+        if (this.stopped || this.timer)
+            return;
+        this.timer = setTimeout(() => { this.timer = undefined; void this.drain(); }, 3000);
+        this.timer.unref();
+    }
+    async drainOnce() {
+        if (this.stopped)
+            return;
+        const pending = pendingTaskNotifications(this.warn);
+        if (!pending.length)
+            return;
+        const start = this.offset % pending.length;
+        const batch = [...pending.slice(start), ...pending.slice(0, start)].slice(0, 32);
+        this.offset = start + batch.length;
+        // Retry only outstanding work; never run an idle discovery timer.
+        this.retry();
+        for (let index = 0; index < batch.length; index += 4) {
+            if (this.stopped)
+                break;
+            await Promise.all(batch.slice(index, index + 4).map(async (task) => {
+                for (const event of task.events.slice(0, 32)) {
+                    if (this.stopped)
+                        break;
+                    try {
+                        const response = await this.request(this.config.origin + '/api/v1/send', {
+                            method: 'POST', redirect: 'error', signal: AbortSignal.any([this.shutdown.signal, AbortSignal.timeout(10_000)]),
+                            headers: { ...(this.config.gatewayCredential
+                                    ? { 'X-Ours-Api-Token': this.config.gatewayCredential, 'X-Ours-Notifications-Producer': this.config.token }
+                                    : { Authorization: `Bearer ${this.config.token}` }), 'Content-Type': 'application/json' },
+                            body: JSON.stringify(event),
+                        });
+                        await response.body?.cancel();
+                        if (!response.ok) {
+                            this.warn(`Task notification delivery pending (HTTP ${response.status}); committed event retained`);
+                            break;
+                        }
+                        acknowledgeTaskNotification(task.taskId, event.eventId);
+                    }
+                    catch {
+                        this.warn('Task notification delivery pending; committed event retained');
+                        break;
+                    }
+                }
+            }));
+        }
+    }
+    async close() { this.stopped = true; clearTimeout(this.timer); this.shutdown.abort(); await this.running; }
+}

package/dist/ops.d.ts

@@ -54,5 +54,7 @@ export declare function up(cfg: FleetConfig, names: string[], deps: OpsDeps, con
 export declare function down(cfg: FleetConfig, names: string[], deps: OpsDeps): Promise<void>;
 /** Re-sync from config + bounce. mode 'keep' resumes context; 'fresh' wipes it. */
 export declare function restartRoles(cfg: FleetConfig, names: string[], deps: OpsDeps, mode: 'keep' | 'fresh', configPath?: string): Promise<void>;
+/** Read-only admission check shared by CLI removal and web preview, before any archive. */
+export declare function assertStandaloneRoleRemoval(cfg: FleetConfig, name: string): void;
 /** Stop + forget a role: unit, state dir, and its fleet.d file when spawned. */
 export declare function rmRole(cfg: FleetConfig, name: string, deps: OpsDeps): Promise<void>;