firedrill-run 2.2.5 → 3.0.0

package/dist/cli/src/native/companion-channel.js

@@ -0,0 +1,421 @@
+/**
+ * The companion's one shared, resumable wake channel (Plan20 §7.3).
+ *
+ * Root's `firedrill.native-companion-stream.v1` route
+ * (`GET /api/workspaces/{workspace}/native/devices/{device}/campaigns/{campaign}/stream`, the terminal bearer,
+ * no query) multiplexes invalidation hints for one campaign: its offers, admission and progress, and this
+ * device's assignments in it. Root keeps one open channel per runner device, so this client keeps exactly one
+ * connection for the selected campaign, shared by the session's offer loop and every drill:
+ *  - consumers read root and act only while the channel is `live`, that is, once the current connection
+ *    delivered a valid `snapshot` (first connection) or `resync` (every reconnection). Either one wakes every
+ *    consumer for a canonical read; `changed` wakes only the consumers of the resources it names (an empty one
+ *    only moves the cursor);
+ *  - the cursor (`Last-Event-ID`) is an invalidation watermark, never an acknowledgment. Claims, controls,
+ *    observations and closes stay signed requests whose own receipts are authoritative, and a wake-up never
+ *    dispatches anything by itself;
+ *  - root rotates a connection after 55 s (`retry_later`, or the end of the stream); it is reopened at once.
+ *    Busy (409), unavailable (503), rate-limited, network, stalled and malformed connections retry with a
+ *    capped backoff. Until the next valid resync nothing is woken, so nothing reads root on a timer;
+ *  - `access_changed` and answered refusals (401, 403, 404, 422) put every consumer on hold (`refused`):
+ *    no claim, step or start until a fresh authenticated connection delivers a valid resync, which root grants
+ *    only with current session, device, membership and campaign access. Revocation itself is still decided by
+ *    the device heartbeat;
+ *  - a refused cursor (`native_stream_cursor_invalid`) is dropped and the channel starts from a snapshot.
+ * The bearer stays in memory and request headers (no cookie, redirect, query or log). Frames are bounded
+ * (16 KiB, 64 resources) and carry no input.
+ */
+import { setTimeout as delay } from 'node:timers/promises';
+import { access } from '../auth.js';
+import { CliError, endpoint, responseError } from '../http.js';
+export const COMPANION_STREAM_SCHEMA = 'firedrill.native-companion-stream.v1';
+const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
+const SEQUENCE = /^(0|[1-9][0-9]{0,18})$/;
+const MAX_SEQUENCE = 9223372036854775807n;
+const PAYLOAD_KEYS = ['campaign_id', 'device_id', 'resources', 'schema', 'sequence', 'workspace_id'];
+const MAX_RESOURCES = 64;
+const MAX_FRAME_BYTES = 16_384;
+const MAX_BUFFERED_CHARS = 65_536;
+const MAX_REFUSAL_BYTES = 16_384;
+/** Root writes a frame or a keepalive every 5 s; a connection silent this long is stalled. */
+const STALL_MS = 20_000;
+/** Root rotates a connection after 55 s; one still open after this is ended here. */
+const CONNECTION_MS = 70_000;
+/** A rotated connection is reopened after 250 ms to 1 s. */
+const ROTATE_MS = 250;
+const RETRY_BASE_MS = 1_000;
+const RETRY_CAP_MS = 15_000;
+/** Access is asked for again more slowly after an answered refusal. */
+const REFUSED_BASE_MS = 5_000;
+const REFUSED_CAP_MS = 60_000;
+/** A generation counter with waiters: wake-ups coalesce, and one that lands between a read and its wait is never lost. */
+export class Wakeup {
+    #generation = 0;
+    #closed = false;
+    #waiters = new Set();
+    #listeners = new Set();
+    get generation() { return this.#generation; }
+    get closed() { return this.#closed; }
+    wake() {
+        if (this.#closed)
+            return;
+        this.#generation++;
+        for (const waiter of [...this.#waiters])
+            waiter();
+        for (const listener of [...this.#listeners])
+            listener();
+    }
+    onWake(listener) {
+        if (this.#closed)
+            return () => undefined;
+        this.#listeners.add(listener);
+        return () => { this.#listeners.delete(listener); };
+    }
+    close() {
+        if (this.#closed)
+            return;
+        this.#closed = true;
+        this.#listeners.clear();
+        for (const waiter of [...this.#waiters])
+            waiter();
+    }
+    /** True once the generation passed `after`; false when `signal` aborts or this closed first. Never throws. */
+    wait(after, signal) {
+        return settled(() => this.#generation > after ? true : this.#closed ? false : null, this.#waiters, signal);
+    }
+}
+class Subscription extends Wakeup {
+    resource;
+    #release;
+    constructor(resource, release) {
+        super();
+        this.resource = resource;
+        this.#release = release;
+    }
+    close() {
+        if (this.closed)
+            return;
+        super.close();
+        this.#release(this);
+    }
+}
+/**
+ * The selected campaign's companion channel. It connects with the first subscription and closes with the
+ * last one (or `close()`); subscriptions are the session's offer loop and each running drill.
+ */
+export function companionChannel(scope) {
+    const campaign = scope.campaign.toLowerCase();
+    const workspace = scope.workspace.toLowerCase();
+    if (!UUID.test(campaign))
+        throw new CliError('native_campaign_required', 'Select the campaign to run with --campaign <campaign-id>. Nothing was started.', 2);
+    const subscriptions = new Set();
+    const liveWaiters = new Set();
+    const stateListeners = new Set();
+    let connection = null;
+    let state = 'connecting';
+    let stateCode = null;
+    const setState = (next, code) => {
+        if (state === 'closed' || (state === next && stateCode === code))
+            return;
+        state = next;
+        stateCode = code;
+        for (const waiter of [...liveWaiters])
+            waiter();
+        for (const listener of [...stateListeners])
+            listener(next, code);
+    };
+    /** Wake the named resources' consumers, or every consumer (null). */
+    const wake = (resources) => {
+        for (const subscription of [...subscriptions]) {
+            if (resources === null || resources.some((item) => item.kind === subscription.resource.kind && item.id === subscription.resource.id)) {
+                subscription.wake();
+            }
+        }
+    };
+    const release = (subscription) => {
+        subscriptions.delete(subscription);
+        if (subscriptions.size === 0 && connection) {
+            connection.abort();
+            connection = null;
+            setState('connecting', null);
+        }
+    };
+    /** One connection, until it rotates, fails or is refused. Never reports `rotated` without a delivered snapshot or resync. */
+    const connect = async (cursor, signal, deliver) => {
+        const device = scope.device().toLowerCase();
+        const token = await access(async (value) => value);
+        signal.throwIfAborted();
+        const stalled = new AbortController();
+        let watchdog = setTimeout(() => stalled.abort(), STALL_MS);
+        let reader;
+        try {
+            const response = await fetch(endpoint(`/api/workspaces/${scope.workspace}/native/devices/${device}/campaigns/${campaign}/stream`), {
+                method: 'GET', redirect: 'error', credentials: 'omit',
+                headers: { Authorization: `Bearer ${token}`, Accept: 'text/event-stream', 'X-Firedrill-Workspace': scope.workspace,
+                    ...(scope.actor ? { 'X-Firedrill-Actor': scope.actor } : {}), ...(cursor !== null ? { 'Last-Event-ID': cursor } : {}) },
+                signal: AbortSignal.any([signal, stalled.signal, AbortSignal.timeout(CONNECTION_MS)]),
+            });
+            if (!response.ok)
+                throw await refusal(response);
+            if (!response.headers.get('Content-Type')?.toLowerCase().startsWith('text/event-stream') || !response.body) {
+                await response.body?.cancel().catch(() => undefined);
+                throw invalid();
+            }
+            reader = response.body.getReader();
+            const decoder = new TextDecoder('utf-8', { fatal: true });
+            let buffer = '';
+            let delivered = false;
+            for (;;) {
+                const part = await reader.read();
+                signal.throwIfAborted();
+                if (part.done)
+                    return delivered ? 'rotated' : 'failed';
+                clearTimeout(watchdog);
+                watchdog = setTimeout(() => stalled.abort(), STALL_MS);
+                buffer = (buffer + decoder.decode(part.value, { stream: true })).replace(/\r\n/g, '\n');
+                let boundary;
+                while ((boundary = buffer.indexOf('\n\n')) !== -1) {
+                    const frame = buffer.slice(0, boundary);
+                    buffer = buffer.slice(boundary + 2);
+                    if (Buffer.byteLength(frame, 'utf8') > MAX_FRAME_BYTES)
+                        throw invalid();
+                    const { event, id, data } = fields(frame);
+                    if (event === 'access_changed')
+                        return 'refused';
+                    if (event === 'retry_later')
+                        return delivered ? 'rotated' : 'failed';
+                    // Keepalive comments and unknown events carry no authority and wake nothing.
+                    if (event !== 'snapshot' && event !== 'resync' && event !== 'changed')
+                        continue;
+                    // Root opens every connection with one snapshot (no cursor sent) or resync (cursor sent), which requires a canonical read.
+                    if ((event === 'changed') === !delivered || (event !== 'changed' && (event === 'snapshot') !== (cursor === null)))
+                        throw invalid();
+                    deliver(event, notification(data, id, { workspace, device, campaign }));
+                    delivered = true;
+                }
+                if (buffer.length > MAX_BUFFERED_CHARS)
+                    throw invalid();
+            }
+        }
+        finally {
+            clearTimeout(watchdog);
+            await reader?.cancel().catch(() => undefined);
+        }
+    };
+    /** Keep one connection open while anyone is subscribed. Never throws. */
+    const maintain = async (signal) => {
+        let cursor = null;
+        let failures = 0;
+        while (!signal.aborted) {
+            let ending;
+            let code = null;
+            try {
+                ending = await connect(cursor, signal, (event, value) => {
+                    // A buffered response from a replaced connection has no authority.
+                    signal.throwIfAborted();
+                    if (connection?.signal !== signal)
+                        throw invalid();
+                    cursor = value.sequence;
+                    failures = 0;
+                    if (event === 'changed') {
+                        wake(value.resources);
+                        return;
+                    }
+                    // A valid snapshot or resync: consumers may read and act again, and every one of them reads now.
+                    setState('live', null);
+                    wake(null);
+                });
+                if (ending === 'refused')
+                    code = 'native_stream_access_changed';
+            }
+            catch (error) {
+                if (signal.aborted)
+                    return;
+                code = error instanceof CliError ? error.code : 'native_stream_disconnected';
+                if (code === 'native_stream_cursor_invalid')
+                    cursor = null;
+                ending = refused(error) ? 'refused' : 'failed';
+            }
+            if (signal.aborted)
+                return;
+            if (ending === 'rotated') {
+                setState('reconnecting', null);
+                await pause(ROTATE_MS + Math.floor(Math.random() * ROTATE_MS * 3), signal);
+                continue;
+            }
+            // Hold every consumer until a later connection delivers a valid resync; nothing is woken meanwhile.
+            if (ending === 'refused')
+                cursor = null;
+            setState(ending === 'refused' ? 'refused' : 'reconnecting', code);
+            await pause(ending === 'refused' ? backoff(failures, REFUSED_BASE_MS, REFUSED_CAP_MS) : backoff(failures, RETRY_BASE_MS, RETRY_CAP_MS), signal);
+            failures++;
+        }
+    };
+    return {
+        campaign_id: campaign,
+        get state() { return state; },
+        live(signal) {
+            return settled(() => state === 'live' ? true : state === 'closed' ? false : null, liveWaiters, signal);
+        },
+        onState(listener) {
+            stateListeners.add(listener);
+            return () => { stateListeners.delete(listener); };
+        },
+        subscribe(resource) {
+            const id = typeof resource?.id === 'string' ? resource.id.toLowerCase() : '';
+            if ((resource?.kind !== 'campaign' && resource?.kind !== 'assignment') || !UUID.test(id) || (resource.kind === 'campaign' && id !== campaign)) {
+                throw new CliError('native_stream_subscription_invalid', 'The wake-up subscription names no assignment of the selected campaign. Nothing was opened.');
+            }
+            const subscription = new Subscription({ kind: resource.kind, id }, release);
+            if (state === 'closed') {
+                subscription.close();
+                return subscription;
+            }
+            subscriptions.add(subscription);
+            if (!connection) {
+                const controller = new AbortController();
+                connection = controller;
+                void maintain(controller.signal).finally(() => { if (connection === controller)
+                    connection = null; });
+            }
+            return subscription;
+        },
+        close() {
+            connection?.abort();
+            connection = null;
+            for (const subscription of [...subscriptions])
+                subscription.close();
+            setState('closed', null);
+            stateListeners.clear();
+        },
+    };
+}
+/**
+ * Resolve with `outcome()` once it is not null, or false when `signal` aborts first. `waiters` holds the
+ * re-check until the owner calls it on a change. Never throws.
+ */
+function settled(outcome, waiters, signal) {
+    return new Promise((resolve) => {
+        const check = () => {
+            const value = signal.aborted ? false : outcome();
+            if (value === null)
+                return;
+            waiters.delete(check);
+            signal.removeEventListener('abort', check);
+            resolve(value);
+        };
+        const first = signal.aborted ? false : outcome();
+        if (first !== null) {
+            resolve(first);
+            return;
+        }
+        waiters.add(check);
+        signal.addEventListener('abort', check, { once: true });
+    });
+}
+/** SSE fields of one frame; comment lines (keepalives) are skipped. */
+function fields(frame) {
+    let event = '';
+    let id = null;
+    const data = [];
+    for (const line of frame.split('\n')) {
+        if (!line || line.startsWith(':'))
+            continue;
+        const colon = line.indexOf(':');
+        const name = colon === -1 ? line : line.slice(0, colon);
+        let value = colon === -1 ? '' : line.slice(colon + 1);
+        if (value.startsWith(' '))
+            value = value.slice(1);
+        if (name === 'event')
+            event = value;
+        else if (name === 'id')
+            id = value;
+        else if (name === 'data')
+            data.push(value);
+    }
+    return { event, id, data: data.join('\n') };
+}
+/** One `snapshot`, `resync` or `changed` payload (root's companion-stream.v1 schema), confirmed to be this device's channel for this campaign. */
+function notification(data, id, expected) {
+    let value;
+    try {
+        value = JSON.parse(data);
+    }
+    catch {
+        throw invalid();
+    }
+    const body = value !== null && typeof value === 'object' && !Array.isArray(value) ? value : null;
+    if (!body || Object.keys(body).sort().join() !== PAYLOAD_KEYS.join() || body.schema !== COMPANION_STREAM_SCHEMA
+        || lower(body.workspace_id) !== expected.workspace || lower(body.device_id) !== expected.device
+        || lower(body.campaign_id) !== expected.campaign || !position(body.sequence) || (id !== null && id !== body.sequence)
+        || !Array.isArray(body.resources) || body.resources.length > MAX_RESOURCES)
+        throw invalid();
+    const seen = new Set();
+    const resources = body.resources.map((item) => {
+        const row = item !== null && typeof item === 'object' && !Array.isArray(item) ? item : null;
+        const resourceId = lower(row?.id);
+        if (!row || Object.keys(row).sort().join() !== 'id,kind' || (row.kind !== 'campaign' && row.kind !== 'assignment')
+            || resourceId === null || !UUID.test(resourceId) || (row.kind === 'campaign' && resourceId !== expected.campaign)
+            || seen.has(`${row.kind}:${resourceId}`))
+            throw invalid();
+        seen.add(`${row.kind}:${resourceId}`);
+        return { kind: row.kind, id: resourceId };
+    });
+    return { sequence: body.sequence, resources };
+}
+function lower(value) {
+    return typeof value === 'string' ? value.toLowerCase() : null;
+}
+function position(value) {
+    return typeof value === 'string' && SEQUENCE.test(value) && BigInt(value) <= MAX_SEQUENCE;
+}
+function invalid() {
+    return new CliError('native_stream_invalid', 'The companion channel sent a frame that could not be confirmed. Consumers wait for a valid resync; nothing was dispatched.');
+}
+/** Answered refusals of this device's access; busy (409), unavailable (503), rate limits and network failures are retried. */
+function refused(error) {
+    if (!(error instanceof CliError))
+        return false;
+    if (error.code === 'native_stream_access_changed')
+        return true;
+    const status = error.httpStatus;
+    return status !== null && status >= 400 && status < 500 && ![408, 409, 425, 429].includes(status);
+}
+/** Root's coded refusal, read from a bounded body. */
+async function refusal(response) {
+    let body = null;
+    const reader = response.body?.getReader();
+    if (reader) {
+        const chunks = [];
+        let size = 0;
+        try {
+            for (;;) {
+                const part = await reader.read();
+                if (part.done)
+                    break;
+                size += part.value.byteLength;
+                if (size > MAX_REFUSAL_BYTES) {
+                    chunks.length = 0;
+                    break;
+                }
+                chunks.push(part.value);
+            }
+            if (chunks.length)
+                body = JSON.parse(Buffer.concat(chunks).toString('utf8'));
+        }
+        catch {
+            body = null;
+        }
+        finally {
+            await reader.cancel().catch(() => undefined);
+        }
+    }
+    return responseError(body, response.status);
+}
+/** Capped exponential backoff with jitter (half to all of the step). */
+function backoff(attempt, base, cap) {
+    const ceiling = Math.min(cap, base * 2 ** Math.min(attempt, 16));
+    return Math.floor(ceiling / 2 + Math.random() * (ceiling / 2));
+}
+async function pause(ms, signal) {
+    await delay(ms, undefined, { signal }).catch(() => undefined);
+}