@aywengo/mercury-fleet 0.0.1-bootstrap

package/dist/server.js

@@ -0,0 +1,540 @@
+/**
+ * The Fleet service: an HTTP server over the registry and the prober.
+ *
+ * The route table is enumerated rather than grown ad hoc (docs/fleet-design.md section 15.4). The rule that
+ * matters is that nothing here accepts a URL to fetch: a caller names a host by its registry id and Fleet
+ * resolves it. That is what keeps section 9's "do not proxy arbitrary paths to children" true as the surface
+ * grows, instead of eroding one convenience at a time.
+ */
+import { createServer } from 'node:http';
+import { readFileSync } from 'node:fs';
+import * as https from 'node:https';
+import { HostRegistry, RegistryError } from "./registry.js";
+import { BindingStore, UNKNOWN } from "./bindings.js";
+import { createChildClient } from "./child.js";
+import { DispatchError, recoverPending, refreshStates, submitRun } from "./dispatch.js";
+import { startSweeper } from "./sweep.js";
+import { listMirroredEvents } from "./events.js";
+import { startEventStream } from "./stream.js";
+import { loadRepoUrlMap, routeRun, RoutingError } from "./routing.js";
+import { cancelRun, retryRun, sendInput } from "./interact.js";
+import { mergeRollup, scrapeAll } from "./metrics.js";
+import { createProber } from "./prober.js";
+import { CredentialError } from "./credentials.js";
+import { parseCallerTokens, hostAllowed } from "./auth.js";
+import { authenticate, HttpError, matchRoute, readJsonBody, sendJson } from "./http.js";
+import { FLEET_PRODUCT, FLEET_VERSION } from "./version.js";
+function bodyObject(body) {
+    if (body === null || typeof body !== 'object' || Array.isArray(body)) {
+        throw new HttpError(400, 'request body must be a JSON object');
+    }
+    return body;
+}
+/** The standard Idempotency-Key header, accepted alongside an in-body equivalent. */
+function headerIdempotency(ctx) {
+    const v = ctx.headers?.['idempotency-key'];
+    return typeof v === 'string' && v.length > 0 ? v : null;
+}
+function scopedHosts(caller, registry) {
+    if (caller.isAdmin || caller.allowedHosts === '*')
+        return '*';
+    return caller.allowedHosts;
+}
+/** Parse a query parameter as a non-negative integer, falling back when it is anything else. */
+function nonNegativeInt(raw, fallback) {
+    if (raw === null || raw === undefined || raw.trim() === '')
+        return fallback;
+    const n = Number(raw);
+    if (!Number.isFinite(n) || !Number.isInteger(n) || n < 0)
+        return fallback;
+    return n;
+}
+/**
+ * Unwrap the optional { input: ... } wrapper without losing a genuine null, and without rejecting a body that
+ * is a bare string, number, array or null -- the child accepts any JSON value.
+ */
+function unwrapInput(body) {
+    if (body !== null && typeof body === 'object' && !Array.isArray(body) && 'input' in body) {
+        return body.input;
+    }
+    return body;
+}
+/** Narrow to a plain object without accepting arrays or null, both of which `typeof` would let through. */
+function isRecord(v) {
+    return typeof v === 'object' && v !== null && !Array.isArray(v);
+}
+function str(body, key) {
+    const v = body[key];
+    if (typeof v !== 'string' || !v)
+        throw new HttpError(400, `${key} is required and must be a non-empty string`);
+    return v;
+}
+/** Hosts this caller may see. Admins and `*` callers see everything. */
+function visibleHosts(registry, caller) {
+    const all = registry.listWithProbe();
+    if (caller.isAdmin || caller.allowedHosts === '*')
+        return all;
+    return all.filter((h) => caller.allowedHosts.includes(h.id));
+}
+export function buildRoutes(deps) {
+    const registry = new HostRegistry(deps.db);
+    // Loaded once at startup. A malformed map must fail the process loudly rather than degrade into "no
+    // mapping", which would present as routing refusing work it should have accepted.
+    const resolveCloneUrl = loadRepoUrlMap(deps.config.repoUrlsFile);
+    const dispatch = {
+        registry,
+        bindings: new BindingStore(deps.db),
+        child: createChildClient({ timeoutMs: deps.config.probeTimeoutMs }),
+        resolveToken: (ref) => deps.credentials.secret(ref),
+    };
+    const interaction = {
+        registry: dispatch.registry,
+        bindings: dispatch.bindings,
+        child: dispatch.child,
+        resolveToken: dispatch.resolveToken,
+    };
+    const metricsDeps = {
+        registry: dispatch.registry,
+        child: dispatch.child,
+        resolveToken: dispatch.resolveToken,
+        timeoutMs: deps.config.probeTimeoutMs,
+    };
+    const prober = createProber({
+        registry,
+        resolveToken: (ref) => deps.credentials.secret(ref),
+        intervalMs: deps.config.probeIntervalMs,
+        timeoutMs: deps.config.probeTimeoutMs,
+    });
+    const routes = [
+        {
+            method: 'GET', pattern: ['healthz'], public: true,
+            handle: (_ctx, res) => sendJson(res, 200, {
+                ok: true,
+                ts: new Date().toISOString(),
+                product: FLEET_PRODUCT,
+                version: FLEET_VERSION,
+            }),
+        },
+        {
+            // GET /metrics -- the fleet rollup. Mounted at the root like a Mercury's own, so a Prometheus job can
+            // point at either with the same path.
+            //
+            // Scoped to the caller's visible hosts for the same reason reads are: a caller limited to one host must
+            // not learn another host's queue depth, run counts and worker topology from one aggregate endpoint.
+            method: 'GET', pattern: ['metrics'],
+            handle: async (ctx, res) => {
+                const hosts = visibleHosts(registry, ctx.caller);
+                const scrapes = await scrapeAll(metricsDeps, hosts.map((h) => h.id));
+                const { text, dropped } = mergeRollup(scrapes);
+                if (dropped.length > 0) {
+                    // Reported rather than swallowed: a rollup that quietly loses a family under-reports, and the whole
+                    // point of the endpoint is to be trusted.
+                    deps.logger.warn('metrics rollup dropped data', { dropped: dropped.length, examples: dropped.slice(0, 20) });
+                }
+                res.writeHead(200, {
+                    'content-type': 'text/plain; version=0.0.4; charset=utf-8',
+                    'cache-control': 'no-cache',
+                });
+                res.end(text);
+            },
+        },
+        {
+            // Registry reads. A caller sees only the hosts it may act on; naming a host it may not touch is a
+            // 403 elsewhere, never a silent substitution.
+            method: 'GET', pattern: ['fleet', 'hosts'],
+            handle: (ctx, res) => sendJson(res, 200, { hosts: visibleHosts(registry, ctx.caller) }),
+        },
+        {
+            method: 'POST', pattern: ['fleet', 'hosts'], admin: true,
+            handle: (ctx, res) => {
+                const b = bodyObject(ctx.body);
+                // Reject an unknown ref at registration. Accepting a typo defers the failure to a probe that reports
+                // auth-fail, which points at the host when the mistake is in this request.
+                const ref = str(b, 'credentialRef');
+                deps.credentials.secret(ref);
+                const host = registry.add({
+                    id: str(b, 'id'),
+                    baseUrl: str(b, 'baseUrl'),
+                    credentialRef: ref,
+                    labels: (b.labels ?? {}),
+                    localPaths: (b.localPaths ?? []),
+                    enabled: b.enabled !== false,
+                });
+                sendJson(res, 201, { host });
+            },
+        },
+        {
+            method: 'POST', pattern: ['fleet', 'hosts', ':id', 'enable'], admin: true,
+            handle: (ctx, res) => {
+                const b = bodyObject(ctx.body);
+                const host = registry.setEnabled(ctx.params[0], b.enabled !== false);
+                sendJson(res, 200, { host });
+            },
+        },
+        {
+            method: 'DELETE', pattern: ['fleet', 'hosts', ':id'], admin: true,
+            handle: (ctx, res) => {
+                // ?force=1 discards the bindings to this host's Runs. Offered because refusing outright would give
+                // no way forward, but never implied: the default protects the one table that cannot be rebuilt.
+                const force = ctx.query.get('force') === '1';
+                let removed = false;
+                try {
+                    removed = registry.remove(ctx.params[0], { force });
+                }
+                catch (err) {
+                    if (err instanceof RegistryError && /still owns/.test(err.message)) {
+                        throw new HttpError(409, err.message);
+                    }
+                    throw err;
+                }
+                if (!removed)
+                    throw new HttpError(404, 'no such host');
+                sendJson(res, 200, { removed: ctx.params[0], forced: force });
+            },
+        },
+        {
+            method: 'POST', pattern: ['fleet', 'hosts', ':id', 'probe'], admin: true,
+            handle: async (ctx, res) => {
+                const id = ctx.params[0];
+                const host = registry.get(id);
+                if (!host)
+                    throw new HttpError(404, 'no such host');
+                const { probeAndRecord } = await import("./probe.js");
+                const rec = await probeAndRecord({
+                    hostId: id, baseUrl: host.baseUrl,
+                    token: deps.credentials.secret(host.credentialRef),
+                    timeoutMs: deps.config.probeTimeoutMs,
+                });
+                registry.recordProbe(rec);
+                sendJson(res, 200, { probe: rec });
+            },
+        },
+        {
+            // Dispatch. The caller names a host; the allowlist is checked against THAT host and a mismatch is 403,
+            // never a silent substitution to a host the caller may use.
+            method: 'POST', pattern: ['fleet', 'runs'],
+            handle: async (ctx, res) => {
+                const b = bodyObject(ctx.body);
+                const namedHost = typeof b.host === 'string' && b.host ? b.host : undefined;
+                // Routing runs over the hosts THIS caller may see, not the whole fleet. A router that could place work
+                // on a hidden host would turn an allowlist into a suggestion.
+                let decision;
+                try {
+                    decision = routeRun(visibleHosts(registry, ctx.caller), {
+                        host: namedHost,
+                        agent: typeof b.agent === 'string' && b.agent ? b.agent : undefined,
+                        // Passed through rather than quietly dropped when the shape is wrong: an array or a string here
+                        // must be a 400 about the caller's body, not a request that silently loses its filter.
+                        labels: b.labels,
+                        repository: b.repository,
+                    }, { resolveCloneUrl });
+                }
+                catch (err) {
+                    // A routing failure names every host considered and why each was excluded. Swallowing that into a
+                    // generic 400 is exactly how a five-second mistake becomes an hour of confusion.
+                    if (err instanceof RoutingError)
+                        throw new HttpError(err.status, err.message, undefined, err.exclusions);
+                    throw err;
+                }
+                if (!hostAllowed(ctx.caller, decision.hostId)) {
+                    // Unreachable given the scoping above, kept because a router change must not silently widen access.
+                    throw new HttpError(403, `caller ${ctx.caller.ownerId} may not submit work to host ${decision.hostId}`);
+                }
+                const requested = { ...b, ...(decision.repository ? { repository: decision.repository } : {}) };
+                if (!decision.repository)
+                    delete requested.repository;
+                delete requested.host;
+                delete requested.idempotency;
+                const idem = typeof b.idempotency === 'string' && b.idempotency
+                    ? b.idempotency
+                    : headerIdempotency(ctx);
+                // ownerId comes from the authenticated caller, never the body: idempotency scoping is a security
+                // boundary, and a caller that could choose it could claim another caller's binding.
+                const outcome = await submitRun(dispatch, {
+                    hostId: decision.hostId, ownerId: ctx.caller.ownerId, requested, clientToken: idem ?? null,
+                });
+                sendJson(res, outcome.reused ? 200 : 201, {
+                    fleetRunId: outcome.binding.fleetRunId,
+                    hostId: outcome.binding.hostId,
+                    // Say so when the localPath was replaced: a caller who asked for a path and got a clone should be
+                    // able to see that the constraint was lifted, not discover it from a missing working tree.
+                    ...(decision.rewroteLocalPath ? { rewroteLocalPath: true } : {}),
+                    childRunId: outcome.binding.childRunId,
+                    pending: outcome.pending,
+                    reused: outcome.reused,
+                    status: dispatch.bindings.state(outcome.binding.fleetRunId)?.status ?? UNKNOWN,
+                    ...(outcome.note ? { note: outcome.note } : {}),
+                });
+            },
+        },
+        {
+            method: 'GET', pattern: ['fleet', 'runs'],
+            handle: (ctx, res) => {
+                const runs = dispatch.bindings.list(scopedHosts(ctx.caller, registry));
+                sendJson(res, 200, { runs });
+            },
+        },
+        {
+            method: 'GET', pattern: ['fleet', 'runs', ':id'],
+            handle: async (ctx, res) => {
+                const id = ctx.params[0];
+                const binding = dispatch.bindings.get(id);
+                if (!binding)
+                    throw new HttpError(404, 'no such fleet run');
+                if (!hostAllowed(ctx.caller, binding.hostId)) {
+                    // Same status as "does not exist" would be for a different reason; 403 is honest here because the
+                    // caller demonstrably knows an id that it may not read.
+                    throw new HttpError(403, `caller ${ctx.caller.ownerId} may not read runs on host ${binding.hostId}`);
+                }
+                await refreshStates(dispatch, [binding.hostId]);
+                sendJson(res, 200, {
+                    ...binding,
+                    state: dispatch.bindings.state(id),
+                });
+            },
+        },
+        // Interaction verbs. Each is scoped by the same host allowlist as reads: a caller who may not see a host
+        // must not be able to cancel or feed work to a Run on it.
+        ...['input', 'cancel', 'retry'].map((verb) => ({
+            method: 'POST',
+            pattern: ['fleet', 'runs', ':id', verb],
+            handle: async (ctx, res) => {
+                const id = ctx.params[0];
+                const binding = dispatch.bindings.get(id);
+                if (!binding)
+                    throw new HttpError(404, 'no such fleet run');
+                if (!hostAllowed(ctx.caller, binding.hostId)) {
+                    throw new HttpError(403, `caller ${ctx.caller.ownerId} may not act on runs on host ${binding.hostId}`);
+                }
+                const outcome = verb === 'input'
+                    // The child takes an arbitrary JSON value; Fleet forwards it rather than inventing a shape.
+                    // The child takes any JSON value, so Fleet must not demand an object. A body of {"input": ...} is
+                    // unwrapped -- but by key presence, not truthiness, because {input: null} is a caller genuinely
+                    // sending null rather than one who omitted it.
+                    ? await sendInput(interaction, id, unwrapInput(ctx.body))
+                    : verb === 'cancel'
+                        ? await cancelRun(interaction, id)
+                        : await retryRun(interaction, id);
+                // 200 even when unknown: the request was accepted and understood, and the uncertainty is about the
+                // child's answer rather than about the caller's request. The body says which it was.
+                sendJson(res, 200, outcome);
+            },
+        })),
+        {
+            // GET /fleet/runs/:id/stream -- Fleet-side SSE, a view onto the mirror rather than a second path to the
+            // child. Losing this stream costs latency only: the client reconnects with ?after=<cursor> and resumes
+            // exactly where it was, because the cursor is the correctness mechanism and SSE is the optimisation.
+            method: 'GET', pattern: ['fleet', 'runs', ':id', 'stream'],
+            handle: async (ctx, res) => {
+                const id = ctx.params[0];
+                const binding = dispatch.bindings.get(id);
+                if (!binding)
+                    throw new HttpError(404, 'no such fleet run');
+                if (!hostAllowed(ctx.caller, binding.hostId)) {
+                    throw new HttpError(403, `caller ${ctx.caller.ownerId} may not read runs on host ${binding.hostId}`);
+                }
+                res.writeHead(200, {
+                    'content-type': 'text/event-stream',
+                    'cache-control': 'no-cache',
+                    connection: 'keep-alive',
+                    // Without this a buffering proxy turns a live stream into one long download.
+                    'x-accel-buffering': 'no',
+                });
+                // Flush the headers now. Otherwise a client that connected but receives nothing waits on the socket
+                // and cannot tell a working stream from a hung request.
+                res.flushHeaders?.();
+                startEventStream(res, {
+                    db: deps.db, bindings: dispatch.bindings, fleetRunId: id,
+                    pollIntervalMs: deps.config.streamPollMs,
+                    // Passed in rather than applied afterwards: the backlog is written synchronously, so a handle method
+                    // would run after the client had already been sent events it asked to skip.
+                    after: nonNegativeInt(ctx.query?.get('after'), 0),
+                });
+            },
+        },
+        {
+            // GET /fleet/runs/:id/events?after=<cursor>&limit=<n>
+            // Reads Fleet's mirror, so it answers at the freshness of the last sweep rather than this instant, and
+            // costs the child nothing. That is deliberate: section 8 makes the cursor the correctness mechanism and
+            // polling the baseline, so a client that pages from nextCursor sees every event whether or not a child
+            // connection is open.
+            method: 'GET', pattern: ['fleet', 'runs', ':id', 'events'],
+            handle: async (ctx, res) => {
+                const id = ctx.params[0];
+                const binding = dispatch.bindings.get(id);
+                if (!binding)
+                    throw new HttpError(404, 'no such fleet run');
+                if (!hostAllowed(ctx.caller, binding.hostId)) {
+                    throw new HttpError(403, `caller ${ctx.caller.ownerId} may not read runs on host ${binding.hostId}`);
+                }
+                // Parsed as integers before anything reaches SQLite. `after=Infinity` and `limit=2.5` are both
+                // accepted by Number() and both produce pagination nobody can reason about -- and Infinity is a
+                // perfectly truthy value, so the usual `|| 0` fallback would not have caught it.
+                const after = nonNegativeInt(ctx.query?.get('after'), 0);
+                // Bounded rather than trusted: an unbounded limit would let one caller pull an entire mirrored
+                // transcript into memory in one request.
+                const limit = Math.min(nonNegativeInt(ctx.query?.get('limit'), 200) || 200, 1000);
+                sendJson(res, 200, listMirroredEvents(deps.db, id, after, limit));
+            },
+        },
+        {
+            method: 'POST', pattern: ['fleet', 'probe'], admin: true,
+            handle: async (_ctx, res) => {
+                const results = await prober.sweepOnce();
+                sendJson(res, 200, { probed: results.length, hosts: registry.listWithProbe() });
+            },
+        },
+    ];
+    return { routes, prober, dispatch };
+}
+export function createFleetServer(deps) {
+    const { routes, prober, dispatch } = buildRoutes(deps);
+    const registry = new HostRegistry(deps.db);
+    // Built from the same dispatch the routes use, so mirroring resolves hosts and tokens exactly the way
+    // dispatch and reconciliation do rather than through a second, subtly different wiring.
+    const eventMirror = {
+        db: deps.db,
+        bindings: dispatch.bindings,
+        registry: dispatch.registry,
+        child: dispatch.child,
+        resolveToken: dispatch.resolveToken,
+    };
+    // Parsed once, not per request: the token set is fixed at startup, and re-splitting it on every call
+    // would let a caller's request rate scale the cost of an operation that never changes.
+    const authDeps = { callers: parseCallerTokens(deps.config.apiTokens), adminToken: deps.config.adminToken };
+    const handler = async (req, res) => {
+        const url = new URL(req.url ?? '/', 'http://internal');
+        const path = url.pathname.replace(/^\/+|\/+$/g, '');
+        const matched = matchRoute(routes, req.method ?? 'GET', path);
+        if (!matched) {
+            sendJson(res, 404, { error: 'not found' });
+            return;
+        }
+        const { route, params } = matched;
+        try {
+            let caller = { ownerId: 'anonymous', isAdmin: false, allowedHosts: [] };
+            if (!route.public) {
+                const resolved = authenticate(req, authDeps);
+                if (!resolved) {
+                    sendJson(res, 401, { error: 'authentication required' });
+                    return;
+                }
+                caller = resolved;
+                if (route.admin && !caller.isAdmin) {
+                    sendJson(res, 403, { error: 'administrator token required' });
+                    return;
+                }
+            }
+            const body = route.method === 'POST' ? await readJsonBody(req) : {};
+            await route.handle({ caller, params, query: url.searchParams, headers: req.headers, body, log: deps.logger }, res);
+        }
+        catch (err) {
+            // Each of these is a caller mistake rather than a server fault, and each carries or maps to a 4xx:
+            // HttpError carries its own status, DispatchError carries 404 or 409, and RegistryError plus
+            // CredentialError are always 400. CredentialError in particular names a credential_ref that is not in the
+            // file -- answering 500 there sends the operator to the service logs when the answer is in their own
+            // request body. Anything NOT listed here falls through to the 500 below, which is the point: an
+            // unrecognised exception is a server fault and must not be dressed up as a client error.
+            if (err instanceof HttpError || err instanceof RegistryError || err instanceof CredentialError
+                || err instanceof DispatchError) {
+                // DispatchError's status is chosen where the failure is understood, so it must survive to the caller:
+                // 404 for an unknown Fleet Run, 409 for a disabled host or an idempotency conflict.
+                const status = err instanceof HttpError ? err.status
+                    : err instanceof DispatchError ? err.status
+                        : 400;
+                sendJson(res, status, {
+                    error: err.message,
+                    ...(err instanceof HttpError && err.details !== undefined ? { details: err.details } : {}),
+                });
+                return;
+            }
+            // The logger redacts, so an exception message carrying a header cannot reach the log intact.
+            deps.logger.error('request failed', { method: req.method, path, err: err });
+            sendJson(res, 500, { error: 'internal error' });
+        }
+    };
+    const tls = Boolean(deps.config.tlsCert && deps.config.tlsKey);
+    const server = tls
+        ? https.createServer({ cert: readFileSync(deps.config.tlsCert), key: readFileSync(deps.config.tlsKey) }, handler)
+        : createServer(handler);
+    // A dashboard holding an open stream must not stall `systemctl stop` until SIGKILL.
+    server.keepAliveTimeout = 5000;
+    server.headersTimeout = 10000;
+    const sweeperRef = { handle: null };
+    return {
+        server,
+        prober,
+        routes,
+        get sweeper() {
+            return sweeperRef.handle;
+        },
+        async listen() {
+            // Startup order from section 15.6: credentials were loaded by the caller, the database is open, and
+            // reconciliation happens here. Binding LAST means a client can never get a response from an endpoint
+            // that has not finished loading the registry.
+            const hosts = registry.list();
+            // Recovery BEFORE binding, per section 15.6: a client must not be able to ask about a Run while the
+            // set of Runs is still being worked out. Pending bindings are those where Fleet asked a child for a Run
+            // and never recorded the answer -- typically because Fleet died in between.
+            const pending = dispatch.bindings.pending();
+            if (pending.length > 0) {
+                const rec = await recoverPending(dispatch);
+                deps.logger.info('recovered pending bindings', {
+                    found: pending.length, resolved: rec.resolved, stillPending: rec.stillPending,
+                });
+            }
+            deps.logger.info('fleet starting', {
+                hosts: hosts.length,
+                bindHost: deps.config.bindHost,
+                port: deps.config.port,
+                tls,
+            });
+            await new Promise((resolve, reject) => {
+                server.once('error', reject);
+                server.listen(deps.config.port, deps.config.bindHost, () => resolve());
+            });
+            const addr = server.address();
+            prober.start();
+            // Sweep once immediately so the first GET /fleet/hosts is not all "never-probed".
+            void prober.sweepOnce().catch((err) => deps.logger.error('initial sweep failed', { err }));
+            // Reconciliation on its own timer, from startup, whether or not anyone is asking (section 7). A sweep
+            // driven by reads would report exactly the staleness it exists to prevent: a Run that finishes while the
+            // dashboard is closed would stay RUNNING until somebody opens the page again.
+            sweeperRef.handle = startSweeper(dispatch, {
+                intervalMs: deps.config.sweepIntervalMs,
+                // Event metadata rides with reconciliation rather than on a third timer: both walk the same set of
+                // live bindings, and a Run worth a status read is worth an event page.
+                events: eventMirror,
+                onEvent: (event) => {
+                    // LOST is an operator event, not a Run outcome: the binding asserts a Run exists and the child
+                    // denies it. Loud in the log; a webhook belongs with the alerting work in a later phase.
+                    deps.logger.error('binding lost: child has no such Run', {
+                        fleetRunId: event.fleetRunId, hostId: event.hostId, childRunId: event.childRunId,
+                        detail: event.detail,
+                    });
+                },
+                onError: (err) => deps.logger.error('reconciliation sweep failed', {
+                    err: err instanceof Error ? err : new Error(String(err)),
+                }),
+            });
+            return {
+                host: deps.config.bindHost,
+                port: typeof addr === 'object' && addr ? addr.port : deps.config.port,
+                tls,
+            };
+        },
+        async close() {
+            // In-flight probes are abandoned, not awaited: the sweep writes cache rows and cache is cheap to lose.
+            prober.stop();
+            // Stopped before the socket closes so a pass cannot start against a half-torn-down process and write a
+            // cache row after the database is on its way out.
+            sweeperRef.handle?.stop();
+            sweeperRef.handle = null;
+            await new Promise((resolve) => {
+                server.closeAllConnections?.();
+                server.close(() => resolve());
+            });
+        },
+    };
+}
+export { hostAllowed };

package/dist/stream.js

@@ -0,0 +1,132 @@
+import { listMirroredEvents } from "./events.js";
+import { isTerminal } from "./sweep.js";
+function write(res, event, data, id) {
+    // writeHead must precede the first write; the caller does that so status and headers go out together.
+    // Framing is explicit rather than relying on newlines inside a template literal, which reads as a bug even
+    // when it happens to produce the right bytes.
+    const idLine = id === undefined ? '' : `id: ${id}\n`;
+    return res.write(`event: ${event}\n${idLine}data: ${JSON.stringify(data)}\n\n`);
+}
+export function startEventStream(res, opts) {
+    const pollMs = opts.pollIntervalMs ?? 1000;
+    const keepAliveMs = opts.keepAliveMs ?? 15000;
+    const drainTimeoutMs = opts.drainTimeoutMs ?? 30000;
+    let closed = false;
+    // Set before the first pump. Doing this by calling back into the handle afterwards cannot work: the backlog
+    // is written synchronously inside startEventStream, so it would already have been sent.
+    let cursor = opts.after ?? 0;
+    let backlogDone = false;
+    let pollTimer = null;
+    let keepTimer = null;
+    let drainTimer = null;
+    const clearTimers = () => {
+        if (pollTimer)
+            clearInterval(pollTimer);
+        if (keepTimer)
+            clearInterval(keepTimer);
+        if (drainTimer)
+            clearTimeout(drainTimer);
+        pollTimer = null;
+        keepTimer = null;
+        drainTimer = null;
+    };
+    const stop = (reason) => {
+        if (closed)
+            return;
+        closed = true;
+        clearTimers();
+        opts.onEnd?.(reason);
+        // res.end() throws on a destroyed socket, which is often the very reason this path is running. Nothing
+        // can act on it, and letting it escape would surface as an uncaught exception during teardown.
+        try {
+            if (!res.writableEnded)
+                res.end();
+        }
+        catch {
+            /* already gone */
+        }
+    };
+    /**
+     * A subscriber that has stopped reading gets its socket torn down rather than a graceful end. Ending lets
+     * the kernel keep buffering a transcript nobody will read, and the event whose write returned false has not
+     * been recorded in the cursor, so a reconnect could receive it twice. Same reasoning as Mercury's own SSE
+     * backpressure handling.
+     */
+    const abortUnreadable = (reason) => {
+        if (closed)
+            return;
+        closed = true;
+        clearTimers();
+        opts.onEnd?.(reason);
+        try {
+            res.destroy();
+        }
+        catch {
+            /* already gone */
+        }
+    };
+    const armDrainWatch = () => {
+        if (drainTimer)
+            clearTimeout(drainTimer);
+        drainTimer = setTimeout(() => stop('subscriber stopped reading'), drainTimeoutMs);
+        // A stream nobody is watching must not be the reason the process stays alive.
+        drainTimer.unref?.();
+    };
+    const pump = () => {
+        if (closed)
+            return;
+        // A throw inside a timer callback is an uncaught exception, not a failed request, and would take the whole
+        // service down over one bad read. Every other periodic task here (sweep, prober) wraps its body likewise.
+        try {
+            pumpInner();
+        }
+        catch (err) {
+            opts.onError?.(err);
+            stop('read failed');
+        }
+    };
+    const pumpInner = () => {
+        const page = listMirroredEvents(opts.db, opts.fleetRunId, cursor, 500);
+        for (const ev of page.events) {
+            if (!write(res, 'event', ev, ev.sequence)) {
+                // The kernel buffer is full: the subscriber has stopped reading.
+                abortUnreadable('backpressure');
+                return;
+            }
+            cursor = ev.sequence;
+        }
+        armDrainWatch();
+        if (!backlogDone) {
+            backlogDone = true;
+            // Tell the client where it stands before any live events, so a UI can distinguish "caught up" from
+            // "still replaying".
+            write(res, 'snapshot', { cursor, hasMore: page.hasMore });
+        }
+        // A terminal Run whose log is drained has nothing left to send; end rather than hold the socket.
+        const state = opts.bindings.state(opts.fleetRunId);
+        if (!page.hasMore && state && isTerminal(state.status) && state.eventsDrained) {
+            write(res, 'done', { cursor, status: state.status });
+            stop('terminal');
+        }
+    };
+    res.on('close', () => stop('client disconnected'));
+    armDrainWatch();
+    pump();
+    if (!closed) {
+        pollTimer = setInterval(pump, pollMs);
+        pollTimer.unref?.();
+        keepTimer = setInterval(() => {
+            if (closed)
+                return;
+            if (!res.write(': keep-alive\n\n'))
+                stop('backpressure');
+        }, keepAliveMs);
+        keepTimer.unref?.();
+    }
+    return {
+        stop: () => stop('stopped'),
+        get closed() {
+            return closed;
+        },
+    };
+}