@nylorun/runtime 0.9.0-beta → 0.11.0-beta

package/dist/tenant/ephemeral.js

@@ -0,0 +1,224 @@
+/**
+ * `startEphemeralRuntime`: a private, in-process Host on port 0 with one Tenant, for tests and
+ * embeds that need the Runtime's HTTP API without the Docker stack. Nothing in it is durable:
+ * its Tenants live in memory (the memory Session Store and memory Durable Streams) and are gone
+ * after `close()`, or in the Postgres database a test passes. Scheduling is the in-process
+ * execution each Tenant starts for itself.
+ *
+ * It is not the temporary Tenant the smoke checks use (scripts/lib/temporary-tenant.mjs):
+ * that one is created on the running stack, with the Tenant-level fixture model
+ * (`model-setting.ts`), and deleted afterwards.
+ */
+import { existsSync, mkdirSync, rmSync, writeFileSync } from "node:fs";
+import { randomBytes } from "node:crypto";
+import { createRequire } from "node:module";
+import { newTenantId } from "@nylorun/core/compatibility";
+import { hashToken, mintBearerToken } from "../core/executors.js";
+import { createHost } from "../host/create-host.js";
+import { createKekFile } from "../vault/kek.js";
+import { MemorySessionStore } from "../store/memory.js";
+import { createTenantModule } from "./module.js";
+import { createMemoryTenantStore } from "./store-memory.js";
+import { createPostgresTenantStore } from "./store-pg.js";
+import { MemoryStreams } from "../streams/memory.js";
+import { createTenantLogger } from "./logger.js";
+import { hostPaths, tenantPaths } from "./paths.js";
+import { bootstrapPrincipal } from "./principals.js";
+import { openTenantRuntime } from "./runtime.js";
+const nodeRequire = createRequire(import.meta.url);
+function coreVersion() {
+    try {
+        return nodeRequire("@nylorun/core/package.json").version;
+    }
+    catch {
+        return "unknown";
+    }
+}
+function newHostId() {
+    return `host_${newTenantId().slice(3)}`;
+}
+/**
+ * A Tenant store whose Tenants live in memory: one memory Session Store per Tenant, created
+ * with its bootstrap principals, kept across closes and re-opens, dropped on delete.
+ */
+function memoryTenants(options) {
+    const stores = new Map();
+    const inner = createMemoryTenantStore({
+        configFor: options.configFor,
+        logger: options.logger,
+        openRuntime: async (config) => {
+            const store = stores.get(config.tenantId);
+            if (!store)
+                throw new Error(`Tenant ${config.tenantId} has no Session Store`);
+            return openTenantRuntime(config, {
+                createKekIfMissing: true,
+                streams: options.streams,
+                store: kept(store),
+                envelope: await inner.readEnvelope(config.tenantId),
+            });
+        },
+    });
+    const store = {
+        ...inner,
+        async create(envelope, bootstrap) {
+            const created = await inner.create(envelope, bootstrap);
+            if (created === "created") {
+                const sessions = new MemorySessionStore({ tenantId: envelope.id });
+                await sessions.tx((t) => bootstrapPrincipal(t, bootstrap));
+                stores.set(envelope.id, sessions);
+            }
+            return created;
+        },
+        async trash(id, now) {
+            await inner.trash(id, now);
+            await stores.get(id)?.close();
+            stores.delete(id);
+        },
+        async removePartial(id) {
+            await inner.removePartial(id);
+            await stores.get(id)?.close();
+            stores.delete(id);
+        },
+    };
+    return {
+        store,
+        async close() {
+            for (const sessions of stores.values())
+                await sessions.close();
+            stores.clear();
+        },
+    };
+}
+/** The Tenant closes its store on close; a memory store must outlive that for a re-open. */
+function kept(store) {
+    return {
+        tenantId: store.tenantId,
+        tx: (fn) => store.tx(fn),
+        onCommit: (listener) => store.onCommit(listener),
+        health: () => store.health(),
+        close: async () => { },
+    };
+}
+/**
+ * Private Host on port 0 with one Tenant (D9 / A19). Not durable (see the module comment).
+ * Exported from `@nylorun/runtime` and `@nylorun/runtime/core`.
+ */
+export async function startEphemeralRuntime(options) {
+    const hostRoot = options.hostRoot;
+    const paths = hostPaths(hostRoot);
+    mkdirSync(paths.home, { recursive: true });
+    mkdirSync(paths.tmp, { recursive: true });
+    mkdirSync(paths.tenants, { recursive: true });
+    mkdirSync(paths.trash, { recursive: true });
+    const hostId = newHostId();
+    const adminKey = options.adminKey ?? randomBytes(32).toString("hex");
+    const hostConfig = {
+        hostId,
+        host: "127.0.0.1",
+        port: 0,
+    };
+    const credentials = { adminKey };
+    writeFileSync(paths.config, `${JSON.stringify(hostConfig, null, 2)}\n`, {
+        mode: 0o600,
+    });
+    writeFileSync(paths.credentials, `${JSON.stringify(credentials, null, 2)}\n`, { mode: 0o600 });
+    const logger = options.logger ??
+        {
+            info() { },
+            warn() { },
+            error() { },
+        };
+    const baseline = options.baseline ?? {};
+    const model = options.model ?? { kind: "scripted", output: "ok" };
+    const configFor = (tenantId) => {
+        const tenant = tenantPaths(hostRoot, tenantId);
+        // A seeded `sandbox.backend` setting overrides this when the Tenant opens.
+        const sandboxBackend = options.sandboxBackend ?? "virtual";
+        return {
+            tenantId,
+            mode: "ephemeral",
+            paths: tenant,
+            sandbox: { backend: sandboxBackend },
+            model,
+            childEnv: Object.freeze({
+                ...baseline,
+                HOME: tenant.home,
+                TMPDIR: tenant.tmp,
+            }),
+            logger: createTenantLogger({
+                tenantId,
+                logPath: tenant.log,
+            }),
+        };
+    };
+    const streams = new MemoryStreams();
+    const memory = options.database
+        ? undefined
+        : memoryTenants({ configFor, streams, logger });
+    const store = options.database
+        ? createPostgresTenantStore({
+            hostRoot,
+            sql: options.database,
+            configFor,
+            logger,
+            openRuntime: (config, opened) => openTenantRuntime(config, {
+                createKekIfMissing: true,
+                streams,
+                ...opened,
+            }),
+        })
+        : memory.store;
+    const module = createTenantModule({
+        store,
+        logger,
+        onDeleted: (tenantId) => streams.deleteTenant(tenantId),
+    });
+    await module.start();
+    const tenantId = options.tenantId ?? newTenantId();
+    const applicationKey = options.applicationKey ?? mintBearerToken();
+    const principalId = options.principalId ?? `principal_${randomBytes(8).toString("hex")}`;
+    const credentialHash = hashToken(applicationKey);
+    await module.create({
+        tenantId,
+        name: options.name ?? "ephemeral",
+        principalId,
+        credentialHash,
+        idempotencyKey: `ephemeral-${tenantId}`,
+        ...(options.studioCredentialHash
+            ? { studioCredentialHash: options.studioCredentialHash }
+            : {}),
+    });
+    // KEK for first vault write; openTenantRuntime also creates when hooks allow.
+    const kekPath = tenantPaths(hostRoot, tenantId).kek;
+    if (!existsSync(kekPath))
+        createKekFile(kekPath);
+    const host = createHost({
+        hostRoot,
+        module,
+        config: hostConfig,
+        credentials,
+        logger,
+        coreVersion: coreVersion(),
+    });
+    await host.listen();
+    let closed = false;
+    return {
+        url: host.url,
+        tenantId,
+        applicationKey,
+        adminKey,
+        principalId,
+        hostRoot,
+        async close() {
+            if (closed)
+                return;
+            closed = true;
+            await host.close();
+            await module.close();
+            await memory?.close();
+            await streams.close();
+            if (!options.retainRoot)
+                rmSync(hostRoot, { recursive: true, force: true });
+        },
+    };
+}

package/dist/tenant/http.d.ts

@@ -0,0 +1,32 @@
+/**
+ * HTTP plumbing shared by the Tenant route modules: typed rejections, the opaque 404 for
+ * unknown credentials (D5), request-body parsing and the client-abort signal.
+ *
+ * Later waves: stable; the streams seam (Wave 2 / Y) keeps using these helpers.
+ */
+import type { IncomingMessage } from "node:http";
+/** D5 opaque failure for unknown/rejected credentials on Tenant routes. */
+export declare const OPAQUE_NOT_FOUND: {
+    status: "rejected";
+    code: "not_found";
+    message: string;
+};
+export declare class HttpError extends Error {
+    readonly status: number;
+    constructor(status: number, message: string);
+}
+export declare const fail: (status: number, message: string) => never;
+export declare class OpaqueAuthError extends Error {
+    readonly status = 404;
+    readonly body: {
+        status: "rejected";
+        code: "not_found";
+        message: string;
+    };
+    constructor();
+}
+export declare const failOpaque: () => never;
+/** AbortSignal tied to the HTTP request being closed by the client. */
+export declare function requestAborted(request: IncomingMessage): AbortSignal;
+/** Read a JSON request body, capped at 1 MiB. */
+export declare function readBody(request: IncomingMessage): Promise<unknown>;

package/dist/tenant/http.js

@@ -0,0 +1,51 @@
+/** D5 opaque failure for unknown/rejected credentials on Tenant routes. */
+export const OPAQUE_NOT_FOUND = {
+    status: "rejected",
+    code: "not_found",
+    message: "Not found",
+};
+export class HttpError extends Error {
+    status;
+    constructor(status, message) {
+        super(message);
+        this.status = status;
+    }
+}
+export const fail = (status, message) => {
+    throw new HttpError(status, message);
+};
+export class OpaqueAuthError extends Error {
+    status = 404;
+    body = OPAQUE_NOT_FOUND;
+    constructor() {
+        super(OPAQUE_NOT_FOUND.message);
+        this.name = "OpaqueAuthError";
+    }
+}
+export const failOpaque = () => {
+    throw new OpaqueAuthError();
+};
+/** AbortSignal tied to the HTTP request being closed by the client. */
+export function requestAborted(request) {
+    const controller = new AbortController();
+    request.on("close", () => {
+        if (!request.complete)
+            controller.abort();
+    });
+    return controller.signal;
+}
+/** Read a JSON request body, capped at 1 MiB. */
+export async function readBody(request) {
+    let data = "";
+    for await (const chunk of request) {
+        data += chunk;
+        if (Buffer.byteLength(data) > 1024 * 1024)
+            fail(413, "Request too large");
+    }
+    try {
+        return JSON.parse(data);
+    }
+    catch {
+        return fail(400, "Invalid JSON");
+    }
+}

package/dist/tenant/live.d.ts

@@ -0,0 +1,92 @@
+/**
+ * Live delivery over Durable Streams (architecture §12.4, "Reading"): session history
+ * (`GET /v1/sessions/:id/items`), session SSE (`GET /v1/sessions/:id/events`) and executor
+ * work streams (`GET /v1/executors/connect`).
+ *
+ * - **Streams.** A session's events are in the stream of its current incarnation
+ *   (`streamOfSession`, `sessions/<id>/<incarnation>`), resolved from the session on every
+ *   request; cursors carry only the session id and sequence.
+ * - **History** reads the session's stream from the cursor up to the tail seen when the read
+ *   starts. The `agent` filter runs here, and the response cursor is the last record read. If
+ *   the streams fail, history answers `503`.
+ * - **SSE** shares one stream read per observed session in this process (a `SessionFeed`)
+ *   among that session's observers. Each observer keeps the next sequence it needs and skips
+ *   what it already has, so a client resuming from `Last-Event-ID` sees no gap and no
+ *   duplicate. An observer behind the feed restarts the shared read from its own position.
+ *   A feed follows one incarnation: it ends once its session is gone or has a new
+ *   incarnation (a reset, possibly on another node), checked when a client joins, when its
+ *   read fails, on a `sessions.reset` signal, and periodically (`checkFeeds`).
+ * - **Executor streams** receive `work_available` from the `tenant/work` reader that
+ *   `tenant/streams.ts` runs (one per Tenant per process); discovery stays `GET /v1/actions`.
+ *
+ * Business code never writes here: events reach streams only through the relay, after commit.
+ */
+import type { IncomingMessage, ServerResponse } from "node:http";
+import type { LiveEvent } from "@nylorun/core/contracts";
+import type { TenantContext } from "./context.js";
+import type { StreamsWiring } from "./streams.js";
+/** One SSE client of a session: the next sequence it needs. */
+interface Observer {
+    readonly response: ServerResponse;
+    next: number;
+}
+/** The shared stream read of one observed session. */
+interface SessionFeed {
+    readonly sessionId: string;
+    /** The stream of the session's incarnation when the feed started. */
+    readonly stream: string;
+    /** The sequence the shared read yields next. Every observer's `next` is at or past it. */
+    next: number;
+    readonly observers: Set<Observer>;
+    /** Aborts the current read; replaced when the read restarts from an earlier sequence. */
+    read: AbortController;
+}
+export interface LiveHub {
+    /** Set once by `wireStreams`. */
+    wiring: StreamsWiring | undefined;
+    /** Session id → its shared read and observers. */
+    readonly feeds: Map<string, SessionFeed>;
+    readonly executorStreams: Map<string, Set<ServerResponse>>;
+}
+export declare function createLiveHub(): LiveHub;
+export declare function send(response: ServerResponse, data: string): void;
+/** Start an SSE response with keepalives; `onClose` runs once the client goes away. */
+export declare function openSse(request: IncomingMessage, response: ServerResponse, onClose: () => void): void;
+/** The resume cursor: `?cursor=` wins over `Last-Event-ID`. */
+export declare function requestCursor(request: IncomingMessage, url: URL): string | undefined;
+/**
+ * `GET /v1/sessions/:id/items`: the session stream after the cursor, up to its tail when the
+ * read starts. The cursor returned is the last record read (the request's cursor when none
+ * was), whatever the `agent` filter kept.
+ */
+export declare function readHistory(ctx: TenantContext, sessionId: string, cursor: string | undefined, agent: string | undefined): Promise<{
+    items: LiveEvent[];
+    cursor: string | null;
+}>;
+/**
+ * `GET /v1/sessions/:id/events`: joins the session's shared read at the client's cursor and
+ * follows it until the client leaves, the session is reset or deleted, or the Tenant closes.
+ */
+export declare function streamSessionEvents(ctx: TenantContext, request: IncomingMessage, response: ServerResponse, sessionId: string, cursor: string | undefined): Promise<void>;
+/**
+ * Ends every feed in this process whose session is gone or has a new incarnation. Run on a
+ * `sessions.reset` signal and periodically, so feeds on nodes other than the one that reset
+ * the Tenant end too.
+ */
+export declare function checkFeeds(ctx: TenantContext): Promise<void>;
+/** `GET /v1/executors/connect`: an executor's work stream, primed with one `work_available`. */
+export declare function streamExecutorWork(hub: LiveHub, request: IncomingMessage, response: ServerResponse, tokenHash: string): void;
+/** Called by the `tenant/work` reader for each signal: wake every connected executor. */
+export declare function announceWork(hub: LiveHub): void;
+export declare function executorConnected(hub: LiveHub, tokenHash: string): boolean;
+export declare function connectedExecutorCount(hub: LiveHub): number;
+/** End the streams a token owns (rotation or deregistration). */
+export declare function endExecutorStreams(hub: LiveHub, tokenHash: string): void;
+/** Reset or close: end every session observer and stop every shared read. */
+export declare function clearObservers(hub: LiveHub): void;
+/** Reset: end and forget every executor stream. */
+export declare function clearExecutorStreams(hub: LiveHub): void;
+/** Close: end every executor stream, then every session observer. */
+export declare function endAllStreams(hub: LiveHub): void;
+export declare function sleep(ms: number, signal: AbortSignal): Promise<void>;
+export {};

package/dist/tenant/live.js

@@ -0,0 +1,290 @@
+import { decodeCursor, encodeCursor } from "../store/cursor.js";
+import { streamOfSession, } from "../streams/types.js";
+import { fail } from "./http.js";
+export function createLiveHub() {
+    return { wiring: undefined, feeds: new Map(), executorStreams: new Map() };
+}
+const WORK_AVAILABLE = 'event: work_available\ndata: {"type":"work_available"}\n\n';
+const RETRY_MIN_MS = 100;
+const RETRY_MAX_MS = 2000;
+function frame(event) {
+    return `id: ${event.cursor}\nevent: ${event.type}\ndata: ${JSON.stringify(event)}\n\n`;
+}
+export function send(response, data) {
+    if (!response.write(data))
+        response.destroy();
+}
+/** Start an SSE response with keepalives; `onClose` runs once the client goes away. */
+export function openSse(request, response, onClose) {
+    response.writeHead(200, {
+        "content-type": "text/event-stream",
+        "cache-control": "no-cache",
+        connection: "keep-alive",
+        "x-accel-buffering": "no",
+    });
+    response.flushHeaders();
+    const timer = setInterval(() => send(response, ": keepalive\n\n"), 15000);
+    timer.unref();
+    request.on("close", () => {
+        clearInterval(timer);
+        onClose();
+    });
+}
+/** The resume cursor: `?cursor=` wins over `Last-Event-ID`. */
+export function requestCursor(request, url) {
+    return (url.searchParams.get("cursor") ??
+        (typeof request.headers["last-event-id"] === "string"
+            ? request.headers["last-event-id"]
+            : undefined));
+}
+/** The first sequence to read: after the cursor's, or the session's first event. */
+function startSeq(sessionId, cursor) {
+    return cursor ? decodeCursor(sessionId, cursor) + 1 : 0;
+}
+/** `agent` keeps only events of one agent used as a tool, by delegationId or path. */
+function belongsTo(event, agent) {
+    const ref = event.payload?.agent;
+    return ref?.delegationId === agent || ref?.path === agent;
+}
+/** The stream of the session's current incarnation, or undefined when the session is gone. */
+async function currentStream(ctx, sessionId) {
+    const session = await ctx.store.tx((t) => t.get("sessions", sessionId));
+    return session && streamOfSession(session);
+}
+function streamsOf(ctx) {
+    return (ctx.live.wiring?.streams ?? fail(503, "Session streams are unavailable"));
+}
+/**
+ * `GET /v1/sessions/:id/items`: the session stream after the cursor, up to its tail when the
+ * read starts. The cursor returned is the last record read (the request's cursor when none
+ * was), whatever the `agent` filter kept.
+ */
+export async function readHistory(ctx, sessionId, cursor, agent) {
+    const from = startSeq(sessionId, cursor);
+    const streams = streamsOf(ctx);
+    const stream = (await currentStream(ctx, sessionId)) ?? fail(404, "Session not found");
+    const items = [];
+    let last;
+    try {
+        for await (const record of streams.read(ctx.config.tenantId, stream, from, { follow: false })) {
+            last = record.seq;
+            if (agent === undefined || belongsTo(record.body, agent))
+                items.push(record.body);
+        }
+    }
+    catch (error) {
+        ctx.config.logger.warn("session history read failed", {
+            sessionId,
+            message: error instanceof Error ? error.message : String(error),
+        });
+        fail(503, "Session history is unavailable");
+    }
+    return {
+        items,
+        cursor: last !== undefined ? encodeCursor(sessionId, last) : cursor ?? null,
+    };
+}
+/**
+ * `GET /v1/sessions/:id/events`: joins the session's shared read at the client's cursor and
+ * follows it until the client leaves, the session is reset or deleted, or the Tenant closes.
+ */
+export async function streamSessionEvents(ctx, request, response, sessionId, cursor) {
+    const from = startSeq(sessionId, cursor);
+    streamsOf(ctx);
+    const stream = (await currentStream(ctx, sessionId)) ?? fail(404, "Session not found");
+    const hub = ctx.live;
+    const observer = { response, next: from };
+    let feed = hub.feeds.get(sessionId);
+    if (feed && feed.stream !== stream) {
+        // The session was created again since that feed started: it follows an abandoned stream.
+        endFeed(hub, feed);
+        feed = undefined;
+    }
+    if (!feed) {
+        feed = {
+            sessionId,
+            stream,
+            next: from,
+            observers: new Set(),
+            read: new AbortController(),
+        };
+        hub.feeds.set(sessionId, feed);
+        runFeed(ctx, feed);
+    }
+    else if (from < feed.next) {
+        // Behind the shared read: restart it here; observers ahead skip what they have.
+        feed.read.abort();
+        feed.read = new AbortController();
+        feed.next = from;
+        runFeed(ctx, feed);
+    }
+    const joined = feed;
+    joined.observers.add(observer);
+    openSse(request, response, () => leave(hub, joined, observer));
+}
+function leave(hub, feed, observer) {
+    feed.observers.delete(observer);
+    if (feed.observers.size > 0)
+        return;
+    feed.read.abort();
+    if (hub.feeds.get(feed.sessionId) === feed)
+        hub.feeds.delete(feed.sessionId);
+}
+function deliver(observer, record) {
+    if (record.seq < observer.next)
+        return;
+    observer.next = record.seq + 1;
+    send(observer.response, frame(record.body));
+}
+/**
+ * Follows the feed's stream from `feed.next` for the current `feed.read`, retrying failed
+ * reads from where it stopped while the session still has that stream. A read that ends by
+ * itself (the stream's Tenant is gone or the streams closed), or a session that is gone or
+ * has a new incarnation, ends the observers.
+ */
+function runFeed(ctx, feed) {
+    const read = feed.read;
+    const signal = read.signal;
+    void (async () => {
+        let delay = RETRY_MIN_MS;
+        while (!signal.aborted) {
+            const streams = ctx.live.wiring?.streams;
+            if (!streams)
+                break;
+            try {
+                for await (const record of streams.read(ctx.config.tenantId, feed.stream, feed.next, { signal })) {
+                    // A restart replaced this read: the new one owns `feed.next`.
+                    if (signal.aborted || feed.read !== read)
+                        return;
+                    if (record.seq < feed.next)
+                        continue;
+                    feed.next = record.seq + 1;
+                    delay = RETRY_MIN_MS;
+                    for (const observer of feed.observers)
+                        deliver(observer, record);
+                }
+                if (signal.aborted)
+                    return;
+                break;
+            }
+            catch (error) {
+                if (signal.aborted)
+                    return;
+                ctx.config.logger.warn("session stream read failed; retrying", {
+                    sessionId: feed.sessionId,
+                    message: error instanceof Error ? error.message : String(error),
+                });
+            }
+            if (!(await stillCurrent(ctx, feed)))
+                break;
+            await sleep(delay, signal);
+            delay = Math.min(delay * 2, RETRY_MAX_MS);
+        }
+        if (signal.aborted || feed.read !== read)
+            return;
+        endFeed(ctx.live, feed);
+    })();
+}
+/** False when the feed's session is gone or has a new incarnation. True when unsure. */
+async function stillCurrent(ctx, feed) {
+    try {
+        return (await currentStream(ctx, feed.sessionId)) === feed.stream;
+    }
+    catch {
+        return true;
+    }
+}
+/**
+ * Ends every feed in this process whose session is gone or has a new incarnation. Run on a
+ * `sessions.reset` signal and periodically, so feeds on nodes other than the one that reset
+ * the Tenant end too.
+ */
+export async function checkFeeds(ctx) {
+    const feeds = [...ctx.live.feeds.values()];
+    if (feeds.length === 0)
+        return;
+    const current = await ctx.store.tx(async (t) => {
+        const streams = new Map();
+        for (const feed of feeds) {
+            const session = await t.get("sessions", feed.sessionId);
+            streams.set(feed.sessionId, session && streamOfSession(session));
+        }
+        return streams;
+    });
+    for (const feed of feeds)
+        if (ctx.live.feeds.get(feed.sessionId) === feed &&
+            current.get(feed.sessionId) !== feed.stream)
+            endFeed(ctx.live, feed);
+}
+function endFeed(hub, feed) {
+    feed.read.abort();
+    if (hub.feeds.get(feed.sessionId) === feed)
+        hub.feeds.delete(feed.sessionId);
+    for (const observer of feed.observers)
+        observer.response.end();
+    feed.observers.clear();
+}
+/** `GET /v1/executors/connect`: an executor's work stream, primed with one `work_available`. */
+export function streamExecutorWork(hub, request, response, tokenHash) {
+    let streams = hub.executorStreams.get(tokenHash);
+    if (!streams)
+        hub.executorStreams.set(tokenHash, (streams = new Set()));
+    const set = streams;
+    set.add(response);
+    openSse(request, response, () => {
+        set.delete(response);
+        if (!set.size && hub.executorStreams.get(tokenHash) === set)
+            hub.executorStreams.delete(tokenHash);
+    });
+    send(response, WORK_AVAILABLE);
+}
+/** Called by the `tenant/work` reader for each signal: wake every connected executor. */
+export function announceWork(hub) {
+    for (const streams of hub.executorStreams.values())
+        for (const response of streams)
+            send(response, WORK_AVAILABLE);
+}
+export function executorConnected(hub, tokenHash) {
+    return (hub.executorStreams.get(tokenHash)?.size ?? 0) > 0;
+}
+export function connectedExecutorCount(hub) {
+    return [...hub.executorStreams.values()].filter((set) => set.size > 0)
+        .length;
+}
+/** End the streams a token owns (rotation or deregistration). */
+export function endExecutorStreams(hub, tokenHash) {
+    for (const r of hub.executorStreams.get(tokenHash) ?? [])
+        r.end();
+    hub.executorStreams.delete(tokenHash);
+}
+/** Reset or close: end every session observer and stop every shared read. */
+export function clearObservers(hub) {
+    for (const feed of [...hub.feeds.values()])
+        endFeed(hub, feed);
+    hub.feeds.clear();
+}
+/** Reset: end and forget every executor stream. */
+export function clearExecutorStreams(hub) {
+    for (const streams of hub.executorStreams.values())
+        for (const r of streams)
+            r.end();
+    hub.executorStreams.clear();
+}
+/** Close: end every executor stream, then every session observer. */
+export function endAllStreams(hub) {
+    clearExecutorStreams(hub);
+    clearObservers(hub);
+}
+export function sleep(ms, signal) {
+    return new Promise((resolve) => {
+        if (signal.aborted)
+            return resolve();
+        const done = () => {
+            clearTimeout(timer);
+            signal.removeEventListener("abort", done);
+            resolve();
+        };
+        const timer = setTimeout(done, ms);
+        signal.addEventListener("abort", done, { once: true });
+    });
+}