@nylorun/runtime 0.9.0-beta → 0.11.0-beta

package/dist/tenant/streams.d.ts

@@ -0,0 +1,126 @@
+import type { SessionStore } from "../store/types.js";
+import { type Relay } from "../streams/relay.js";
+import { type DurableStreams } from "../streams/types.js";
+import type { TenantContext } from "./context.js";
+export interface WireStreamsOptions {
+    tenantId: string;
+    store: SessionStore;
+    streams: DurableStreams;
+    /** Close `streams` with the wiring (streams created for this Tenant alone). */
+    ownsStreams?: boolean;
+    /** An existing relay to use instead of creating one; the caller closes it. */
+    relay?: Relay;
+}
+/** The state of the Tenant's basin as this process last saw it. */
+export interface BasinStatus {
+    /** `ensureTenant` succeeded and no append has failed since. */
+    ready: boolean;
+    /** Failed `ensureTenant` attempts since the basin was last ready. */
+    failures: number;
+    /** The last failure's message, cleared once the basin is ready. */
+    lastError: string | null;
+}
+/** What one `collectSessionStreams` pass did. */
+export interface CollectResult {
+    /** Abandoned session streams deleted. */
+    deleted: number;
+    /** Session streams of existing sessions, kept. */
+    kept: number;
+    /** Deletions that failed; the collection stays pending. */
+    failed: number;
+}
+/** The handles `wireStreams` returns (also kept on `ctx.live.wiring`). */
+export interface StreamsWiring {
+    readonly streams: DurableStreams;
+    readonly relay: Relay;
+    /** The basin's state (`streamsStatus`). */
+    basin(): BasinStatus;
+    /** Appends leftover outbox rows, at most `limit`; resolves with the number relayed. */
+    drain(limit?: number): Promise<number>;
+    /**
+     * Deletes abandoned session streams when a collection is pending; undefined when none was.
+     * One pass at a time.
+     */
+    collect(): Promise<CollectResult | undefined>;
+    /** Stops the readers, waits for in-flight relays, and closes what `wireStreams` created. */
+    close(): Promise<void>;
+}
+/** The Tenant setting that holds a pending collection's token ("" when none is pending). */
+export declare const COLLECT_SETTING = "streams.collect";
+/**
+ * Wires the Tenant to Durable Streams: checks the basin, creates the relay (unless given),
+ * starts the work and control readers, and records the handles on `ctx.live.wiring`. Call it
+ * once, after `ctx` is built and before the Tenant serves requests.
+ */
+export declare function wireStreams(ctx: TenantContext, options: WireStreamsOptions): Promise<StreamsWiring>;
+/** The Tenant sweep's outbox recovery: appends leftover rows, at most `limit`. */
+export declare function drainOutbox(ctx: TenantContext, limit?: number): Promise<number>;
+/**
+ * Appends `session.cancel` for `sessionId` to `tenant/control`, so the process running its
+ * advance aborts it. Call it from `t.afterCommit` after a cancel commits. A lost signal costs
+ * latency only (the advance checks the Session Store before every effect), so failures are
+ * logged, never thrown.
+ */
+export declare function signalSessionCancel(ctx: TenantContext, sessionId: string): void;
+/**
+ * Before a reset that deletes sessions: records a pending collection of their streams in the
+ * Tenant settings, so it is retried by the sweep even if this process stops before it runs.
+ */
+export declare function requestStreamCollection(ctx: TenantContext): Promise<void>;
+/**
+ * After a reset deleted sessions: tells every process with the Tenant open to check its
+ * session feeds (`sessions.reset`), and deletes the abandoned streams now, best effort.
+ * Failures are logged; the sweep retries the collection.
+ */
+export declare function sessionStreamsAbandoned(ctx: TenantContext): void;
+/**
+ * Deletes the Tenant's session streams that no session uses any more: streams of deleted
+ * sessions and of earlier incarnations. Runs only while a collection is pending
+ * (`requestStreamCollection`), and clears it once every deletion succeeded, unless another
+ * reset requested a new one meanwhile.
+ *
+ * Streams are listed before their sessions are read, so a session created meanwhile keeps
+ * its stream: its row committed before the relay created the stream.
+ */
+export declare function collectSessionStreams(ctx: TenantContext, streams: DurableStreams): Promise<CollectResult | undefined>;
+/** Tenant status of the streams seam (for `GET /v1/tenant` and the Admin API). */
+export interface StreamsStatus {
+    /** The streams' service answered a probe (`probeStreams`) within the timeout. */
+    reachable: boolean;
+    basin: BasinStatus;
+    outbox: {
+        /** Events committed but not yet in their streams. */
+        depth: number;
+        /** Age of the oldest of them, or null when the outbox is empty. */
+        oldestAgeMs: number | null;
+    };
+    /** How far the relay is behind: the oldest unrelayed event's age, 0 when none is waiting. */
+    relayLagMs: number;
+    /** Abandoned session streams are waiting to be deleted. */
+    collectionPending: boolean;
+}
+/**
+ * The streams seam's status for an open Tenant: S2 reachability, the basin, outbox depth and
+ * relay lag. Reads the Session Store and probes the streams (at most `probeTimeoutMs`).
+ */
+export declare function streamsStatus(ctx: TenantContext, options?: {
+    probeTimeoutMs?: number;
+    now?: () => number;
+}): Promise<StreamsStatus>;
+/**
+ * Creates a new Tenant's basin. Call it when the Tenant is created, after its schema commits;
+ * retries transient failures, then throws. A failure need not fail the creation: opening the
+ * Tenant repairs a missing basin.
+ */
+export declare function createTenantStreams(streams: DurableStreams, tenantId: string, options?: {
+    attempts?: number;
+}): Promise<void>;
+/**
+ * Deletes a Tenant's basin and every stream in it. Call it when the Tenant is deleted (never
+ * on reset). Idempotent; retries transient failures, then throws.
+ */
+export declare function deleteTenantStreams(streams: DurableStreams, tenantId: string, options?: {
+    attempts?: number;
+}): Promise<void>;
+/** Close: stops the readers and relay; the observers were ended by `endAllStreams`. */
+export declare function closeStreams(ctx: TenantContext): Promise<void>;

package/dist/tenant/streams.js

@@ -0,0 +1,419 @@
+/**
+ * The Durable Streams seam of an open Tenant (architecture §12.4), wired by one
+ * `wireStreams(ctx, …)` call in the composition root:
+ *
+ * - **Basin.** The Tenant's basin is created when the Tenant is created (`createTenantStreams`)
+ *   and checked when it opens. A missing basin (S2 down at creation, a crash in between) is
+ *   repaired on first use: a failed `ensureTenant` or relay append retries `ensureTenant` in the
+ *   background with backoff, and `streamsStatus` reports the basin's state. Only Tenant deletion
+ *   deletes the basin (`deleteTenantStreams`); a reset never does (s2-lite keeps a deleted basin's
+ *   name for about a minute).
+ * - **Relay.** The outbox relay (`streams/relay.ts`) subscribes to the Session Store's commits,
+ *   appends committed events to the session's stream, and appends `work_available` to
+ *   `tenant/work` after a commit that called `t.signalWork()`. It is the only writer of either;
+ *   business code never publishes or notifies.
+ * - **Readers.** History and session SSE read the session's stream (`tenant/live.ts`). This
+ *   module runs one `tenant/work` reader per Tenant per process, which wakes connected
+ *   executors, and one `tenant/control` reader, which calls `ctx.abortLocal` for each
+ *   `session.cancel` and checks the session feeds for each `sessions.reset`.
+ * - **Incarnations.** A session's stream is `sessions/<id>/<incarnation>`, with the incarnation
+ *   stored on the session when it is created (`streams/types.ts`). A reset abandons the
+ *   streams of the sessions it deletes instead of deleting and re-creating them, so a session
+ *   created again with the same id (during or after the reset, on any node) starts a new stream
+ *   at sequence 0. Abandoned streams are deleted best effort by `collectSessionStreams`: right
+ *   after the reset, and by the Tenant sweep until a collection succeeds. The pending
+ *   collection is a Tenant setting written before the reset, so it survives a crash.
+ * - **Recovery.** `drainOutbox(ctx)` appends leftover outbox rows (after a crash or an S2
+ *   outage). The Tenant sweep calls it.
+ *
+ * The caller passes the streams: the Host's S2 streams, or `MemoryStreams` for tests and a
+ * local development Host (not durable).
+ */
+import { randomBytes } from "node:crypto";
+import { createRelay, signalCancel, signalSessionsReset, StreamGapError, } from "../streams/relay.js";
+import { CONTROL_STREAM, SESSION_STREAM_PREFIX, WORK_STREAM, parseSessionStream, streamOfSession, } from "../streams/types.js";
+import { announceWork, checkFeeds, sleep } from "./live.js";
+const RETRY_MIN_MS = 100;
+const RETRY_MAX_MS = 2000;
+/** The longest wait between two `ensureTenant` attempts while the basin is being repaired. */
+const BASIN_RETRY_MAX_MS = 5000;
+/** A repair started within this long after the last one succeeded waits for the next failure. */
+const BASIN_REPAIR_COOLDOWN_MS = 1000;
+/** Outbox rows per drain page at open. */
+const OPEN_DRAIN_PAGE = 1000;
+/** How often every session feed is checked against its session (backstop for lost signals). */
+const FEED_CHECK_MS = 30_000;
+/** The Tenant setting that holds a pending collection's token ("" when none is pending). */
+export const COLLECT_SETTING = "streams.collect";
+/** Sessions looked up per transaction while collecting. */
+const COLLECT_LOOKUP_PAGE = 500;
+/** Stream deletions in flight at once while collecting. */
+const COLLECT_PARALLEL = 8;
+const messageOf = (error) => error instanceof Error ? error.message : String(error);
+/**
+ * Wires the Tenant to Durable Streams: checks the basin, creates the relay (unless given),
+ * starts the work and control readers, and records the handles on `ctx.live.wiring`. Call it
+ * once, after `ctx` is built and before the Tenant serves requests.
+ */
+export async function wireStreams(ctx, options) {
+    if (ctx.live.wiring)
+        throw new Error("Streams are already wired");
+    const { store, streams, tenantId } = options;
+    const logger = ctx.config.logger;
+    const stop = new AbortController();
+    // Nothing is reported once the wiring stops: the Tenant (and its log directory) may be gone,
+    // and a throwing report in a detached loop would be an unhandled rejection.
+    const report = (what) => (error) => {
+        if (!stop.signal.aborted)
+            logger.warn(what, { message: messageOf(error) });
+    };
+    const basin = basinKeeper(streams, tenantId, stop.signal, report("stream basin unavailable; retrying"));
+    // Wait for the first check so the first commits find the basin; a failure repairs it later.
+    await basin.ensure().catch((error) => {
+        report("stream basin unavailable; relaying later")(error);
+        basin.repair();
+    });
+    const relay = options.relay ??
+        createRelay({
+            store,
+            streams,
+            tenantId,
+            onError: (error) => {
+                report("event relay failed; events stay in the outbox")(error);
+                // The basin may be missing (never created, or deleted underneath); make sure of it.
+                if (!(error instanceof StreamGapError))
+                    basin.repair();
+            },
+        });
+    const work = follow(streams, tenantId, WORK_STREAM, stop.signal, () => announceWork(ctx.live), report("work stream read failed; retrying"));
+    const control = follow(streams, tenantId, CONTROL_STREAM, stop.signal, (body) => {
+        const signal = body;
+        if (signal?.type === "session.cancel" && typeof signal.sessionId === "string")
+            ctx.abortLocal(signal.sessionId);
+        else if (signal?.type === "sessions.reset")
+            void checkFeeds(ctx).catch(report("session feed check failed"));
+    }, report("control stream read failed; retrying"));
+    // Signals appended from here on reach this process.
+    await Promise.all([work, control]);
+    // Feeds whose session was reset end on the `sessions.reset` signal; this catches lost ones.
+    const feedCheck = setInterval(() => void checkFeeds(ctx).catch(report("session feed check failed")), FEED_CHECK_MS);
+    feedCheck.unref();
+    let collecting;
+    const wiring = {
+        streams,
+        relay,
+        basin: () => basin.status(),
+        async drain(limit) {
+            await basin.ensure();
+            return relay.drain(limit);
+        },
+        collect() {
+            collecting ??= collectSessionStreams(ctx, streams).finally(() => {
+                collecting = undefined;
+            });
+            return collecting;
+        },
+        async close() {
+            stop.abort();
+            clearInterval(feedCheck);
+            unregisterSweep();
+            if (!options.relay)
+                await relay.close();
+            await collecting?.catch(() => undefined);
+            if (options.ownsStreams)
+                await streams.close();
+        },
+    };
+    ctx.live.wiring = wiring;
+    // The sweep retries a collection a reset left pending (or one a crash interrupted).
+    const unregisterSweep = ctx.onSweep(async () => {
+        await wiring.collect();
+    });
+    // Rows a previous process committed but did not relay; the sweep picks up what this misses.
+    void (async () => {
+        while ((await wiring.drain(OPEN_DRAIN_PAGE)) >= OPEN_DRAIN_PAGE)
+            ;
+    })().catch(report("outbox drain at open failed"));
+    return wiring;
+}
+/** The Tenant sweep's outbox recovery: appends leftover rows, at most `limit`. */
+export function drainOutbox(ctx, limit) {
+    const wiring = ctx.live.wiring;
+    return wiring ? wiring.drain(limit) : Promise.resolve(0);
+}
+/**
+ * Appends `session.cancel` for `sessionId` to `tenant/control`, so the process running its
+ * advance aborts it. Call it from `t.afterCommit` after a cancel commits. A lost signal costs
+ * latency only (the advance checks the Session Store before every effect), so failures are
+ * logged, never thrown.
+ */
+export function signalSessionCancel(ctx, sessionId) {
+    const streams = ctx.live.wiring?.streams;
+    if (!streams)
+        return;
+    void signalCancel(streams, ctx.config.tenantId, sessionId).catch((error) => ctx.config.logger.warn("cancel signal failed", {
+        sessionId,
+        message: messageOf(error),
+    }));
+}
+/**
+ * Before a reset that deletes sessions: records a pending collection of their streams in the
+ * Tenant settings, so it is retried by the sweep even if this process stops before it runs.
+ */
+export async function requestStreamCollection(ctx) {
+    const token = `${Date.now()}-${randomBytes(6).toString("hex")}`;
+    await ctx.store.tx((t) => t.putSetting(COLLECT_SETTING, token));
+}
+/**
+ * After a reset deleted sessions: tells every process with the Tenant open to check its
+ * session feeds (`sessions.reset`), and deletes the abandoned streams now, best effort.
+ * Failures are logged; the sweep retries the collection.
+ */
+export function sessionStreamsAbandoned(ctx) {
+    const wiring = ctx.live.wiring;
+    if (!wiring)
+        return;
+    const warn = (what) => (error) => ctx.config.logger.warn(what, { message: messageOf(error) });
+    void signalSessionsReset(wiring.streams, ctx.config.tenantId).catch(warn("sessions.reset signal failed"));
+    void wiring.collect().catch(warn("session stream collection failed; the sweep retries"));
+}
+/**
+ * Deletes the Tenant's session streams that no session uses any more: streams of deleted
+ * sessions and of earlier incarnations. Runs only while a collection is pending
+ * (`requestStreamCollection`), and clears it once every deletion succeeded, unless another
+ * reset requested a new one meanwhile.
+ *
+ * Streams are listed before their sessions are read, so a session created meanwhile keeps
+ * its stream: its row committed before the relay created the stream.
+ */
+export async function collectSessionStreams(ctx, streams) {
+    const { store } = ctx;
+    const tenantId = ctx.config.tenantId;
+    const token = await store.tx((t) => t.getSetting(COLLECT_SETTING));
+    if (!token)
+        return undefined;
+    const bySession = new Map();
+    for (const name of await streams.listStreams(tenantId, SESSION_STREAM_PREFIX)) {
+        const parsed = parseSessionStream(name);
+        if (!parsed)
+            continue;
+        let names = bySession.get(parsed.sessionId);
+        if (!names)
+            bySession.set(parsed.sessionId, (names = []));
+        names.push(name);
+    }
+    const ids = [...bySession.keys()];
+    const current = new Map();
+    for (let i = 0; i < ids.length; i += COLLECT_LOOKUP_PAGE) {
+        const page = ids.slice(i, i + COLLECT_LOOKUP_PAGE);
+        await store.tx(async (t) => {
+            for (const id of page) {
+                const session = await t.get("sessions", id);
+                if (session)
+                    current.set(id, streamOfSession(session));
+            }
+        });
+    }
+    const abandoned = [...bySession].flatMap(([id, names]) => names.filter((name) => current.get(id) !== name));
+    const result = {
+        deleted: 0,
+        kept: [...bySession.values()].flat().length - abandoned.length,
+        failed: 0,
+    };
+    let next = 0;
+    await Promise.all(Array.from({ length: Math.min(COLLECT_PARALLEL, abandoned.length) }, async () => {
+        while (next < abandoned.length) {
+            const name = abandoned[next++];
+            try {
+                await streams.deleteStream(tenantId, name);
+                result.deleted += 1;
+            }
+            catch (error) {
+                result.failed += 1;
+                ctx.config.logger.warn("session stream delete failed; the sweep retries", {
+                    stream: name,
+                    message: messageOf(error),
+                });
+            }
+        }
+    }));
+    if (result.failed === 0)
+        await store.tx(async (t) => {
+            if ((await t.getSetting(COLLECT_SETTING)) === token)
+                await t.putSetting(COLLECT_SETTING, "");
+        });
+    return result;
+}
+/**
+ * The streams seam's status for an open Tenant: S2 reachability, the basin, outbox depth and
+ * relay lag. Reads the Session Store and probes the streams (at most `probeTimeoutMs`).
+ */
+export async function streamsStatus(ctx, options = {}) {
+    const wiring = ctx.live.wiring;
+    const now = options.now ?? Date.now;
+    const [reachable, stored] = await Promise.all([
+        wiring
+            ? probe(wiring.streams, options.probeTimeoutMs ?? 2000)
+            : Promise.resolve(false),
+        ctx.store.tx(async (t) => ({
+            outbox: await t.outboxStats(),
+            collect: await t.getSetting(COLLECT_SETTING),
+        })),
+    ]);
+    const oldest = stored.outbox.oldestCreatedAt;
+    const oldestAgeMs = oldest === null ? null : Math.max(0, now() - Date.parse(oldest));
+    return {
+        reachable,
+        basin: wiring?.basin() ?? { ready: false, failures: 0, lastError: null },
+        outbox: { depth: stored.outbox.depth, oldestAgeMs },
+        relayLagMs: oldestAgeMs ?? 0,
+        collectionPending: !!stored.collect,
+    };
+}
+/** Resolves true when the streams answer a probe within `timeoutMs` (in-process ones always do). */
+async function probe(streams, timeoutMs) {
+    try {
+        await streams.probe?.(AbortSignal.timeout(timeoutMs));
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+/**
+ * Creates a new Tenant's basin. Call it when the Tenant is created, after its schema commits;
+ * retries transient failures, then throws. A failure need not fail the creation: opening the
+ * Tenant repairs a missing basin.
+ */
+export async function createTenantStreams(streams, tenantId, options = {}) {
+    await withRetries(() => streams.ensureTenant(tenantId), options.attempts ?? 3);
+}
+/**
+ * Deletes a Tenant's basin and every stream in it. Call it when the Tenant is deleted (never
+ * on reset). Idempotent; retries transient failures, then throws.
+ */
+export async function deleteTenantStreams(streams, tenantId, options = {}) {
+    await withRetries(() => streams.deleteTenant(tenantId), options.attempts ?? 5);
+}
+/** Close: stops the readers and relay; the observers were ended by `endAllStreams`. */
+export async function closeStreams(ctx) {
+    await ctx.live.wiring?.close();
+}
+// ---------------------------------------------------------------------------
+async function withRetries(task, attempts) {
+    let delay = RETRY_MIN_MS;
+    for (let attempt = 1;; attempt += 1) {
+        try {
+            return await task();
+        }
+        catch (error) {
+            if (attempt >= attempts)
+                throw error;
+            await new Promise((resolve) => setTimeout(resolve, delay));
+            delay = Math.min(delay * 2, RETRY_MAX_MS);
+        }
+    }
+}
+function basinKeeper(streams, tenantId, signal, onError) {
+    let ready = false;
+    let failures = 0;
+    let lastError = null;
+    let attempt;
+    let repairing = false;
+    let repairedAt = 0;
+    const ensure = () => {
+        if (ready)
+            return Promise.resolve();
+        attempt ??= streams
+            .ensureTenant(tenantId)
+            .then(() => {
+            ready = true;
+            failures = 0;
+            lastError = null;
+        }, (error) => {
+            failures += 1;
+            lastError = messageOf(error);
+            throw error;
+        })
+            .finally(() => {
+            attempt = undefined;
+        });
+        return attempt;
+    };
+    return {
+        ensure,
+        repair() {
+            if (repairing || signal.aborted)
+                return;
+            if (ready && Date.now() - repairedAt < BASIN_REPAIR_COOLDOWN_MS)
+                return;
+            repairing = true;
+            ready = false;
+            void (async () => {
+                let delay = RETRY_MIN_MS;
+                while (!signal.aborted) {
+                    try {
+                        await ensure();
+                        repairedAt = Date.now();
+                        break;
+                    }
+                    catch (error) {
+                        if (failures === 1 || failures % 10 === 0)
+                            onError(error);
+                    }
+                    await sleep(delay, signal);
+                    delay = Math.min(delay * 2, BASIN_RETRY_MAX_MS);
+                }
+                repairing = false;
+            })();
+        },
+        status: () => ({ ready, failures, lastError }),
+    };
+}
+/**
+ * Follows a signal stream from its tail at start, calling `onRecord` for each record, until
+ * `signal` aborts. Failed reads retry with backoff; a read that ends by itself (the basin was
+ * deleted) starts again from the new tail. Resolves once the first tail is known (or failed).
+ */
+function follow(streams, tenantId, stream, signal, onRecord, onError) {
+    let started;
+    const ready = new Promise((resolve) => (started = resolve));
+    void (async () => {
+        let from;
+        let delay = RETRY_MIN_MS;
+        while (!signal.aborted) {
+            try {
+                try {
+                    from ??= await streams.tail(tenantId, stream);
+                }
+                finally {
+                    started();
+                }
+                for await (const record of streams.read(tenantId, stream, from, {
+                    signal,
+                })) {
+                    from = record.seq + 1;
+                    delay = RETRY_MIN_MS;
+                    try {
+                        onRecord(record.body);
+                    }
+                    catch (error) {
+                        onError(error);
+                    }
+                }
+                if (signal.aborted)
+                    return;
+                from = undefined;
+            }
+            catch (error) {
+                if (signal.aborted)
+                    return;
+                onError(error);
+            }
+            await sleep(delay, signal);
+            delay = Math.min(delay * 2, RETRY_MAX_MS);
+        }
+        started();
+    })();
+    return ready;
+}

package/dist/tenant/sweep.d.ts

@@ -0,0 +1,15 @@
+import type { TenantContext } from "./context.js";
+export declare function sweep(ctx: TenantContext, options?: {
+    afterOpen?: boolean;
+}): Promise<void>;
+/**
+ * Expire lapsed Action claims: hooks, fn and verify are offered again, tools become
+ * `uncertain`. One transaction per claim, so each locks only its own session.
+ */
+export declare function expireClaims(ctx: Pick<TenantContext, "store">, now?: Date): Promise<number>;
+/** Offer claimed `fn`/`verify` Actions again, one transaction each. Returns how many. */
+export declare function reofferFnVerifyClaims(ctx: Pick<TenantContext, "store">): Promise<number>;
+/** Settle pending workflow `agent` effects whose linked turn already finished. */
+export declare function reconcileLinkedAgents(ctx: Pick<TenantContext, "store" | "wake">): Promise<void>;
+/** Wake `running`/`runnable` sessions without a live owner. Returns the ids woken. */
+export declare function wakeOrphanedSessions(ctx: Pick<TenantContext, "store" | "wake">, now?: Date): Promise<string[]>;

package/dist/tenant/sweep.js

@@ -0,0 +1,122 @@
+import { claimedFnVerifyActions, pendingAgentEffects, reconcilePendingAgentEffect, reofferFnVerifyClaim, } from "../core/flow-host.js";
+const BATCH = 100;
+export async function sweep(ctx, options = {}) {
+    if (ctx.closing || ctx.closed)
+        return;
+    const now = new Date();
+    const steps = [
+        ...(options.afterOpen
+            ? [["reoffer", () => reofferFnVerifyClaims(ctx)]]
+            : []),
+        ["claims", () => expireClaims(ctx, now)],
+        ["linked", () => reconcileLinkedAgents(ctx)],
+        ["orphans", () => wakeOrphanedSessions(ctx, now)],
+        [
+            "sandboxes",
+            () => ctx.sandbox.sweep({
+                now: now.getTime(),
+                sessionExists: async (id) => !!(await ctx.store.tx((t) => t.get("sessions", id))),
+            }),
+        ],
+        ...[...ctx.sweepHooks].map((hook) => ["hook", hook]),
+    ];
+    let failure;
+    for (const [step, run] of steps) {
+        if (ctx.closing || ctx.closed)
+            return;
+        try {
+            await run();
+        }
+        catch (error) {
+            ctx.config.logger.warn("tenant sweep step failed", {
+                step,
+                message: error instanceof Error ? error.message : String(error),
+            });
+            failure ??= { error };
+        }
+    }
+    if (failure)
+        throw failure.error;
+}
+/**
+ * Expire lapsed Action claims: hooks, fn and verify are offered again, tools become
+ * `uncertain`. One transaction per claim, so each locks only its own session.
+ */
+export async function expireClaims(ctx, now = new Date()) {
+    const { store } = ctx;
+    const seen = new Set();
+    let expiredCount = 0;
+    for (;;) {
+        const expired = (await store.tx((t) => t.expiredClaims(now, BATCH))).filter((action) => !seen.has(action.actionId));
+        for (const found of expired) {
+            seen.add(found.actionId);
+            const changed = await store.tx(async (t) => {
+                const s = await t.lockSession(found.sessionId);
+                // Read again under the session lock: a heartbeat or result may have won.
+                const action = await t.get("actions", found.actionId);
+                if (!action ||
+                    action.status !== "claimed" ||
+                    Date.parse(action.leaseExpiresAt) > now.getTime())
+                    return false;
+                if (action.kind === "hook" ||
+                    action.kind === "fn" ||
+                    action.kind === "verify") {
+                    action.status = "pending";
+                    action.claimId = null;
+                    action.leaseExpiresAt = null;
+                    await t.put("actions", action.actionId, action);
+                    t.signalWork();
+                    return true;
+                }
+                action.status = "uncertain";
+                await t.put("actions", action.actionId, action);
+                const effect = await t.get("effects", action.actionId);
+                if (effect) {
+                    effect.status = "uncertain";
+                    await t.put("effects", action.actionId, effect);
+                }
+                if (s && s.status !== "cancelled" && s.activeTurnId === action.turnId) {
+                    s.status = "uncertain";
+                    await t.put("sessions", s.id, s);
+                    await t.event(s.id, s.activeTurnId, "action.uncertain", {
+                        actionId: action.actionId,
+                        ...(action.agent ? { agent: action.agent } : {}),
+                    });
+                }
+                return true;
+            });
+            if (changed)
+                expiredCount += 1;
+        }
+        if (expired.length < BATCH)
+            return expiredCount;
+    }
+}
+/** Offer claimed `fn`/`verify` Actions again, one transaction each. Returns how many. */
+export async function reofferFnVerifyClaims(ctx) {
+    const claimed = await ctx.store.tx((t) => claimedFnVerifyActions(t));
+    let count = 0;
+    for (const action of claimed)
+        if (await ctx.store.tx((t) => reofferFnVerifyClaim(t, action.actionId)))
+            count += 1;
+    return count;
+}
+/** Settle pending workflow `agent` effects whose linked turn already finished. */
+export async function reconcileLinkedAgents(ctx) {
+    const { store } = ctx;
+    const effects = await store.tx((t) => pendingAgentEffects(t));
+    for (const effect of effects)
+        await store.tx((t) => reconcilePendingAgentEffect({
+            t,
+            effectId: effect.request.effectId,
+            schedule: ctx.wake,
+        }));
+}
+/** Wake `running`/`runnable` sessions without a live owner. Returns the ids woken. */
+export async function wakeOrphanedSessions(ctx, now = new Date()) {
+    const orphans = await ctx.store.tx((t) => t.orphanedSessions(now, BATCH));
+    const ids = orphans.map((session) => session.id);
+    for (const id of ids)
+        await ctx.wake(id, { reason: "recover" });
+    return ids;
+}