cloving 1.3.0 → 1.5.0

package/dist/src/daemon/issue-queue.js

@@ -1,57 +1,14 @@
-import { activeQueueRuns } from "./queue-status.js";
+import { queueStatus, spendNotice } from "./queue-status.js";
 import { randomUUID } from "node:crypto";
-import { readFile, rename, rm, writeFile } from "node:fs/promises";
+import { readFile, rm, writeFile } from "node:fs/promises";
 import { join } from "node:path";
 import { setTimeout as delay } from "node:timers/promises";
 import { send } from "../cli/client.js";
 import { record } from "../protocol.js";
 import { runIdentity } from "../service/submission-handler.js";
-function savedHold(value) {
-    if (value === undefined)
-        return undefined;
-    const held = record(value);
-    if (typeof held?.["mark"] !== "string" ||
-        typeof held["startedAt"] !== "number" ||
-        !Number.isFinite(held["startedAt"]) ||
-        held["startedAt"] <= 0)
-        throw new Error("invalid-queue-commands");
-    return { mark: held["mark"], startedAt: held["startedAt"] };
-}
-function identifiers(value) {
-    if (!Array.isArray(value) ||
-        !value.every((id) => typeof id === "string" && id.length > 0))
-        throw new Error("invalid-queue-commands");
-    return value.map(String);
-}
-async function journal(directory) {
-    const file = join(directory, "queue-commands.json");
-    const text = await readFile(file, "utf8").catch((error) => {
-        if (error instanceof Error && "code" in error && error.code === "ENOENT")
-            return "[]";
-        throw error;
-    });
-    const value = JSON.parse(text);
-    const saved = record(value);
-    const commands = new Set(identifiers(Array.isArray(value) ? value : saved?.["commands"]));
-    const failures = Array.isArray(value) ? [] : identifiers(saved?.["failures"]);
-    if (failures.length > 3)
-        throw new Error("invalid-queue-commands");
-    const pending = {
-        commands,
-        failures,
-        retryCommandId: undefined,
-        selfUpdateHold: savedHold(saved?.["selfUpdateHold"]),
-        save: async () => {
-            await writeFile(`${file}.tmp`, JSON.stringify({
-                commands: [...commands],
-                failures,
-                selfUpdateHold: pending.selfUpdateHold,
-            }) + "\n", { mode: 0o600, flush: true });
-            await rename(`${file}.tmp`, file);
-        },
-    };
-    return pending;
-}
+import { queueFailureLimit } from "../queue-state.js";
+import { queueJournal } from "./queue-journal.js";
+import { queueControlPort, queueStopped, } from "./queue-control.js";
 function terminal(run) {
     return ["completed", "failed", "cancelled"].includes(String(run["state"]));
 }
@@ -111,11 +68,18 @@ function handedOff(run) {
         outcome?.["kind"] === "handoff_requested" &&
         outcome["issueUpdate"] !== "failed");
 }
-class QueueFailureLimit extends Error {
+function limited(pending) {
+    return pending.failures.length >= queueFailureLimit;
 }
-function checkFailureLimit(pending) {
-    if (pending.failures.length >= 3)
-        throw new QueueFailureLimit(`3 runs in a row ended without handoff: ${pending.failures.join(", ")}`);
+function stopNotice() {
+    let noticed = false;
+    return (pending) => {
+        const stopped = limited(pending);
+        if (stopped && !noticed)
+            process.stderr.write(`Queue stopped: 3 runs in a row ended without handoff: ${pending.failures.join(", ")}\n`);
+        noticed = stopped;
+        return stopped;
+    };
 }
 async function retired(pending, ending) {
     const { commandId, run } = ending;
@@ -125,14 +89,12 @@ async function retired(pending, ending) {
     else
         pending.failures.push(runIdentity(commandId));
     await pending.save();
-    checkFailureLimit(pending);
-    if (handedOff(run))
+    if (handedOff(run) || limited(pending))
         return;
     const outcome = record(run["outcome"]);
     process.stderr.write(`Queue run ended ${String(outcome?.["kind"] ?? run["state"])}${outcome?.["issueUpdate"] === "failed" ? "; issueUpdate: failed" : ""}: ${runIdentity(commandId)}; continuing\n`);
 }
 async function completed(settings, pending) {
-    checkFailureLimit(pending);
     let pendingRuns = 0;
     const endings = [];
     const tracked = [...pending.commands].filter((commandId) => commandId !== pending.retryCommandId);
@@ -145,7 +107,8 @@ async function completed(settings, pending) {
     endings.sort((first, second) => Number(first.run["terminalSequence"] ?? first.run["latestSequence"] ?? 0) -
         Number(second.run["terminalSequence"] ?? second.run["latestSequence"] ?? 0));
     for (const ending of endings)
-        await retired(pending, ending);
+        if (!limited(pending))
+            await retired(pending, ending);
     return pendingRuns;
 }
 async function submit(settings, pending) {
@@ -155,7 +118,7 @@ async function submit(settings, pending) {
     const reply = await send(settings.runtimeDirectory, {
         operation: "run",
         parameters: {
-            workflowId: settings.workflowId ?? "develop-issue",
+            workflowId: settings.workflowId,
             source: "ticket",
             submission: { ticket: "next" },
             commandId,
@@ -178,6 +141,7 @@ async function submit(settings, pending) {
         "workflow-active",
         "ticket-active",
         "ticket-dependency-open",
+        "queue-spend-limit",
     ].includes(reply.error.code))
         throw new Error(reply.error.code);
     pending.commands.delete(commandId);
@@ -229,20 +193,54 @@ async function admission(settings, attempt) {
     const file = join(settings.directory, "queue-submission");
     await writeFile(file, "", { mode: 0o600 });
     try {
-        if (!(await attempt.holding()) && !attempt.signal.aborted)
+        if (!(await attempt.holding()) &&
+            !attempt.signal.aborted &&
+            attempt.pending.drain === undefined)
             await submit(settings, attempt.pending);
     }
     finally {
         await rm(file, { force: true });
     }
 }
-async function cycle(settings, signal) {
-    const pending = await journal(settings.directory);
+async function drained(pending, busy) {
+    const { drain } = pending;
+    if (drain === undefined || drain.drainedAt !== null || busy > 0)
+        return;
+    pending.drain = { ...drain, drainedAt: Date.now() };
+    await pending.save();
+    process.stderr.write("Queue drained\n");
+}
+async function admissible(settings, checks) {
+    const { pending, stop } = checks;
+    const pendingRuns = stop(pending) ? 0 : await completed(settings, pending);
+    if (stop(pending) && pending.drain === undefined)
+        return false;
+    const status = await queueStatus(settings);
+    await drained(pending, status.busy);
+    if (stop(pending) || pending.drain !== undefined)
+        return false;
+    const spend = checks.notice(status);
+    const updating = await checks.update(status.busy);
+    const lanes = settings.lanes ?? 1;
+    if (Math.max(pendingRuns, status.busy) >= lanes || status.busy < pendingRuns)
+        return false;
+    return !updating && !spend;
+}
+async function cycle(settings, run) {
+    const { signal, pending } = run;
     if (settings.resetQueue === true) {
         pending.failures.length = 0;
+        pending.drain = undefined;
         await pending.save();
     }
-    const holding = updateHold(settings.directory, pending);
+    const update = updateHold(settings.directory, pending);
+    const notice = spendNotice();
+    const stop = stopNotice();
+    const holding = async (activeRuns = 0) => {
+        const updating = await update(activeRuns);
+        const spend = notice(await queueStatus(settings));
+        return updating || spend;
+    };
     await rm(join(settings.directory, "queue-submission"), { force: true });
     await recover(settings, pending);
     const stopped = () => signal.aborted;
@@ -250,17 +248,7 @@ async function cycle(settings, signal) {
         await delay(settings.intervalMilliseconds, undefined, { signal }).catch(() => undefined);
         if (stopped())
             return;
-        const pendingRuns = await completed(settings, pending);
-        const lanes = settings.lanes ?? 1;
-        if (pendingRuns >= lanes) {
-            await holding(pendingRuns);
-            continue;
-        }
-        const activeRuns = await activeQueueRuns(settings);
-        const held = await holding(activeRuns);
-        if (activeRuns >= lanes || activeRuns < pendingRuns)
-            continue;
-        if (held)
+        if (!(await admissible(settings, { pending, stop, notice, update })))
             continue;
         if (stopped())
             return;
@@ -268,16 +256,26 @@ async function cycle(settings, signal) {
     }
 }
 export function issueQueue(settings) {
-    const stopping = new AbortController();
-    const running = cycle(settings, stopping.signal).catch((error) => {
-        const reason = error instanceof Error ? error.message : String(error);
-        const diagnostic = error instanceof QueueFailureLimit ? reason : JSON.stringify(reason);
-        process.stderr.write(`Queue stopped: ${diagnostic}\n`);
+    const runtime = {
+        pending: undefined,
+        error: undefined,
+        stopping: new AbortController(),
+    };
+    const loaded = queueJournal(settings.directory).then((pending) => {
+        runtime.pending = pending;
+        return pending;
+    });
+    const running = loaded
+        .then((pending) => cycle(settings, { signal: runtime.stopping.signal, pending }))
+        .catch((error) => {
+        queueStopped(runtime, error);
     });
     return {
         stop: async () => {
-            stopping.abort();
+            runtime.stopping.abort();
             await running;
         },
+        port: queueControlPort(runtime),
+        ready: loaded.then(() => undefined, () => undefined),
     };
 }

package/dist/src/daemon/merge-turns.d.ts

@@ -0,0 +1,10 @@
+export interface Turn {
+    readonly key: string;
+    readonly signal: AbortSignal;
+}
+export interface MergeTurn {
+    readonly repository: string;
+    readonly signal: AbortSignal;
+}
+export declare function inTurn<Result>(turn: Turn, work: () => Promise<Result>): Promise<Result>;
+export declare function inMergeTurn<Result>(turn: MergeTurn, work: () => Promise<Result>): Promise<Result>;

package/dist/src/daemon/merge-turns.js

@@ -0,0 +1,37 @@
+const turns = new Map();
+function waited(previous, signal) {
+    signal.throwIfAborted();
+    return new Promise((resolve, reject) => {
+        const aborted = () => {
+            const reason = signal.reason;
+            reject(reason instanceof Error ? reason : new Error("merge-turn-aborted"));
+        };
+        signal.addEventListener("abort", aborted, { once: true });
+        previous.then(() => {
+            signal.removeEventListener("abort", aborted);
+            resolve(undefined);
+        }, reject);
+    });
+}
+export async function inTurn(turn, work) {
+    const previous = turns.get(turn.key) ?? Promise.resolve(undefined);
+    const ended = Promise.withResolvers();
+    const queued = previous
+        .then(() => ended.promise)
+        .then(() => {
+        if (turns.get(turn.key) === queued)
+            turns.delete(turn.key);
+        return undefined;
+    });
+    turns.set(turn.key, queued);
+    try {
+        await waited(previous, turn.signal);
+        return await work();
+    }
+    finally {
+        ended.resolve(undefined);
+    }
+}
+export function inMergeTurn(turn, work) {
+    return inTurn({ key: turn.repository, signal: turn.signal }, work);
+}

package/dist/src/daemon/notify-events.d.ts

@@ -0,0 +1,6 @@
+import { type NotifyEvent } from "./notify-message.ts";
+export type OutcomeCache = Map<string, {
+    readonly latestSequence: number;
+    readonly event: NotifyEvent | undefined;
+}>;
+export declare function notificationEvents(runtimeDirectory: string, cache: OutcomeCache): Promise<readonly NotifyEvent[]>;

package/dist/src/daemon/notify-events.js

@@ -0,0 +1,84 @@
+import { send } from "../cli/client.js";
+import { record } from "../protocol.js";
+import { decisionNotification, outcomeNotification, scheduleStopNotifications, } from "./notify-message.js";
+async function request(runtimeDirectory, request) {
+    const reply = await send(runtimeDirectory, request);
+    if (reply.kind === "error")
+        throw new Error(reply.error.code);
+    return reply.result;
+}
+function rows(value) {
+    if (!Array.isArray(value))
+        throw new Error("invalid-notify-list");
+    return value.map((item) => {
+        const row = record(item);
+        if (row === undefined)
+            throw new Error("invalid-notify-row");
+        return row;
+    });
+}
+async function scheduleNotifications(runtimeDirectory) {
+    const reply = await send(runtimeDirectory, {
+        operation: "schedule",
+        parameters: { action: "list" },
+    });
+    if (reply.kind !== "error")
+        return rows(reply.result["stops"]).flatMap(scheduleStopNotifications);
+    if (reply.error.code === "schedules-unavailable")
+        return [];
+    throw new Error(reply.error.code);
+}
+async function observedOutcome(summary, source) {
+    const { runtimeDirectory, cache } = source;
+    const latestSequence = summary["latestSequence"];
+    if (typeof latestSequence !== "number")
+        throw new Error("invalid-notify-sequence");
+    const runId = String(summary["runId"]);
+    const previous = cache.get(runId);
+    if (previous !== undefined && previous.latestSequence === latestSequence)
+        return previous.event;
+    const status = await request(runtimeDirectory, {
+        operation: "status",
+        parameters: { runId },
+    });
+    const run = record(status["run"]);
+    if (run === undefined)
+        throw new Error("invalid-notify-status");
+    const event = outcomeNotification(run);
+    cache.set(runId, { latestSequence, event });
+    return event;
+}
+export async function notificationEvents(runtimeDirectory, cache) {
+    const found = [];
+    const runs = new Map();
+    let after = "";
+    do {
+        const page = await request(runtimeDirectory, {
+            operation: "status",
+            parameters: { after, limit: 500 },
+        });
+        for (const summary of rows(page["runs"])) {
+            runs.set(String(summary["runId"]), summary);
+            const outcome = await observedOutcome(summary, {
+                runtimeDirectory,
+                cache,
+            });
+            if (outcome !== undefined)
+                found.push(outcome);
+        }
+        const next = page["next"];
+        if (next !== null && (typeof next !== "string" || next <= after))
+            throw new Error("invalid-notify-page");
+        after = next;
+    } while (after !== null);
+    const questions = await request(runtimeDirectory, {
+        operation: "questions",
+        parameters: {},
+    });
+    for (const decision of rows(questions["decisions"])) {
+        const run = runs.get(String(decision["runId"]));
+        if (run !== undefined)
+            found.push(decisionNotification(run, decision));
+    }
+    return [...found, ...(await scheduleNotifications(runtimeDirectory))];
+}

package/dist/src/daemon/notify-journal.js

@@ -46,7 +46,7 @@ function notification(value) {
     const message = record(value);
     if (message === undefined)
         return false;
-    return (["decision", "blocked", "failed"].includes(String(message["event"])) &&
+    return (["decision", "blocked", "failed", "schedule-stopped"].includes(String(message["event"])) &&
         ["event", "runId", "workflowId", "subject", "body"].every((field) => typeof message[field] === "string"));
 }
 function deliveryState(value) {

package/dist/src/daemon/notify-message.d.ts

@@ -1,10 +1,11 @@
 import { type Fields } from "../protocol.ts";
 export interface Notification {
-    readonly event: "decision" | "blocked" | "failed";
+    readonly event: "decision" | "blocked" | "failed" | "schedule-stopped";
     readonly runId: string;
     readonly workflowId: string;
     readonly subject: string;
     readonly body: string;
+    readonly failedRunIds?: readonly string[];
 }
 export interface NotifyEvent {
     readonly key: string;
@@ -12,3 +13,5 @@ export interface NotifyEvent {
 }
 export declare function decisionNotification(run: Fields, decision: Fields): NotifyEvent;
 export declare function outcomeNotification(run: Fields): NotifyEvent | undefined;
+export declare function stoppedScheduleNotification(schedule: Fields): NotifyEvent | undefined;
+export declare function scheduleStopNotifications(schedule: Fields): readonly NotifyEvent[];

package/dist/src/daemon/notify-message.js

@@ -47,3 +47,39 @@ export function outcomeNotification(run) {
         }),
     };
 }
+export function stoppedScheduleNotification(schedule) {
+    if (schedule["state"] !== "stopped")
+        return undefined;
+    const failedRunIds = schedule["failedRunIds"];
+    if (!Array.isArray(failedRunIds) || failedRunIds.length !== 3)
+        throw new Error("invalid-stopped-schedule");
+    const ids = failedRunIds.map((runId) => String(runId));
+    const workflowId = String(schedule["workflowId"]);
+    return {
+        key: `schedule-stopped:${workflowId}:${String(schedule["stoppedSlot"])}`,
+        message: {
+            ...message({ workflowId, runId: ids[2] }, {
+                event: "schedule-stopped",
+                body: body(`Schedule stopped after three failed runs:\n${ids.join("\n")}`, [`cloving schedule resume ${workflowId}`]),
+            }),
+            failedRunIds: ids,
+        },
+    };
+}
+export function scheduleStopNotifications(schedule) {
+    const stops = schedule["stops"];
+    if (stops === undefined) {
+        const event = stoppedScheduleNotification(schedule);
+        return event === undefined ? [] : [event];
+    }
+    if (!Array.isArray(stops))
+        throw new Error("invalid-stopped-schedule");
+    return stops.flatMap((stop) => {
+        const event = stoppedScheduleNotification({
+            ...record(stop),
+            workflowId: schedule["workflowId"],
+            state: "stopped",
+        });
+        return event === undefined ? [] : [event];
+    });
+}

package/dist/src/daemon/notify.d.ts

@@ -1,4 +1,4 @@
-import { type Notification } from "./notify-message.ts";
+import type { Notification } from "./notify-message.ts";
 interface NotifySettings {
     readonly directory: string;
     readonly runtimeDirectory: string;

package/dist/src/daemon/notify.js

@@ -1,27 +1,9 @@
 import { recordedNotifications, saveNotifications, } from "./notify-journal.js";
 import { spawn } from "node:child_process";
 import { join } from "node:path";
-import { send } from "../cli/client.js";
 import { deliveryBackoffMilliseconds } from "../delivery-backoff.js";
-import { record } from "../protocol.js";
-import { decisionNotification, outcomeNotification, } from "./notify-message.js";
+import { notificationEvents } from "./notify-events.js";
 const retentionMilliseconds = 7 * 24 * 60 * 60_000;
-async function request(runtimeDirectory, request) {
-    const reply = await send(runtimeDirectory, request);
-    if (reply.kind === "error")
-        throw new Error(reply.error.code);
-    return reply.result;
-}
-function rows(value) {
-    if (!Array.isArray(value))
-        throw new Error("invalid-notify-list");
-    return value.map((item) => {
-        const row = record(item);
-        if (row === undefined)
-            throw new Error("invalid-notify-row");
-        return row;
-    });
-}
 async function flushNotifications(journal) {
     if (journal.unsaved.size === 0)
         return;
@@ -36,60 +18,6 @@ async function finishDelivery(journal, entry) {
     journal.unsaved.set(entry.key, entry);
     await flushNotifications(journal);
 }
-async function observedOutcome(summary, source) {
-    const { runtimeDirectory, cache } = source;
-    const latestSequence = summary["latestSequence"];
-    if (typeof latestSequence !== "number")
-        throw new Error("invalid-notify-sequence");
-    const runId = String(summary["runId"]);
-    const previous = cache.get(runId);
-    if (previous !== undefined && previous.latestSequence === latestSequence)
-        return previous.event;
-    const status = await request(runtimeDirectory, {
-        operation: "status",
-        parameters: { runId },
-    });
-    const run = record(status["run"]);
-    if (run === undefined)
-        throw new Error("invalid-notify-status");
-    const event = outcomeNotification(run);
-    cache.set(runId, { latestSequence, event });
-    return event;
-}
-async function events(runtimeDirectory, cache) {
-    const found = [];
-    const runs = new Map();
-    let after = "";
-    do {
-        const page = await request(runtimeDirectory, {
-            operation: "status",
-            parameters: { after, limit: 500 },
-        });
-        for (const summary of rows(page["runs"])) {
-            runs.set(String(summary["runId"]), summary);
-            const outcome = await observedOutcome(summary, {
-                runtimeDirectory,
-                cache,
-            });
-            if (outcome !== undefined)
-                found.push(outcome);
-        }
-        const next = page["next"];
-        if (next !== null && (typeof next !== "string" || next <= after))
-            throw new Error("invalid-notify-page");
-        after = next;
-    } while (after !== null);
-    const questions = await request(runtimeDirectory, {
-        operation: "questions",
-        parameters: {},
-    });
-    for (const decision of rows(questions["decisions"])) {
-        const run = runs.get(String(decision["runId"]));
-        if (run !== undefined)
-            found.push(decisionNotification(run, decision));
-    }
-    return found;
-}
 export function notifyCommand(command, notification) {
     return new Promise((resolve, reject) => {
         const [executable = "", ...argumentsList] = command;
@@ -210,7 +138,7 @@ async function check(settings, journal) {
             journal.retiredKeys.add(key);
         }
     }
-    for (const event of await events(settings.runtimeDirectory, journal.cache)) {
+    for (const event of await notificationEvents(settings.runtimeDirectory, journal.cache)) {
         if (journal.entries.has(event.key) || journal.retiredKeys.has(event.key))
             continue;
         const entry = {
@@ -237,12 +165,12 @@ export async function notifyRuns(settings) {
         unsaved: new Map(),
     };
     if (recorded === undefined) {
-        for (const event of await events(settings.runtimeDirectory, cache))
+        for (const event of await notificationEvents(settings.runtimeDirectory, cache))
             journal.entries.set(event.key, {
                 ...event,
-                state: "sent",
+                state: event.message.event === "schedule-stopped" ? "pending" : "sent",
                 attempts: 0,
-                lastAttemptAt: Date.now(),
+                lastAttemptAt: event.message.event === "schedule-stopped" ? null : Date.now(),
                 lastFailure: null,
             });
     }

package/dist/src/daemon/operation-journal.d.ts

@@ -11,6 +11,7 @@ export interface WorkerReport {
     readonly reason?: string;
 }
 export interface Recorded {
+    readonly stoppedBy?: "daemon-shutdown";
     readonly outcome: WorkerReport["outcome"];
     readonly result?: string;
     readonly reason?: string;

package/dist/src/daemon/queue-control.d.ts

@@ -0,0 +1,13 @@
+import type { QueuePort } from "../queue-state.ts";
+import { type QueueJournal } from "./queue-journal.ts";
+export interface QueueRuntime {
+    pending: QueueJournal | undefined;
+    error: string | undefined;
+    readonly stopping: AbortController;
+}
+export declare function queueConnection(): {
+    readonly port: QueuePort;
+    readonly connect: (port: QueuePort) => void;
+};
+export declare function queueStopped(runtime: QueueRuntime, error: unknown): string;
+export declare function queueControlPort(runtime: QueueRuntime): QueuePort;

package/dist/src/daemon/queue-control.js

@@ -0,0 +1,86 @@
+import { queueControlLimit } from "./queue-journal.js";
+const unavailable = {
+    kind: "unavailable",
+    detail: "The queue is not running.",
+};
+export function queueConnection() {
+    let connected;
+    return {
+        port: {
+            control: () => connected?.control(),
+            change: (request) => connected?.change(request) ?? Promise.resolve(unavailable),
+        },
+        connect: (port) => {
+            connected = port;
+        },
+    };
+}
+export function queueStopped(runtime, error) {
+    const reason = error instanceof Error ? error.message : String(error);
+    runtime.error = reason;
+    runtime.stopping.abort();
+    process.stderr.write(`Queue stopped: ${JSON.stringify(reason)}\n`);
+    return reason;
+}
+function stoppedChange(error) {
+    return {
+        kind: "unavailable",
+        detail: `The queue stopped: ${error}. Restart the service.`,
+    };
+}
+function recorded(pending, request) {
+    const prior = pending.controls.find((control) => control.commandId === request.commandId);
+    if (prior === undefined)
+        return undefined;
+    return {
+        kind: prior.command === request.command ? "replayed" : "conflicting-command",
+    };
+}
+function applied(pending, command) {
+    if (command === "drain") {
+        pending.drain ??= { startedAt: Date.now(), drainedAt: null };
+        return;
+    }
+    pending.failures.length = 0;
+    pending.drain = undefined;
+}
+async function changed(runtime, request) {
+    const { pending, error } = runtime;
+    if (error !== undefined)
+        return stoppedChange(error);
+    if (pending === undefined)
+        return unavailable;
+    const prior = recorded(pending, request);
+    if (prior !== undefined)
+        return prior;
+    applied(pending, request.command);
+    pending.controls.push(request);
+    pending.controls.splice(0, pending.controls.length - queueControlLimit);
+    try {
+        await pending.save();
+        return { kind: "changed" };
+    }
+    catch (failure) {
+        return stoppedChange(queueStopped(runtime, failure));
+    }
+}
+export function queueControlPort(runtime) {
+    let changing = Promise.resolve();
+    return {
+        control: () => {
+            const { pending, error } = runtime;
+            if (pending === undefined && error === undefined)
+                return undefined;
+            return {
+                failedRunIds: [...(pending?.failures ?? [])],
+                drain: pending?.drain,
+                error,
+            };
+        },
+        change: (request) => {
+            const next = changing.then(() => changed(runtime, request));
+            changing = next;
+            return next;
+        },
+    };
+}