@wardby/cli 0.5.3 → 0.7.0

package/dist/core/pg-auth.js

@@ -0,0 +1,152 @@
+/**
+ * How wardby authenticates to Postgres (DATABASE_AUTH).
+ *
+ * `password` (default): the URL carries everything, exactly as before — pgConnectionConfig returns
+ * `{ connectionString }` and nothing else, so unset DATABASE_AUTH changes no connection.
+ *
+ * `rds-iam`: Amazon RDS IAM database authentication. The password is a short-lived token signed
+ * with the workload's AWS credentials (EKS Pod Identity via the default provider chain), minted per
+ * new connection and reused for RDS_IAM_TOKEN_REUSE_MS — tokens are valid 15 minutes, and only for
+ * establishing a connection, so an open connection outlives its token. TLS with certificate
+ * verification is mandatory: the URL must say sslmode=verify-full and name the RDS CA bundle in
+ * sslrootcert.
+ *
+ * Under rds-iam the URL is parsed here into explicit fields rather than passed as connectionString:
+ * pg merges a parsed connection string over explicit config, and a password-less URL would replace
+ * the token function with an empty password. Nothing in the URL is dropped silently: each query
+ * parameter is consumed here (sslmode, sslrootcert), carried over as the same field pg would have
+ * read from the connection string (SESSION_PARAMS), read from the URL by other code
+ * (OTHER_READERS_PARAMS), or refused by name.
+ *
+ * Refusals never include the URL, its credentials, or a token.
+ */
+import { readFileSync } from "node:fs";
+import { Signer } from "@aws-sdk/rds-signer";
+export const DATABASE_AUTH_ERROR = "database_auth_invalid";
+export const RDS_IAM_TOKEN_REUSE_MS = 600_000;
+export function databaseAuthFromEnv(env = process.env) {
+    const value = env.DATABASE_AUTH?.trim() || "password";
+    if (value === "password" || value === "rds-iam")
+        return value;
+    throw new Error(`${DATABASE_AUTH_ERROR}: DATABASE_AUTH must be "password" or "rds-iam" (found "${value}")`);
+}
+/** pg reads these from a connection string as same-named config fields (strings); carried over as-is. */
+const SESSION_PARAMS = [
+    "application_name",
+    "fallback_application_name",
+    "options",
+    "client_encoding",
+    "statement_timeout",
+    "lock_timeout",
+    "idle_in_transaction_session_timeout",
+    "query_timeout",
+];
+/**
+ * Read from the URL elsewhere, so allowed through: connection_limit and pool_timeout (poolSettings
+ * in db.ts), connect_timeout (the DBOS system pool, as DBOS reads it for its own; pg's JavaScript
+ * client never reads it), schema (Prisma's CLI; the pg adapter ignores it, so it has no effect at
+ * run time under either auth mode).
+ */
+const OTHER_READERS_PARAMS = ["connection_limit", "pool_timeout", "connect_timeout", "schema"];
+const ACCEPTED_PARAMS = new Set([
+    "sslmode",
+    "sslrootcert",
+    ...SESSION_PARAMS,
+    ...OTHER_READERS_PARAMS,
+]);
+function refuse(detail) {
+    throw new Error(`${DATABASE_AUTH_ERROR}: DATABASE_AUTH=rds-iam ${detail}`);
+}
+/** One token at a time: concurrent connects share a signing call; a failure is never cached. */
+export function cachedToken(signer, now = Date.now, reuseMs = RDS_IAM_TOKEN_REUSE_MS) {
+    let token;
+    let mintedAt = 0;
+    let inflight;
+    return async () => {
+        if (token !== undefined && now() - mintedAt < reuseMs)
+            return token;
+        if (!inflight) {
+            const startedAt = now();
+            inflight = signer
+                .getAuthToken()
+                .then((fresh) => {
+                token = fresh;
+                mintedAt = startedAt;
+                return fresh;
+            })
+                .finally(() => {
+                inflight = undefined;
+            });
+        }
+        return inflight;
+    };
+}
+function decode(component, what) {
+    try {
+        return decodeURIComponent(component);
+    }
+    catch {
+        return refuse(`needs a correctly percent-encoded ${what} in DATABASE_URL`);
+    }
+}
+export function pgConnectionConfig(url, auth, options = {}) {
+    if (auth === "password")
+        return { connectionString: url };
+    const env = options.env ?? process.env;
+    let parsed;
+    try {
+        parsed = new URL(url);
+    }
+    catch {
+        refuse("needs a valid postgresql:// DATABASE_URL");
+    }
+    if (parsed.protocol !== "postgresql:" && parsed.protocol !== "postgres:") {
+        refuse("needs a postgresql:// (or postgres://) DATABASE_URL");
+    }
+    for (const name of parsed.searchParams.keys()) {
+        // The name only: a refused parameter's value can be a key path or a secret.
+        if (!ACCEPTED_PARAMS.has(name))
+            refuse(`does not support the DATABASE_URL parameter "${name}"`);
+    }
+    if (parsed.password)
+        refuse("takes no password in DATABASE_URL (the IAM token is the password)");
+    const user = decode(parsed.username, "user name");
+    if (!user)
+        refuse("needs a user name in DATABASE_URL (the database user mapped to the IAM role)");
+    const region = env.DATABASE_AWS_REGION?.trim() || env.AWS_REGION?.trim();
+    if (!region)
+        refuse("needs AWS_REGION (or DATABASE_AWS_REGION)");
+    const rootCert = parsed.searchParams.get("sslrootcert");
+    if (parsed.searchParams.get("sslmode") !== "verify-full" || !rootCert) {
+        refuse("needs sslmode=verify-full and sslrootcert=<path to the RDS CA bundle> in DATABASE_URL");
+    }
+    const host = parsed.hostname;
+    const port = parsed.port ? Number(parsed.port) : 5432;
+    const database = decode(parsed.pathname.replace(/^\//, ""), "database name") || undefined;
+    const makeSigner = options.makeSigner ?? ((params) => new Signer(params));
+    const readFile = options.readFile ?? ((path) => readFileSync(path, "utf8"));
+    let ca;
+    try {
+        ca = readFile(rootCert);
+    }
+    catch (err) {
+        // The error code only (ENOENT, EACCES, ...): the message repeats the path.
+        const code = err.code;
+        refuse(`could not read the sslrootcert file named in DATABASE_URL${code ? ` (${code})` : ""}`);
+    }
+    const session = {};
+    for (const name of SESSION_PARAMS) {
+        const value = parsed.searchParams.get(name);
+        if (value !== null)
+            session[name] = value;
+    }
+    return {
+        ...session,
+        host,
+        port,
+        user,
+        database,
+        password: cachedToken(makeSigner({ hostname: host, port, username: user, region }), options.now),
+        ssl: { ca, rejectUnauthorized: true, servername: host },
+    };
+}

package/dist/core/power-composition.d.ts

@@ -0,0 +1,16 @@
+/**
+ * Builds the process's single low-power `Power` from the environment. Every
+ * composition root (serve, mcp, scheduler) calls this once and hands the same
+ * instance to both executors, so wake and sleep agree on state.
+ */
+import type { PrismaClient } from "#prisma";
+import { type Power } from "./power.js";
+/**
+ * Whether buildPower would return an enabled Power for this environment (WARDBY_LOW_POWER=1 and
+ * JOB_LAUNCHER=kubernetes), without building one. Throws on an invalid WARDBY_LOW_POWER.
+ */
+export declare function lowPowerActive(env?: NodeJS.ProcessEnv): boolean;
+export declare function buildPower({ env, db }: {
+    env?: NodeJS.ProcessEnv;
+    db: PrismaClient;
+}): Power;

package/dist/core/power-composition.js

@@ -0,0 +1,37 @@
+import { DEFAULT_NATIVE_GATEWAY_SERVICE, loadKubernetesJobConfig } from "../config/providers.js";
+import { loadLowPowerConfig } from "../config/low-power.js";
+import { ClientNodeKubernetesApi } from "../providers/jobs/kubernetes-client.js";
+import { logger } from "./logger.js";
+import { createPower, DISABLED_POWER } from "./power.js";
+import { powerBlockers, powerBusy } from "./power-idle.js";
+const log = logger.child({ module: "power-composition" });
+/** Low-power mode only scales Kubernetes Deployments; with any other job launcher it is ignored. */
+const kubernetesJobs = (env) => env.JOB_LAUNCHER === "kubernetes";
+/**
+ * Whether buildPower would return an enabled Power for this environment (WARDBY_LOW_POWER=1 and
+ * JOB_LAUNCHER=kubernetes), without building one. Throws on an invalid WARDBY_LOW_POWER.
+ */
+export function lowPowerActive(env = process.env) {
+    return loadLowPowerConfig(env).enabled && kubernetesJobs(env);
+}
+export function buildPower({ env = process.env, db }) {
+    const config = loadLowPowerConfig(env);
+    if (!config.enabled)
+        return DISABLED_POWER;
+    if (!kubernetesJobs(env)) {
+        log.warn("low_power_ignored: WARDBY_LOW_POWER requires JOB_LAUNCHER=kubernetes");
+        return DISABLED_POWER;
+    }
+    const kubernetes = loadKubernetesJobConfig(env);
+    return createPower({
+        config,
+        api: new ClientNodeKubernetesApi({ context: kubernetes.context }),
+        namespace: kubernetes.namespace,
+        deployments: {
+            proxy: kubernetes.proxyService,
+            gateway: env.NATIVE_GATEWAY_SERVICE?.trim() || DEFAULT_NATIVE_GATEWAY_SERVICE,
+        },
+        busy: powerBusy(db),
+        blockers: (target) => powerBlockers(db, target),
+    });
+}

package/dist/core/power-idle.d.ts

@@ -0,0 +1,14 @@
+/**
+ * Low-power mode's idle rules: whether the coding proxy or native gateway has
+ * had work since `since`. A target is busy while any of its runs is pending
+ * (queued included) or running, or finished at or after `since`; the gateway
+ * is also busy while any gateway session is active. powerBlockers lists the
+ * run ids that keep a target busy right now (recent finishes do not count).
+ */
+import type { PrismaClient } from "#prisma";
+import type { PowerTarget } from "./power.js";
+export declare function proxyBusy(db: PrismaClient, since: Date): Promise<boolean>;
+export declare function gatewayBusy(db: PrismaClient, since: Date): Promise<boolean>;
+export declare function powerBusy(db: PrismaClient): (target: PowerTarget, since: Date) => Promise<boolean>;
+/** Run ids keeping `target` busy right now: active or queued runs (plus active gateway sessions for the gateway). */
+export declare function powerBlockers(db: PrismaClient, target: PowerTarget): Promise<string[]>;

package/dist/core/power-idle.js

@@ -0,0 +1,30 @@
+const ACTIVE = { in: ["pending", "running"] };
+export async function proxyBusy(db, since) {
+    const n = await db.codingRun.count({
+        where: { run: { OR: [{ status: ACTIVE }, { finishedAt: { gte: since } }] } },
+    });
+    return n > 0;
+}
+export async function gatewayBusy(db, since) {
+    const runs = await db.run.count({
+        where: { nativeExecutionMode: "sandbox", OR: [{ status: ACTIVE }, { finishedAt: { gte: since } }] },
+    });
+    if (runs > 0)
+        return true;
+    return (await db.nativeGatewaySession.count({ where: { status: "active" } })) > 0;
+}
+export function powerBusy(db) {
+    return (target, since) => (target === "proxy" ? proxyBusy(db, since) : gatewayBusy(db, since));
+}
+/** Run ids keeping `target` busy right now: active or queued runs (plus active gateway sessions for the gateway). */
+export async function powerBlockers(db, target) {
+    if (target === "proxy") {
+        const rows = await db.codingRun.findMany({ where: { run: { status: ACTIVE } }, select: { runId: true } });
+        return [...new Set(rows.map((r) => r.runId))].sort();
+    }
+    const [runs, sessions] = await Promise.all([
+        db.run.findMany({ where: { nativeExecutionMode: "sandbox", status: ACTIVE }, select: { id: true } }),
+        db.nativeGatewaySession.findMany({ where: { status: "active" }, select: { runId: true } }),
+    ]);
+    return [...new Set([...runs.map((r) => r.id), ...sessions.map((s) => s.runId)])].sort();
+}

package/dist/core/power.d.ts

@@ -0,0 +1,71 @@
+import type { LowPowerConfig } from "../config/low-power.js";
+import { type KubernetesApi } from "../providers/jobs/kubernetes-api.js";
+export type PowerTarget = "proxy" | "gateway";
+export declare const LOW_POWER_WAKE_TIMEOUT = "low_power_wake_timeout";
+export declare const LOW_POWER_SCALE_FORBIDDEN = "low_power_scale_forbidden";
+export declare const LOW_POWER_DISABLED = "low_power_disabled";
+/** ISO-8601 time the control plane last scaled the Deployment up from zero. */
+export declare const POWER_WOKE_AT = "wardby.io/power-woke-at";
+/**
+ * The least time a launcher waits for a woken target: a cold node can take a minute or two to
+ * come up. A launcher's own ready timeout above this raises the bound.
+ */
+export declare const LOW_POWER_WAKE_FLOOR_MS = 300000;
+/** How long a launcher with this ready timeout lets ensureAwake wait. */
+export declare const wakeTimeoutMs: (readyTimeoutMs: number) => number;
+export interface PowerOptions {
+    config: LowPowerConfig;
+    /** Required when config.enabled. */
+    api?: KubernetesApi;
+    namespace: string;
+    /** Deployment name per target; each has the same name as its Service. */
+    deployments: Record<PowerTarget, string>;
+    /** Whether the target has had work since `since` (used by sleepIfIdle). */
+    busy?: (target: PowerTarget, since: Date) => Promise<boolean>;
+    /** Run ids keeping the target busy right now (active or queued; ignores recent finishes). Needed by sleepNow. */
+    blockers?: (target: PowerTarget) => Promise<string[]>;
+    now?: () => Date;
+    sleep?: (ms: number) => Promise<void>;
+    /** How often ensureAwake polls for a ready endpoint. Default 2000. */
+    pollMs?: number;
+}
+export interface Power {
+    readonly enabled: boolean;
+    /** Scales the target up from zero if needed and waits for a ready endpoint. */
+    ensureAwake(target: PowerTarget, opts: {
+        timeoutMs: number;
+        runId: string;
+    }): Promise<void>;
+    isAsleep(target: PowerTarget): Promise<boolean>;
+    sleepIfIdle(): Promise<void>;
+    /** Missing permissions, e.g. "patch deployments/wardby-coding-proxy". */
+    checkAccess(): Promise<string[]>;
+    /** Each target's state. A target whose Deployment does not exist is left out. Disabled: `{ enabled: false }`, no API call. */
+    status(): Promise<PowerStatus>;
+    /**
+     * Scales each target to zero now, skipping the idle window and woke-at grace,
+     * unless it has blockers (then it is reported, untouched). Never stops a run.
+     * A target whose Deployment does not exist is neither slept nor blocked.
+     */
+    sleepNow(targets: PowerTarget[]): Promise<SleepNowResult>;
+}
+export interface PowerTargetStatus {
+    target: PowerTarget;
+    replicas: number;
+    ready: boolean;
+    wokeAt: string | null;
+    blockers: string[];
+}
+export interface PowerStatus {
+    enabled: boolean;
+    targets?: PowerTargetStatus[];
+}
+export interface SleepNowResult {
+    slept: PowerTarget[];
+    blocked: Array<{
+        target: PowerTarget;
+        runIds: string[];
+    }>;
+}
+export declare const DISABLED_POWER: Power;
+export declare function createPower(options: PowerOptions): Power;

package/dist/core/power.js

@@ -0,0 +1,202 @@
+import { KubernetesForbiddenError } from "../providers/jobs/kubernetes-api.js";
+import { logger } from "./logger.js";
+const powerLog = logger.child({ module: "power" });
+export const LOW_POWER_WAKE_TIMEOUT = "low_power_wake_timeout";
+export const LOW_POWER_SCALE_FORBIDDEN = "low_power_scale_forbidden";
+export const LOW_POWER_DISABLED = "low_power_disabled";
+/** ISO-8601 time the control plane last scaled the Deployment up from zero. */
+export const POWER_WOKE_AT = "wardby.io/power-woke-at";
+/**
+ * The least time a launcher waits for a woken target: a cold node can take a minute or two to
+ * come up. A launcher's own ready timeout above this raises the bound.
+ */
+export const LOW_POWER_WAKE_FLOOR_MS = 300_000;
+/** How long a launcher with this ready timeout lets ensureAwake wait. */
+export const wakeTimeoutMs = (readyTimeoutMs) => Math.max(readyTimeoutMs, LOW_POWER_WAKE_FLOOR_MS);
+/** sleepIfIdle does real work at most this often. */
+const SLEEP_SWEEP_INTERVAL_MS = 60_000;
+export const DISABLED_POWER = {
+    enabled: false,
+    async ensureAwake() { },
+    async isAsleep() {
+        return false;
+    },
+    async sleepIfIdle() { },
+    async checkAccess() {
+        return [];
+    },
+    async status() {
+        return { enabled: false };
+    },
+    async sleepNow() {
+        throw new Error(`${LOW_POWER_DISABLED}: WARDBY_LOW_POWER is not 1 on this deployment.`);
+    },
+};
+const ready = (ep) => (ep?.subsets ?? []).some((s) => (s.addresses ?? []).length > 0);
+/** Rethrows a Forbidden read or patch as low_power_scale_forbidden; anything else unchanged. */
+function scaleError(error, name, access = "patch Deployment") {
+    if (!(error instanceof KubernetesForbiddenError))
+        return error;
+    return new Error(`${LOW_POWER_SCALE_FORBIDDEN}: the control plane may not ${access} "${name}"; apply the wardby-power Role (up.sh with WARDBY_LOW_POWER=1).`, { cause: error });
+}
+export function createPower(options) {
+    if (!options.config.enabled)
+        return DISABLED_POWER;
+    const { api, namespace, deployments } = options;
+    if (!api)
+        throw new Error("Low-power mode needs a Kubernetes API client.");
+    const now = options.now ?? (() => new Date());
+    const sleep = options.sleep ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms)));
+    const pollMs = options.pollMs ?? 2000;
+    const { busy, blockers } = options;
+    const idleMinutes = options.config.idleMinutes;
+    const idleMs = idleMinutes * 60_000;
+    // The idle window counts from the later of the Deployment's woke-at stamp and this process's
+    // start: a stamp left from before a restart (or none at all) never sleeps a target sooner.
+    const startedAt = now().getTime();
+    let lastSweep;
+    const readDeployment = async (name) => {
+        try {
+            return await api.readDeployment(namespace, name);
+        }
+        catch (error) {
+            throw scaleError(error, name, "get Deployment");
+        }
+    };
+    const readEndpoints = async (name) => {
+        try {
+            return await api.readEndpoints(namespace, name);
+        }
+        catch (error) {
+            throw scaleError(error, name, "get Endpoints");
+        }
+    };
+    async function sleepTarget(target, at) {
+        if (!api || !busy)
+            return;
+        const name = deployments[target];
+        const dep = await readDeployment(name);
+        if (!dep || (dep.spec?.replicas ?? 1) === 0)
+            return;
+        const stamped = Date.parse(dep.metadata?.annotations?.[POWER_WOKE_AT] ?? "");
+        const wokeAt = Number.isNaN(stamped) ? startedAt : Math.max(stamped, startedAt);
+        if (at - wokeAt < idleMs)
+            return;
+        const since = new Date(at - idleMs);
+        if (await busy(target, since))
+            return;
+        // A run may have started since the first check; look again right before scaling down.
+        if (await busy(target, since))
+            return;
+        await api.patchDeployment(namespace, name, { replicas: 0 });
+        powerLog.info({ target }, `power: ${target} slept (idle ${idleMinutes}m)`);
+    }
+    return {
+        enabled: true,
+        async ensureAwake(target, { timeoutMs, runId }) {
+            const name = deployments[target];
+            const dep = await readDeployment(name);
+            if (!dep)
+                throw new Error(`${LOW_POWER_WAKE_TIMEOUT}: Deployment "${name}" not found in "${namespace}".`);
+            if ((dep.spec?.replicas ?? 1) === 0) {
+                try {
+                    await api.patchDeployment(namespace, name, {
+                        replicas: 1,
+                        annotations: { [POWER_WOKE_AT]: now().toISOString() },
+                    });
+                }
+                catch (error) {
+                    throw scaleError(error, name);
+                }
+                powerLog.info({ target, runId }, `power: ${target} woke`);
+            }
+            const deadline = now().getTime() + timeoutMs;
+            while (!ready(await readEndpoints(name))) {
+                if (now().getTime() >= deadline) {
+                    throw new Error(`${LOW_POWER_WAKE_TIMEOUT}: ${target} "${name}" had no ready endpoint within ${timeoutMs} ms.`);
+                }
+                await sleep(pollMs);
+            }
+        },
+        async isAsleep(target) {
+            return ((await readDeployment(deployments[target]))?.spec?.replicas ?? 1) === 0;
+        },
+        async sleepIfIdle() {
+            const at = now().getTime();
+            if (lastSweep !== undefined && at - lastSweep < SLEEP_SWEEP_INTERVAL_MS)
+                return;
+            lastSweep = at;
+            for (const target of Object.keys(deployments)) {
+                try {
+                    await sleepTarget(target, at);
+                }
+                catch (err) {
+                    powerLog.warn({ err, target }, "power_sleep_failed");
+                }
+            }
+        },
+        async status() {
+            const targets = [];
+            for (const target of Object.keys(deployments)) {
+                const name = deployments[target];
+                const dep = await readDeployment(name);
+                if (!dep)
+                    continue;
+                targets.push({
+                    target,
+                    replicas: dep.spec?.replicas ?? 1,
+                    ready: ready(await readEndpoints(name)),
+                    wokeAt: dep.metadata?.annotations?.[POWER_WOKE_AT] ?? null,
+                    blockers: blockers ? await blockers(target) : [],
+                });
+            }
+            return { enabled: true, targets };
+        },
+        async sleepNow(targets) {
+            // Without a blockers check this could scale a target out from under a run.
+            if (!blockers)
+                throw new Error("power: sleepNow needs a blockers check.");
+            const result = { slept: [], blocked: [] };
+            for (const target of targets) {
+                const name = deployments[target];
+                const dep = await readDeployment(name);
+                if (!dep)
+                    continue;
+                if ((dep.spec?.replicas ?? 1) === 0) {
+                    result.slept.push(target);
+                    continue;
+                }
+                let runIds = await blockers(target);
+                // A run may have started since the first check; look again right before scaling down.
+                if (runIds.length === 0)
+                    runIds = await blockers(target);
+                if (runIds.length > 0) {
+                    result.blocked.push({ target, runIds });
+                    continue;
+                }
+                try {
+                    await api.patchDeployment(namespace, name, { replicas: 0 });
+                }
+                catch (error) {
+                    throw scaleError(error, name);
+                }
+                powerLog.info({ target }, `power: ${target} slept (forced)`);
+                result.slept.push(target);
+            }
+            return result;
+        },
+        async checkAccess() {
+            const missing = [];
+            for (const name of Object.values(deployments)) {
+                for (const verb of ["get", "patch"]) {
+                    if (!(await api.canI(namespace, verb, "apps", "deployments", name)))
+                        missing.push(`${verb} deployments/${name}`);
+                }
+                // ensureAwake and status poll the Service's Endpoints (same name as the Deployment).
+                if (!(await api.canI(namespace, "get", "", "endpoints", name)))
+                    missing.push(`get endpoints/${name}`);
+            }
+            return missing;
+        },
+    };
+}

package/dist/core/pull-request-state-sync.js

@@ -6,8 +6,8 @@ export const PR_STATE_SYNC_BATCH = 10;
 /** A row is re-checked at most this often. */
 export const PR_STATE_SYNC_MIN_AGE_MS = 10 * 60_000;
 export async function syncOpenPullRequestStates(db, hosts, trackers, now = new Date()) {
-    // handlePullRequestClosed needs trackers; without them no IssuePullRequest rows are written anyway.
-    if (!hosts || !trackers)
+    // Without trackers the rows (a GitHub issue's, say) are still settled; only tracker actions are skipped.
+    if (!hosts)
         return 0;
     let settled = 0;
     try {

package/dist/core/reconciler.d.ts

@@ -36,15 +36,18 @@
  * it also catches a run that ended before its comment was posted.
  *
  * Given host-event deps, each pass also starts reviews deferred until CI
- * finished (waitForCi links) whose CI has not finished in time, and drops
- * deferred reviews that are too old (host-events.ts startDeferredReviews).
+ * finished (waitForCi links) whose CI has not finished in time, releases
+ * reviews that waited for a delegating run that ended without its finalizer
+ * (lost, including leads this pass just marked lost), and drops deferred
+ * reviews that are too old (host-events.ts startDeferredReviews).
  */
 import type { PrismaClient } from "#prisma";
 import type { Executor } from "../providers/executor/types.js";
 import type { ReviewHostRegistry } from "../providers/review-host/types.js";
 import type { IssueTrackerRegistry } from "../providers/issue-tracker/types.js";
+import { type HostStatusDb } from "./host-status.js";
 import { type ReviewStartDeps } from "./host-events.js";
-export type ReconcilerDb = Pick<PrismaClient, "run" | "runHostCheck" | "runHostStatus" | "runIssueStatus" | "agentIssueProject" | "issuePullRequest" | "codingRun" | "agent" | "$transaction" | "$queryRaw">;
+export type ReconcilerDb = Pick<PrismaClient, "run" | "runHostCheck" | "runBudgetEvent" | "runHostStatus" | "runIssueStatus" | "agentIssueProject" | "issuePullRequest" | "codingRun" | "registryFetch" | "registryAuditRecord" | "agent" | "agentRepository" | "$transaction" | "$queryRaw">;
 /**
  * How long after a run finishes before its still-open check counts as
  * orphaned: long enough that the sweep never races `executeRun`'s own
@@ -64,14 +67,14 @@ export declare const ORPHANED_CHECK_MAX_AGE_MS: number;
  * than retried forever; its Re-run button still works. Never queries when
  * no host is configured. `closeOpenHostCheck` is best-effort and never throws.
  */
-export declare function closeOrphanedHostChecks(db: Pick<PrismaClient, "runHostCheck" | "run">, hosts: ReviewHostRegistry | undefined, now?: Date): Promise<void>;
+export declare function closeOrphanedHostChecks(db: Pick<PrismaClient, "runHostCheck" | "run" | "runBudgetEvent">, hosts: ReviewHostRegistry | undefined, now?: Date): Promise<void>;
 /**
  * Completes open mention status comments on the same terms as
  * closeOrphanedHostChecks: the run ended between ORPHANED_CHECK_GRACE_MS and
  * ORPHANED_CHECK_MAX_AGE_MS ago, newest first, at most ORPHANED_CHECK_BATCH per
  * pass. `completeHostStatus` is best-effort and never throws.
  */
-export declare function closeOrphanedHostStatuses(db: Pick<PrismaClient, "runHostStatus" | "run">, hosts: ReviewHostRegistry | undefined, now?: Date): Promise<void>;
+export declare function closeOrphanedHostStatuses(db: HostStatusDb, hosts: ReviewHostRegistry | undefined, now?: Date): Promise<void>;
 /** Runs one reconciliation pass. Returns the number of runs marked `lost`. */
 export declare function reconcileOnce(db: ReconcilerDb, now?: Date, heartbeatTimeoutMs?: number, executor?: Executor, reviewHosts?: ReviewHostRegistry, issueTrackers?: IssueTrackerRegistry, 
 /** What starting a deferred review needs; absent = no deferred-review sweep. */

package/dist/core/reconciler.js

@@ -36,17 +36,22 @@
  * it also catches a run that ended before its comment was posted.
  *
  * Given host-event deps, each pass also starts reviews deferred until CI
- * finished (waitForCi links) whose CI has not finished in time, and drops
- * deferred reviews that are too old (host-events.ts startDeferredReviews).
+ * finished (waitForCi links) whose CI has not finished in time, releases
+ * reviews that waited for a delegating run that ended without its finalizer
+ * (lost, including leads this pass just marked lost), and drops deferred
+ * reviews that are too old (host-events.ts startDeferredReviews).
  */
 import { HEARTBEAT_TIMEOUT_MS, RECONCILE_INTERVAL_MS } from "./timing.js";
 import { prisma as defaultDb } from "./db.js";
 import { logger } from "./logger.js";
+import { stopChildrenOfLostRun } from "./lost-run-children.js";
 import { closeOpenHostCheck } from "./review-host-checks.js";
 import { completeHostStatus } from "./host-status.js";
 import { closeOrphanedIssueStatuses } from "./issue-status.js";
 import { fileSelfDefect } from "./self-defects.js";
+import { emitRunFinishedEvents } from "./workflow-run-events.js";
 import { syncOpenPullRequestStates } from "./pull-request-state-sync.js";
+import { sweepMergeOrderChecks } from "./merge-order-check.js";
 import { startDeferredReviews } from "./host-events.js";
 const reconcilerLog = logger.child({ module: "reconciler" });
 /** A pass waits this long per self-defect, then moves on; the filing finishes (or logs) in the background. */
@@ -206,7 +211,11 @@ deferredReviews) {
                 continue;
             let recovered;
             try {
-                recovered = await executor.recover({ runId: run.id, backend: run.executionBackend, id: run.id });
+                recovered = await executor.recover({
+                    runId: run.id,
+                    backend: run.executionBackend,
+                    id: run.executionWorkflowId ?? run.id,
+                });
             }
             catch (err) {
                 reconcilerLog.error({ err, runId: run.id }, "durable run recovery failed");
@@ -220,8 +229,17 @@ deferredReviews) {
                 data: { status: "lost", error: reason, finishedAt: now },
             });
             lost += result.count;
-            if (result.count > 0)
+            if (result.count > 0) {
+                await stopChildrenOfLostRun(db, executor, run.id);
                 await fileSelfDefect(db, issueTrackers, { ...run, status: "lost", error: reason, finishedAt: now }, SELF_DEFECT_WAIT);
+                await emitRunFinishedEvents(db, {
+                    id: run.id,
+                    agentId: run.agentId,
+                    status: "lost",
+                    error: reason,
+                    parentRunId: run.parentRunId,
+                });
+            }
             continue;
         }
         const result = await db.run.updateMany({
@@ -230,17 +248,30 @@ deferredReviews) {
         });
         lost += result.count;
         // Only after this pass made the row lost (not when another instance won the race); bounded, never throws.
-        if (result.count > 0)
+        if (result.count > 0) {
+            await stopChildrenOfLostRun(db, executor, run.id);
             await fileSelfDefect(db, issueTrackers, { ...run, status: "lost", error: reason, finishedAt: now }, SELF_DEFECT_WAIT);
+            await emitRunFinishedEvents(db, {
+                id: run.id,
+                agentId: run.agentId,
+                status: "lost",
+                error: reason,
+                parentRunId: run.parentRunId,
+            });
+        }
     }
     await closeOrphanedHostChecks(db, reviewHosts, now);
     await closeOrphanedHostStatuses(db, reviewHosts, now);
     await closeOrphanedIssueStatuses(db, issueTrackers, now);
     // Settle stored pull requests whose merge/close webhook was missed; bounded per pass, never throws.
     await syncOpenPullRequestStates(db, reviewHosts, issueTrackers, now);
-    // Start reviews whose head's CI never finished (waitForCi links); bounded per pass, never throws.
+    // Start reviews whose head's CI never finished (waitForCi links), and reviews whose delegating run
+    // ended without releasing them (after the lost marking above); bounded per pass, never throws.
     if (deferredReviews)
         await startDeferredReviews(deferredReviews, now);
+    // Re-post `wardby merge order` checks whose pr_closed or new-head event was missed; after the
+    // deferred-review start, so no review waits on it. Throttled and bounded per pass, never throws.
+    await sweepMergeOrderChecks({ db, hosts: reviewHosts }, now);
     return lost;
 }
 export function startReconciler(options = {}) {