@nanobpm/nano-workforce 0.206.4 → 0.208.0

package/app/service.ts

@@ -288,6 +288,14 @@ export const MERGEABLE_REPOLL_INTERVAL = mergeableRepollInterval(
  * to one attempt per window. Set via `NANO_PR_REVIEW_NUDGE_MINUTES` (minutes). */
 export const REVIEW_NUDGE_MS = clampNudgeMinutes(process.env.NANO_PR_REVIEW_NUDGE_MINUTES) * 60_000;
 
+/** Merge-loop head-current review request (issue #939). When the merge backstop finds no
+ * head-current Copilot review (the loop's own rebase/fix push moved the head), the merge loop
+ * requests the review itself and waits `REVIEW_REQUEST_WAIT` before re-checking — derived from the
+ * SAME nudge window the convergence poller uses, so there is one knob for "how long to give Copilot".
+ * It re-requests up to `MAX_REVIEW_REQUESTS` times before escalating (stating what was tried). */
+export const REVIEW_REQUEST_WAIT = `PT${Math.round(REVIEW_NUDGE_MS / 60_000)}M`;
+export const MAX_REVIEW_REQUESTS = 3;
+
 /** Grace window (ms) before the `escalated`→`running` self-heal (below) may act on a row. The sole
  * writer of `status="escalated"` — `record-feature-escalation` — runs on the `escalated` arm and stamps
  * `updated_at` IMMEDIATELY BEFORE the engine creates the `feature-escalation` user task. A poll landing
@@ -388,13 +396,16 @@ export interface PullRequest {
   // running agent curls (GET /app/api/hooks/abandon?token=…) to learn whether this run was cancelled before
   // it performs a side effect. Minted at submit, reused across the convergence + merge instances.
   abandon_token: string | null;
-  // Technical-incident surfacing (017_pr_incident.sql, issue #94), written by the poller's
-  // `pollIncidents` pass. `incident_key` is the engine incidentKey of the ACTIVE incident parking
-  // this PR's instance and `incident_message` its errorMessage; both NULL when the instance has no
-  // active incident. Orthogonal to `status` — an incident is a cross-cutting liveness fault, not a
-  // workflow stage.
+  // Technical-incident surfacing (017_pr_incident.sql, issue #94; enriched 126, issue #931), written
+  // by the poller's `pollIncidents` pass. `incident_key` is the engine incidentKey of the ACTIVE
+  // incident parking this PR's instance, `incident_type` its errorType, `incident_message` its
+  // errorMessage and `incident_since` its creationTime; all NULL when the instance has no active
+  // incident. Orthogonal to `status` — an incident is a cross-cutting liveness fault, not a
+  // workflow stage. `/status` derives the `incident` overlay from these columns (see `activePrs`).
   incident_key: string | null;
+  incident_type: string | null;
   incident_message: string | null;
+  incident_since: string | null;
   // Lineage projection (037_lineage.sql, issue #245): the stable ORIGIN identity (the issue =
   // feature_key / plan_key) threaded onto this PR by `submitPr`, and passed as a `createInstance`
   // variable onto the convergence + merge instances so every descendant carries the root. For a
@@ -629,9 +640,11 @@ export async function submitPr(
   humanApproval = false,
   autoResolveEscalations = false,
   autoMerge?: boolean,
+  handoffFrom: string | null = null,
+  headShaOverride: string | null = null,
 ) {
   return withPrSubmitLock(parsed.prKey, () =>
-    submitPrCritical(data, engine, parsed, dependsOn, maxRounds, convergeOnly, rootRequestKey, humanApproval, autoResolveEscalations, autoMerge),
+    submitPrCritical(data, engine, parsed, dependsOn, maxRounds, convergeOnly, rootRequestKey, humanApproval, autoResolveEscalations, autoMerge, handoffFrom, headShaOverride),
   );
 }
 
@@ -646,6 +659,8 @@ async function submitPrCritical(
   humanApproval = false,
   autoResolveEscalations = false,
   autoMerge?: boolean,
+  handoffFrom: string | null = null,
+  headShaOverride: string | null = null,
 ) {
   // Merge intent (issue #916): `autoMerge` is the CANONICAL positive form end to end — the same name
   // the feature/convergence-loop/plan-fanout doors accept. The legacy inverted `convergeOnly`
@@ -677,7 +692,28 @@ async function submitPrCritical(
   // would skip it as non-terminal, never re-enrolling). Treat a keyless non-terminal row as
   // RESUBMITTABLE — fall through and re-enroll, exactly as `startFeature`'s intake guard (feature.ts)
   // treats a keyless `running` feature row. A terminal row (any `process_key`) already falls through.
-  if (trackedExisting && existing?.process_key != null && !TERMINAL_STATUSES.includes(trackedExisting.derived_status)) {
+  // Merge-loop → convergence hand-back (issue #939): the merge-loop instance that currently OWNS the
+  // row (`process_key`) re-enters convergence when a head-current review reports findings. That
+  // instance is live (it is the caller, and ends right after this submit), so it must not read as
+  // "already running" — only the exact owning instance may hand back; any other key still dedupes.
+  // The bypass additionally requires the row to be LIVE (non-terminal, #941 review): a redelivered
+  // `pr.reconverge` can arrive after its successor has already settled the PR (merged/abandoned),
+  // and the settled row's `process_key` still names that stale job's instance — an owner match alone
+  // would let the dead hand-off restart convergence and clear the settled state. So an internal
+  // hand-off against a TERMINAL row is fenced off entirely (below): it dedupes as `alreadyRunning`
+  // instead of resubmitting. Only a DELIBERATE PUBLIC submission (no `handoffFrom`) may re-open a
+  // settled PR — the internal hand-off channel never may. A partially-failed hand-off's retry
+  // against a keyless NON-terminal row still works (that row is not terminal, so it falls through
+  // and re-enrolls via the phantom-row path).
+  const trackedTerminal = trackedExisting != null && TERMINAL_STATUSES.includes(trackedExisting.derived_status);
+  const handingBack =
+    handoffFrom != null && existing?.process_key != null && existing.process_key === handoffFrom && !trackedTerminal;
+  if (handoffFrom != null && trackedTerminal) {
+    // Stale internal hand-off against a settled row: report the existing (terminal) owner without
+    // minting a fresh run or touching the settled state. The redelivered job completes idempotently.
+    return { prKey: parsed.prKey, alreadyRunning: true };
+  }
+  if (!handingBack && trackedExisting && existing?.process_key != null && !trackedTerminal) {
     // Human approval is monotonic (issue #826): a gated caller adopting a live loop — possibly an
     // ungated one another run started — narrows it to gated instead of inheriting its auto-merge.
     // Idempotent, so a retry against our own gated loop is a no-op. A loop the engine no longer reports
@@ -798,7 +834,14 @@ async function submitPrCritical(
   // fleet advertising `durable-resume`: with no participant it stays null, so the round redrives from
   // scratch (graceful degradation). Absent (null) on a first submit, which leaves the envelope
   // unchanged.
-  const worldSha = await worldRestoreSha(data, parsed.prKey);
+  //
+  // `headShaOverride` (#941 round-4 review) WINS over the checkpoint: the checkpoint is written only
+  // by the convergence loop's `pr.persist-round`, so a merge-stage rebase/fix-ci push advances the PR
+  // head WITHOUT updating it. A merge→convergence hand-off (`pr.reconverge`) that restored the
+  // checkpoint would provision the fresh run at the PRE-MERGE tree, not the head whose review raised
+  // the findings — so the hand-off passes the live head SHA it read, and a fresh hand-off restores
+  // the CURRENT head. `null` (every other caller) keeps the checkpoint resume.
+  const worldSha = headShaOverride ?? (await worldRestoreSha(data, parsed.prKey));
   const { processInstanceKey } = await engine.createInstance({
     processDefinitionId: PROCESS_ID,
     variables: {
@@ -874,6 +917,11 @@ async function submitPrCritical(
   // run auto-applies stale decisions, and (crucially) the row stays in its OLD terminal shape until
   // the atomic reopen write lands, so no straggler ever sees a fresh-run row still owned by the old
   // instance.
+  let reopened = false;
+  // Snapshot BEFORE the reopen write — the row object may be a live reference the update mutates.
+  const prior = existing
+    ? { status: existing.status, current_round: existing.current_round, process_key: existing.process_key }
+    : null;
   try {
     if (existing) {
       // ATOMIC REOPEN (Copilot review of #819): reset the row to the fresh run's shape AND publish the
@@ -928,6 +976,7 @@ async function submitPrCritical(
         process_key: processKey,
         updated_at: ts,
       });
+      reopened = true;
     } else if (processKey != null) {
       // First submit: the row was INSERTED keyless before `createInstance` (the phantom-row pattern,
       // #704/#497); publish its `process_key` now that the instance exists.
@@ -946,6 +995,19 @@ async function submitPrCritical(
       await resetAdjudications(data, parsed.prKey);
     }
   } catch (err) {
+    // Restore the prior row (#941 review): the atomic reopen already advanced `process_key` to the
+    // instance we are about to cancel. Leaving that cancelled key on the row would make a dead run its
+    // "owner" — once tracking derives `abandoned`, a retried merge-loop hand-off would be fenced as
+    // stale (settled row) and the merge loop would end with no live convergence owner. Restoring the
+    // pre-reopen shape (owner key + status) keeps the retry the owning hand-off, and a public retry
+    // still sees the old terminal row as resubmittable.
+    if (reopened && prior) {
+      try {
+        await table.update(parsed.prKey, { ...prior, updated_at: now() });
+      } catch (restoreErr) {
+        console.warn(`[submit] ${parsed.prKey} rollback-restore failed: ${restoreErr}`);
+      }
+    }
     if (processInstanceKey != null) {
       try {
         await engine.cancelInstance({ processInstanceKey: String(processInstanceKey) });
@@ -1022,6 +1084,11 @@ export async function startMerge(
       mergeableRepollInterval: MERGEABLE_REPOLL_INTERVAL,
       mergeStallRounds: 0,
       mergeStallMax: MAX_MERGE_STALL_ROUNDS,
+      // Head-current review request budget (issue #939): see REVIEW_REQUEST_WAIT.
+      reviewRequestRound: 0,
+      reviewRequestMax: MAX_REVIEW_REQUESTS,
+      reviewRequestWait: REVIEW_REQUEST_WAIT,
+      reviewRequest: null,
       // Lineage (issue #245): thread the origin identity onto the merge instance (see startMerge).
       rootRequestKey,
       abandonUrl: abUrl,
@@ -1057,6 +1124,40 @@ export interface MergePolicy {
   blockedBy: "env" | "human-approval" | null;
 }
 
+/** The status `/status` reports for a run whose instance is parked on an ACTIVE engine incident
+ * (issue #931). Derived at READ time from the mirrored `incident_*` columns — the base `status`
+ * workflow transient is never overwritten, so the overlay can never wedge the status machine and
+ * lifts the moment the incident clears. */
+export const INCIDENT_STATUS = "incident";
+
+/** The incident block every `/status` row carries (issue #931): the ACTIVE engine incident the
+ * poller mirrored onto the record, else null. `key` is the engine incidentKey (the handle the
+ * `retryIncident` action resolves), `type` the errorType (e.g. `JOB_NO_RETRIES`), `message` the
+ * errorMessage, `since` the incident's creationTime. */
+export interface StatusIncident {
+  key: string;
+  type: string | null;
+  message: string | null;
+  since: string | null;
+}
+
+/** Derive the `/status` incident block from a row's mirrored incident columns (issue #931). Null
+ * when the row carries no active incident. */
+function statusIncidentFor(row: {
+  incident_key: string | null;
+  incident_type: string | null;
+  incident_message: string | null;
+  incident_since: string | null;
+}): StatusIncident | null {
+  if (!row.incident_key) return null;
+  return {
+    key: row.incident_key,
+    type: row.incident_type ?? null,
+    message: row.incident_message ?? null,
+    since: row.incident_since ?? null,
+  };
+}
+
 /** A PR currently in flight, as reported by the status endpoint. */
 export interface ActivePr {
   prKey: string;
@@ -1081,6 +1182,9 @@ export interface ActivePr {
   /** The effective merge policy (issue #916): what the run intends (`requested`), what the
    * deployment allows (`enabled`), and what will actually happen (`effective`/`blockedBy`). */
   merge: MergePolicy;
+  /** The ACTIVE engine incident parking this PR's instance (issue #931), mirrored by the poller's
+   * `pollIncidents` pass; null when none. While set, `status` reads `incident`. */
+  incident: StatusIncident | null;
 }
 
 /** Every tracked PR not in a terminal state (converged/abandoned), newest-updated first. Backs
@@ -1135,23 +1239,30 @@ export async function activePrs(
       mergeByPr.set(p.pr_key, await mergePolicyFor(p.process_key ?? null, engine));
     }),
   );
-  return active.map((p) => ({
-    prKey: p.pr_key,
-    repo: p.repo,
-    number: p.number,
-    url: p.url,
-    title: p.title ?? null,
-    status: p.status,
-    round: p.current_round,
-    processKey: p.process_key ?? null,
-    waitingSince: p.waiting_since ?? null,
-    openEscalation: openEscByPr.get(p.pr_key) ?? null,
-    updatedAt: p.updated_at,
-    activeWorker: p.active_worker ?? null,
-    leaseUntil: p.lease_until ?? null,
-    autoResolveEscalations: p.auto_resolve_escalations === 1,
-    merge: mergeByPr.get(p.pr_key) ?? mergePolicyDefault(),
-  }));
+  return active.map((p) => {
+    // Issue #931: an ACTIVE engine incident parked on the loop's instance must be VISIBLE — the row
+    // kept reading `converging` for hours while nothing ran. Overlay the derived `incident` status
+    // (the base `status` column is never touched) and carry the mirrored incident block.
+    const incident = statusIncidentFor(p);
+    return {
+      prKey: p.pr_key,
+      repo: p.repo,
+      number: p.number,
+      url: p.url,
+      title: p.title ?? null,
+      status: incident ? INCIDENT_STATUS : p.status,
+      round: p.current_round,
+      processKey: p.process_key ?? null,
+      waitingSince: p.waiting_since ?? null,
+      openEscalation: openEscByPr.get(p.pr_key) ?? null,
+      updatedAt: p.updated_at,
+      activeWorker: p.active_worker ?? null,
+      leaseUntil: p.lease_until ?? null,
+      autoResolveEscalations: p.auto_resolve_escalations === 1,
+      merge: mergeByPr.get(p.pr_key) ?? mergePolicyDefault(),
+      incident,
+    };
+  });
 }
 
 /** The default merge policy when no instance variables are readable (mid-handoff row, engine
@@ -1225,6 +1336,9 @@ export interface ActiveDelivery {
    * eligible for steward auto-resolution. Derived from the durable `auto_resolve_escalations` flag,
    * never a clobbered process variable. Shown in `/status` per delivery. */
   autoResolveEscalations: boolean;
+  /** The ACTIVE engine incident parking this run's instance (issue #931), mirrored by the poller's
+   * `pollIncidents` pass; null when none. While set, `status` reads `incident`. */
+  incident: StatusIncident | null;
 }
 
 /** Every delivery-graph run not in a terminal state (done/failed/abandoned), newest-updated first.
@@ -1248,16 +1362,22 @@ export async function activeDeliveries(data: DataLayer): Promise<ActiveDelivery[
     if (openEscByRun.has(t.subject_key)) continue;
     openEscByRun.set(t.subject_key, toOpenEscalation(t));
   }
-  return active.map((r) => ({
-    runKey: r.run_key,
-    title: r.title ?? null,
-    status: r.status,
-    phase: r.phase ?? null,
-    processKey: r.process_key ?? null,
-    openEscalation: openEscByRun.get(r.run_key) ?? null,
-    updatedAt: r.updated_at,
-    autoResolveEscalations: r.auto_resolve_escalations === 1,
-  }));
+  return active.map((r) => {
+    // Issue #931: same incident overlay as the PR rows — a delivery run parked on an engine
+    // incident must not keep reading `running`.
+    const incident = statusIncidentFor(r);
+    return {
+      runKey: r.run_key,
+      title: r.title ?? null,
+      status: incident ? INCIDENT_STATUS : r.status,
+      phase: r.phase ?? null,
+      processKey: r.process_key ?? null,
+      openEscalation: openEscByRun.get(r.run_key) ?? null,
+      updatedAt: r.updated_at,
+      autoResolveEscalations: r.auto_resolve_escalations === 1,
+      incident,
+    };
+  });
 }
 
 /** One review-ready poll pass (SPEC §10): for every PR waiting on a review, fetch its GitHub
@@ -2351,30 +2471,88 @@ export async function pollJobActivationImpl(
 }
 
 /** The subset of a Camunda-8 `/v2/incidents/search` result item this app reads. `incidentKey` is
- * the unique incident id; `errorMessage` is the human-readable fault; `state` is the incident
- * lifecycle (`ACTIVE` while it parks the token, `RESOLVED` once cleared); `creationTime` orders
- * concurrent incidents. */
-interface IncidentSearchItem {
+ * the unique incident id; `errorType` the machine fault class (e.g. `JOB_NO_RETRIES`);
+ * `errorMessage` is the human-readable fault; `state` is the incident lifecycle (`ACTIVE` while it
+ * parks the token, `RESOLVED` once cleared); `creationTime` orders concurrent incidents and is
+ * surfaced as the incident's `since`. */
+export interface IncidentSearchItem {
   incidentKey?: string;
+  errorType?: string | null;
   errorMessage?: string | null;
   state?: string;
   creationTime?: string | null;
+  /** The job the incident sits on (a `JOB_NO_RETRIES` incident parks until that job carries
+   * retries again) — the handle the `retryIncident` action bumps. */
+  jobKey?: string;
 }
 
-/** Incident-surfacing poll pass (issue #94). A convergence or merge process instance can hit a
- * *technical* incident — an unhandled engine error that parks the token — and nothing on the PR
- * row reflected it: the grid kept showing the last workflow status (`converging`, `merging`, …)
- * while the run was actually dead in the water (a PR sat "converging" all day on an incident).
+/** The four mirrored incident columns every incident-watched aggregate carries (PRs since #94/#931,
+ * delivery-graph runs since #931). All-or-nothing: a row with no ACTIVE incident reads NULL on all
+ * four. */
+interface IncidentMirror {
+  incident_key: string | null;
+  incident_type: string | null;
+  incident_message: string | null;
+  incident_since: string | null;
+}
+
+/** Pick the ACTIVE incident to mirror for one instance: the OLDEST active one (the first thing that
+ * broke — a stable choice if the instance somehow parks more than one). Re-filters on state
+ * defensively in case the wire filter is ignored. An incident with no `creationTime` sorts *last*,
+ * so a missing timestamp can never masquerade as the oldest. Returns null when none is active. */
+export function oldestActiveIncident(items: IncidentSearchItem[] | undefined): IncidentSearchItem | null {
+  return (
+    (items ?? [])
+      .filter((i) => (i.state ?? "ACTIVE") === "ACTIVE")
+      .sort((a, b) => (a.creationTime ?? "\uffff").localeCompare(b.creationTime ?? "\uffff"))[0] ?? null
+  );
+}
+
+/** Query the engine for the OLDEST ACTIVE incident on one process instance. Returns the incident,
+ * `null` when the instance has no active incident, or `undefined` when the transport failed (the
+ * caller keeps the last-known mirror and retries next pass — a failed query must never clear a real
+ * incident nor invent one). */
+export async function fetchActiveIncident(
+  base: string,
+  headers: Record<string, string>,
+  processInstanceKey: string,
+): Promise<IncidentSearchItem | null | undefined> {
+  try {
+    const res = await fetch(`${base}/incidents/search`, {
+      method: "POST",
+      headers,
+      body: JSON.stringify({
+        filter: { processInstanceKey, state: "ACTIVE" },
+        page: { limit: 20 },
+      }),
+    });
+    if (!res.ok) return undefined; // engine unhappy → unknown, retry next pass
+    // biome-ignore lint/plugin: runtime/framework contract boundary for external data shape
+    const body = (await res.json()) as { items?: IncidentSearchItem[] };
+    return oldestActiveIncident(body.items);
+  } catch {
+    return undefined;
+  }
+}
+
+/** Incident-surfacing poll pass (issue #94; enriched and extended to delivery-graph runs in issue
+ * #931). A convergence, merge, or delivery-graph process instance can hit a *technical* incident —
+ * an unhandled engine error that parks the token — and nothing on the record row reflected it: the
+ * grid kept showing the last workflow status (`converging`, `merging`, `running`, …) while the run
+ * was actually dead in the water (a PR sat "converging" all day on an incident; on merlin two loops
+ * sat dead for 1.3h/5h before anyone found them via a hand-run engine query).
  *
- * This pass reads the engine's Camunda-8 `/v2/incidents/search` for each PR that still has a live
- * instance (has a `process_key`, non-terminal status) and mirrors an ACTIVE incident onto two
- * orthogonal columns — `incident_key` + `incident_message` — leaving `status` untouched. An
- * incident is a cross-cutting liveness fault, not a workflow stage, so it must not overload the
- * status machine. Clearing is idempotent: when the instance has no active incident (resolved, or
- * never had one) the columns are nulled, so an incident raised or resolved out-of-band converges
- * to the truth on the next pass. Best-effort transport: a failed query leaves the last-known
- * values untouched and the next pass retries. Updates (and bumps `updated_at`) only on an actual
- * change so a steady state doesn't churn the grid. */
+ * This pass reads the engine's Camunda-8 `/v2/incidents/search` for each tracked record that still
+ * has a live instance (has a `process_key`, non-terminal derived status) and mirrors an ACTIVE
+ * incident onto four orthogonal columns — `incident_key`, `incident_type`, `incident_message`,
+ * `incident_since` — leaving `status` untouched. An incident is a cross-cutting liveness fault, not
+ * a workflow stage, so it must not overload the status machine (`/status` derives the `incident`
+ * overlay at READ time — see `activePrs`/`activeDeliveries`). Clearing is idempotent: when the
+ * instance has no active incident (resolved, or never had one) the columns are nulled, so an
+ * incident raised or resolved out-of-band converges to the truth on the next pass. Best-effort
+ * transport: a failed query leaves the last-known values untouched and the next pass retries.
+ * Updates (and bumps `updated_at`) only on an actual change so a steady state doesn't churn the
+ * grid. */
 async function pollIncidents(
   data: DataLayer,
   restAddress: string,
@@ -2387,12 +2565,24 @@ async function pollIncidents(
 }
 
 /** Testable core of {@link pollIncidents}: given the normalised `base` URL and prepared auth
- * `headers`, reconcile every PR row against the engine's active incidents. Split out so tests can
- * exercise the reconciliation with a stubbed `fetch` without re-deriving transport wiring. */
+ * `headers`, reconcile every incident-watched record row (PRs and delivery-graph runs) against the
+ * engine's active incidents. Split out so tests can exercise the reconciliation with a stubbed
+ * `fetch` without re-deriving transport wiring. */
 export async function pollIncidentsImpl(
   data: DataLayer,
   base: string,
   headers: Record<string, string>,
+) {
+  await reconcilePrIncidents(data, base, headers);
+  await reconcileDeliveryIncidents(data, base, headers);
+}
+
+/** The PR half of {@link pollIncidentsImpl}: mirror the oldest ACTIVE incident of each live PR
+ * instance onto the PR row's four incident columns. */
+async function reconcilePrIncidents(
+  data: DataLayer,
+  base: string,
+  headers: Record<string, string>,
 ) {
   const all = await prsTracking(data).all();
   for (const pr of all) {
@@ -2404,58 +2594,83 @@ export async function pollIncidentsImpl(
     // instance. Reuses the canonical `TERMINAL_STATUSES` so incident logic can't drift from the rest
     // of the status machine. Writes stay on the base `prs`.
     if (!pr.process_key || TERMINAL_STATUSES.includes(pr.derived_status)) {
-      if (pr.incident_key || pr.incident_message) {
-        await prs(data).update(pr.pr_key, {
-          incident_key: null,
-          incident_message: null,
-          updated_at: now(),
-        });
-      }
+      await clearIncidentMirror(pr, (patch) => prs(data).update(pr.pr_key, patch));
       continue;
     }
 
-    let incidentKey: string | null = null;
-    let incidentMessage: string | null = null;
+    let active: IncidentSearchItem | null | undefined;
     try {
-      const res = await fetch(`${base}/incidents/search`, {
-        method: "POST",
-        headers,
-        body: JSON.stringify({
-          filter: { processInstanceKey: pr.process_key, state: "ACTIVE" },
-          page: { limit: 20 },
-        }),
-      });
-      if (!res.ok) continue; // engine unhappy → keep last-known, retry next pass
-      // biome-ignore lint/plugin: runtime/framework contract boundary for external data shape
-      const body = (await res.json()) as { items?: IncidentSearchItem[] };
-      // Surface the oldest ACTIVE incident (the first thing that broke — a stable choice if the
-      // instance somehow parks more than one). Re-filter on state defensively in case the wire
-      // filter is ignored. An incident with no `creationTime` sorts *last*, so a missing timestamp
-      // can never masquerade as the oldest.
-      const active = (body.items ?? [])
-        .filter((i) => (i.state ?? "ACTIVE") === "ACTIVE")
-        .sort((a, b) =>
-          (a.creationTime ?? "\uffff").localeCompare(b.creationTime ?? "\uffff")
-        )[0];
-      if (active) {
-        incidentKey = active.incidentKey ?? null;
-        incidentMessage = active.errorMessage ?? null;
-      }
+      active = await fetchActiveIncident(base, headers, pr.process_key);
     } catch (err) {
       console.error(`[poller] incidents ${pr.pr_key}: ${err}`);
       continue;
     }
+    if (active === undefined) continue; // transport unhappy → keep last-known, retry next pass
 
-    if (
-      incidentKey !== (pr.incident_key ?? null) ||
-      incidentMessage !== (pr.incident_message ?? null)
-    ) {
-      await prs(data).update(pr.pr_key, {
-        incident_key: incidentKey,
-        incident_message: incidentMessage,
-        updated_at: now(),
-      });
+    await writeIncidentMirror(pr, active, (patch) => prs(data).update(pr.pr_key, patch));
+  }
+}
+
+/** The delivery-graph half of {@link pollIncidentsImpl} (issue #931): a delivery RUN's instance
+ * parks on an incident exactly as a convergence loop does, and was just as invisible — `/status`
+ * kept reporting `running`. Mirror onto the run row's four incident columns. */
+async function reconcileDeliveryIncidents(
+  data: DataLayer,
+  base: string,
+  headers: Record<string, string>,
+) {
+  const terminal: readonly string[] = DELIVERY_GRAPH_TERMINAL_STATUSES;
+  const all = await deliveryGraphRunsTracking(data).all();
+  for (const run of all) {
+    if (!run.process_key || terminal.includes(run.derived_status)) {
+      await clearIncidentMirror(run, (patch) => deliveryGraphRuns(data).update(run.run_key, patch));
+      continue;
+    }
+
+    let active: IncidentSearchItem | null | undefined;
+    try {
+      active = await fetchActiveIncident(base, headers, run.process_key);
+    } catch (err) {
+      console.error(`[poller] incidents ${run.run_key}: ${err}`);
+      continue;
     }
+    if (active === undefined) continue;
+
+    await writeIncidentMirror(run, active, (patch) => deliveryGraphRuns(data).update(run.run_key, patch));
+  }
+}
+
+/** Null all four incident columns when any is set (a terminal / instance-less record must not keep
+ * a stale mirror). Shared by both reconcile halves so the clearing shape can never drift. */
+async function clearIncidentMirror(
+  row: IncidentMirror,
+  update: (patch: IncidentMirror & { updated_at: string }) => Promise<unknown>,
+) {
+  if (row.incident_key || row.incident_type || row.incident_message || row.incident_since) {
+    await update({ incident_key: null, incident_type: null, incident_message: null, incident_since: null, updated_at: now() });
+  }
+}
+
+/** Write the mirrored incident onto the row — or clear it when `active` is null — updating (and
+ * bumping `updated_at`) only on an actual change so a steady state doesn't churn the grid. */
+async function writeIncidentMirror(
+  row: IncidentMirror,
+  active: IncidentSearchItem | null,
+  update: (patch: IncidentMirror & { updated_at: string }) => Promise<unknown>,
+) {
+  const next: IncidentMirror = {
+    incident_key: active?.incidentKey ?? null,
+    incident_type: active?.errorType ?? null,
+    incident_message: active?.errorMessage ?? null,
+    incident_since: active?.creationTime ?? null,
+  };
+  if (
+    next.incident_key !== (row.incident_key ?? null) ||
+    next.incident_type !== (row.incident_type ?? null) ||
+    next.incident_message !== (row.incident_message ?? null) ||
+    next.incident_since !== (row.incident_since ?? null)
+  ) {
+    await update({ ...next, updated_at: now() });
   }
 }
 

package/build-info.json

@@ -1,6 +1,6 @@
 {
-  "version": "0.206.4",
-  "gitSha": "b8c5e4e5cb59ce42bb2d6573e107d1865393fa27",
-  "gitTag": "v0.206.4",
-  "builtAt": "2026-11-09T21:50:28.782Z"
+  "version": "0.208.0",
+  "gitSha": "0e2d50e8610beb5e939cd3a143d28e62d690f272",
+  "gitTag": "v0.208.0",
+  "builtAt": "2026-11-10T03:09:52.732Z"
 }

package/db/migrations/126_pull_requests_incident_type_since.sql

@@ -0,0 +1,8 @@
+-- Incident enrichment (issue #931). The #94 mirror carried only `incident_key` + `incident_message`
+-- — enough to light a badge, not enough to answer "WHAT broke and SINCE WHEN" from the row, which is
+-- what an operator triaging a parked loop needs (the merlin incidents were `JOB_NO_RETRIES` idle
+-- timeouts; the errorType is the tell). `incident_type` is the engine incident's `errorType`;
+-- `incident_since` its `creationTime` (ISO). Both NULL when the instance has no active incident —
+-- the poller's `pollIncidents` pass clears all four columns together, idempotently.
+ALTER TABLE pull_requests ADD COLUMN incident_type TEXT;   -- the active incident's errorType (e.g. JOB_NO_RETRIES); NULL when none
+ALTER TABLE pull_requests ADD COLUMN incident_since TEXT;  -- the active incident's creationTime (ISO); NULL when none

package/db/migrations/127_delivery_graph_run_incidents.sql

@@ -0,0 +1,60 @@
+-- Delivery-graph incident mirroring (issue #931). A delivery-graph RUN's engine instance can park
+-- on a technical incident exactly as a convergence loop can — and was just as invisible: `/status`
+-- kept reporting `running` while nothing moved. These four columns are the delivery-side twin of
+-- the PR incident mirror (017 + 126), written by the SAME `pollIncidents` pass and surfaced on the
+-- `/status` delivery rows. All NULL when the run's instance has no active incident.
+ALTER TABLE delivery_graph_runs ADD COLUMN incident_key TEXT;      -- engine incidentKey of the active incident parking this run's instance; NULL when none
+ALTER TABLE delivery_graph_runs ADD COLUMN incident_type TEXT;     -- the active incident's errorType; NULL when none
+ALTER TABLE delivery_graph_runs ADD COLUMN incident_message TEXT;  -- the active incident's errorMessage; NULL when none
+ALTER TABLE delivery_graph_runs ADD COLUMN incident_since TEXT;    -- the active incident's creationTime (ISO); NULL when none
+
+-- Mirror the new columns through the `delivery_units` aggregate (ADR 0006) so the
+-- `delivery_graph_runs__units` compat VIEW stays row-for-row with the base table (as 120 did for
+-- `auto_resolve_escalations`): add the columns, then re-create the delivery-graph sync triggers and
+-- the compat VIEW with them.
+ALTER TABLE delivery_units ADD COLUMN incident_key TEXT;
+ALTER TABLE delivery_units ADD COLUMN incident_type TEXT;
+ALTER TABLE delivery_units ADD COLUMN incident_message TEXT;
+ALTER TABLE delivery_units ADD COLUMN incident_since TEXT;
+
+DROP TRIGGER IF EXISTS delivery_graph_runs__du_ai;
+CREATE TRIGGER delivery_graph_runs__du_ai AFTER INSERT ON delivery_graph_runs
+BEGIN
+  INSERT OR REPLACE INTO delivery_units (unit_id, kind, legacy_key, legacy_id, parent_unit_id, node_index, delivery_status, dispatch_status, process_key, process_definition_id, digest, status, side_effecting, node_count, human_node_count, side_effect_count, title, phase, phase_node_id, human_labels, acknowledged_at, auto_resolve_escalations, incident_key, incident_type, incident_message, incident_since, created_at, updated_at)
+  VALUES ('delivery-graph:' || NEW.run_key, 'delivery-graph', NEW.run_key, NULL, NULL, NULL, CASE WHEN NEW.status = 'awaiting-approval' THEN 'requested' WHEN NEW.status = 'running' THEN 'running' WHEN NEW.status = 'done' THEN 'done' WHEN NEW.status = 'failed' THEN 'failed' WHEN NEW.status = 'abandoned' THEN 'abandoned' ELSE NULL END, CASE WHEN NEW.status IN ('awaiting-approval') THEN 'pending' WHEN NEW.status IN ('done', 'failed', 'abandoned') THEN 'settled' WHEN NEW.status IN ('running') THEN 'dispatched' ELSE NULL END, NEW.process_key, NEW.process_definition_id, NEW.digest, NEW.status, NEW.side_effecting, NEW.node_count, NEW.human_node_count, NEW.side_effect_count, NEW.title, NEW.phase, NEW.phase_node_id, NEW.human_labels, NEW.acknowledged_at, NEW.auto_resolve_escalations, NEW.incident_key, NEW.incident_type, NEW.incident_message, NEW.incident_since, NEW.created_at, NEW.updated_at);
+END;
+
+DROP TRIGGER IF EXISTS delivery_graph_runs__du_au;
+CREATE TRIGGER delivery_graph_runs__du_au AFTER UPDATE ON delivery_graph_runs
+BEGIN
+  INSERT OR REPLACE INTO delivery_units (unit_id, kind, legacy_key, legacy_id, parent_unit_id, node_index, delivery_status, dispatch_status, process_key, process_definition_id, digest, status, side_effecting, node_count, human_node_count, side_effect_count, title, phase, phase_node_id, human_labels, acknowledged_at, auto_resolve_escalations, incident_key, incident_type, incident_message, incident_since, created_at, updated_at)
+  VALUES ('delivery-graph:' || NEW.run_key, 'delivery-graph', NEW.run_key, NULL, NULL, NULL, CASE WHEN NEW.status = 'awaiting-approval' THEN 'requested' WHEN NEW.status = 'running' THEN 'running' WHEN NEW.status = 'done' THEN 'done' WHEN NEW.status = 'failed' THEN 'failed' WHEN NEW.status = 'abandoned' THEN 'abandoned' ELSE NULL END, CASE WHEN NEW.status IN ('awaiting-approval') THEN 'pending' WHEN NEW.status IN ('done', 'failed', 'abandoned') THEN 'settled' WHEN NEW.status IN ('running') THEN 'dispatched' ELSE NULL END, NEW.process_key, NEW.process_definition_id, NEW.digest, NEW.status, NEW.side_effecting, NEW.node_count, NEW.human_node_count, NEW.side_effect_count, NEW.title, NEW.phase, NEW.phase_node_id, NEW.human_labels, NEW.acknowledged_at, NEW.auto_resolve_escalations, NEW.incident_key, NEW.incident_type, NEW.incident_message, NEW.incident_since, NEW.created_at, NEW.updated_at);
+END;
+
+DROP VIEW IF EXISTS delivery_graph_runs__units;
+CREATE VIEW delivery_graph_runs__units AS
+SELECT
+  du.legacy_key AS run_key,
+  du.process_key AS process_key,
+  du.process_definition_id AS process_definition_id,
+  du.digest AS digest,
+  du.status AS status,
+  du.side_effecting AS side_effecting,
+  du.node_count AS node_count,
+  du.human_node_count AS human_node_count,
+  du.side_effect_count AS side_effect_count,
+  du.title AS title,
+  du.phase AS phase,
+  du.phase_node_id AS phase_node_id,
+  du.human_labels AS human_labels,
+  du.created_at AS created_at,
+  du.updated_at AS updated_at,
+  du.acknowledged_at AS acknowledged_at,
+  du.auto_resolve_escalations AS auto_resolve_escalations,
+  du.incident_key AS incident_key,
+  du.incident_type AS incident_type,
+  du.incident_message AS incident_message,
+  du.incident_since AS incident_since
+FROM delivery_units du
+WHERE du.kind = 'delivery-graph';
+