@nanobpm/nano-workforce 0.206.4 → 0.207.0

package/CHANGELOG.md

@@ -1,3 +1,9 @@
+## [0.207.0](https://github.com/nanobpm/nano-workforce/compare/v0.206.4...v0.207.0) (2026-11-09)
+
+### Features
+
+* **status:** surface engine incidents on loop and delivery rows with a retry action ([#938](https://github.com/nanobpm/nano-workforce/issues/938)) ([e3955c8](https://github.com/nanobpm/nano-workforce/commit/e3955c8145a966a29806400872b7a9c362d1d992)), closes [#931](https://github.com/nanobpm/nano-workforce/issues/931) [#931](https://github.com/nanobpm/nano-workforce/issues/931)
+
 ## [0.206.4](https://github.com/nanobpm/nano-workforce/compare/v0.206.3...v0.206.4) (2026-11-09)
 
 ### Bug Fixes

package/app/deliveryGraphReadModel.test.ts

@@ -39,7 +39,7 @@ import { applyMigrationSet, readMigrationSetFromDisk } from "../test/migrations.
 const MIG = (name: string) => readFileSync(fileURLToPath(new URL(`../db/migrations/${name}`, import.meta.url)), "utf8");
 const PAGE = (name: string) => JSON.parse(readFileSync(fileURLToPath(new URL(`../pages/${name}`, import.meta.url)), "utf8"));
 
-const READ_MODEL_MIGRATION = "121_delivery_graph_read_model_auto_resolve.sql";
+const READ_MODEL_MIGRATION = "129_delivery_graph_read_model_incidents.sql";
 
 // A minimal in-memory DB carrying the base `delivery_graph_runs` / `pull_requests` shapes the VIEW
 // reads, plus stand-ins for the managed `<table>__tracking` derived VIEWs urban provisions at mount
@@ -53,7 +53,9 @@ function viewDb(): DatabaseSync {
        run_key TEXT PRIMARY KEY, process_key TEXT, process_definition_id TEXT, digest TEXT,
        status TEXT, side_effecting INTEGER, node_count INTEGER, human_node_count INTEGER,
        side_effect_count INTEGER, title TEXT, phase TEXT, phase_node_id TEXT, human_labels TEXT,
-       created_at TEXT, updated_at TEXT, acknowledged_at TEXT, auto_resolve_escalations INTEGER, derived_status_override TEXT);
+       created_at TEXT, updated_at TEXT, acknowledged_at TEXT, auto_resolve_escalations INTEGER,
+       incident_key TEXT, incident_type TEXT, incident_message TEXT, incident_since TEXT,
+       derived_status_override TEXT);
      CREATE TABLE pull_requests (pr_key TEXT PRIMARY KEY, root_request_key TEXT, status TEXT,
        derived_status_override TEXT);`,
   );

package/app/deliveryGraphRun.ts

@@ -56,6 +56,14 @@ export interface DeliveryGraphRun {
   /** Opt-in escalation steward (120_escalation_auto_resolutions.sql, issue #889): 1 ⇒ this run's
    * escalations are eligible for steward auto-resolution. Default 0. */
   auto_resolve_escalations: number;
+  /** Technical-incident mirror (127_delivery_graph_run_incidents.sql, issue #931), written by the
+   *  poller's `pollIncidents` pass — the delivery-side twin of the PR incident columns (017/126).
+   *  All NULL when the run's instance has no active incident; `/status` derives the `incident`
+   *  overlay from these columns (see `activeDeliveries` in app/service.ts). */
+  incident_key: string | null;
+  incident_type: string | null;
+  incident_message: string | null;
+  incident_since: string | null;
 }
 
 /** The run lifecycle. `awaiting-approval` is RESERVED but no longer produced (issue #460 moved dispatch
@@ -530,5 +538,9 @@ export function buildDeliveryGraphRunRow(input: {
     created_at: input.createdAt ?? at,
     updated_at: at,
     acknowledged_at: null,
+    incident_key: null,
+    incident_type: null,
+    incident_message: null,
+    incident_since: null,
   };
 }

package/app/incidentStatus.test.ts

@@ -0,0 +1,175 @@
+// Red/green for issue #931 — an engine incident parked on a convergence loop's job must be VISIBLE
+// on the `/status` row, not hide behind a healthy-looking `status: "converging"`.
+//
+// The defect (observed on merlin, 2026-11-09): a loop whose `senior:pr-review` job exhausted its
+// retries (`JOB_NO_RETRIES`, the agent idle-timeout) sat dead for hours while `/status` reported
+// `status: "converging"`, `openEscalation: null`, `activeWorker: null` — indistinguishable from a
+// healthy queued run. `pollIncidents` mirrored the incident onto `incident_key`/`incident_message`
+// (issue #94), but the `/status` projection never READ those columns, so the only discovery path
+// was a hand-run `POST /v2/incidents/search`.
+//
+// The fix surfaces the mirrored incident on every ActivePr/ActiveDelivery row:
+//   status   — overlaid to "incident" while the row carries an ACTIVE incident (the base
+//              `pull_requests.status` workflow transient is untouched; the overlay is derived at
+//              read time so it can never wedge the status machine), and
+//   incident — { key, type, message, since }: the engine incidentKey, errorType, errorMessage and
+//              creationTime the poller mirrored, so an operator sees WHAT broke and WHEN without
+//              opening the engine.
+import { test } from "node:test";
+import { assertEquals } from "#test-assert";
+import { withTrackingViews } from "../test/trackingViews.ts";
+import { activeDeliveries, activePrs } from "./service.ts";
+
+function memTable(rows: any[], key: string) {
+  return {
+    get: (k: any) => Promise.resolve(rows.find((r) => r[key] === k) ?? null),
+    all: () => Promise.resolve([...rows]),
+    find: (q: any) =>
+      Promise.resolve(rows.filter((r) => Object.entries(q).every(([f, v]) => r[f] === v))),
+    findOne: (q: any) =>
+      Promise.resolve(rows.find((r) => Object.entries(q).every(([f, v]) => r[f] === v)) ?? null),
+    insert: (r: any) => {
+      rows.push(r);
+      return Promise.resolve(r);
+    },
+    update: (k: any, patch: any) => {
+      const r = rows.find((x) => x[key] === k);
+      if (r) Object.assign(r, patch);
+      return Promise.resolve(r);
+    },
+    delete: (k: any) => {
+      for (let i = rows.length - 1; i >= 0; i--) if (rows[i][key] === k) rows.splice(i, 1);
+      return Promise.resolve();
+    },
+  };
+}
+
+type Stores = Record<string, { rows: any[]; key: string }>;
+
+function memData(stores: Stores) {
+  return {
+    table: withTrackingViews((name: string, key: string) => memTable(stores[name]?.rows ?? [], stores[name]?.key ?? key)),
+  } as any;
+}
+
+/** A fake engine whose `searchVariables` serves the given per-instance variables (none here — the
+ *  merge policy degrades to its historical default, which these tests do not exercise). */
+function fakeEngine() {
+  return {
+    searchVariables: () => Promise.resolve([]),
+  } as any;
+}
+
+function prRow(over: Record<string, unknown>) {
+  return {
+    pr_key: "owner/repo#1",
+    repo: "owner/repo",
+    number: 1,
+    url: "https://github.com/owner/repo/pull/1",
+    title: "t",
+    status: "converging",
+    current_round: 1,
+    process_key: "PI-1",
+    updated_at: "2024-01-01",
+    ...over,
+  };
+}
+
+async function statusOf(rowOver: Record<string, unknown> = {}) {
+  const stores: Stores = {
+    pull_requests: { rows: [prRow(rowOver)], key: "pr_key" },
+    user_tasks: { rows: [], key: "id" },
+  };
+  const [pr] = await activePrs(memData(stores), fakeEngine());
+  return pr;
+}
+
+test("incident surfacing: a PR row carrying an ACTIVE incident reads status 'incident', not 'converging' (the #931 defect)", async () => {
+  const pr = await statusOf({
+    incident_key: "598235",
+    incident_type: "JOB_NO_RETRIES",
+    incident_message: 'agent "nano-coder" failed: no output and no subtree progress for >= 300000ms (idle)',
+    incident_since: "2026-11-09T01:00:00.000Z",
+  });
+  assertEquals(pr.status, "incident");
+  assertEquals(pr.incident, {
+    key: "598235",
+    type: "JOB_NO_RETRIES",
+    message: 'agent "nano-coder" failed: no output and no subtree progress for >= 300000ms (idle)',
+    since: "2026-11-09T01:00:00.000Z",
+  });
+});
+
+test("incident surfacing: a healthy PR row keeps its workflow status and a null incident", async () => {
+  const pr = await statusOf({
+    incident_key: null,
+    incident_type: null,
+    incident_message: null,
+    incident_since: null,
+  });
+  assertEquals(pr.status, "converging");
+  assertEquals(pr.incident, null);
+});
+
+test("incident surfacing: the overlay applies to any live workflow status (e.g. waiting_merge), never just converging", async () => {
+  const pr = await statusOf({
+    status: "waiting_merge",
+    incident_key: "INC-9",
+    incident_type: "EXTRACT_VALUE_ERROR",
+    incident_message: "failed to evaluate expression",
+    incident_since: "2026-11-09T02:00:00.000Z",
+  });
+  assertEquals(pr.status, "incident");
+  assertEquals(pr.incident?.key, "INC-9");
+});
+
+test("incident surfacing: a delivery run carrying an ACTIVE incident reads status 'incident' with the incident block", async () => {
+  const run = {
+    run_key: "dg-1",
+    title: "ship it",
+    status: "running",
+    phase: "Running",
+    process_key: "PI-DG-1",
+    updated_at: "2024-01-01",
+    auto_resolve_escalations: 0,
+    incident_key: "INC-DG-1",
+    incident_type: "JOB_NO_RETRIES",
+    incident_message: "delivery node job exhausted retries",
+    incident_since: "2026-11-09T03:00:00.000Z",
+  };
+  const stores: Stores = {
+    delivery_graph_runs: { rows: [run], key: "run_key" },
+    user_tasks: { rows: [], key: "id" },
+  };
+  const [delivery] = await activeDeliveries(memData(stores));
+  assertEquals(delivery.status, "incident");
+  assertEquals(delivery.incident, {
+    key: "INC-DG-1",
+    type: "JOB_NO_RETRIES",
+    message: "delivery node job exhausted retries",
+    since: "2026-11-09T03:00:00.000Z",
+  });
+});
+
+test("incident surfacing: a healthy delivery run keeps its status and a null incident", async () => {
+  const run = {
+    run_key: "dg-2",
+    title: "ship it",
+    status: "running",
+    phase: "Running",
+    process_key: "PI-DG-2",
+    updated_at: "2024-01-01",
+    auto_resolve_escalations: 0,
+    incident_key: null,
+    incident_type: null,
+    incident_message: null,
+    incident_since: null,
+  };
+  const stores: Stores = {
+    delivery_graph_runs: { rows: [run], key: "run_key" },
+    user_tasks: { rows: [], key: "id" },
+  };
+  const [delivery] = await activeDeliveries(memData(stores));
+  assertEquals(delivery.status, "running");
+  assertEquals(delivery.incident, null);
+});

package/app/mcpExclusions.test.ts

@@ -31,9 +31,10 @@ function projectedOperations() {
 }
 
 // The operator-only cockpit doors: the staged delivery-graph lifecycle (stage -> dispatch ->
-// dismiss) whose approval is a human click in the cockpit (ADR 0005 Decision 7). These — and ONLY
-// these — must be excluded from the projected MCP tool surface.
-const EXPECTED_EXCLUDED = ["stageDeliveryGraph", "dispatchDeliveryGraph", "dismissProposal"];
+// dismiss) whose approval is a human click in the cockpit (ADR 0005 Decision 7), plus the
+// incident-retry remediation door (issue #931) — the human unstick action on an incident-parked
+// run. These — and ONLY these — must be excluded from the projected MCP tool surface.
+const EXPECTED_EXCLUDED = ["stageDeliveryGraph", "dispatchDeliveryGraph", "dismissProposal", "retryIncident"];
 
 // A representative set of agent-facing operations that MUST stay exposed as tools — the drive doors
 // (submit PR / epic / delivery-graph set), the pure compile/preview doors, and the read/orientation

package/app/mcpToolSurface.ts

@@ -87,8 +87,12 @@ export const FRAMEWORK_TOOL_PREFIX = "urban_debug_";
  * regression guard, not the reduction itself: the reduction an agent actually experiences comes from
  * importing {@link CURATED_MCP_TOOLS} rather than `["*"]`, and the transport-level shrink of the
  * framework tool family is tracked upstream (nano-ide#488).
+ *
+ * RAISE PROVENANCE — 60 → 61 (#931): the read-only `listIncidents` door (sibling of
+ * `listEscalations`) stays on the surface per the `x-mcp` convention that read/orient doors are
+ * exposed; its operator-only remedy `retryIncident` is excluded. Deliberate, documented growth.
  */
-export const MCP_TOOL_COUNT_BUDGET = 60;
+export const MCP_TOOL_COUNT_BUDGET = 61;
 
 /**
  * Hard CEILING on the serialized byte size of the full `tools/list` payload (the schema bytes a
@@ -138,8 +142,13 @@ export const MCP_TOOL_COUNT_BUDGET = 60;
  * intended description text (a correctness fix, not new surface). Measured 90,420 bytes serialized
  * (over the old ceiling's 920-byte headroom). Deliberate, documented growth of restored description
  * text — not schema fat.
+ *
+ * RAISE PROVENANCE — 91_000 → 92_000 (#931): the read-only `listIncidents` tool (engine-incident
+ * liveness sibling of `listEscalations`) plus the `incident` field on the status row schemas.
+ * Measured 91,069 bytes serialized (over the old ceiling's headroom). Deliberate, documented growth
+ * of a real new read surface — not schema fat.
  */
-export const MCP_SURFACE_BYTES_BUDGET = 91_000;
+export const MCP_SURFACE_BYTES_BUDGET = 92_000;
 
 /**
  * The eagerly-loaded curated subset MUST stay materially smaller than the full surface — otherwise it

package/app/pullRequestReadModel.test.ts

@@ -33,7 +33,7 @@ import {
 const MIG = (name: string) => readFileSync(fileURLToPath(new URL(`../db/migrations/${name}`, import.meta.url)), "utf8");
 const PAGE = (name: string) => JSON.parse(readFileSync(fileURLToPath(new URL(`../pages/${name}`, import.meta.url)), "utf8"));
 
-const READ_MODEL_MIGRATION = "122_pull_requests_read_model_auto_resolve.sql";
+const READ_MODEL_MIGRATION = "128_pull_requests_read_model_incidents.sql";
 
 // The real base `pull_requests` columns, in schema order — DERIVED from the migration chain (not a
 // hand-kept list that could silently omit one), used by both the drift guard and the e2e stand-in.

package/app/service.test.ts

@@ -675,6 +675,123 @@ test("pollIncidents picks the oldest incident by creationTime, sorting a missing
   assertEquals(row.incident_message, "the first fault");
 });
 
+// Issue #931: the mirror must carry the incident's `errorType` and `creationTime` (not just key +
+// message) so `/status` can show WHAT broke and SINCE WHEN, and it must cover a delivery-graph
+// run's instance too — a delivery parked on an engine incident was just as invisible as a PR.
+test("pollIncidents mirrors errorType + creationTime alongside the key/message, and clears them with the incident", async () => {
+  const row = {
+    pr_key: "owner/repo#12",
+    status: "converging",
+    process_key: "PI-12",
+    incident_key: null as string | null,
+    incident_type: null as string | null,
+    incident_message: null as string | null,
+    incident_since: null as string | null,
+    updated_at: "t0",
+  };
+  const stores: Record<string, { rows: unknown[]; key: string }> = {
+    pull_requests: { rows: [row], key: "pr_key" },
+  };
+  const data = {
+    table: withTrackingViews((name: string, key: string) => memTable(stores[name]?.rows ?? [], stores[name]?.key ?? key)),
+    open: () => memOpen(stores),
+  } as any;
+  const headers = { "content-type": "application/json" };
+
+  const prevFetch = globalThis.fetch;
+  globalThis.fetch = incidentFetch({
+    "PI-12": [{
+      incidentKey: "598235",
+      errorType: "JOB_NO_RETRIES",
+      errorMessage: 'agent "nano-coder" failed: no output and no subtree progress for >= 300000ms (idle)',
+      state: "ACTIVE",
+      creationTime: "2026-11-09T01:00:00.000Z",
+    }],
+  }) as typeof fetch;
+  try {
+    await pollIncidentsImpl(data, "http://engine/v2", headers);
+  } finally {
+    globalThis.fetch = prevFetch;
+  }
+  assertEquals(row.incident_key, "598235");
+  assertEquals(row.incident_type, "JOB_NO_RETRIES");
+  assertEquals(row.incident_message, 'agent "nano-coder" failed: no output and no subtree progress for >= 300000ms (idle)');
+  assertEquals(row.incident_since, "2026-11-09T01:00:00.000Z");
+  assertEquals(row.status, "converging"); // orthogonal: the workflow transient is never touched
+
+  // Once resolved, ALL FOUR columns clear idempotently — a stale `incident_since` must not linger.
+  globalThis.fetch = incidentFetch({ "PI-12": [] }) as typeof fetch;
+  try {
+    await pollIncidentsImpl(data, "http://engine/v2", headers);
+  } finally {
+    globalThis.fetch = prevFetch;
+  }
+  assertEquals(row.incident_key, null);
+  assertEquals(row.incident_type, null);
+  assertEquals(row.incident_message, null);
+  assertEquals(row.incident_since, null);
+});
+
+test("pollIncidents mirrors an ACTIVE incident onto a delivery-graph run row, and skips a terminal run", async () => {
+  const running = {
+    run_key: "dg-1",
+    status: "running",
+    process_key: "PI-DG-1",
+    incident_key: null as string | null,
+    incident_type: null as string | null,
+    incident_message: null as string | null,
+    incident_since: null as string | null,
+    updated_at: "t0",
+  };
+  const done = {
+    run_key: "dg-2",
+    status: "done",
+    process_key: "PI-DG-2",
+    incident_key: "STALE-DG",
+    incident_type: "JOB_NO_RETRIES",
+    incident_message: "left over",
+    incident_since: "2026-11-08T00:00:00.000Z",
+    updated_at: "t0",
+  };
+  const stores: Record<string, { rows: unknown[]; key: string }> = {
+    delivery_graph_runs: { rows: [running, done], key: "run_key" },
+  };
+  const data = {
+    table: withTrackingViews((name: string, key: string) => memTable(stores[name]?.rows ?? [], stores[name]?.key ?? key)),
+    open: () => memOpen(stores),
+  } as any;
+  const headers = { "content-type": "application/json" };
+
+  const prevFetch = globalThis.fetch;
+  globalThis.fetch = incidentFetch({
+    "PI-DG-1": [{
+      incidentKey: "INC-DG-1",
+      errorType: "JOB_NO_RETRIES",
+      errorMessage: "delivery node job exhausted retries",
+      state: "ACTIVE",
+      creationTime: "2026-11-09T03:00:00.000Z",
+    }],
+    // A terminal run's instance is gone — the pass must never query it (the stub would serve an
+    // incident, so a query would wrongly overwrite the clearing below).
+    "PI-DG-2": [{ incidentKey: "INC-DG-2", errorMessage: "ghost", state: "ACTIVE", creationTime: "2026-11-09T04:00:00.000Z" }],
+  }) as typeof fetch;
+  try {
+    await pollIncidentsImpl(data, "http://engine/v2", headers);
+  } finally {
+    globalThis.fetch = prevFetch;
+  }
+  assertEquals(running.incident_key, "INC-DG-1");
+  assertEquals(running.incident_type, "JOB_NO_RETRIES");
+  assertEquals(running.incident_message, "delivery node job exhausted retries");
+  assertEquals(running.incident_since, "2026-11-09T03:00:00.000Z");
+  assertEquals(running.status, "running"); // orthogonal: status untouched
+  // Terminal run: the stale incident is cleared without ever consulting the engine.
+  assertEquals(done.incident_key, null);
+  assertEquals(done.incident_type, null);
+  assertEquals(done.incident_message, null);
+  assertEquals(done.incident_since, null);
+});
+
 
 // Red/green regression (nano-workforce#845 Copilot review): the convergence loop runs TWO external
 // agent task types while the PR reads `converging` — `review-round` (`senior:pr-review`) and, since