@nanobpm/nano-workforce 0.209.0 → 0.210.0

package/workers/merge-stall-probe/worker.test.ts

@@ -0,0 +1,119 @@
+// pr.merge-stall-probe — issue #951: the probe must re-read PR state before the merge loop can
+// escalate. A PR that already MERGED (a human queued it while the loop waited) must surface `ready`
+// so `gw-merge-stall` routes it to `attempt-merge` (whose already-merged short-circuit ends the loop
+// as merged) instead of exhausting the stall budget into a "merge-blocked" escalation.
+import { test } from "node:test";
+import { assertEquals } from "#test-assert";
+import type { PrState } from "../../app/github.ts";
+import { DEFAULT_MERGE_PROTOCOL, type MergeProtocol } from "../../app/mergeProtocol.ts";
+import { noopLog } from "../../test/log.ts";
+import { makeHandler } from "./worker.ts";
+
+function state(over: Partial<PrState>): PrState {
+  return {
+    merged: false,
+    state: "open",
+    mergeStateStatus: "BLOCKED",
+    failingChecks: 0,
+    failingCheckNames: [],
+    presentCheckNames: ["build", "Mergify Merge Queue"],
+    pendingCheckNames: [],
+    checkConclusions: { build: "SUCCESS", "Mergify Merge Queue": "NEUTRAL" },
+    totalChecks: 2,
+    isDraft: false,
+    headRefOid: "abc",
+    mergeQueueEntry: null,
+    unresolvedThreads: 0,
+    ...over,
+  };
+}
+
+const MERGIFY: MergeProtocol = {
+  ...DEFAULT_MERGE_PROTOCOL,
+  land: { method: "mergify-queue", comment: "@mergifyio queue" },
+  requiredChecks: [{ name: "build", acceptedConclusions: ["success"] }],
+};
+
+async function run(st: PrState, protocol: MergeProtocol | null, isUpdatePending = false): Promise<Record<string, unknown>> {
+  const handler = makeHandler({
+    readState: () => Promise.resolve(st),
+    readProtocol: () => Promise.resolve(protocol),
+    isUpdatePending: () => isUpdatePending,
+  });
+  const job: any = { variables: { prKey: "o/r#1", repo: "o/r", prNumber: 1 } };
+  const app: any = { log: noopLog() };
+  return (await handler(job, app)) as Record<string, unknown>;
+}
+
+async function runNullRead(isUpdatePending = false): Promise<Record<string, unknown>> {
+  const handler = makeHandler({
+    readState: () => Promise.reject(new Error("GitHub unreachable")),
+    readProtocol: () => Promise.resolve(null),
+    isUpdatePending: () => isUpdatePending,
+  });
+  const job: any = { variables: { prKey: "o/r#1", repo: "o/r", prNumber: 1 } };
+  const app: any = { log: noopLog() };
+  return (await handler(job, app)) as Record<string, unknown>;
+}
+
+test("merge-stall-probe: a PR already MERGED surfaces `ready` (never an escalation) (#951)", async () => {
+  const out = await run(state({ merged: true, state: "merged", mergeStateStatus: "UNKNOWN" }), MERGIFY);
+  assertEquals(out.mergeState, "ready");
+});
+
+test("merge-stall-probe: a CLOSED PR surfaces `ready` so attempt-merge abandons it (#951)", async () => {
+  const out = await run(state({ state: "closed", mergeStateStatus: "UNKNOWN" }), null);
+  assertEquals(out.mergeState, "ready");
+});
+
+test("merge-stall-probe: green PR awaiting `@mergifyio queue` is `ready`, not a review stall (#951)", async () => {
+  const out = await run(state({}), MERGIFY);
+  assertEquals(out.mergeState, "ready");
+});
+
+test("merge-stall-probe: BLOCKED gh-merge PR with green checks still waits (#951)", async () => {
+  const out = await run(state({}), { ...MERGIFY, land: { method: "gh-merge" } });
+  assertEquals(out.mergeState, "waiting");
+});
+
+// Issue #951 review: while an accepted (async) update-branch still exposes the old head, the probe
+// must NOT classify that head's stale check failures as `blocked` (which starts fix-ci / escalates).
+// It defers to `waiting` (bounded re-poll) until the head moves or the remedy's window lapses.
+test("merge-stall-probe: a blocked verdict on a pending-update old head defers to waiting (#951)", async () => {
+  const blocked = state({ failingChecks: 1, failingCheckNames: ["build"], checkConclusions: { build: "FAILURE" } });
+  // Sanity: without a pending update this IS blocked (would route to fix-ci).
+  assertEquals((await run(blocked, MERGIFY, false)).mergeState, "blocked");
+  // With the update still pending on this head, the probe holds remediation.
+  const held = await run(blocked, MERGIFY, true);
+  assertEquals(held.mergeState, "waiting");
+  // The hold is surfaced so gw-merge-stall re-polls past the stall cap (only for this hold).
+  assertEquals(held.updatePending, true);
+  assertEquals((await run(blocked, MERGIFY, false)).updatePending, false);
+});
+
+test("merge-stall-probe: a non-blocked verdict is never downgraded by a pending update (#951)", async () => {
+  // A ready PR stays ready even if (spuriously) an update is pending — the hold only masks `blocked`.
+  const out = await run(state({}), MERGIFY, true);
+  assertEquals(out.mergeState, "ready");
+});
+
+// PR #953 review: a probe that emits `updatePending: true` is followed (next re-poll) by a read that
+// does NOT recompute the hold — a failed/null state read, or a terminal (merged/closed) liveness
+// short-circuit. Those paths must explicitly clear the flag: an omitted field would leave the stale
+// `true` standing, and `gw-merge-stall` would keep bypassing the stall cap on it even after the
+// remedy's bounded hold expired — a GitHub outage would turn the bounded hold into an indefinite
+// re-poll.
+test("merge-stall-probe: a failed/null state read clears updatePending (no stale cap bypass) (#953)", async () => {
+  const out = await runNullRead(true);
+  assertEquals(out.mergeState, "waiting");
+  assertEquals(out.updatePending, false);
+});
+
+test("merge-stall-probe: a terminal (merged/closed) read clears updatePending (#953)", async () => {
+  const merged = await run(state({ merged: true, state: "merged", mergeStateStatus: "UNKNOWN" }), MERGIFY, true);
+  assertEquals(merged.mergeState, "ready");
+  assertEquals(merged.updatePending, false);
+  const closed = await run(state({ state: "closed", mergeStateStatus: "UNKNOWN" }), null, true);
+  assertEquals(closed.mergeState, "ready");
+  assertEquals(closed.updatePending, false);
+});

package/workers/merge-stall-probe/worker.ts

@@ -17,8 +17,9 @@
 // (incremented by this task's output mapping, capped by `mergeStallMax`) stops the probe from
 // re-arming forever: once exhausted, `gw-merge-stall` routes to human escalation instead.
 import type { AppJobHandler } from "@nanobpm/urban";
-import { classifyMergeability, fetchPrState } from "../../app/github.ts";
-import { loadMergeProtocol } from "../../app/mergeProtocol.ts";
+import { isBehindBaseUpdatePending } from "../../app/behindBaseRemedy.ts";
+import { classifyMergeability, classifyPrLiveness, fetchPrState, type PrState } from "../../app/github.ts";
+import { loadMergeProtocol, type MergeProtocol } from "../../app/mergeProtocol.ts";
 import type { WorkerInputs } from "../../nano-generated/worker-io.d.ts";
 
 // Input typed off the model data envelope (`MergeStallProbeIn` in merge-loop.bpmn) — ADR 0040.
@@ -43,7 +44,25 @@ interface Out extends Record<string, unknown> {
   pendingChecks: number;
 }
 
-const handler: AppJobHandler<In, Out> = async (job, app) => {
+/** Injectable GitHub readers (tests stub them); the default export binds the real ones. */
+export interface StallProbeDeps {
+  readState: (repo: string, prNumber: number, token: string) => Promise<PrState | null>;
+  readProtocol: (repo: string, token: string) => Promise<MergeProtocol | null>;
+  /** Issue #951 review: is an accepted (async) update-branch still holding this head? While it is,
+   * the old head's stale check failures must NOT classify as `blocked` (which would start fix-ci /
+   * escalate) — the branch update is in flight and will move the head. Bounded by the remedy's own
+   * window, so a never-applied update stops holding. */
+  isUpdatePending: (repo: string, prNumber: number, headOid: string) => boolean;
+}
+
+const realDeps: StallProbeDeps = {
+  readState: (repo, prNumber, token) => fetchPrState(repo, prNumber, token, { withReviewThreads: true }),
+  readProtocol: (repo, token) => loadMergeProtocol(repo, token),
+  isUpdatePending: (repo, prNumber, headOid) => isBehindBaseUpdatePending(repo, prNumber, headOid),
+};
+
+export function makeHandler(deps: StallProbeDeps): AppJobHandler<In, Out> {
+  return async (job, app) => {
   const { repo, prNumber } = job.variables;
   const token = process.env.GITHUB_TOKEN ?? "";
 
@@ -57,12 +76,50 @@ const handler: AppJobHandler<In, Out> = async (job, app) => {
   // classified by its real cause (issue #921) — unresolved review threads surface as the
   // `review-threads` verdict and route to their own arm instead of re-polling until the budget
   // exhausts into the misleading "mergeability never settled" escalation.
-  const st = await fetchPrState(repo, prNumber, token, { withReviewThreads: true }).catch(() => null);
+  const st = await deps.readState(repo, prNumber, token).catch(() => null);
   if (st === null) {
-    return { mergeState: "waiting", failingChecks: 0, failingChecksList: "", reviewThreads: -1, pendingChecks: 0 };
+    // Explicitly clear `updatePending` (PR #953 review): a failed/null read returns `waiting`
+    // WITHOUT recomputing the hold, and an omitted field would leave any earlier `true` standing —
+    // `gw-merge-stall` would then keep bypassing the stall cap on the stale flag even after the
+    // remedy's bounded hold expired. Returning `false` keeps a GitHub outage from turning the
+    // bounded hold into an indefinite re-poll.
+    return { mergeState: "waiting", updatePending: false, failingChecks: 0, failingChecksList: "", reviewThreads: -1, pendingChecks: 0 };
+  }
+  // Issue #951: re-read PR liveness BEFORE any verdict. A PR that already MERGED (e.g. a human
+  // queued it while the loop waited) or was CLOSED must never reach an escalation: surface `ready`,
+  // which `gw-merge-stall` always routes (even past the stall budget) to `attempt-merge`, whose
+  // already-merged / closed short-circuits finish the loop as merged / abandoned.
+  const liveness = classifyPrLiveness(st);
+  if (liveness === "merged" || liveness === "closed") {
+    app.log.info("merge-stall-probe: PR already terminal — routing to attempt-merge short-circuit", {
+      prKey: job.variables.prKey,
+      liveness,
+    });
+    // `updatePending: false` here too: a terminal PR short-circuits the merge, so a stale hold
+    // from an earlier round must not survive into `gw-merge-stall`'s bypass-cap condition.
+    return { mergeState: "ready", updatePending: false, failingChecks: 0, failingChecksList: "", reviewThreads: 0, pendingChecks: 0 };
+  }
+  const protocol = await deps.readProtocol(repo, token).catch(() => null);
+  let mergeState = classifyMergeability(st, protocol ?? undefined);
+
+  // Issue #951 review: a `blocked` verdict on the OLD head while an accepted update-branch is still
+  // pending is a stale-read, not a real block — the poller's behind-base remedy is holding that head
+  // (no fix-ci on its stale failures) until GitHub applies the update and the head moves. Mirroring
+  // that hold here, the probe downgrades the verdict to `waiting` so `gw-mergeable` routes to the
+  // bounded re-poll (`wait-mergeable-repoll`) instead of dispatching fix-ci / escalating against a
+  // head that is about to change. The hold shares the remedy's bounded window, so a never-applied
+  // update stops holding and the next probe judges the (still-red) head afresh.
+  // `updatePending` is surfaced so `gw-merge-stall` keeps re-polling inside the remedy's bounded hold
+  // window even once `mergeStallMax` is spent (an ordinary `waiting` still respects the stall cap).
+  let updatePending = false;
+  if (mergeState === "blocked" && st.headRefOid && deps.isUpdatePending(repo, prNumber, st.headRefOid)) {
+    updatePending = true;
+    app.log.info("merge-stall-probe: behind-base update pending — deferring remediation until the head moves", {
+      prKey: job.variables.prKey,
+      head: st.headRefOid,
+    });
+    mergeState = "waiting";
   }
-  const protocol = await loadMergeProtocol(repo, token).catch(() => null);
-  const mergeState = classifyMergeability(st, protocol ?? undefined);
 
   app.log.info("merge-stall-probe: poller stalled — re-derived mergeability", {
     prKey: job.variables.prKey,
@@ -72,6 +129,7 @@ const handler: AppJobHandler<In, Out> = async (job, app) => {
 
   return {
     mergeState,
+    updatePending,
     // Carried for the CI-fix arm (mergeState "blocked" = a failed required check), mirroring the
     // poller's `merge-ready` payload so fix-ci knows which gates to green.
     failingChecks: st.failingChecks,
@@ -79,6 +137,9 @@ const handler: AppJobHandler<In, Out> = async (job, app) => {
     reviewThreads: st.unresolvedThreads,
     pendingChecks: st.pendingCheckNames.length,
   };
-};
+  };
+}
+
+const handler: AppJobHandler<In, Out> = makeHandler(realDeps);
 
 export default handler;

package/workers/reconverge/worker.test.ts

@@ -108,3 +108,53 @@ test("a head read that resolves no SHA degrades to the checkpoint restore (no ov
   await makeHandler(submit, head)({ processInstanceKey: "4242", variables: { prKey: "acme/w#7" } } as any, app);
   assertEquals(args[11], null, "no live head SHA → no override (checkpoint restore, the pre-fix behaviour)");
 });
+
+// Issue #956 review (Copilot thread 4547353262): a merge-stage reconverge re-enters `submitPr`
+// WITHOUT the run's operator custom instructions, so the replacement convergence loop runs
+// unsteered (and `submitPr`'s run-scoped reopen resets `custom_instructions` to 0). The worker must
+// read the durable text off the PR row (`custom_instructions_text`, stamped by `submitPr`) and pass
+// it through, so convergence → merge → reconvergence keeps the steering.
+test("preserves the run's custom instructions across the merge→reconvergence handoff (#956)", async () => {
+  const app = {
+    data: {
+      table: (name: string) => ({
+        get: async () =>
+          name === "pull_requests"
+            ? { auto_resolve_escalations: 1, custom_instructions_text: "  commit as josh.wulf@camunda.com  " }
+            : undefined,
+        insert: async () => 1,
+      }),
+    },
+    engine: {},
+    log: noopLog(),
+  } as any;
+  let args: unknown[] = [];
+  const submit = (async (...a: unknown[]) => {
+    args = a;
+    return { prKey: "acme/w#7", processKey: "999" };
+  }) as unknown as Submit;
+  const head = (async () => null) as unknown as FetchPrHead;
+  await makeHandler(submit, head)({ processInstanceKey: "4242", variables: { prKey: "acme/w#7" } } as any, app);
+  assertEquals(args[12], "  commit as josh.wulf@camunda.com  ", "the durable instruction text rides the handoff into the fresh convergence run");
+});
+
+test("a PR row with no stored instructions hands off with null (no steering)", async () => {
+  const app = {
+    data: {
+      table: (name: string) => ({
+        get: async () => (name === "pull_requests" ? { auto_resolve_escalations: 0, custom_instructions_text: null } : undefined),
+        insert: async () => 1,
+      }),
+    },
+    engine: {},
+    log: noopLog(),
+  } as any;
+  let args: unknown[] = [];
+  const submit = (async (...a: unknown[]) => {
+    args = a;
+    return { prKey: "acme/w#7", processKey: "999" };
+  }) as unknown as Submit;
+  const head = (async () => null) as unknown as FetchPrHead;
+  await makeHandler(submit, head)({ processInstanceKey: "4242", variables: { prKey: "acme/w#7" } } as any, app);
+  assertEquals(args[12], null, "no stored text → null (the fresh run's prompts stay byte-identical to no-instructions)");
+});

package/workers/reconverge/worker.ts

@@ -27,6 +27,12 @@ export function makeHandler(submit: Submit, fetchHead: FetchPrHead = fetchPrHead
     const row = await app.data.table("pull_requests", "pr_key").get(prKey);
     // biome-ignore lint/plugin: runtime/framework contract boundary for the untyped row shape
     const optIn = (row as { auto_resolve_escalations?: number } | undefined)?.auto_resolve_escalations === 1;
+    // Operator steering (issue #956 review): re-supply the run's durable instruction text so the
+    // merge→reconvergence handoff never starts an unsteered replacement loop (and never resets the
+    // `custom_instructions` projection). Read the SAME `pull_requests` row the opt-in came from.
+    // biome-ignore lint/plugin: runtime/framework contract boundary for the untyped row shape
+    const rowInstructions = (row as { custom_instructions_text?: string | null } | undefined)?.custom_instructions_text;
+    const customInstructions = typeof rowInstructions === "string" && rowInstructions.trim() !== "" ? rowInstructions : null;
     await app.data.table("merges", "id").insert({
       pr_key: prKey,
       outcome: "reconverge",
@@ -63,6 +69,7 @@ export function makeHandler(submit: Submit, fetchHead: FetchPrHead = fetchPrHead
       true,
       job.processInstanceKey != null ? String(job.processInstanceKey) : null,
       headSha,
+      customInstructions,
     );
     // `submitPr` can return `processKey: null` after rewriting the row to `converging` with no owner
     // (the create-instance failure window). Completing HERE would end the merge loop at

package/workers/request-review/worker.test.ts

@@ -21,6 +21,7 @@ function ops(over: Partial<ReviewRequestOps>, calls: string[]): ReviewRequestOps
     hasPending: async () => (calls.push("pending?"), false),
     remove: async () => calls.push("remove"),
     request: async () => (calls.push("request"), "requested"),
+    awaitStarted: async () => true,
     ...over,
   };
 }
@@ -52,3 +53,29 @@ test("Copilot not assignable reports unavailable; a transport error reports fail
   );
   assertEquals(await makeHandler(ops({ request: async () => null }, []))(job(0), app), { reviewRequest: "failed" });
 });
+
+// Issue #951: a request that produced `review_requested` but never `copilot_work_started` is stuck —
+// verify via the timeline and withdraw + retry ONCE before leaving it to the bounded wait.
+test("a request that never starts Copilot is withdrawn and retried once (#951)", async () => {
+  const calls: string[] = [];
+  const { app } = fakeApp();
+  const out = await makeHandler(
+    ops({ awaitStarted: async () => (calls.push("started?"), false) }, calls),
+  )(job(0), app);
+  assertEquals(out, { reviewRequest: "requested" });
+  assertEquals(calls, ["request", "started?", "remove", "request", "started?"]);
+});
+
+test("a request that starts Copilot is not retried (#951)", async () => {
+  const calls: string[] = [];
+  const { app } = fakeApp();
+  await makeHandler(ops({ awaitStarted: async () => (calls.push("started?"), true) }, calls))(job(0), app);
+  assertEquals(calls, ["request", "started?"]);
+});
+
+test("an unverifiable timeline (null) is not treated as a dropped request (#951)", async () => {
+  const calls: string[] = [];
+  const { app } = fakeApp();
+  await makeHandler(ops({ awaitStarted: async () => (calls.push("started?"), null) }, calls))(job(0), app);
+  assertEquals(calls, ["request", "started?"]);
+});

package/workers/request-review/worker.ts

@@ -11,7 +11,12 @@
 // second POST is then a no-op. On a re-request (round > 0) with Copilot still pending, withdraw the
 // request first and re-add it — the equivalent that re-triggers the review.
 import type { AppJobHandler } from "@nanobpm/urban";
-import { hasPendingCopilotReviewer, removeCopilotReviewer, requestCopilotReview } from "../../app/github.ts";
+import {
+  hasCopilotWorkStarted,
+  hasPendingCopilotReviewer,
+  removeCopilotReviewer,
+  requestCopilotReview,
+} from "../../app/github.ts";
 import type { WorkerInputs } from "../../nano-generated/worker-io.d.ts";
 
 // Input typed off the model data envelope (`RequestReviewIn` in merge-loop.bpmn) — ADR 0040.
@@ -28,6 +33,29 @@ export interface ReviewRequestOps {
   hasPending: (repo: string, prNumber: number) => Promise<boolean | null>;
   remove: (repo: string, prNumber: number) => Promise<unknown>;
   request: (repo: string, prNumber: number) => Promise<"requested" | "unavailable" | null>;
+  /** Issue #951: poll the PR timeline until Copilot STARTS the requested review
+   * (`copilot_work_started` at/after `sinceIso`) or the verify window (~2 min) elapses. `true` =
+   * started, `false` = the request was silently dropped, `null` = unverifiable (treated as started). */
+  awaitStarted: (repo: string, prNumber: number, sinceIso: string) => Promise<boolean | null>;
+}
+
+const VERIFY_WINDOW_MS = 120_000;
+const VERIFY_POLL_MS = 15_000;
+
+async function pollWorkStarted(repo: string, n: number, sinceIso: string): Promise<boolean | null> {
+  const deadline = Date.now() + VERIFY_WINDOW_MS;
+  for (;;) {
+    // Bound each read by the REMAINING verification budget (issue #951 review): the deadline is
+    // otherwise checked only AFTER the read returns, so a stalled `gh`/REST read could run past it
+    // and starve the BPMN review-wait timer. A read that exhausts its budget returns null (unknown)
+    // — surfaced as "started" (no spurious retry), and the loop stops because the deadline is met.
+    const remaining = deadline - Date.now();
+    if (remaining <= 0) return false;
+    const started = await hasCopilotWorkStarted(repo, n, token(), sinceIso, { timeoutMs: remaining });
+    if (started !== false) return started;
+    if (Date.now() >= deadline) return false;
+    await new Promise((r) => setTimeout(r, Math.min(VERIFY_POLL_MS, deadline - Date.now())));
+  }
 }
 
 const token = () => process.env.GITHUB_TOKEN ?? "";
@@ -35,6 +63,7 @@ const defaultOps: ReviewRequestOps = {
   hasPending: (repo, n) => hasPendingCopilotReviewer(repo, n, token()),
   remove: (repo, n) => removeCopilotReviewer(repo, n, token()),
   request: (repo, n) => requestCopilotReview(repo, n, token()),
+  awaitStarted: pollWorkStarted,
 };
 
 export function makeHandler(ops: ReviewRequestOps): AppJobHandler<In, Out> {
@@ -47,7 +76,18 @@ export function makeHandler(ops: ReviewRequestOps): AppJobHandler<In, Out> {
       if (round > 0 && (await ops.hasPending(repo, prNumber)) === true) {
         await ops.remove(repo, prNumber);
       }
-      const res = await ops.request(repo, prNumber);
+      let since = new Date().toISOString();
+      let res = await ops.request(repo, prNumber);
+      // Issue #951: verify the request actually STARTED a review. A request that only produced
+      // `review_requested` (no `copilot_work_started`) is stuck — withdraw it and retry ONCE before
+      // leaving it to the bounded wait (and, past budget, the escalation).
+      if (res === "requested" && (await ops.awaitStarted(repo, prNumber, since)) === false) {
+        app.log.warn(`request-review: ${prKey} Copilot review request did not start — withdrawing and retrying once`);
+        await ops.remove(repo, prNumber);
+        since = new Date().toISOString();
+        res = await ops.request(repo, prNumber);
+        if (res === "requested") await ops.awaitStarted(repo, prNumber, since);
+      }
       result = res === "requested" ? "requested" : res === "unavailable" ? "unavailable" : "failed";
     } catch (err) {
       app.log.warn(`request-review: ${prKey} Copilot review request failed: ${err}`);