@nanobpm/nano-workforce 0.200.9 → 0.201.0

package/workers/progress-check/worker.ts

@@ -18,6 +18,7 @@
 import type { AgentInstanceSummary, AppApi, AppJobHandler } from "@nanobpm/urban";
 import { type HeadReader, makeDefaultReadHead } from "../../app/currentHead.ts";
 import { compareCommits, fetchBranchHead, fetchPrHead, updateBranchRef } from "../../app/github.ts";
+import { CHURN_WINDOW, type ChurnRound, detectChurn } from "../../app/roundChurn.ts";
 import { decideProgress, isAddressedStatus } from "../../app/roundProgress.ts";
 import { parsePr } from "../../app/service.ts";
 import { WorldStore } from "../../app/world/index.ts";
@@ -238,6 +239,53 @@ function agentWorkFromEngine(engine: AppApi["engine"]): AgentWorkReader {
   };
 }
 
+/** Reads a PR's durable round history (its `rounds` rows) projected to the fields {@link detectChurn}
+ * consumes, SCOPED to one convergence run. Injectable so churn tests never touch the datasource; the
+ * default binds the app's `rounds` table and filters by the calling job's `processInstanceKey`. */
+export type RoundsReader = (prKey: string, processInstanceKey: string | null | undefined) => Promise<ChurnRound[]>;
+
+/** Project raw `rounds` rows to the {@link ChurnRound} shape {@link detectChurn} consumes, SCOPED to
+ * one convergence run. Pure and exported so the run-scoping rule can be tested directly (with
+ * current-run, prior-run, and legacy NULL-key rows) without a datasource — the injectable reader in
+ * {@link detectChurn}'s tests returns a canned array and so never exercises this filter.
+ *
+ * When `processInstanceKey` is known, keep ONLY rows stamped with it; a legacy NULL-key row (pre-`102`
+ * history) or any OTHER run's row is dropped — a fail-OPEN policy, since dropping a row only ever
+ * REDUCES the churn signal (it can never fabricate one). When the key is absent (a testkit/synthetic
+ * job that persisted no key), fall back to the whole history, matching `persist-round`'s own null-key
+ * fallback. */
+export function projectRunScopedRounds(
+  rows: ReadonlyArray<{ round_no: number; status: string; summary?: string | null; process_instance_key?: string | null }>,
+  processInstanceKey: string | null | undefined,
+): ChurnRound[] {
+  const key = processInstanceKey != null ? String(processInstanceKey) : null;
+  const scoped = key === null ? rows : rows.filter((r) => r.process_instance_key === key);
+  return scoped.map((r) => ({ roundNo: r.round_no, status: r.status, summary: r.summary ?? null }));
+}
+
+/** Default round-history reader — the canonical `rounds` table over `app.data`, SCOPED to the current
+ * convergence run via {@link projectRunScopedRounds}. `rounds` history is retained across resubmissions
+ * (`submitPr` re-opens a PR at `current_round = 1` in a NEW process instance WITHOUT deleting prior-run
+ * rows — `migration 102`, `workers/persist-round/worker.ts`), so a bare `pr_key` read mixes runs: a
+ * prior run's higher-numbered `converged`/`blocked` row could permanently break the trailing run, and a
+ * prior run's trailing `addressed` streak could fabricate an early churn escalation. The projection
+ * drops every row not stamped with the caller's `processInstanceKey` (and falls back to the whole
+ * history when the key is absent). Any read failure surfaces as a rejected promise the caller `.catch`es
+ * to an empty history (churn then never fires, failing OPEN exactly like the head read). */
+function defaultReadRounds(app: AppApi): RoundsReader {
+  return async (prKey, processInstanceKey) => {
+    const tbl = app.data.table<{
+      pr_key: string;
+      round_no: number;
+      status: string;
+      summary?: string;
+      process_instance_key?: string | null;
+    }>("rounds", "id");
+    const rows = await tbl.find({ pr_key: prKey });
+    return projectRunScopedRounds(rows, processInstanceKey);
+  };
+}
+
 /** Build the handler with injectable readers (see {@link HeadReader} / {@link AgentWorkReader}). The
  * default export binds the real GitHub reader; the agent-work reader defaults to the engine's
  * AgentInstance channel when not injected. Tests inject stubs. */
@@ -245,6 +293,7 @@ export function makeHandler(deps: {
   readHead: HeadReader;
   readAgentWork?: AgentWorkReader;
   selfHeal?: SelfHealFn;
+  readRounds?: RoundsReader;
 }): AppJobHandler<In, Out> {
   return async (job, app) => {
     const { prKey, status, repo, prNumber, round, huskRetries, roundEntryHead } = job.variables;
@@ -259,6 +308,7 @@ export function makeHandler(deps: {
       last_progress_job_key: string | null;
       last_progress_result: string | null;
       last_progress_agent_watermark: string | null;
+      churn_escalated_round: number | null;
     }>("pull_requests", "pr_key");
     const row = await prs.get(prKey);
 
@@ -301,7 +351,12 @@ export function makeHandler(deps: {
     };
     const commit = async (
       out: Out,
-      opts: { head?: string | null; status?: "waiting_review" | "converging"; agentWatermark?: string | null },
+      opts: {
+        head?: string | null;
+        status?: "waiting_review" | "converging";
+        agentWatermark?: string | null;
+        churnEscalatedRound?: number;
+      },
     ): Promise<Out> => {
       const ts = new Date().toISOString();
       // Re-check ownership immediately before the write and drop it when this job no longer owns the
@@ -320,6 +375,10 @@ export function makeHandler(deps: {
         // unreadable head, a non-addressed round, or an injected bare-verdict reader), so the next
         // round still correlates against the last real attempt.
         ...(opts.agentWatermark !== undefined ? { last_progress_agent_watermark: opts.agentWatermark } : {}),
+        // Stamp the run-scoped churn-reset watermark when a churn escalation is raised (#870), so the
+        // human-answer resume — which re-enters the SAME numeric round and writes no `blocked` row —
+        // restarts the churn clock instead of re-raising the identical question on the resumed round.
+        ...(opts.churnEscalatedRound !== undefined ? { churn_escalated_round: opts.churnEscalatedRound } : {}),
         ...(jobKey ? { last_progress_job_key: jobKey, last_progress_result: JSON.stringify(out) } : {}),
         updated_at: ts,
       });
@@ -429,6 +488,44 @@ export function makeHandler(deps: {
       return commit(out, { head: currentHead, status: "converging", agentWatermark });
     }
     if (decision.progressed) {
+      // NON-CONVERGING CHURN guard (#870). The head DID advance (real commit), so the no-progress
+      // guard correctly continues — but a contested surface whose findings never end keeps the loop
+      // "progressing" every round without ever reaching a fixed point (e.g. a fail-closed analyzer
+      // whose bypass forms are unbounded). Before parking for the next review, check the durable round
+      // history: when the trailing run of consecutive `addressed` rounds keeps landing findings in the
+      // SAME file over a window, escalate for a human SCOPE decision instead of looping to maxRounds.
+      // Only on an addressed round with a readable head (a GitHub outage fails OPEN like everywhere
+      // else — never fabricate a churn escalation); the round-history read fails open to no-churn too.
+      if (currentHead && isAddressedStatus(status)) {
+        const history: ChurnRound[] = await (deps.readRounds ?? defaultReadRounds(app))(prKey, job.processInstanceKey).catch(
+          () => [],
+        );
+        const churn = detectChurn(history, CHURN_WINDOW, row?.churn_escalated_round ?? 0);
+        if (churn.churning && churn.question) {
+          app.log.info("non-converging churn detected — escalating for a human scope decision (#870)", {
+            prKey,
+            round: roundNo,
+            file: churn.file,
+            rounds: churn.rounds,
+          });
+          // Route to the SAME human escalation path the no-advance guard uses (gw-husk → default →
+          // persist-escalation-noprogress, which renders `noProgressQuestion`): progressed:false with
+          // huskRetry:false escapes the husk auto-retry straight to the human. Leave the status unset
+          // (the escalation worker owns it) and advance the baseline + watermark, exactly as the
+          // no-advance escalation `commit` below does. Also stamp the run-scoped churn-reset watermark
+          // at THIS round so the human-answer resume restarts the churn clock (see detectChurn).
+          return commit(
+            {
+              progressed: false,
+              huskRetries: 0,
+              huskRetry: false,
+              noProgressReason: "churn",
+              noProgressQuestion: churn.question,
+            },
+            { head: currentHead, agentWatermark, churnEscalatedRound: roundNo },
+          );
+        }
+      }
       // Genuine progress → the loop parks at wait-review. THIS is the review-wait park, written only
       // once the husk retry has been ruled out, so a husk-retry round never transits `waiting_review`.
       return commit(out, { head: currentHead, status: "waiting_review", agentWatermark });

package/workers/readiness-probe/worker.test.ts

@@ -3,7 +3,7 @@ import { test } from "node:test";
 import { assert, assertEquals, assertRejects } from "#test-assert";
 import type { CommandResult, HttpResponse, ProbeExec, ReadinessProbe } from "../../app/readiness.ts";
 import { parseProbe } from "../../app/readiness.ts";
-import handler, { probeSingleShot, READINESS_READY_MESSAGE, readGateVars, safeBind } from "./worker.ts";
+import handler, { __setProbeExecForTest, probeSingleShot, READINESS_READY_MESSAGE, readGateVars, safeBind } from "./worker.ts";
 
 function execReturning(seq: Array<HttpResponse>, counts: { http: number } = { http: 0 }): ProbeExec {
   let i = 0;
@@ -239,6 +239,77 @@ test("handler: a blank gateKey fails fast (an empty correlationKey would never r
   );
 });
 
+test("#876 review: a DOTTED command target (e.g. check.sh) is NOT an unresolved fact-ref — the worker probes it, never parks the gate", async () => {
+  // The #872 fail-closed guard must key on the compiler's NULL provenance only — a runtime fact-ref
+  // SYNTAX test cannot distinguish an unresolved `<node>.<fact>` literal from a valid dotted command
+  // target, and misclassifying `check.sh` would park the gate forever without ever running the probe.
+  const ran: string[] = [];
+  const exec: ProbeExec = {
+    async run(command: string): Promise<CommandResult> {
+      ran.push(command);
+      return { code: 0, stdout: "", stderr: "" };
+    },
+    async httpGet(): Promise<HttpResponse> {
+      throw new Error("not an http probe");
+    },
+  };
+  const restore = __setProbeExecForTest(exec);
+  const published: string[] = [];
+  try {
+    const job = { variables: { probe: { kind: "command", target: "check.sh" }, gateKey: "g1", probeTimeout: "PT30M" } };
+    const app = {
+      log: { info() {}, warn() {} },
+      engine: {
+        async publishMessage(msg: { name: string }) {
+          published.push(msg.name);
+        },
+      },
+    };
+    // biome-ignore lint/suspicious/noExplicitAny: minimal job/app stub over the real handler.
+    const out = await handler(job as any, app as any);
+    assertEquals(out.ready, true, "a dotted command target probes normally and reports ready");
+  } finally {
+    __setProbeExecForTest(restore);
+  }
+  assertEquals(ran, ["check.sh"], "the probe RAN the dotted command target (it was not misclassified as an unresolved fact-ref)");
+  assertEquals(published, [READINESS_READY_MESSAGE], "a green dotted-target probe publishes readiness");
+});
+
+test("#872: a NULL/blank late-bound target still fails closed (the gate parks and escalates, never probes a null handle)", async () => {
+  for (const target of [null, undefined, "  "]) {
+    // Intermediate poll activation (no lastAttempt): fails closed, logs at INFO, never WARNs — the
+    // boundary-only WARN policy (PR #876 review) keeps a 30-minute gate from emitting ~120 WARN lines.
+    const job = { variables: { probe: { kind: "command", target }, gateKey: "g1", probeTimeout: "PT30M" } };
+    const infos: string[] = [];
+    const warnings: string[] = [];
+    const app = {
+      log: { info(msg: string) { infos.push(msg); }, warn(msg: string) { warnings.push(msg); } },
+      engine: { async publishMessage() { throw new Error("must never publish"); } },
+    };
+    // biome-ignore lint/suspicious/noExplicitAny: minimal job/app stub over the real handler.
+    const out = await handler(job as any, app as any);
+    assertEquals(out.ready, false, `a ${JSON.stringify(target)} target is not ready`);
+    assert(String(out.detail).includes("unresolved target"), "the detail names the unresolved target");
+    assertEquals(warnings.length, 0, `an intermediate ${JSON.stringify(target)} activation must NOT WARN`);
+    assertEquals(infos.length, 1, "the fail-closed park is logged at info on intermediate attempts");
+
+    // Final (boundary) activation: the SAME fail-closed park, but now it IS the escalation trigger, so
+    // it WARNs exactly once (and does not duplicate at info).
+    const lastJob = { variables: { ...job.variables, lastAttempt: true } };
+    const lastInfos: string[] = [];
+    const lastWarnings: string[] = [];
+    const lastApp = {
+      log: { info(msg: string) { lastInfos.push(msg); }, warn(msg: string) { lastWarnings.push(msg); } },
+      engine: { async publishMessage() { throw new Error("must never publish"); } },
+    };
+    // biome-ignore lint/suspicious/noExplicitAny: minimal job/app stub over the real handler.
+    const lastOut = await handler(lastJob as any, lastApp as any);
+    assertEquals(lastOut.ready, false, `a ${JSON.stringify(target)} target is not ready on the last attempt`);
+    assertEquals(lastWarnings.length, 1, "the boundary (lastAttempt) fail-closed park WARNs exactly once");
+    assertEquals(lastInfos.length, 0, "the boundary park does not also log at info");
+  }
+});
+
 test("READINESS_READY_MESSAGE is the name the gate correlates", () => {
   assertEquals(READINESS_READY_MESSAGE, "readiness-ready");
 });

package/workers/readiness-probe/worker.ts

@@ -49,6 +49,27 @@ function errorDetail(err: unknown): string {
   return `probe error: ${err instanceof Error ? err.name : "Error"}`;
 }
 
+/** Fail-closed guard for an UNRESOLVED wait-gate target (issue #872, PR #876 review). A `wait` node's
+ * probe `target` is late-bound from an upstream emit: the compiler rewrites it to the observed value
+ * when that fact resolved to a non-null value, otherwise it writes NULL (the compiler owns the
+ * resolved/unresolved provenance — it is the only component that knows whether the target was actually
+ * bound). A probe left with a null/blank target MUST NOT pass through — it is not a real handle, so the
+ * gate is NOT satisfiable yet. Returning not-ready keeps the gate parked so its bounded timeout
+ * escalates to a human, instead of either incident-ing inside `parseProbe` (which throws on a blank
+ * target) or — worse — silently treating the null target as "ready". Returns `null` when the target is
+ * a usable literal (let `parseProbe` run).
+ *
+ * Deliberately NO fact-ref SYNTAX test here: a dotted, hash-free literal (`check.sh`) is a VALID
+ * `command` target yet is indistinguishable from an unresolved `<node>.<fact>` reference by shape
+ * alone — guarding on syntax parked such a gate forever without ever running its probe (PR #876
+ * review). Only the compiler's null provenance marks an unresolved target. */
+function unresolvedTargetDetail(rawProbe: unknown): string | null {
+  const isRecord = (v: unknown): v is Record<string, unknown> => typeof v === "object" && v !== null;
+  const target = isRecord(rawProbe) ? rawProbe.target : undefined;
+  if (typeof target !== "string" || target.trim() === "") return "wait gate: unresolved target (not ready)";
+  return null;
+}
+
 /** Single activation of the readiness probe. The engine owns retry cadence and the timeout boundary;
  * this function performs one `probeOnce`, publishes only when ready, and optionally performs exactly
  * one empirical fallback on a model-marked last attempt. */
@@ -149,6 +170,24 @@ function resolveProbeExec(): ProbeExec {
 }
 
 const handler: AppJobHandler<In, Out> = async (job, app) => {
+  // Fail closed on an unresolved late-bound target (issue #872): a null/blank target means the
+  // upstream emit this gate waits on never resolved — the gate is not satisfiable, so stay parked (and
+  // let the gate's bounded timeout escalate) rather than throw in parseProbe or pass a null target
+  // through as if ready. (The compiler writes the null provenance; a dotted LITERAL target — e.g. a
+  // `check.sh` command — is never unresolved and reaches `parseProbe` verbatim, PR #876 review.)
+  const unresolved = unresolvedTargetDetail(job.variables.probe);
+  if (unresolved !== null) {
+    // Boundary-only WARN policy (mirrors probeSingleShot, PR #876 review): an unresolved late-bound
+    // target stays unresolved for the WHOLE bounded wait, so the engine re-activates this job on every
+    // poll tick. Warning on each activation would emit ~120 WARN lines over a 30-minute gate at the
+    // 15s default cadence and drown the one that matters. Reserve WARN for the final attempt (the
+    // escalation trigger) and log intermediate activations at info — the same boundary-only policy
+    // probeSingleShot applies to a not-ready probe.
+    const msg = `readiness gate not ready: ${unresolved}`;
+    if (job.variables.lastAttempt === true) app.log.warn(msg);
+    else app.log.info(msg);
+    return { ready: false, detail: unresolved };
+  }
   const probe = parseProbe(job.variables.probe);
   const { gateKey } = readGateVars(job.variables);
   const exec = resolveProbeExec();