@nanobpm/nano-workforce 0.193.1 → 0.194.0

package/app/service.ts

@@ -106,6 +106,9 @@ import {
   latestTrialMergeQuestion,
   type OpenEscalation,
   PLAN_REVIEW_ELEMENT,
+  PR_ESCALATION_PRODUCER_ELEMENTS,
+  PR_RECORD_ANSWER_ELEMENT,
+  PR_RECORD_MERGE_ANSWER_ELEMENT,
   PR_WAIT_ANSWER_ELEMENT,
   PR_WAIT_MERGE_ANSWER_ELEMENT,
   prEscalations,
@@ -251,6 +254,15 @@ export const REVIEW_NUDGE_MS = clampNudgeMinutes(process.env.NANO_PR_REVIEW_NUDG
  * still reconciling genuinely-stranded rows. */
 export const FEATURE_ESCALATION_HEAL_GRACE_MS = 60_000;
 
+/** Grace window (ms) before the `escalated`→`converging` PR self-heal (below) may act on a row — the
+ * convergence-path twin of `FEATURE_ESCALATION_HEAL_GRACE_MS` (issue #828). The sole writer of a PR's
+ * `status="escalated"` — `pr.persist-escalation` — stamps the row IMMEDIATELY BEFORE the engine creates
+ * the `wait-answer` / `wait-merge-answer` user task. A poll landing in that window would see
+ * `openUserTasks` report none and wrongly heal the just-raised escalation back to `converging`, hiding
+ * it before the operator can answer. Only heal rows whose escalation is older than this window, by when
+ * the user task must already be observable. */
+export const PR_ESCALATION_HEAL_GRACE_MS = 60_000;
+
 /** Whether a converged PR is automatically driven to merge (the merge-loop). Default on; set
  * `NANO_PR_AUTO_MERGE=0` to stop at `converged` (review-only mode). */
 export const AUTO_MERGE = !["0", "false", "off", "no"].includes(
@@ -3325,6 +3337,152 @@ export async function pollUserTasks(
     if (queryErrored || stillParked) continue;
     await featureRuns(data).update(run.feature_key, { status: "running", updated_at: at });
   }
+
+  // ── Self-heal: a convergence PR row holds `escalated` ONLY while parked (issue #828) ─────────────
+  // The convergence-path twin of the feature self-heal above (issue #642). `pr.persist-escalation` is
+  // the sole writer of a PR's `status="escalated"`, and the sole writer that moves it back off is the
+  // `pr.answer-escalation` worker (the guarded `UPDATE … SET status='converging'` on user-task
+  // completion). When that write is lost — e.g. the app restarts while a `wait-answer` completion /
+  // `answer-escalation` job is in flight — the engine resumes the loop (token back on `review-round`,
+  // no open user task) but the PR row stays `escalated` forever, since NOTHING on the PR path repairs
+  // it (unlike the feature path). `/status` then shows `status="escalated"` with a null derived
+  // `openEscalation` indefinitely, inviting a human to answer an already-answered question.
+  //
+  // Heal it engine-first, on the SAME positive-evidence contract as the feature sweep: a PR at
+  // `status="escalated"` is reconciled back to `converging` only when the engine confirms NO open PR
+  // escalation task (`wait-answer` / `wait-merge-answer`) for its instance. As above, presence in THIS
+  // pass's `desired` set is itself positive evidence of parking (truncation only ever DROPS tasks), so a
+  // PR whose escalation task was already swept is genuinely parked — skip its per-instance RPC. Only a
+  // PR not confirmed parked by the sweep falls through to the per-instance `openUserTasks` check, and a
+  // query error is negative evidence (skip the heal), so durable state flips only on positive proof.
+  //
+  // Candidate set: read through the ADR-0065 derived tracking VIEW and require `derived_status="escalated"`
+  // (Copilot review of #829). A PR TERMINATED while its base status was `escalated` keeps that frozen base
+  // status forever — the reconciler never rewrites it (app/instanceTracking.ts) — so a raw `prs.find({
+  // status:"escalated" })` scan would reselect every such HISTORICAL row on every poll and burn up to three
+  // sequential engine RPCs on each, indefinitely. The VIEW folds the terminal edge to `derived_status`
+  // (out-of-band terminate → `abandoned`), so filtering on `derived_status="escalated"` bounds the pass to
+  // LIVE/UNKNOWN candidates and leaves terminal rows to tracking as intended; the ACTIVE-instance gate below
+  // is the correctness backstop for a candidate the VIEW hasn't yet reconciled. Writes still target `prs`.
+  const sweptParkedPrEscalations = new Set<string>();
+  for (const r of desired) {
+    if (r.element_id !== PR_WAIT_ANSWER_ELEMENT && r.element_id !== PR_WAIT_MERGE_ANSWER_ELEMENT) continue;
+    if (r.subject_type !== "pr") continue;
+    if (r.process_key) sweptParkedPrEscalations.add(r.process_key);
+    if (r.subject_key) sweptParkedPrEscalations.add(r.subject_key);
+  }
+  for (const pr of await prsTracking(data).find({ derived_status: "escalated" })) {
+    if (!pr.process_key) continue;
+    if (sweptParkedPrEscalations.has(pr.process_key) || sweptParkedPrEscalations.has(pr.pr_key)) continue; // parked this pass — no RPC, no heal
+    // Don't race a just-raised escalation: `pr.persist-escalation` stamps `updated_at` immediately
+    // before the engine creates the user task, so heal only rows past the grace window.
+    const escalatedAt = Date.parse(pr.updated_at ?? "");
+    if (Number.isFinite(escalatedAt) && Date.now() - escalatedAt < PR_ESCALATION_HEAL_GRACE_MS) continue;
+    // Snapshot the PR generation (`process_key` + `updated_at`) and the `open` escalation rows we observe
+    // NOW — BEFORE the remote engine reads below. The repair writes are fenced on this snapshot, so a
+    // FRESH escalation the SAME instance opens in the read→write window is neither clobbered nor retired.
+    // The convergence loop REUSES one process instance across rounds, so a re-escalation keeps the same
+    // `process_key`; the discriminator is `updated_at`, which `pr.persist-escalation` re-stamps every time
+    // it writes `status="escalated"`. Capturing the open-escalation ids here (not re-reading `status=open`
+    // after the write) means a fresh escalation row — a NEW id inserted after this point — is never retired
+    // to `stale` even though it is `open`, since it was not part of the snapshot we healed (issue #829).
+    const snapshotProcessKey = pr.process_key;
+    const snapshotUpdatedAt = pr.updated_at ?? null;
+    const snapshotOpenEscalationIds = (await escs(data).find({ pr_key: pr.pr_key, status: "open" })).map((e) => e.id);
+    let stillParked = false;
+    let escalationInFlight = false;
+    let instanceActive = false;
+    let queryErrored = false;
+    try {
+      const openTasks = await engine.openUserTasks({ processInstanceKey: pr.process_key });
+      stillParked = openTasks.some((t) => t.elementId === PR_WAIT_ANSWER_ELEMENT || t.elementId === PR_WAIT_MERGE_ANSWER_ELEMENT);
+      // An empty open-task set is NOT yet proof the escalation is dead. Two distinct DB/engine boundaries
+      // each leave the row `escalated` with NO open user task while an escalation is genuinely live:
+      //
+      //   1. The ANSWER side. `pr.answer-escalation` (`record-answer` / `record-merge-answer`) is the sole
+      //      writer that moves a PR OFF `escalated`, and both BPMN models flow the completed `wait-answer` /
+      //      `wait-merge-answer` user task DIRECTLY to that service task. So a normal answer has a window
+      //      where the user task is already gone (this query returns none) while the answer-recording job is
+      //      still queued or running.
+      //   2. The PRODUCER side. `pr.persist-escalation` (the `PR_ESCALATION_PRODUCER_ELEMENTS`) COMMITS the
+      //      `open` row and `status="escalated"` IMMEDIATELY BEFORE the engine creates the `wait-answer` /
+      //      `wait-merge-answer` user task. The raise-time grace window covers the normal case, but if that
+      //      producer service task stays ACTIVE past the grace (e.g. an app restart / lease delay between
+      //      the DB commit and the job completing) the row is old enough to heal yet no user task exists.
+      //   3. The PRODUCER→USER-TASK TOCTOU BETWEEN THE TWO READS. `openUserTasks` (read 1) can return empty
+      //      while a persist producer is still ACTIVE; that producer then COMPLETES and the engine creates
+      //      the `wait-answer` / `wait-merge-answer` user task BEFORE `searchElementInstances` (read 2). Read 2
+      //      then sees an ACTIVE WAIT element — not the producer. Because user-task creation does NOT re-stamp
+      //      the PR row's `updated_at` (only `pr.persist-escalation` does, before the task exists), the row is
+      //      past the grace, so neither the grace nor the producer/answer-recorder check covers it. Treat an
+      //      ACTIVE wait element as positive evidence here too, or the CAS wins and stales a live escalation.
+      //
+      // In EITHER window the escalation is arbitrarily old (past the raise-time grace), so the grace above
+      // does not cover it. Healing here flips the row to `converging` and retires the still-`open`
+      // escalation to `stale`, and the live escalation is lost — the answer-recorder finds no target and
+      // silently drops the operator's answer, or the just-raised question never surfaces (issue #829). An
+      // ACTIVE answer-recording, persist-escalation, OR wait-answer/wait-merge-answer element instance is
+      // positive evidence an escalation is in flight, so treat it exactly like an open task and skip the
+      // heal. But even a PR with NEITHER an open task NOR any ACTIVE escalation element is NOT yet
+      // heal-eligible — it must ALSO be a confirmed-ACTIVE instance (the positive-liveness gate below);
+      // a terminal/absent instance matches this negative shape too but never resumed (the #828 tear).
+      if (!stillParked) {
+        const elements = await engine.searchElementInstances({ processInstanceKey: pr.process_key });
+        escalationInFlight = elements.some(
+          (el) =>
+            el.state === "ACTIVE" &&
+            (el.elementId === PR_RECORD_ANSWER_ELEMENT ||
+              el.elementId === PR_RECORD_MERGE_ANSWER_ELEMENT ||
+              el.elementId === PR_WAIT_ANSWER_ELEMENT ||
+              el.elementId === PR_WAIT_MERGE_ANSWER_ELEMENT ||
+              (el.elementId != null && PR_ESCALATION_PRODUCER_ELEMENTS.includes(el.elementId))),
+        );
+      }
+      // "No open task and no in-flight escalation element" is necessary but NOT sufficient to heal
+      // (Copilot review of #829). A TERMINAL or ABSENT process instance — one cancelled, completed, or
+      // dropped from the read model — ALSO reports no open task and no ACTIVE escalation element, yet its
+      // loop never resumed: flipping such a frozen `escalated` row to `converging` and retiring its audit
+      // rows fabricates a live loop the engine will never advance. Terminal/absent PRs are owned by the
+      // ADR-0065 derived tracking + reconciliation (they read `abandoned`/settled on `derived_status`), so
+      // leave them be. Require POSITIVE engine-truth that the instance is `ACTIVE` before healing — the
+      // same tri-state contract as the feature-handoff probe (search ~L2562) and `makeEngineActiveProbe`
+      // (app/reconcile.ts): ONLY a confirmed `ACTIVE` state heals; a known-terminal or genuinely-absent
+      // instance is left to tracking; a missing/empty/unknown `state` (or a query error) is spared, never
+      // healed off a wire shape we misread. So a healed row is provably the #828 lost-write tear: an
+      // ACTIVE loop that resumed but whose `status="converging"` flip was lost.
+      const snapshots = await engine.searchProcessInstances({ processInstanceKeys: [pr.process_key] });
+      const match = snapshots.find((s) => String(s.processInstanceKey) === pr.process_key);
+      instanceActive = match != null && String(match.state ?? "").trim().toUpperCase() === "ACTIVE";
+    } catch (err) {
+      console.error(`[poller] escalated-pr self-heal (${pr.pr_key} @ ${pr.process_key}): ${err}`);
+      queryErrored = true;
+    }
+    if (queryErrored || stillParked || escalationInFlight || !instanceActive) continue;
+    // Repair conditionally and atomically (Copilot review of #829). The read→write window between the
+    // engine reads above and here is wide (two remote RPCs), so the PR may have re-escalated on the same
+    // instance meanwhile; a blind `update` would then overwrite that fresh `escalated` status and retire
+    // its new `open` escalation. The PR flip is therefore a SINGLE guarded UPDATE fenced on the inspected
+    // snapshot — still `escalated`, same `process_key`, same `updated_at` — an atomic compare-and-swap
+    // (SQLite serialises it against the worker's guarded writes in the other process). Only when that CAS
+    // WON (`changed === 1`) do we retire the snapshot's own orphan escalation rows, in the SAME
+    // transaction: a crash can never leave a healed PR with a dangling `open` row, nor an un-healed PR
+    // with retired rows — a rolled-back heal is simply retried by a later pass. Retiring by the captured
+    // snapshot ids (never a fresh `status=open` re-read) means a just-opened escalation is left untouched.
+    const db = data.open();
+    await db.tx(async (t) => {
+      const flipped = await t.exec(
+        `UPDATE "pull_requests" SET "status" = 'converging', "updated_at" = ? WHERE "pr_key" = ? AND "status" = 'escalated' AND "process_key" IS ? AND "updated_at" IS ?`,
+        [at, pr.pr_key, snapshotProcessKey, snapshotUpdatedAt],
+      );
+      if (flipped.changed !== 1) return; // snapshot moved (a fresh escalation re-stamped the row) — leave it for a later pass
+      // Retire the orphaned `open` escalation rows so the audit trail matches the healed PR row — the
+      // `answer-escalation` write that would have answered/retired them was the very write we lost. We
+      // mark them `stale` (not `answered`) because no operator answer was recorded for them.
+      for (const escId of snapshotOpenEscalationIds) {
+        await t.exec(`UPDATE "escalations" SET "status" = 'stale' WHERE "id" = ? AND "status" = 'open'`, [escId]);
+      }
+    });
+  }
 }
 
 /** One full poll pass: advance the review stage, the merge stage, the wave-merge barrier, and

package/app/userTasks.ts

@@ -52,6 +52,40 @@ export const PR_WAIT_ANSWER_ELEMENT = "wait-answer";
  *  canonical `completeUserTask` door and surfaced in this same Tasks inbox. */
 export const PR_WAIT_MERGE_ANSWER_ELEMENT = "wait-merge-answer";
 
+/** The PR review-loop answer-recording service task (convergence-loop.bpmn) — the `pr.answer-escalation`
+ *  job the token moves to IMMEDIATELY when `wait-answer` completes. Its ACTIVE element instance is the
+ *  positive-evidence marker that an operator's answer is in-flight (recorded but the loop hasn't looped
+ *  yet), so the escalation self-heal must NOT steal the row while it runs (issue #829). */
+export const PR_RECORD_ANSWER_ELEMENT = "record-answer";
+
+/** The PR merge-loop answer-recording service task (merge-loop.bpmn) — the merge-path twin of
+ *  `PR_RECORD_ANSWER_ELEMENT`, reached when `wait-merge-answer` completes (issue #829). */
+export const PR_RECORD_MERGE_ANSWER_ELEMENT = "record-merge-answer";
+
+/** Every `pr.persist-escalation` PRODUCER element across the two PR loops — the service tasks that write
+ *  `status="escalated"` (and INSERT the `open` escalation row) IMMEDIATELY BEFORE the engine creates the
+ *  `wait-answer` / `wait-merge-answer` user task. An ACTIVE instance of any of these is positive evidence
+ *  an escalation is being RAISED right now: the row is already `escalated` but no user task exists yet, so
+ *  neither the open-task probe nor the answer-recorder probe covers it. If such a producer stays ACTIVE
+ *  past the raise-time grace window (e.g. an app restart / lease delay between the DB commit and the job
+ *  completing), the escalated-PR self-heal would otherwise see no parked task and wrongly retire the live
+ *  escalation. So the heal must treat an ACTIVE producer as in-flight too (issue #829). This is the single
+ *  source of truth for that set; `app/persistEscalationElements.test.ts` fails if it drifts from the
+ *  `pr.persist-escalation` tasks actually modelled in the BPMN. */
+export const PR_ESCALATION_PRODUCER_ELEMENTS: readonly string[] = [
+  // convergence-loop.bpmn
+  "persist-escalation",
+  "persist-escalation-noprogress",
+  "persist-escalation-maxrounds",
+  "persist-escalation-blockedcomments",
+  "persist-escalation-scope",
+  "persist-review-stalled",
+  // merge-loop.bpmn
+  "merge-esc-conflict",
+  "merge-esc-attempt",
+  "merge-esc-landed",
+];
+
 /** The ACP permission-prompt escalation (issue #559, ADR 0056) — the Tasks-inbox kind a bridged
  *  `session/request_permission` surfaces under when an escalate-policy agent asks a human to Allow/Deny
  *  a proposed action. Unlike the other escalation elements this is NOT a BPMN user-task element; it is

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nanobpm/nano-workforce",
-  "version": "0.193.1",
+  "version": "0.194.0",
   "description": "Nano Workforce — an Agent Graph Orchestration application for Agentic SDLC: durable BPMN processes that coordinate a graph of AI agents across the software delivery lifecycle.",
   "type": "module",
   "main": "main.ts",
@@ -83,7 +83,7 @@
     "yaml": "^2.9.0"
   },
   "overrides": {
-    "@nanobpm/engine-wasm": "^0.9.0",
+    "@nanobpm/engine-wasm": "^0.10.0",
     "@semantic-release/npm": "^13.1.5"
   }
 }

package/pages/cockpit/cockpit.css

@@ -316,9 +316,24 @@
   display: flex;
   gap: 8px;
   align-items: baseline;
-  margin-bottom: 4px;
+  cursor: pointer;
+  list-style: none;
+  user-select: none;
+}
+
+.cockpit-agent-turn-meta::-webkit-details-marker { display: none; }
+
+.cockpit-agent-turn-meta::before {
+  content: "\25B8";
+  color: var(--cockpit-muted);
+  font-size: 11px;
+  width: 10px;
 }
 
+.cockpit-agent-turn[open] > .cockpit-agent-turn-meta::before { content: "\25BE"; }
+
+.cockpit-agent-turn-body { margin-top: 4px; padding-left: 18px; }
+
 .cockpit-agent-turn-role {
   font-size: 11px;
   text-transform: uppercase;
@@ -327,6 +342,7 @@
 }
 
 .cockpit-agent-turn-iter,
+.cockpit-agent-turn-time,
 .cockpit-agent-turn-metrics {
   font-size: 11px;
   color: var(--cockpit-muted);

package/pages/cockpit/mount.js

@@ -574,6 +574,15 @@ function turnMetrics(m) {
   return dur != null ? `${base} \u00b7 ${dur}` : base;
 }
 
+function turnTime(iso) {
+  if (iso == null || iso === "") return undefined;
+  const ms = Date.parse(iso);
+  if (!Number.isFinite(ms)) return undefined;
+  const d = new Date(ms);
+  const pad = (n) => String(n).padStart(2, "0");
+  return `${pad(d.getUTCHours())}:${pad(d.getUTCMinutes())}:${pad(d.getUTCSeconds())}`;
+}
+
 function agentHistoryView(report) {
   const turns = (report.records ?? []).map((r) => ({
     historyItemKey: r.historyItemKey,
@@ -582,6 +591,8 @@ function agentHistoryView(report) {
     text: turnText(r),
     toolCalls: (r.toolCalls ?? []).map((c) => ({ toolCallId: c.toolCallId, toolName: c.toolName, elementId: c.elementId })),
     metrics: turnMetrics(r.metrics),
+    producedAt: r.producedAt != null && r.producedAt !== "" ? r.producedAt : undefined,
+    time: turnTime(r.producedAt),
   }));
   return {
     agentInstanceKey: report.agentInstanceKey,
@@ -639,17 +650,28 @@ function renderAgentSessions(host, doc, view, onSelect, activeInstanceKey) {
   host.appendChild(root);
 }
 
+// One turn as a native collapsible <details> (collapsed by default); its <summary> heading (role,
+// iteration, timestamp, metrics) toggles the details body on click (#831).
 function agentTurnBlock(doc, t) {
-  const block = el(doc, "div", "cockpit-agent-turn");
+  const block = el(doc, "details", "cockpit-agent-turn");
   block.setAttribute("data-history-item-key", t.historyItemKey);
   block.setAttribute("data-role", t.role);
   block.setAttribute("data-loop-iteration", String(t.loopIteration));
-  const meta = el(doc, "div", "cockpit-agent-turn-meta");
+  const meta = el(doc, "summary", "cockpit-agent-turn-meta");
   meta.appendChild(el(doc, "span", "cockpit-agent-turn-role", t.role));
   meta.appendChild(el(doc, "span", "cockpit-agent-turn-iter", `#${t.loopIteration}`));
+  if (t.time != null) {
+    const time = el(doc, "time", "cockpit-agent-turn-time", t.time);
+    if (t.producedAt != null) {
+      time.setAttribute("datetime", t.producedAt);
+      time.setAttribute("title", t.producedAt);
+    }
+    meta.appendChild(time);
+  }
   if (t.metrics != null) meta.appendChild(el(doc, "span", "cockpit-agent-turn-metrics", t.metrics));
   block.appendChild(meta);
-  if (t.text !== "") block.appendChild(el(doc, "pre", "cockpit-agent-turn-text", t.text));
+  const body = el(doc, "div", "cockpit-agent-turn-body");
+  if (t.text !== "") body.appendChild(el(doc, "pre", "cockpit-agent-turn-text", t.text));
   if (t.toolCalls.length > 0) {
     const tools = el(doc, "ul", "cockpit-agent-turn-tools");
     for (const call of t.toolCalls) {
@@ -657,8 +679,9 @@ function agentTurnBlock(doc, t) {
       li.setAttribute("data-tool-call-id", call.toolCallId);
       tools.appendChild(li);
     }
-    block.appendChild(tools);
+    body.appendChild(tools);
   }
+  block.appendChild(body);
   return block;
 }
 

package/workers/persist-escalation/worker.ts

@@ -111,20 +111,35 @@ const handler: AppJobHandler<In> = async (job, app) => {
       process_instance_key: job.processInstanceKey != null ? String(job.processInstanceKey) : null,
     });
   }
-  const escalationId = await app.data.table("escalations", "id").insert({
-    pr_key: prKey,
-    round_no: round,
-    kind,
-    question,
-    transcript,
-    worker,
-    status: "open",
-    asked_at: now,
-  });
-  await app.data.table("pull_requests", "pr_key").update(prKey, {
-    status: "escalated",
-    current_round: round,
-    updated_at: now,
+  // Open the escalation row and stamp the PR to `escalated` ATOMICALLY, in ONE transaction (Copilot
+  // review of #829). `pr.persist-escalation` is the SOLE producer of a PR's escalated generation, and the
+  // escalated-PR self-heal (`pollUserTasks` in app/service.ts) fences its repair on a snapshot of the PR
+  // generation (`process_key` + `updated_at`) plus the `open` escalation ids it observes. If the fresh
+  // `open` row were INSERTED before the PR's `updated_at` re-stamp — two separate writes, as this worker
+  // used to do — a heal snapshot landing between them would capture the new escalation while the PR still
+  // matched the OLD generation; its compare-and-swap, still matching the un-bumped `updated_at`, could then
+  // WIN and retire the just-opened escalation to `stale`, orphaning a live user task and dropping the
+  // operator's next answer. Committing the insert and the generation re-stamp together closes that window
+  // categorically: the heal can only ever observe the new escalation AND the bumped `updated_at` atomically,
+  // so its snapshot-fenced CAS misses (the discriminator moved) and never clobbers. The same transaction
+  // also means a crash mid-write rolls back cleanly — never an `open` escalation row with no `escalated` PR.
+  let escalationId: number | bigint = 0;
+  await app.data.open().tx(async (t) => {
+    escalationId = await t.table("escalations", "id").insert({
+      pr_key: prKey,
+      round_no: round,
+      kind,
+      question,
+      transcript,
+      worker,
+      status: "open",
+      asked_at: now,
+    });
+    await t.table("pull_requests", "pr_key").update(prKey, {
+      status: "escalated",
+      current_round: round,
+      updated_at: now,
+    });
   });
 
   return { escalationId: Number(escalationId), escalated: true, question };