@nanobpm/nano-workforce 0.191.1 → 0.192.0

package/app/service.test.ts

@@ -338,7 +338,136 @@ test("re-submit advances process_key BEFORE resetting adjudications (fence order
   });
 });
 
-// Red/green regression (Copilot review): the durable adjudication RESET runs AFTER the new instance is
+// Red/green regression (Copilot review of #819): the reopen must publish the fresh run's identity
+// (`process_key`) in the SAME write that flips the row to `converging`/round 1. If the status flip
+// landed while `process_key` still named the OLD instance, a delayed old-run `pr.progress-check`
+// straggler would read its own key as the current owner, pass `isSuperseded`/`stillOwns`, and
+// fast-forward the PR head onto its stale checkpoint under the fresh run's identity. Asserting the
+// atomic write closes that "converging-but-old-owner" window.
+test("re-submit flips status to converging and advances process_key in the SAME write (no old-owner window, #819)", async () => {
+  await withGithubOff(async () => {
+    const PR_KEY = "owner/repo#42";
+    const patches: Array<Record<string, unknown>> = [];
+    const stores: Record<string, { rows: any[]; key: string }> = {
+      pull_requests: {
+        rows: [{ pr_key: PR_KEY, repo: "owner/repo", number: 42, url: "https://github.com/owner/repo/pull/42", title: "t", status: "converged", process_key: "PI-OLD" }],
+        key: "pr_key",
+      },
+      escalations: { rows: [], key: "id" },
+      pr_adjudications: { rows: [], key: "id" },
+      pr_dependencies: { rows: [], key: "pr_key" },
+    };
+    const wrap = (name: string, key: string) => {
+      const t = memTable(stores[name]?.rows ?? [], stores[name]?.key ?? key);
+      return {
+        ...t,
+        update: (k: any, patch: any) => {
+          if (name === "pull_requests") patches.push({ ...patch });
+          return t.update(k, patch);
+        },
+      };
+    };
+    const data = { table: withTrackingViews(wrap), open: () => memOpen(stores) } as any;
+    const engine = { createInstance: () => Promise.resolve({ processInstanceKey: "PI-NEW" }) } as any;
+
+    await submitPr(data, engine, { repo: "owner/repo", number: 42, url: "https://github.com/owner/repo/pull/42", prKey: PR_KEY });
+
+    // No update may set status to "converging" while leaving process_key at (or below) the old key.
+    const convergingPatch = patches.find((p) => p.status === "converging");
+    assertEquals(convergingPatch !== undefined, true, "the reopen flips the row to converging");
+    assertEquals(convergingPatch?.process_key, "PI-NEW", "the SAME write that flips to converging advances the owner to the new run");
+    assertEquals(
+      patches.some((p) => p.status === "converging" && p.process_key === "PI-OLD"),
+      false,
+      "the row never presents as the fresh run while still owned by the old instance",
+    );
+    const pr = stores.pull_requests.rows[0] as Record<string, unknown>;
+    assertEquals(pr.status, "converging");
+    assertEquals(pr.current_round, 1);
+    assertEquals(pr.process_key, "PI-NEW");
+  });
+});
+
+// Red/green regression (Copilot review of #819, follow-up): deferring the terminal→`converging`
+// transition until AFTER `createInstance()` (the atomic-reopen fix above) left the row in its OLD
+// terminal shape across the create await. A second `submitPr` for the same PR racing that await would
+// read the still-terminal row, pass the RESUBMITTABLE idempotency gate, and create a SECOND
+// convergence instance for one PR. `submitPr` serializes its create→reopen critical section per
+// `prKey`, so the loser observes the winner's claim (`converging` + the new key, derived-active) and
+// returns `alreadyRunning` — exactly one instance is created. Without the lock both concurrent calls
+// enter `createInstance` (calls === 2) and this fails.
+test("concurrent submitPr for one reopened PR mints only ONE convergence instance (#819 follow-up)", async () => {
+  await withGithubOff(async () => {
+    const PR_KEY = "owner/repo#42";
+    const stores: Record<string, { rows: any[]; key: string }> = {
+      pull_requests: {
+        rows: [{ pr_key: PR_KEY, repo: "owner/repo", number: 42, url: "https://github.com/owner/repo/pull/42", title: "t", status: "converged", current_round: 3, process_key: "PI-OLD" }],
+        key: "pr_key",
+      },
+      escalations: { rows: [], key: "id" },
+      pr_adjudications: { rows: [], key: "id" },
+      pr_dependencies: { rows: [], key: "pr_key" },
+    };
+    const data = { table: withTrackingViews((name: string, key: string) => memTable(stores[name]?.rows ?? [], stores[name]?.key ?? key)), open: () => memOpen(stores) } as any;
+    let calls = 0;
+    const engine = {
+      createInstance: async () => {
+        const id = ++calls;
+        // Hold the critical section open long enough for a racing submit to interleave at this await.
+        await new Promise((r) => setTimeout(r, 5));
+        return { processInstanceKey: `PI-${id}` };
+      },
+    } as any;
+    const parsed = { repo: "owner/repo", number: 42, url: "https://github.com/owner/repo/pull/42", prKey: PR_KEY };
+
+    const results = await Promise.all([submitPr(data, engine, parsed), submitPr(data, engine, parsed)]);
+
+    assertEquals(calls, 1, "concurrent submits of the same PR create exactly one convergence instance");
+    assertEquals(
+      results.filter((r: any) => r.alreadyRunning === true).length,
+      1,
+      "the second concurrent submit observes the first's claim and short-circuits at the alreadyRunning gate",
+    );
+    const pr = stores.pull_requests.rows[0] as Record<string, unknown>;
+    assertEquals(pr.status, "converging");
+    assertEquals(pr.process_key, "PI-1", "the single created instance owns the reopened row");
+  });
+});
+
+// A distinct key must never block another PR's submit — the per-`prKey` lock serializes only same-PR
+// calls, so two DIFFERENT PRs submitted concurrently both proceed and each mints its own instance.
+test("submitPr serialization is per-prKey — different PRs never block each other (#819 follow-up)", async () => {
+  await withGithubOff(async () => {
+    const mkStores = (n: number): Record<string, { rows: any[]; key: string }> => ({
+      pull_requests: { rows: [{ pr_key: `owner/repo#${n}`, repo: "owner/repo", number: n, url: `https://github.com/owner/repo/pull/${n}`, title: "t", status: "converged", current_round: 1, process_key: `PI-OLD-${n}` }], key: "pr_key" },
+      escalations: { rows: [], key: "id" },
+      pr_adjudications: { rows: [], key: "id" },
+      pr_dependencies: { rows: [], key: "pr_key" },
+    });
+    const s1 = mkStores(1);
+    const s2 = mkStores(2);
+    const mkData = (s: Record<string, { rows: any[]; key: string }>) => ({ table: withTrackingViews((name: string, key: string) => memTable(s[name]?.rows ?? [], s[name]?.key ?? key)), open: () => memOpen(s) } as any);
+    let calls = 0;
+    const engine = {
+      createInstance: async () => {
+        const id = ++calls;
+        await new Promise((r) => setTimeout(r, 5));
+        return { processInstanceKey: `PI-${id}` };
+      },
+    } as any;
+
+    await Promise.all([
+      submitPr(mkData(s1), engine, { repo: "owner/repo", number: 1, url: "https://github.com/owner/repo/pull/1", prKey: "owner/repo#1" }),
+      submitPr(mkData(s2), engine, { repo: "owner/repo", number: 2, url: "https://github.com/owner/repo/pull/2", prKey: "owner/repo#2" }),
+    ]);
+
+    assertEquals(calls, 2, "two different PRs each mint their own convergence instance");
+    assertEquals(s1.pull_requests.rows[0].status, "converging");
+    assertEquals(s2.pull_requests.rows[0].status, "converging");
+  });
+});
+
+
 // created and `process_key` is advanced. If the reset DELETE fails, the new convergence instance is
 // already live while the OLD adjudications remain — and because the new instance is ACTIVE the
 // `alreadyRunning` idempotency gate short-circuits every retry, so the reset is never re-run and the
@@ -1024,7 +1153,7 @@ test("repoEnvelopeVars emits branch.create only for a non-blank pre-PR branch (#
 // so the round redrives from scratch (exactly as today). Proven against a REAL in-memory SQLite db
 // with the world (049) + enrolment (052) schemas applied.
 test("worldRestoreSha is gated on the durable-resume enrolment: participant → SHA, none → null", async () => {
-  const { data } = memDataFor(["049_world_checkpoint.sql", "052_worker_durable_resume.sql"]);
+  const { data } = memDataFor(["049_world_checkpoint.sql", "052_worker_durable_resume.sql", "115_world_checkpoint_process_key.sql"]);
   const PR = "owner/repo#7";
   const sha = "77ee0993cc6ad4493da0f7551212ef16722135db";
   await new WorldStore(data).recordCheckpoint({ prKey: PR, roundNo: 1, commitSha: sha });
@@ -1042,7 +1171,7 @@ test("worldRestoreSha is gated on the durable-resume enrolment: participant →
 });
 
 test("worldRestoreSha is null when a participant is enrolled but the PR has no checkpoint yet", async () => {
-  const { data } = memDataFor(["049_world_checkpoint.sql", "052_worker_durable_resume.sql"]);
+  const { data } = memDataFor(["049_world_checkpoint.sql", "052_worker_durable_resume.sql", "115_world_checkpoint_process_key.sql"]);
   await new DurableResumeRegistry(data).recordEnrolment("modern-1", true);
   assertEquals(await worldRestoreSha(data, "owner/repo#8"), null, "nothing to reconstruct on a first activation");
 });

package/app/service.ts

@@ -514,9 +514,45 @@ export async function isPrSettled(data: DataLayer, prKey: string): Promise<boole
   return !!existing && TERMINAL_STATUSES.includes(existing.derived_status);
 }
 
+// Per-`prKey` in-process serialization for `submitPr` (Copilot review of #819). `submitPr` reads the
+// PR row, decides `alreadyRunning`, `createInstance`s, and reopens the row across several `await`s.
+// Two `submitPr` calls for the SAME PR (a poll pass racing a delivery-connector redelivery, a
+// record-wave enrollment, …) could otherwise interleave at the `createInstance` await: both read the
+// pre-submit terminal row, both pass the idempotency gate (a terminal row is RESUBMITTABLE), and both
+// create a convergence instance for one PR. Serializing the whole create→reopen critical section per
+// `prKey` makes the second caller observe the row the first already claimed (`converging` + the new
+// `process_key`, derived-active) and short-circuit at the `alreadyRunning` gate instead of minting a
+// duplicate instance. The lock is deliberately IN-PROCESS, never a durable DB claim: every `submitPr`
+// caller — the poller, record-wave, delivery-connector, converge-feature — runs in THIS one Node
+// process (main.ts hosts the poller and the job workers on a single engine client) against one SQLite
+// store, so an in-memory chain fully serializes them; and an in-memory lock evaporates on crash, so a
+// half-done submit recovers through the existing terminal→resubmittable derive path. A durable non-null
+// claim would instead strand a keyed `converging` row `alreadyRunning` FOREVER on a create-instance
+// crash — exactly the #704/#497 phantom the keyless-row guard above exists to avoid. Distinct keys
+// never block each other. This composes with the atomic reopen below (which fences the old-run
+// straggler): the lock removes the duplicate-instance race, the atomic write removes the stale-owner
+// race, and neither reintroduces the other.
+const submitChains = new Map<string, Promise<unknown>>();
+function withPrSubmitLock<T>(key: string, fn: () => Promise<T>): Promise<T> {
+  const prev = submitChains.get(key) ?? Promise.resolve();
+  // Chain after the previous holder SETTLES (fulfilled OR rejected — both handlers are `fn`), so one
+  // failed/throwing submit never wedges the key for the PR's next submit.
+  const run = prev.then(fn, fn);
+  // The stored tail is error-swallowed so the chain survives a rejection; bound the map by dropping
+  // the key once this tail settles and no later caller has replaced it as the chain's end.
+  const tail = run.then(() => undefined, () => undefined);
+  submitChains.set(key, tail);
+  tail.then(() => {
+    if (submitChains.get(key) === tail) submitChains.delete(key);
+  });
+  return run;
+}
+
 /** Register a PR row (if new) and start the convergence process. Idempotent on prKey. Optional
  * `dependsOn` (explicit refs) is unioned with any `Depends-on:` line parsed from the PR body and
- * recorded as the PR's merge-stage dependency set. */
+ * recorded as the PR's merge-stage dependency set. Concurrent calls for the SAME PR are serialized
+ * (see {@link withPrSubmitLock}) so a poll pass racing a worker redelivery can't mint two convergence
+ * instances for one PR; the loser observes the winner's claim and returns `alreadyRunning`. */
 export async function submitPr(
   data: DataLayer,
   engine: EngineClient,
@@ -525,6 +561,20 @@ export async function submitPr(
   maxRounds: number = MAX_ROUNDS,
   convergeOnly = false,
   rootRequestKey: string | null = null,
+) {
+  return withPrSubmitLock(parsed.prKey, () =>
+    submitPrCritical(data, engine, parsed, dependsOn, maxRounds, convergeOnly, rootRequestKey),
+  );
+}
+
+async function submitPrCritical(
+  data: DataLayer,
+  engine: EngineClient,
+  parsed: ParsedPr,
+  dependsOn: string[] = [],
+  maxRounds: number = MAX_ROUNDS,
+  convergeOnly = false,
+  rootRequestKey: string | null = null,
 ) {
   const table = prs(data);
   const existing = await table.get(parsed.prKey);
@@ -604,44 +654,16 @@ export async function submitPr(
     // while `process_key` still names the OLD instance leaves a window where a delayed old-instance
     // answer job still passes the worker's staleness gate and reinserts its adjudication into the fresh
     // run (Copilot review of #806). Advancing the run identity FIRST, then clearing, fences that job.
-    // Re-open a previously converged/abandoned/merged PR for a fresh convergence run.
-    await table.update(parsed.prKey, {
-      status: "converging",
-      current_round: 1,
-      url: parsed.url,
-      // Coalesce to the key so `pull_requests.title` stays non-blank for the title-led grids
-      // (issue #248): a fresh fetch wins, else the prior title, else the `owner/repo#N` key.
-      // A blank/whitespace title counts as missing (matches the 036 backfill), so an empty
-      // external title never lands as an unlabeled row.
-      title: coalesceTitle(title, existing.title, parsed.prKey),
-      waiting_since: null,
-      last_review_id: null,
-      last_nudge_at: null,
-      // Clear the no-progress head baseline: it is scoped to the PRIOR convergence run, and a fresh
-      // run at round 1 must compare its first addressed round against a clean slate. Leaving a stale
-      // `last_round_head` lets a resubmission whose branch changed read `currentHead !== previousHead`
-      // on its first husked round, mis-route it as progress, and bypass the bounded husk retry (#786).
-      last_round_head: null,
-      // Clear the attempt watermark too: it is scoped to the prior convergence run's `review-round`
-      // instances (Copilot #789). A fresh run mints new, higher-keyed instances so a carried-over
-      // watermark would still be below them, but clearing keeps the per-run husk-correlation state
-      // unambiguous and self-contained.
-      last_progress_agent_watermark: null,
-      // Clear the at-least-once REPLAY stamp too (Copilot PR #789). The idempotency guard replays a
-      // recorded outcome whenever a redelivered job key matches this row; if the stamp survived a
-      // re-open, an OLD `pr.progress-check` delivery redelivered after the NEW convergence instance
-      // starts would still match its job key here and replay a stale escalation/progress effect into
-      // the fresh run. Clearing it makes the new run treat any such straggler as an unknown key (a
-      // normal, freshly-computed decision) rather than replaying the prior run's outcome.
-      last_progress_job_key: null,
-      last_progress_result: null,
-      outcome: null,
-      converged_at: null,
-      merged_at: null,
-      abandon_token: abandonToken,
-      root_request_key: effectiveRoot,
-      updated_at: ts,
-    });
+    // Re-open a previously converged/abandoned/merged PR for a fresh convergence run. The row-reset
+    // that flips the PR to `converging`/round 1 is DEFERRED and applied ATOMICALLY WITH the
+    // `process_key` advance below (after `createInstance`), never here. Flipping the row to the fresh
+    // run's shape while `process_key` still names the OLD (now-superseded) instance opens a window in
+    // which a delayed old-run `pr.progress-check` straggler reads its own key as the CURRENT owner,
+    // passes both `isSuperseded` and the late `stillOwns` guard, and fast-forwards the PR head onto
+    // its own stale checkpoint under the fresh run's identity (Copilot review of #819). Publishing the
+    // new run identity in the SAME write that resets the row eliminates that "converging-but-old-owner"
+    // window: the old generation is invalidated the instant the row becomes the new run's, so no
+    // straggler is ever accepted as owner of a row that already presents as the fresh run.
   } else {
     await table.insert({
       pr_key: parsed.prKey,
@@ -701,18 +723,67 @@ export async function submitPr(
     },
   });
   const processKey = processInstanceKey == null ? null : String(processInstanceKey);
-  // The `process_key` advance and the adjudication reset below are the two writes that MAKE the new
-  // run authoritative. If EITHER throws, the newly created instance is already live but the reopen is
-  // only half-committed — and a retry would short-circuit at the `alreadyRunning` idempotency gate
-  // (the new instance is ACTIVE, so `derived_status` is non-terminal), never re-running the reset. A
-  // failed reset would then leave the fresh run replaying STALE adjudication memory indefinitely
-  // (Copilot review). So roll the just-created run back on failure: terminate it and rethrow, so the
-  // submission is NOT treated as started. Terminating flips the PR's derived tracking status to a
-  // terminal edge (`abandoned`) via the `instanceTracking` reconciler, making the PR resubmittable so
-  // a retry re-creates a fresh instance and re-runs the reset cleanly — no orphaned run auto-applies
-  // stale decisions in the meantime.
+  // The reopen row-reset, the `process_key` advance, and the adjudication reset below are the writes
+  // that MAKE the new run authoritative. If ANY throws, the newly created instance is already live but
+  // the reopen is only half-committed — and a retry would short-circuit at the `alreadyRunning`
+  // idempotency gate (the new instance is ACTIVE, so `derived_status` is non-terminal), never
+  // re-running them. A failed reset would then leave the fresh run replaying STALE adjudication memory
+  // indefinitely (Copilot review). So roll the just-created run back on failure: terminate it and
+  // rethrow, so the submission is NOT treated as started. Terminating flips the PR's derived tracking
+  // status to a terminal edge (`abandoned`) via the `instanceTracking` reconciler, making the PR
+  // resubmittable so a retry re-creates a fresh instance and re-runs everything cleanly — no orphaned
+  // run auto-applies stale decisions, and (crucially) the row stays in its OLD terminal shape until
+  // the atomic reopen write lands, so no straggler ever sees a fresh-run row still owned by the old
+  // instance.
   try {
-    if (processKey != null) {
+    if (existing) {
+      // ATOMIC REOPEN (Copilot review of #819): reset the row to the fresh run's shape AND publish the
+      // new run identity (`process_key`) in a SINGLE write, so the row never presents as `converging`
+      // round 1 while still owned by the OLD instance. A delayed old-run straggler is therefore never
+      // accepted as the current owner of a row that already belongs to the fresh run.
+      await table.update(parsed.prKey, {
+        status: "converging",
+        current_round: 1,
+        url: parsed.url,
+        // Coalesce to the key so `pull_requests.title` stays non-blank for the title-led grids
+        // (issue #248): a fresh fetch wins, else the prior title, else the `owner/repo#N` key.
+        // A blank/whitespace title counts as missing (matches the 036 backfill), so an empty
+        // external title never lands as an unlabeled row.
+        title: coalesceTitle(title, existing.title, parsed.prKey),
+        waiting_since: null,
+        last_review_id: null,
+        last_nudge_at: null,
+        // Clear the no-progress head baseline: it is scoped to the PRIOR convergence run, and a fresh
+        // run at round 1 must compare its first addressed round against a clean slate. Leaving a stale
+        // `last_round_head` lets a resubmission whose branch changed read `currentHead !== previousHead`
+        // on its first husked round, mis-route it as progress, and bypass the bounded husk retry (#786).
+        last_round_head: null,
+        // Clear the attempt watermark too: it is scoped to the prior convergence run's `review-round`
+        // instances (Copilot #789). A fresh run mints new, higher-keyed instances so a carried-over
+        // watermark would still be below them, but clearing keeps the per-run husk-correlation state
+        // unambiguous and self-contained.
+        last_progress_agent_watermark: null,
+        // Clear the at-least-once REPLAY stamp too (Copilot PR #789). The idempotency guard replays a
+        // recorded outcome whenever a redelivered job key matches this row; if the stamp survived a
+        // re-open, an OLD `pr.progress-check` delivery redelivered after the NEW convergence instance
+        // starts would still match its job key here and replay a stale escalation/progress effect into
+        // the fresh run. Clearing it makes the new run treat any such straggler as an unknown key (a
+        // normal, freshly-computed decision) rather than replaying the prior run's outcome.
+        last_progress_job_key: null,
+        last_progress_result: null,
+        outcome: null,
+        converged_at: null,
+        merged_at: null,
+        abandon_token: abandonToken,
+        root_request_key: effectiveRoot,
+        // Publish the fresh run's identity in the SAME write as the status flip (may be null if the
+        // instance could not be created — a phantom row a retry re-enrolls, as on first submit).
+        process_key: processKey,
+        updated_at: ts,
+      });
+    } else if (processKey != null) {
+      // First submit: the row was INSERTED keyless before `createInstance` (the phantom-row pattern,
+      // #704/#497); publish its `process_key` now that the instance exists.
       await table.update(parsed.prKey, { process_key: processKey });
     }
     // Invalidate this PR's durable adjudication memory for the fresh run (issue #806, Copilot review) —
@@ -721,8 +792,7 @@ export async function submitPr(
     // rejected by the worker's staleness gate (its `processInstanceKey` no longer matches the advanced
     // `process_key`), so it cannot reinsert a stale adjudication after the reset; and the worker reads
     // `process_key` as late as possible so it observes this advance. The insert-if-absent record then
-    // re-learns the operator's new answer for the new run. Runs unconditionally (even if `processKey` is
-    // null: the memory must still be clean for the fresh run). The wipe is a SINGLE atomic `DELETE`
+    // re-learns the operator's new answer for the new run. The wipe is a SINGLE atomic `DELETE`
     // (`resetAdjudications`), never a row-by-row loop, so a crash mid-reset cannot leave a partially
     // cleared memory (Copilot review of #806).
     if (existing) {

package/app/world/checkpoint.ts

@@ -45,6 +45,10 @@ export interface WorldCheckpointInput {
   /** The round's irreversible effects, in order. Defaults to a single `push` effect keyed by the
    * commit SHA. */
   readonly effects?: readonly Effect[];
+  /** The convergence RUN (process instance) recording this checkpoint — persisted so the no-advance
+   * self-heal can prove the checkpoint belongs to THIS run and never heals onto a prior run's stale
+   * checkpoint (issue #819). `null`/omitted when the recording job has no process instance. */
+  readonly processKey?: string | null;
 }
 
 /** The result of recording a world checkpoint: the shared offset (mind + world commit at it) and the
@@ -73,6 +77,7 @@ export async function recordWorldCheckpoint(
     roundNo: input.roundNo,
     commitSha: input.commitSha,
     effects,
+    processKey: input.processKey ?? null,
   });
   // The JOIN: hand the mind the identical checkpoint the world just persisted, at the same boundary.
   // Advisory — a sink failure must not undo the durable world record (the world is authoritative for