@nanobpm/nano-workforce 0.191.1 → 0.193.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,194 @@
1
+ // No-advance self-heal (issue #818, the Layer 2 durable fix for the agent PR-head push-divergence
2
+ // class — jwulf/c8ctl-plugin-nano#270).
3
+ //
4
+ // The convergence loop escalates an `addressed` round whose PR head did NOT advance (see
5
+ // app/roundProgress.ts). But the round's work is often ALREADY pushed and reachable on the
6
+ // remote — just not on the PR head. A producer harness that provisions the review shape wrong cuts a
7
+ // throwaway `nano/agent-work/*` fallback branch and pushes the fix THERE, so the agent's "pushed"
8
+ // claim is true while the PR head never moves (this stranded nanobpm/nano-coder#26/#28, recovered by
9
+ // hand). Layer 1 (jwulf/c8ctl-plugin-nano#270) fixes the harness to push the PR head directly AND to
10
+ // report the pushed SHA as a `worldMarker`, which `pr.persist-round` records as a durable
11
+ // push-checkpoint (`world_checkpoints`). This module closes the loop: BEFORE escalating a no-advance
12
+ // round, reconcile that recorded checkpoint SHA against the PR head — if it is a STRICT
13
+ // fast-forward descendant of the head, advance the head onto it and continue the loop as real
14
+ // progress instead of parking a human.
15
+ //
16
+ // It keys off the recorded SHA (canonical world state), not any branch-naming convention, so it heals
17
+ // the whole class: any producer that leaves the round's work reachable-but-off-the-head. Every step
18
+ // fails SAFE — an absent checkpoint, an unreadable comparison, a non-fast-forward (diverged) tip, a
19
+ // cross-repo fork head, or a refused ref update all yield `{ healed: false }`, so the caller escalates
20
+ // exactly as it did before. It is purely additive to the existing no-advance path.
21
+
22
+ /** The push-checkpoint reader — the newest durable `{ commitSha }` recorded by a SPECIFIC RUN of a PR
23
+ * AT A SPECIFIC ROUND, or `null` when that run/round pushed nothing (so there is nothing to
24
+ * reconcile). Scoped by the convergence RUN's `processKey` AND `roundNo` so a heal is anchored to THIS
25
+ * attempt's own checkpoint and can never fast-forward the head onto a stale checkpoint from a prior
26
+ * run — `round_no` alone does not identify a run, since a reopen resets `current_round` while the old
27
+ * run's checkpoints survive (#819). Backed by {@link ../world/store.ts}'s
28
+ * `WorldStore.lastCheckpointForRun`; injectable for tests. */
29
+ export type CheckpointReader = (
30
+ prKey: string,
31
+ processKey: string | null,
32
+ roundNo: number,
33
+ ) => Promise<{ commitSha: string } | null>;
34
+
35
+ /** The PR-head reader — the head branch name + repo of a PR, so the self-heal knows WHICH ref to
36
+ * advance and whether it lives in THIS repo (a fork head is unpushable from the base token). Backed
37
+ * by `fetchPrHead`; injectable for tests. */
38
+ export type PrHeadReader = (
39
+ repo: string,
40
+ prNumber: number,
41
+ ) => Promise<{ headRef: string | null; headRepo: string | null } | null>;
42
+
43
+ /** The ancestry comparator — proves the checkpoint SHA strictly fast-forward-descends the PR head.
44
+ * Backed by `compareCommits`; injectable for tests. `null` means the comparison was unavailable. */
45
+ export type CommitComparator = (
46
+ repo: string,
47
+ base: string,
48
+ head: string,
49
+ ) => Promise<{ status: "ahead" | "behind" | "identical" | "diverged"; aheadBy: number; behindBy: number } | null>;
50
+
51
+ /** The fast-forward ref mover — advances the PR head branch to the checkpoint SHA. Backed by
52
+ * `updateBranchRef` (a NON-force update, so GitHub itself refuses a non-fast-forward). The 4th
53
+ * argument is the COMPARE-AND-SWAP precondition: the exact head SHA the fast-forward proof was
54
+ * computed against (`currentHead`); the mover refuses (`false`) unless the ref still equals it at
55
+ * mutation time, so a head advanced/replaced by a concurrent (superseded) run after the proof is
56
+ * never PATCHed. Returns `false` when the move was refused (non-fast-forward OR lost CAS). Injectable
57
+ * for tests. */
58
+ export type HeadAdvancer = (repo: string, branch: string, sha: string, expectedSha: string) => Promise<boolean>;
59
+
60
+ export interface SelfHealDeps {
61
+ readonly lastCheckpoint: CheckpointReader;
62
+ readonly prHead: PrHeadReader;
63
+ readonly compare: CommitComparator;
64
+ readonly advanceHead: HeadAdvancer;
65
+ /** A LATE ownership guard, re-checked immediately BEFORE the (irreversible) ref mutation to close
66
+ * the check-to-mutation TOCTOU (Copilot review on #819). The worker's pre-call `isSuperseded()`
67
+ * read is not a fence across the async heal: after it returns, this function still awaits the
68
+ * checkpoint lookup, PR-head read, and compare before `advanceHead`, and `submitPr` can install a
69
+ * new `process_key` (a fresh run) in that window. Returns `true` while this run still owns the PR
70
+ * row; when it returns `false` the heal is abandoned WITHOUT touching the ref, so a superseded
71
+ * straggler can never PATCH the branch. Optional — omitted (or absent) means "assume ownership"
72
+ * (the fail-open convention the worker's `isSuperseded` already uses when the identity is unknown),
73
+ * so a caller with no supersession concept keeps the pure #818 behaviour. */
74
+ readonly stillOwns?: () => Promise<boolean>;
75
+ }
76
+
77
+ /** Why a no-advance round did or did not self-heal — `healed` drives the routing (continue vs.
78
+ * escalate); `sha`/`reason` are for the operator log so a skipped heal is diagnosable. */
79
+ export interface SelfHealResult {
80
+ readonly healed: boolean;
81
+ /** The SHA the PR head was advanced to (only when `healed`). */
82
+ readonly sha?: string;
83
+ /** A short machine-readable reason a heal was skipped (only when NOT `healed`). */
84
+ readonly reason?:
85
+ | "no-checkpoint"
86
+ | "already-at-head"
87
+ | "no-head-ref"
88
+ | "fork-head"
89
+ | "compare-unavailable"
90
+ | "not-fast-forward"
91
+ | "superseded"
92
+ /** The ref move was refused: GitHub rejected the non-fast-forward, OR the compare-and-swap lost
93
+ * (the head no longer equals the `currentHead` the fast-forward proof was validated against — a
94
+ * concurrent run moved it). */
95
+ | "advance-refused";
96
+ }
97
+
98
+ /** Attempt to self-heal a no-advance round by fast-forwarding the PR head onto a recorded
99
+ * push-checkpoint SHA. Returns `{ healed: true, sha }` ONLY when a recorded checkpoint strictly
100
+ * fast-forward-descends the current head AND the head was moved onto it; every other outcome is a
101
+ * safe `{ healed: false, reason }` (→ the caller escalates). Never throws: a rejected promise from
102
+ * any dep is caught and degrades to `{ healed: false }` so a transient GitHub/DB outage can never
103
+ * fabricate a heal (or crash the guard).
104
+ *
105
+ * @param currentHead the PR head SHA the no-advance decision was made against (the compare base). It
106
+ * is non-null at the escalation point — a null/unreadable head fails OPEN upstream (continues) and
107
+ * never reaches here — so the comparison is always anchored on the real head.
108
+ * @param roundNo the round being reconciled; the checkpoint lookup is scoped to it (and to
109
+ * `processKey`) so the heal can only fast-forward onto THIS run's own pushed SHA, never a stale
110
+ * prior-run/later-round one.
111
+ * @param processKey the convergence RUN (process instance) reconciling the round. Scopes the
112
+ * checkpoint lookup to this run so a reopened PR (whose `current_round` reset to 1 while the prior
113
+ * run's checkpoints survive) can never select the old run's checkpoint (#819). */
114
+ export async function attemptNoAdvanceSelfHeal(
115
+ deps: SelfHealDeps,
116
+ repo: string,
117
+ prNumber: number,
118
+ prKey: string,
119
+ currentHead: string,
120
+ roundNo: number,
121
+ processKey: string | null,
122
+ ): Promise<SelfHealResult> {
123
+ try {
124
+ const checkpoint = await deps.lastCheckpoint(prKey, processKey, roundNo);
125
+ if (!checkpoint) return { healed: false, reason: "no-checkpoint" };
126
+ const sha = checkpoint.commitSha;
127
+ // The head is already at the checkpoint — nothing stranded, nothing to move. (A no-advance round
128
+ // reaching here with head === checkpoint would be a benign no-op; treat it as "nothing to heal".)
129
+ if (sha === currentHead) return { healed: false, reason: "already-at-head" };
130
+
131
+ const head = await deps.prHead(repo, prNumber);
132
+ const headRef = head?.headRef;
133
+ if (!headRef) return { healed: false, reason: "no-head-ref" };
134
+ // Require a KNOWN, same-repo head before touching any ref. A fork-based PR's head branch lives in
135
+ // ANOTHER repo, and an UNRESOLVABLE head repo (`headRepo == null`, e.g. a deleted fork) is equally
136
+ // unhealable: `fetchPrHead` reports `null` when it cannot resolve the head's owner/name, yet
137
+ // `headRef` can still be a same-named branch that also exists in the base repo. Treating `null` as
138
+ // "in the base repo" would PATCH that unrelated base-repo branch, so classify any non-match (a real
139
+ // fork OR an unresolved head) as an unhealable `fork-head` and escalate instead of risking a
140
+ // cross-repo / wrong-branch ref write. Compare NORMALIZED (case-insensitive) identities: `parsePr`
141
+ // preserves the caller's `owner/repo` spelling while GitHub returns the repo's canonical
142
+ // `head.repo.full_name`, so a valid same-repo PR submitted as `Owner/Repo#7` would otherwise
143
+ // mismatch the canonical `owner/repo`, be misclassified `fork-head`, and silently skip every
144
+ // self-heal (Copilot review of #819). GitHub owner/repo names are case-insensitive, so lowercasing
145
+ // both sides is the correct identity test while retaining the null rejection.
146
+ if (head.headRepo == null || head.headRepo.toLowerCase() !== repo.toLowerCase()) {
147
+ return { healed: false, reason: "fork-head" };
148
+ }
149
+
150
+ // Prove the checkpoint STRICTLY fast-forward-descends the current head before moving anything: a
151
+ // `behind`/`identical`/`diverged` checkpoint is NOT a safe fast-forward and must escalate. Anchor
152
+ // the compare base on `currentHead` (the SHA the no-advance verdict used), not the branch name,
153
+ // so a mid-check head move can't make us fast-forward onto the wrong baseline.
154
+ const cmp = await deps.compare(repo, currentHead, sha);
155
+ if (!cmp) return { healed: false, reason: "compare-unavailable" };
156
+ // Validate the compare COUNTS before trusting the fast-forward proof: a malformed compare
157
+ // response can coerce `ahead_by`/`behind_by` to `NaN` (`Number("not-a-number")`), and because
158
+ // `NaN <= 0` and `NaN !== 0` are BOTH false, a bare `aheadBy <= 0 || behindBy !== 0` gate would
159
+ // let a `{status:"ahead", behindBy:0, aheadBy:NaN}` response slip through and PATCH the ref onto
160
+ // an unproven SHA. Require finite INTEGER counts (a strict fast-forward is `behindBy === 0` AND
161
+ // `aheadBy > 0`); any non-integer count is a corrupt proof and must escalate, not heal.
162
+ if (
163
+ cmp.status !== "ahead" ||
164
+ !Number.isInteger(cmp.aheadBy) ||
165
+ !Number.isInteger(cmp.behindBy) ||
166
+ cmp.behindBy !== 0 ||
167
+ cmp.aheadBy <= 0
168
+ ) {
169
+ return { healed: false, reason: "not-fast-forward" };
170
+ }
171
+
172
+ // LATE OWNERSHIP FENCE (Copilot review on #819). Everything above only READ GitHub/DB state; the
173
+ // ONLY irreversible act is the ref move below. Two guards bind it to THIS run's validated state:
174
+ // (1) the late `stillOwns()` re-check — right before the move — abandons a run that was superseded
175
+ // DURING the awaits above (a concurrent `submitPr` installed a new `process_key`), instead of
176
+ // moving the head and only then being discarded by the downstream `commit` fence (which guards the
177
+ // DB write, not a ref mutation already made). Absent guard ⇒ assume ownership (the same fail-open
178
+ // the worker's `isSuperseded` uses when identity is unknown). (2) the move itself is a
179
+ // COMPARE-AND-SWAP keyed on `currentHead` — the exact base the fast-forward proof was computed
180
+ // against — so the mutation's precondition is BOUND TO the validated state rather than trusted from
181
+ // the separate `stillOwns()` read: a straggler that races past (1) still PATCHes nothing unless the
182
+ // ref is verifiably still at `currentHead` at mutation time. GitHub exposes no server-side
183
+ // expected-old-value precondition, so the CAS is the tightest fence the platform allows; the
184
+ // non-force fast-forward invariant makes the bounded residual window fail-safe (head only ever
185
+ // moves forward onto this run's own reachable checkpoint, never a rewrite).
186
+ if (deps.stillOwns && !(await deps.stillOwns())) return { healed: false, reason: "superseded" };
187
+
188
+ const advanced = await deps.advanceHead(repo, headRef, sha, currentHead);
189
+ if (!advanced) return { healed: false, reason: "advance-refused" };
190
+ return { healed: true, sha };
191
+ } catch {
192
+ return { healed: false };
193
+ }
194
+ }
@@ -33,6 +33,55 @@ test("lastCheckpoint returns the newest push-checkpoint (max offset), or null wh
33
33
  assertEquals(last, { offset: 1, commitSha: "sha-b", roundNo: 2 }, "the newest checkpoint wins");
34
34
  });
35
35
 
36
+ test("lastCheckpointForRun scopes to one RUN's round newest checkpoint (never a stale prior-run round, #819)", async () => {
37
+ const { data } = memWorldData();
38
+ const store = new WorldStore(data);
39
+ const RUN_A = "1001";
40
+ const RUN_B = "2002";
41
+ assertEquals(await store.lastCheckpointForRun(PR, RUN_A, 1), null, "no checkpoint for the run/round yet");
42
+ // Prior run A pushes round 1 and round 2 checkpoints.
43
+ await store.recordCheckpoint({ prKey: PR, roundNo: 1, commitSha: "sha-a", processKey: RUN_A });
44
+ await store.recordCheckpoint({ prKey: PR, roundNo: 2, commitSha: "sha-b", processKey: RUN_A });
45
+ // The PR is reopened: a FRESH run B starts at round 1 and re-pushes (a NEW, higher-offset row).
46
+ await store.recordCheckpoint({ prKey: PR, roundNo: 1, commitSha: "sha-c", processKey: RUN_B });
47
+ assertEquals(
48
+ await store.lastCheckpointForRun(PR, RUN_B, 1),
49
+ { offset: 2, commitSha: "sha-c", roundNo: 1 },
50
+ "run B's round-1 checkpoint wins — never run A's same-numbered round-1 checkpoint",
51
+ );
52
+ assertEquals(
53
+ await store.lastCheckpointForRun(PR, RUN_A, 1),
54
+ { offset: 0, commitSha: "sha-a", roundNo: 1 },
55
+ "run A still resolves to its OWN round-1 checkpoint, not run B's",
56
+ );
57
+ assertEquals(
58
+ await store.lastCheckpointForRun(PR, RUN_A, 2),
59
+ { offset: 1, commitSha: "sha-b", roundNo: 2 },
60
+ "run A round 2 resolves to its own checkpoint",
61
+ );
62
+ // A fresh run at round 1 that has pushed NOTHING never inherits a prior run's checkpoint.
63
+ assertEquals(
64
+ await store.lastCheckpointForRun(PR, "3003", 1),
65
+ null,
66
+ "a run with no checkpoint of its own can never heal onto a prior run's stale checkpoint",
67
+ );
68
+ assertEquals(await store.lastCheckpointForRun(PR, RUN_A, 3), null, "a round that pushed nothing has no checkpoint");
69
+ // A null process key (a job with no process instance) is un-attributable — it matches nothing.
70
+ assertEquals(await store.lastCheckpointForRun(PR, null, 1), null, "a null process key can never resolve a checkpoint");
71
+ });
72
+
73
+ test("lastCheckpointForRun never returns a NULL-process_key (pre-#819 / un-attributable) checkpoint", async () => {
74
+ const { data } = memWorldData();
75
+ const store = new WorldStore(data);
76
+ // A checkpoint recorded without a process key (a legacy row, or a job with no process instance).
77
+ await store.recordCheckpoint({ prKey: PR, roundNo: 1, commitSha: "sha-legacy" });
78
+ assertEquals(
79
+ await store.lastCheckpointForRun(PR, "1001", 1),
80
+ null,
81
+ "an un-attributable checkpoint is never healed onto — it degrades to a safe escalation",
82
+ );
83
+ });
84
+
36
85
  test("recordCheckpoint defaults to a single push effect keyed by the commit SHA", async () => {
37
86
  const { data } = memWorldData();
38
87
  const store = new WorldStore(data);
@@ -20,6 +20,11 @@ interface WorldCheckpointRow {
20
20
  round_no: number;
21
21
  checkpoint_offset: number;
22
22
  commit_sha: string;
23
+ /** The convergence RUN (process instance) that recorded this checkpoint — `String(processInstanceKey)`
24
+ * of the `pr.persist-round` job. NULL on pre-#819 rows or a job without a `processInstanceKey`. The
25
+ * run-scoped self-heal lookup ({@link WorldStore.lastCheckpointForRun}) requires an EXACT non-null
26
+ * match, so a NULL-keyed checkpoint is never attributed to a run and can never heal (issue #819). */
27
+ process_key: string | null;
23
28
  created_at: string;
24
29
  }
25
30
 
@@ -58,6 +63,12 @@ export interface RecordCheckpointInput {
58
63
  * a checkpoint records what a round already DID). A pending tail effect (recorded before it is
59
64
  * performed, for crash-precise replay) is recorded with `applied: false`. */
60
65
  readonly applied?: boolean;
66
+ /** The convergence RUN (process instance) recording this checkpoint — `String(processInstanceKey)`
67
+ * of the persist-round job. Persisted so the no-advance self-heal can prove a checkpoint belongs to
68
+ * THIS run and never fast-forwards the head onto a prior run's stale checkpoint (issue #819).
69
+ * Omitted/`null` for a job without a process instance (the checkpoint is then un-attributable and
70
+ * the run-scoped lookup skips it — a safe escalation, not a heal). */
71
+ readonly processKey?: string | null;
61
72
  }
62
73
 
63
74
  /** A durable store over the `world_checkpoints` + `world_effects` tables. */
@@ -143,6 +154,7 @@ export class WorldStore {
143
154
  prKey: string,
144
155
  roundNo: number,
145
156
  commitSha: string,
157
+ processKey: string | null,
146
158
  now: string,
147
159
  ): Promise<number> {
148
160
  const offset = await WorldStore.#nextOffsetOn(checkpoints, prKey);
@@ -152,6 +164,7 @@ export class WorldStore {
152
164
  round_no: roundNo,
153
165
  checkpoint_offset: offset,
154
166
  commit_sha: commitSha,
167
+ process_key: processKey,
155
168
  created_at: now,
156
169
  });
157
170
  return offset;
@@ -188,6 +201,7 @@ export class WorldStore {
188
201
  */
189
202
  async recordCheckpoint(input: RecordCheckpointInput): Promise<number> {
190
203
  const { prKey, roundNo, commitSha } = input;
204
+ const processKey = input.processKey ?? null;
191
205
  const effects: readonly Effect[] = input.effects ?? [{ kind: "push", idempotencyKey: commitSha }];
192
206
  const applied = input.applied ?? true;
193
207
  const now = new Date().toISOString();
@@ -197,7 +211,7 @@ export class WorldStore {
197
211
  const existing = await checkpoints.findOne({ pr_key: prKey, commit_sha: commitSha });
198
212
  const offset = existing
199
213
  ? existing.checkpoint_offset
200
- : await WorldStore.#insertCheckpointFenced(checkpoints, prKey, roundNo, commitSha, now);
214
+ : await WorldStore.#insertCheckpointFenced(checkpoints, prKey, roundNo, commitSha, processKey, now);
201
215
  let seq = await WorldStore.#nextSeqOn(effectsTable, prKey, offset);
202
216
  for (const effect of effects) {
203
217
  await WorldStore.#appendEffect(effectsTable, prKey, offset, seq++, effect, applied, now);
@@ -258,6 +272,28 @@ export class WorldStore {
258
272
  return { offset: newest.checkpoint_offset, commitSha: newest.commit_sha, roundNo: newest.round_no };
259
273
  }
260
274
 
275
+ /** The newest push-checkpoint recorded by a SPECIFIC RUN of a PR at a SPECIFIC ROUND, or `null` when
276
+ * that run/round pushed nothing. Unlike {@link lastCheckpoint} (newest across ALL runs/rounds), this
277
+ * proves the checkpoint belongs to the attempt being reconciled: the no-advance self-heal (#818,
278
+ * #819) must NOT fast-forward the head onto a checkpoint left by a PRIOR run. `round_no` alone does
279
+ * not identify a run — `submitPr` resets a reopened PR's `current_round` to 1 while the prior run's
280
+ * checkpoints survive, so a fresh run at round 1 would select the OLD run's round-1 checkpoint and
281
+ * could resurrect stale work if it descends the (reset) head. Scoping to the convergence RUN's
282
+ * `process_key` (the instance key advanced on every reopen) AND `round_no` anchors the heal to THIS
283
+ * run's own pushed SHA. A `null` `processKey` (a job with no process instance) matches NOTHING and
284
+ * yields `null`, and a checkpoint row with a `NULL` `process_key` (pre-#819, or an un-attributable
285
+ * record) is likewise never returned — both degrade to a safe escalation, never a heal. Still
286
+ * newest-by-offset within the match, so a genuine re-push of the same round wins. */
287
+ async lastCheckpointForRun(prKey: string, processKey: string | null, roundNo: number): Promise<LastCheckpoint | null> {
288
+ if (processKey == null) return null;
289
+ const rows = (await this.#checkpoints().find({ pr_key: prKey })).filter(
290
+ (r) => r.process_key === processKey && r.round_no === roundNo,
291
+ );
292
+ if (rows.length === 0) return null;
293
+ const newest = rows.reduce((a, b) => (b.checkpoint_offset > a.checkpoint_offset ? b : a));
294
+ return { offset: newest.checkpoint_offset, commitSha: newest.commit_sha, roundNo: newest.round_no };
295
+ }
296
+
261
297
  /** The effect tail recorded at a checkpoint offset, in `seq` order — the sequence the restore path
262
298
  * fence-replays after checking the working tree out to that checkpoint's SHA. `seq` is the intended
263
299
  * order, but it is allocated by a racy read-max-plus-one (`#nextSeqOn`) and the schema has no
@@ -0,0 +1,22 @@
1
+ -- 115_world_checkpoint_process_key.sql — issue #819 (Copilot review): bind a push-checkpoint to the
2
+ -- convergence RUN that produced it, so the no-advance self-heal can prove a checkpoint belongs to the
3
+ -- CURRENT run and never fast-forwards the PR head onto a stale checkpoint left by a PRIOR run.
4
+ --
5
+ -- WHY. The no-advance self-heal (app/world/selfHeal.ts, #818) reconciles a recorded push-checkpoint
6
+ -- against the PR head. #819 scoped that lookup to the current `round_no`, but `round_no` alone does
7
+ -- NOT identify a run: `submitPr` (app/service.ts) resets a reopened PR's `current_round` to 1 while
8
+ -- leaving the prior run's `world_checkpoints` rows in place, so a fresh run at round 1 would still
9
+ -- select the previous run's round-1 checkpoint — and if the branch was reset behind that old SHA the
10
+ -- fast-forward compare passes and the heal resurrects stale work. The convergence instance key
11
+ -- (`pull_requests.process_key`, advanced to the new run on every reopen) is the run generation, so
12
+ -- recording it on each checkpoint lets the self-heal scope by `(process_key, round_no)` — proving the
13
+ -- checkpoint is THIS run's own pushed SHA.
14
+ --
15
+ -- EXPAND (additive) phase: one nullable column, no default, nothing dropped or renamed. `NULL` on the
16
+ -- pre-existing rows (and on any checkpoint recorded by a job without a `processInstanceKey`) is the
17
+ -- SAFE value: the run-scoped lookup requires an EXACT non-null match, so a NULL-keyed checkpoint is
18
+ -- never attributed to a run and can never heal — it degrades to escalation, exactly as an absent
19
+ -- checkpoint does. Numbered after the current highest prefix on origin/main (114). The runner wraps
20
+ -- each file in its own transaction, so this file must NOT contain BEGIN/COMMIT.
21
+
22
+ ALTER TABLE world_checkpoints ADD COLUMN process_key TEXT;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@nanobpm/nano-workforce",
3
- "version": "0.191.1",
3
+ "version": "0.193.0",
4
4
  "description": "Nano Workforce — an Agent Graph Orchestration application for Agentic SDLC: durable BPMN processes that coordinate a graph of AI agents across the software delivery lifecycle.",
5
5
  "type": "module",
6
6
  "main": "main.ts",
package/test/worldDb.ts CHANGED
@@ -118,5 +118,5 @@ export function memDataFor(migrationFiles: readonly string[]): { data: DataLayer
118
118
 
119
119
  /** A `DataLayer` stub over a fresh in-memory db with the world schema applied. */
120
120
  export function memWorldData(): { data: DataLayer; db: DatabaseSync } {
121
- return memDataFor(["049_world_checkpoint.sql"]);
121
+ return memDataFor(["049_world_checkpoint.sql", "115_world_checkpoint_process_key.sql"]);
122
122
  }
@@ -202,6 +202,12 @@ const handler: AppJobHandler<In> = async (job, app) => {
202
202
  prKey,
203
203
  roundNo: round,
204
204
  commitSha: marker.commitSha,
205
+ // Bind the checkpoint to THIS convergence run (issue #819): the no-advance self-heal scopes
206
+ // its lookup by `(process_key, round_no)` so it can never fast-forward the head onto a prior
207
+ // run's stale checkpoint. `processInstanceKey` is this run's instance key — the same value
208
+ // `submitPr` advances into `pull_requests.process_key` and the self-heal's ownership guard
209
+ // compares against.
210
+ processKey: processInstanceKey,
205
211
  ...(marker.effects ? { effects: marker.effects } : {}),
206
212
  });
207
213
  } catch (err) {
@@ -17,9 +17,11 @@
17
17
  // transient GitHub hiccup can never fabricate a no-progress escalation.
18
18
  import type { AgentInstanceSummary, AppApi, AppJobHandler } from "@nanobpm/urban";
19
19
  import { type HeadReader, makeDefaultReadHead } from "../../app/currentHead.ts";
20
- import { fetchBranchHead, fetchPrHead } from "../../app/github.ts";
20
+ import { compareCommits, fetchBranchHead, fetchPrHead, updateBranchRef } from "../../app/github.ts";
21
21
  import { decideProgress, isAddressedStatus } from "../../app/roundProgress.ts";
22
22
  import { parsePr } from "../../app/service.ts";
23
+ import { WorldStore } from "../../app/world/index.ts";
24
+ import { attemptNoAdvanceSelfHeal, type SelfHealResult } from "../../app/world/selfHeal.ts";
23
25
  import type { WorkerInputs, WorkerOutputs } from "../../nano-generated/worker-io.d.ts";
24
26
 
25
27
  // Input/output typed off the model data envelopes (`PrProgressCheckIn` / `PrProgressCheckOut` in
@@ -75,6 +77,45 @@ export { type HeadReader, makeDefaultReadHead };
75
77
 
76
78
  const defaultReadHead: HeadReader = makeDefaultReadHead({ fetchPrHead, fetchBranchHead });
77
79
 
80
+ /** Attempt the no-advance self-heal (#818): reconcile a recorded push-checkpoint against the PR head
81
+ * and fast-forward the head onto it when safe. Injectable so tests never touch git/network; the
82
+ * default binds the real `WorldStore` (over `app.data`) + GitHub transport with the env credential
83
+ * (the same `process.env.GITHUB_TOKEN` default the head reader uses). See {@link
84
+ * ../../app/world/selfHeal.ts}. */
85
+ export type SelfHealFn = (
86
+ app: AppApi,
87
+ repo: string,
88
+ prNumber: number,
89
+ prKey: string,
90
+ currentHead: string,
91
+ roundNo: number,
92
+ processKey: string | null,
93
+ stillOwns: () => Promise<boolean>,
94
+ ) => Promise<SelfHealResult>;
95
+
96
+ const defaultSelfHeal: SelfHealFn = (app, repo, prNumber, prKey, currentHead, roundNo, processKey, stillOwns) => {
97
+ const tok = process.env.GITHUB_TOKEN ?? "";
98
+ const store = new WorldStore(app.data);
99
+ return attemptNoAdvanceSelfHeal(
100
+ {
101
+ lastCheckpoint: (pk, run, rn) => store.lastCheckpointForRun(pk, run, rn),
102
+ prHead: async (r, n) => {
103
+ const h = await fetchPrHead(r, n, tok);
104
+ return h ? { headRef: h.headRef, headRepo: h.headRepo } : null;
105
+ },
106
+ compare: (r, base, head) => compareCommits(r, base, head, tok),
107
+ advanceHead: (r, branch, sha, expected) => updateBranchRef(r, branch, sha, tok, expected),
108
+ stillOwns,
109
+ },
110
+ repo,
111
+ prNumber,
112
+ prKey,
113
+ currentHead,
114
+ roundNo,
115
+ processKey,
116
+ );
117
+ };
118
+
78
119
  /** An agent-instance is "durable work" for husk purposes once it has reached a terminal state — it
79
120
  * carries a `completionDate`, or a terminal lifecycle status. A husked instance never closes (it is
80
121
  * stuck `THINKING` with null dates — jwulf/c8ctl-plugin-nano#230), so it is NOT counted. */
@@ -203,6 +244,7 @@ function agentWorkFromEngine(engine: AppApi["engine"]): AgentWorkReader {
203
244
  export function makeHandler(deps: {
204
245
  readHead: HeadReader;
205
246
  readAgentWork?: AgentWorkReader;
247
+ selfHeal?: SelfHealFn;
206
248
  }): AppJobHandler<In, Out> {
207
249
  return async (job, app) => {
208
250
  const { prKey, status, repo, prNumber, round, huskRetries, roundEntryHead } = job.variables;
@@ -241,25 +283,30 @@ export function makeHandler(deps: {
241
283
  // the single writer of the review-wait park (#786) — persist-round no longer sets
242
284
  // `waiting_review`, so the row rests on its running `converging` status until exactly one write
243
285
  // here resolves it, and the poller never sees a transient `waiting_review` for a husk-retry round.
286
+ // PROCESS-INSTANCE FENCE (Copilot #789). A straggler progress-check from a SUPERSEDED
287
+ // convergence instance can slip past the replay guard (its idempotency stamp was cleared when
288
+ // `submitPr` re-opened the PR) and reach here AFTER `submitPr` has reset the per-run fields and
289
+ // started a NEW convergence instance. Its stale baseline / status / watermark / result must not
290
+ // clobber the fresh run (which could be parked on `waiting_review` or replay stale output) — nor
291
+ // may it mutate GitHub with the old run's checkpoint (see the self-heal fence below).
292
+ // Re-read the row's CURRENT owner and report whether this job's own `processInstanceKey` no
293
+ // longer owns the row — the straggler's token lives in a terminated instance. Fail open (not
294
+ // superseded) when either key is absent (an older instance, or a row whose `process_key` is not
295
+ // yet seeded) so normal single-run behaviour is untouched. Compare as strings — `process_key` is
296
+ // persisted via `String(processInstanceKey)`, and a job key can arrive numeric.
297
+ const isSuperseded = async (): Promise<boolean> => {
298
+ const currentOwner = (await prs.get(prKey))?.process_key ?? null;
299
+ const jobOwner = job.processInstanceKey ?? null;
300
+ return currentOwner !== null && jobOwner !== null && String(currentOwner) !== String(jobOwner);
301
+ };
244
302
  const commit = async (
245
303
  out: Out,
246
304
  opts: { head?: string | null; status?: "waiting_review" | "converging"; agentWatermark?: string | null },
247
305
  ): Promise<Out> => {
248
306
  const ts = new Date().toISOString();
249
- // PROCESS-INSTANCE FENCE (Copilot #789). A straggler progress-check from a SUPERSEDED
250
- // convergence instance can slip past the replay guard (its idempotency stamp was cleared when
251
- // `submitPr` re-opened the PR) and reach here AFTER `submitPr` has reset the per-run fields and
252
- // started a NEW convergence instance. Its stale baseline / status / watermark / result must not
253
- // clobber the fresh run (which could be parked on `waiting_review` or replay stale output).
254
- // Re-read the row's CURRENT owner immediately before the write and drop the write when this
255
- // job's own `processInstanceKey` no longer owns the row — the straggler's token lives in a
256
- // terminated instance, so acking without persisting is correct. Fail open when either key is
257
- // absent (an older instance, or a row whose `process_key` is not yet seeded) so normal
258
- // single-run behaviour is untouched. Compare as strings — `process_key` is persisted via
259
- // `String(processInstanceKey)`, and a job key can arrive numeric.
260
- const currentOwner = (await prs.get(prKey))?.process_key ?? null;
261
- const jobOwner = job.processInstanceKey ?? null;
262
- if (currentOwner && jobOwner && String(currentOwner) !== String(jobOwner)) {
307
+ // Re-check ownership immediately before the write and drop it when this job no longer owns the
308
+ // row — acking without persisting is correct (the straggler's token is in a terminated instance).
309
+ if (await isSuperseded()) {
263
310
  return out;
264
311
  }
265
312
  await prs.update(prKey, {
@@ -386,9 +433,61 @@ export function makeHandler(deps: {
386
433
  // once the husk retry has been ruled out, so a husk-retry round never transits `waiting_review`.
387
434
  return commit(out, { head: currentHead, status: "waiting_review", agentWatermark });
388
435
  }
389
- // The remaining outcome (progressed:false, huskRetry:false) escalates to a human; the
390
- // persist-escalation-noprogress worker owns the status, so leave it unset — but still advance the
391
- // baseline and stamp the idempotency record so a redelivered escalation replays instead of
436
+
437
+ // NO-ADVANCE SELF-HEAL (#818, Layer 2). Before escalating to a human, reconcile the round's work
438
+ // against the remote: a producer harness that provisioned the review shape wrong pushed the fix
439
+ // to a throwaway fallback branch, so it IS reachable on the remote (recorded as a durable
440
+ // push-checkpoint via the harness's `worldMarker`) but left the PR head un-advanced. If that
441
+ // checkpoint SHA STRICTLY fast-forward-descends the current head, advance the head onto it and
442
+ // continue the loop as real progress instead of parking a human — auto-recovering the exact
443
+ // divergence that stranded nanobpm/nano-coder#26/#28. `currentHead` is non-null here (a null head
444
+ // fails OPEN upstream via decideProgress and never reaches this escalation). Every non-heal
445
+ // outcome (no checkpoint, not a fast-forward, a fork head, an unreadable/refused move) degrades
446
+ // to escalation exactly as before — the self-heal is purely additive.
447
+ if (currentHead) {
448
+ // FENCE THE REF MUTATION (Copilot review on #818 / #819). The self-heal PATCHes the PR head on
449
+ // GitHub — an irreversible side effect the atomic `commit` fence below can only guard for the
450
+ // DB write, NOT for a mutation already made. A straggler from a SUPERSEDED instance (its stale
451
+ // checkpoint would resurrect old work on a PR `submitPr` has since reopened under a new
452
+ // `process_key`) must not move the head. Two layers guard this: (1) the run-scoped checkpoint
453
+ // lookup — the self-heal reads `lastCheckpointForRun(prKey, jobProcessKey, roundNo)`, so a fresh
454
+ // run never even sees the prior run's checkpoint; and (2) a cheap early-out here PLUS a LATE
455
+ // ownership re-check threaded INTO the heal (`isSuperseded` inverted → `stillOwns`), evaluated
456
+ // immediately before the ref PATCH, closing the check-to-mutation TOCTOU where a concurrent
457
+ // `submitPr` supersedes this run DURING the heal's awaits. A skipped/abandoned heal falls
458
+ // through to the escalation `commit`, which the same fence turns into a harmless ack-without-write.
459
+ const jobProcessKey = job.processInstanceKey != null ? String(job.processInstanceKey) : null;
460
+ if (!(await isSuperseded())) {
461
+ const heal = await (deps.selfHeal ?? defaultSelfHeal)(
462
+ app,
463
+ ghRepo,
464
+ ghNumber,
465
+ prKey,
466
+ currentHead,
467
+ roundNo,
468
+ jobProcessKey,
469
+ async () => !(await isSuperseded()),
470
+ ).catch((): SelfHealResult => ({ healed: false }));
471
+ if (heal.healed && heal.sha) {
472
+ app.log.info("no-advance self-heal advanced PR head to a reachable push-checkpoint", {
473
+ prKey,
474
+ round: roundNo,
475
+ from: currentHead,
476
+ to: heal.sha,
477
+ });
478
+ // Advance the baseline to the newly-pushed head and park at wait-review as genuine progress
479
+ // (husk counter reset), so the poller solicits the next Copilot review against the healed head.
480
+ return commit(
481
+ { progressed: true, huskRetries: 0 },
482
+ { head: heal.sha, status: "waiting_review", agentWatermark },
483
+ );
484
+ }
485
+ }
486
+ }
487
+
488
+ // The remaining outcome (progressed:false, huskRetry:false, no self-heal) escalates to a human;
489
+ // the persist-escalation-noprogress worker owns the status, so leave it unset — but still advance
490
+ // the baseline and stamp the idempotency record so a redelivered escalation replays instead of
392
491
  // recomputing.
393
492
  return commit(out, { head: currentHead, agentWatermark });
394
493
  };