@nanobpm/nano-workforce 0.191.1 → 0.193.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/app/agentic/families/presence.family.test.ts +315 -11
- package/app/agentic/families/presence.family.ts +189 -8
- package/app/github.test.ts +223 -1
- package/app/github.ts +122 -0
- package/app/roundProgress.test.ts +108 -1
- package/app/service.test.ts +132 -3
- package/app/service.ts +122 -52
- package/app/world/checkpoint.ts +5 -0
- package/app/world/selfHeal.test.ts +296 -0
- package/app/world/selfHeal.ts +194 -0
- package/app/world/store.test.ts +49 -0
- package/app/world/store.ts +37 -1
- package/db/migrations/115_world_checkpoint_process_key.sql +22 -0
- package/package.json +1 -1
- package/test/worldDb.ts +1 -1
- package/workers/persist-round/worker.ts +6 -0
- package/workers/progress-check/worker.ts +117 -18
|
@@ -0,0 +1,194 @@
|
|
|
1
|
+
// No-advance self-heal (issue #818, the Layer 2 durable fix for the agent PR-head push-divergence
|
|
2
|
+
// class — jwulf/c8ctl-plugin-nano#270).
|
|
3
|
+
//
|
|
4
|
+
// The convergence loop escalates an `addressed` round whose PR head did NOT advance (see
|
|
5
|
+
// app/roundProgress.ts). But the round's work is often ALREADY pushed and reachable on the
|
|
6
|
+
// remote — just not on the PR head. A producer harness that provisions the review shape wrong cuts a
|
|
7
|
+
// throwaway `nano/agent-work/*` fallback branch and pushes the fix THERE, so the agent's "pushed"
|
|
8
|
+
// claim is true while the PR head never moves (this stranded nanobpm/nano-coder#26/#28, recovered by
|
|
9
|
+
// hand). Layer 1 (jwulf/c8ctl-plugin-nano#270) fixes the harness to push the PR head directly AND to
|
|
10
|
+
// report the pushed SHA as a `worldMarker`, which `pr.persist-round` records as a durable
|
|
11
|
+
// push-checkpoint (`world_checkpoints`). This module closes the loop: BEFORE escalating a no-advance
|
|
12
|
+
// round, reconcile that recorded checkpoint SHA against the PR head — if it is a STRICT
|
|
13
|
+
// fast-forward descendant of the head, advance the head onto it and continue the loop as real
|
|
14
|
+
// progress instead of parking a human.
|
|
15
|
+
//
|
|
16
|
+
// It keys off the recorded SHA (canonical world state), not any branch-naming convention, so it heals
|
|
17
|
+
// the whole class: any producer that leaves the round's work reachable-but-off-the-head. Every step
|
|
18
|
+
// fails SAFE — an absent checkpoint, an unreadable comparison, a non-fast-forward (diverged) tip, a
|
|
19
|
+
// cross-repo fork head, or a refused ref update all yield `{ healed: false }`, so the caller escalates
|
|
20
|
+
// exactly as it did before. It is purely additive to the existing no-advance path.
|
|
21
|
+
|
|
22
|
+
/** The push-checkpoint reader — the newest durable `{ commitSha }` recorded by a SPECIFIC RUN of a PR
|
|
23
|
+
* AT A SPECIFIC ROUND, or `null` when that run/round pushed nothing (so there is nothing to
|
|
24
|
+
* reconcile). Scoped by the convergence RUN's `processKey` AND `roundNo` so a heal is anchored to THIS
|
|
25
|
+
* attempt's own checkpoint and can never fast-forward the head onto a stale checkpoint from a prior
|
|
26
|
+
* run — `round_no` alone does not identify a run, since a reopen resets `current_round` while the old
|
|
27
|
+
* run's checkpoints survive (#819). Backed by {@link ../world/store.ts}'s
|
|
28
|
+
* `WorldStore.lastCheckpointForRun`; injectable for tests. */
|
|
29
|
+
export type CheckpointReader = (
|
|
30
|
+
prKey: string,
|
|
31
|
+
processKey: string | null,
|
|
32
|
+
roundNo: number,
|
|
33
|
+
) => Promise<{ commitSha: string } | null>;
|
|
34
|
+
|
|
35
|
+
/** The PR-head reader — the head branch name + repo of a PR, so the self-heal knows WHICH ref to
|
|
36
|
+
* advance and whether it lives in THIS repo (a fork head is unpushable from the base token). Backed
|
|
37
|
+
* by `fetchPrHead`; injectable for tests. */
|
|
38
|
+
export type PrHeadReader = (
|
|
39
|
+
repo: string,
|
|
40
|
+
prNumber: number,
|
|
41
|
+
) => Promise<{ headRef: string | null; headRepo: string | null } | null>;
|
|
42
|
+
|
|
43
|
+
/** The ancestry comparator — proves the checkpoint SHA strictly fast-forward-descends the PR head.
|
|
44
|
+
* Backed by `compareCommits`; injectable for tests. `null` means the comparison was unavailable. */
|
|
45
|
+
export type CommitComparator = (
|
|
46
|
+
repo: string,
|
|
47
|
+
base: string,
|
|
48
|
+
head: string,
|
|
49
|
+
) => Promise<{ status: "ahead" | "behind" | "identical" | "diverged"; aheadBy: number; behindBy: number } | null>;
|
|
50
|
+
|
|
51
|
+
/** The fast-forward ref mover — advances the PR head branch to the checkpoint SHA. Backed by
|
|
52
|
+
* `updateBranchRef` (a NON-force update, so GitHub itself refuses a non-fast-forward). The 4th
|
|
53
|
+
* argument is the COMPARE-AND-SWAP precondition: the exact head SHA the fast-forward proof was
|
|
54
|
+
* computed against (`currentHead`); the mover refuses (`false`) unless the ref still equals it at
|
|
55
|
+
* mutation time, so a head advanced/replaced by a concurrent (superseded) run after the proof is
|
|
56
|
+
* never PATCHed. Returns `false` when the move was refused (non-fast-forward OR lost CAS). Injectable
|
|
57
|
+
* for tests. */
|
|
58
|
+
export type HeadAdvancer = (repo: string, branch: string, sha: string, expectedSha: string) => Promise<boolean>;
|
|
59
|
+
|
|
60
|
+
export interface SelfHealDeps {
|
|
61
|
+
readonly lastCheckpoint: CheckpointReader;
|
|
62
|
+
readonly prHead: PrHeadReader;
|
|
63
|
+
readonly compare: CommitComparator;
|
|
64
|
+
readonly advanceHead: HeadAdvancer;
|
|
65
|
+
/** A LATE ownership guard, re-checked immediately BEFORE the (irreversible) ref mutation to close
|
|
66
|
+
* the check-to-mutation TOCTOU (Copilot review on #819). The worker's pre-call `isSuperseded()`
|
|
67
|
+
* read is not a fence across the async heal: after it returns, this function still awaits the
|
|
68
|
+
* checkpoint lookup, PR-head read, and compare before `advanceHead`, and `submitPr` can install a
|
|
69
|
+
* new `process_key` (a fresh run) in that window. Returns `true` while this run still owns the PR
|
|
70
|
+
* row; when it returns `false` the heal is abandoned WITHOUT touching the ref, so a superseded
|
|
71
|
+
* straggler can never PATCH the branch. Optional — omitted (or absent) means "assume ownership"
|
|
72
|
+
* (the fail-open convention the worker's `isSuperseded` already uses when the identity is unknown),
|
|
73
|
+
* so a caller with no supersession concept keeps the pure #818 behaviour. */
|
|
74
|
+
readonly stillOwns?: () => Promise<boolean>;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/** Why a no-advance round did or did not self-heal — `healed` drives the routing (continue vs.
|
|
78
|
+
* escalate); `sha`/`reason` are for the operator log so a skipped heal is diagnosable. */
|
|
79
|
+
export interface SelfHealResult {
|
|
80
|
+
readonly healed: boolean;
|
|
81
|
+
/** The SHA the PR head was advanced to (only when `healed`). */
|
|
82
|
+
readonly sha?: string;
|
|
83
|
+
/** A short machine-readable reason a heal was skipped (only when NOT `healed`). */
|
|
84
|
+
readonly reason?:
|
|
85
|
+
| "no-checkpoint"
|
|
86
|
+
| "already-at-head"
|
|
87
|
+
| "no-head-ref"
|
|
88
|
+
| "fork-head"
|
|
89
|
+
| "compare-unavailable"
|
|
90
|
+
| "not-fast-forward"
|
|
91
|
+
| "superseded"
|
|
92
|
+
/** The ref move was refused: GitHub rejected the non-fast-forward, OR the compare-and-swap lost
|
|
93
|
+
* (the head no longer equals the `currentHead` the fast-forward proof was validated against — a
|
|
94
|
+
* concurrent run moved it). */
|
|
95
|
+
| "advance-refused";
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
/** Attempt to self-heal a no-advance round by fast-forwarding the PR head onto a recorded
|
|
99
|
+
* push-checkpoint SHA. Returns `{ healed: true, sha }` ONLY when a recorded checkpoint strictly
|
|
100
|
+
* fast-forward-descends the current head AND the head was moved onto it; every other outcome is a
|
|
101
|
+
* safe `{ healed: false, reason }` (→ the caller escalates). Never throws: a rejected promise from
|
|
102
|
+
* any dep is caught and degrades to `{ healed: false }` so a transient GitHub/DB outage can never
|
|
103
|
+
* fabricate a heal (or crash the guard).
|
|
104
|
+
*
|
|
105
|
+
* @param currentHead the PR head SHA the no-advance decision was made against (the compare base). It
|
|
106
|
+
* is non-null at the escalation point — a null/unreadable head fails OPEN upstream (continues) and
|
|
107
|
+
* never reaches here — so the comparison is always anchored on the real head.
|
|
108
|
+
* @param roundNo the round being reconciled; the checkpoint lookup is scoped to it (and to
|
|
109
|
+
* `processKey`) so the heal can only fast-forward onto THIS run's own pushed SHA, never a stale
|
|
110
|
+
* prior-run/later-round one.
|
|
111
|
+
* @param processKey the convergence RUN (process instance) reconciling the round. Scopes the
|
|
112
|
+
* checkpoint lookup to this run so a reopened PR (whose `current_round` reset to 1 while the prior
|
|
113
|
+
* run's checkpoints survive) can never select the old run's checkpoint (#819). */
|
|
114
|
+
export async function attemptNoAdvanceSelfHeal(
|
|
115
|
+
deps: SelfHealDeps,
|
|
116
|
+
repo: string,
|
|
117
|
+
prNumber: number,
|
|
118
|
+
prKey: string,
|
|
119
|
+
currentHead: string,
|
|
120
|
+
roundNo: number,
|
|
121
|
+
processKey: string | null,
|
|
122
|
+
): Promise<SelfHealResult> {
|
|
123
|
+
try {
|
|
124
|
+
const checkpoint = await deps.lastCheckpoint(prKey, processKey, roundNo);
|
|
125
|
+
if (!checkpoint) return { healed: false, reason: "no-checkpoint" };
|
|
126
|
+
const sha = checkpoint.commitSha;
|
|
127
|
+
// The head is already at the checkpoint — nothing stranded, nothing to move. (A no-advance round
|
|
128
|
+
// reaching here with head === checkpoint would be a benign no-op; treat it as "nothing to heal".)
|
|
129
|
+
if (sha === currentHead) return { healed: false, reason: "already-at-head" };
|
|
130
|
+
|
|
131
|
+
const head = await deps.prHead(repo, prNumber);
|
|
132
|
+
const headRef = head?.headRef;
|
|
133
|
+
if (!headRef) return { healed: false, reason: "no-head-ref" };
|
|
134
|
+
// Require a KNOWN, same-repo head before touching any ref. A fork-based PR's head branch lives in
|
|
135
|
+
// ANOTHER repo, and an UNRESOLVABLE head repo (`headRepo == null`, e.g. a deleted fork) is equally
|
|
136
|
+
// unhealable: `fetchPrHead` reports `null` when it cannot resolve the head's owner/name, yet
|
|
137
|
+
// `headRef` can still be a same-named branch that also exists in the base repo. Treating `null` as
|
|
138
|
+
// "in the base repo" would PATCH that unrelated base-repo branch, so classify any non-match (a real
|
|
139
|
+
// fork OR an unresolved head) as an unhealable `fork-head` and escalate instead of risking a
|
|
140
|
+
// cross-repo / wrong-branch ref write. Compare NORMALIZED (case-insensitive) identities: `parsePr`
|
|
141
|
+
// preserves the caller's `owner/repo` spelling while GitHub returns the repo's canonical
|
|
142
|
+
// `head.repo.full_name`, so a valid same-repo PR submitted as `Owner/Repo#7` would otherwise
|
|
143
|
+
// mismatch the canonical `owner/repo`, be misclassified `fork-head`, and silently skip every
|
|
144
|
+
// self-heal (Copilot review of #819). GitHub owner/repo names are case-insensitive, so lowercasing
|
|
145
|
+
// both sides is the correct identity test while retaining the null rejection.
|
|
146
|
+
if (head.headRepo == null || head.headRepo.toLowerCase() !== repo.toLowerCase()) {
|
|
147
|
+
return { healed: false, reason: "fork-head" };
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
// Prove the checkpoint STRICTLY fast-forward-descends the current head before moving anything: a
|
|
151
|
+
// `behind`/`identical`/`diverged` checkpoint is NOT a safe fast-forward and must escalate. Anchor
|
|
152
|
+
// the compare base on `currentHead` (the SHA the no-advance verdict used), not the branch name,
|
|
153
|
+
// so a mid-check head move can't make us fast-forward onto the wrong baseline.
|
|
154
|
+
const cmp = await deps.compare(repo, currentHead, sha);
|
|
155
|
+
if (!cmp) return { healed: false, reason: "compare-unavailable" };
|
|
156
|
+
// Validate the compare COUNTS before trusting the fast-forward proof: a malformed compare
|
|
157
|
+
// response can coerce `ahead_by`/`behind_by` to `NaN` (`Number("not-a-number")`), and because
|
|
158
|
+
// `NaN <= 0` and `NaN !== 0` are BOTH false, a bare `aheadBy <= 0 || behindBy !== 0` gate would
|
|
159
|
+
// let a `{status:"ahead", behindBy:0, aheadBy:NaN}` response slip through and PATCH the ref onto
|
|
160
|
+
// an unproven SHA. Require finite INTEGER counts (a strict fast-forward is `behindBy === 0` AND
|
|
161
|
+
// `aheadBy > 0`); any non-integer count is a corrupt proof and must escalate, not heal.
|
|
162
|
+
if (
|
|
163
|
+
cmp.status !== "ahead" ||
|
|
164
|
+
!Number.isInteger(cmp.aheadBy) ||
|
|
165
|
+
!Number.isInteger(cmp.behindBy) ||
|
|
166
|
+
cmp.behindBy !== 0 ||
|
|
167
|
+
cmp.aheadBy <= 0
|
|
168
|
+
) {
|
|
169
|
+
return { healed: false, reason: "not-fast-forward" };
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
// LATE OWNERSHIP FENCE (Copilot review on #819). Everything above only READ GitHub/DB state; the
|
|
173
|
+
// ONLY irreversible act is the ref move below. Two guards bind it to THIS run's validated state:
|
|
174
|
+
// (1) the late `stillOwns()` re-check — right before the move — abandons a run that was superseded
|
|
175
|
+
// DURING the awaits above (a concurrent `submitPr` installed a new `process_key`), instead of
|
|
176
|
+
// moving the head and only then being discarded by the downstream `commit` fence (which guards the
|
|
177
|
+
// DB write, not a ref mutation already made). Absent guard ⇒ assume ownership (the same fail-open
|
|
178
|
+
// the worker's `isSuperseded` uses when identity is unknown). (2) the move itself is a
|
|
179
|
+
// COMPARE-AND-SWAP keyed on `currentHead` — the exact base the fast-forward proof was computed
|
|
180
|
+
// against — so the mutation's precondition is BOUND TO the validated state rather than trusted from
|
|
181
|
+
// the separate `stillOwns()` read: a straggler that races past (1) still PATCHes nothing unless the
|
|
182
|
+
// ref is verifiably still at `currentHead` at mutation time. GitHub exposes no server-side
|
|
183
|
+
// expected-old-value precondition, so the CAS is the tightest fence the platform allows; the
|
|
184
|
+
// non-force fast-forward invariant makes the bounded residual window fail-safe (head only ever
|
|
185
|
+
// moves forward onto this run's own reachable checkpoint, never a rewrite).
|
|
186
|
+
if (deps.stillOwns && !(await deps.stillOwns())) return { healed: false, reason: "superseded" };
|
|
187
|
+
|
|
188
|
+
const advanced = await deps.advanceHead(repo, headRef, sha, currentHead);
|
|
189
|
+
if (!advanced) return { healed: false, reason: "advance-refused" };
|
|
190
|
+
return { healed: true, sha };
|
|
191
|
+
} catch {
|
|
192
|
+
return { healed: false };
|
|
193
|
+
}
|
|
194
|
+
}
|
package/app/world/store.test.ts
CHANGED
|
@@ -33,6 +33,55 @@ test("lastCheckpoint returns the newest push-checkpoint (max offset), or null wh
|
|
|
33
33
|
assertEquals(last, { offset: 1, commitSha: "sha-b", roundNo: 2 }, "the newest checkpoint wins");
|
|
34
34
|
});
|
|
35
35
|
|
|
36
|
+
test("lastCheckpointForRun scopes to one RUN's round newest checkpoint (never a stale prior-run round, #819)", async () => {
|
|
37
|
+
const { data } = memWorldData();
|
|
38
|
+
const store = new WorldStore(data);
|
|
39
|
+
const RUN_A = "1001";
|
|
40
|
+
const RUN_B = "2002";
|
|
41
|
+
assertEquals(await store.lastCheckpointForRun(PR, RUN_A, 1), null, "no checkpoint for the run/round yet");
|
|
42
|
+
// Prior run A pushes round 1 and round 2 checkpoints.
|
|
43
|
+
await store.recordCheckpoint({ prKey: PR, roundNo: 1, commitSha: "sha-a", processKey: RUN_A });
|
|
44
|
+
await store.recordCheckpoint({ prKey: PR, roundNo: 2, commitSha: "sha-b", processKey: RUN_A });
|
|
45
|
+
// The PR is reopened: a FRESH run B starts at round 1 and re-pushes (a NEW, higher-offset row).
|
|
46
|
+
await store.recordCheckpoint({ prKey: PR, roundNo: 1, commitSha: "sha-c", processKey: RUN_B });
|
|
47
|
+
assertEquals(
|
|
48
|
+
await store.lastCheckpointForRun(PR, RUN_B, 1),
|
|
49
|
+
{ offset: 2, commitSha: "sha-c", roundNo: 1 },
|
|
50
|
+
"run B's round-1 checkpoint wins — never run A's same-numbered round-1 checkpoint",
|
|
51
|
+
);
|
|
52
|
+
assertEquals(
|
|
53
|
+
await store.lastCheckpointForRun(PR, RUN_A, 1),
|
|
54
|
+
{ offset: 0, commitSha: "sha-a", roundNo: 1 },
|
|
55
|
+
"run A still resolves to its OWN round-1 checkpoint, not run B's",
|
|
56
|
+
);
|
|
57
|
+
assertEquals(
|
|
58
|
+
await store.lastCheckpointForRun(PR, RUN_A, 2),
|
|
59
|
+
{ offset: 1, commitSha: "sha-b", roundNo: 2 },
|
|
60
|
+
"run A round 2 resolves to its own checkpoint",
|
|
61
|
+
);
|
|
62
|
+
// A fresh run at round 1 that has pushed NOTHING never inherits a prior run's checkpoint.
|
|
63
|
+
assertEquals(
|
|
64
|
+
await store.lastCheckpointForRun(PR, "3003", 1),
|
|
65
|
+
null,
|
|
66
|
+
"a run with no checkpoint of its own can never heal onto a prior run's stale checkpoint",
|
|
67
|
+
);
|
|
68
|
+
assertEquals(await store.lastCheckpointForRun(PR, RUN_A, 3), null, "a round that pushed nothing has no checkpoint");
|
|
69
|
+
// A null process key (a job with no process instance) is un-attributable — it matches nothing.
|
|
70
|
+
assertEquals(await store.lastCheckpointForRun(PR, null, 1), null, "a null process key can never resolve a checkpoint");
|
|
71
|
+
});
|
|
72
|
+
|
|
73
|
+
test("lastCheckpointForRun never returns a NULL-process_key (pre-#819 / un-attributable) checkpoint", async () => {
|
|
74
|
+
const { data } = memWorldData();
|
|
75
|
+
const store = new WorldStore(data);
|
|
76
|
+
// A checkpoint recorded without a process key (a legacy row, or a job with no process instance).
|
|
77
|
+
await store.recordCheckpoint({ prKey: PR, roundNo: 1, commitSha: "sha-legacy" });
|
|
78
|
+
assertEquals(
|
|
79
|
+
await store.lastCheckpointForRun(PR, "1001", 1),
|
|
80
|
+
null,
|
|
81
|
+
"an un-attributable checkpoint is never healed onto — it degrades to a safe escalation",
|
|
82
|
+
);
|
|
83
|
+
});
|
|
84
|
+
|
|
36
85
|
test("recordCheckpoint defaults to a single push effect keyed by the commit SHA", async () => {
|
|
37
86
|
const { data } = memWorldData();
|
|
38
87
|
const store = new WorldStore(data);
|
package/app/world/store.ts
CHANGED
|
@@ -20,6 +20,11 @@ interface WorldCheckpointRow {
|
|
|
20
20
|
round_no: number;
|
|
21
21
|
checkpoint_offset: number;
|
|
22
22
|
commit_sha: string;
|
|
23
|
+
/** The convergence RUN (process instance) that recorded this checkpoint — `String(processInstanceKey)`
|
|
24
|
+
* of the `pr.persist-round` job. NULL on pre-#819 rows or a job without a `processInstanceKey`. The
|
|
25
|
+
* run-scoped self-heal lookup ({@link WorldStore.lastCheckpointForRun}) requires an EXACT non-null
|
|
26
|
+
* match, so a NULL-keyed checkpoint is never attributed to a run and can never heal (issue #819). */
|
|
27
|
+
process_key: string | null;
|
|
23
28
|
created_at: string;
|
|
24
29
|
}
|
|
25
30
|
|
|
@@ -58,6 +63,12 @@ export interface RecordCheckpointInput {
|
|
|
58
63
|
* a checkpoint records what a round already DID). A pending tail effect (recorded before it is
|
|
59
64
|
* performed, for crash-precise replay) is recorded with `applied: false`. */
|
|
60
65
|
readonly applied?: boolean;
|
|
66
|
+
/** The convergence RUN (process instance) recording this checkpoint — `String(processInstanceKey)`
|
|
67
|
+
* of the persist-round job. Persisted so the no-advance self-heal can prove a checkpoint belongs to
|
|
68
|
+
* THIS run and never fast-forwards the head onto a prior run's stale checkpoint (issue #819).
|
|
69
|
+
* Omitted/`null` for a job without a process instance (the checkpoint is then un-attributable and
|
|
70
|
+
* the run-scoped lookup skips it — a safe escalation, not a heal). */
|
|
71
|
+
readonly processKey?: string | null;
|
|
61
72
|
}
|
|
62
73
|
|
|
63
74
|
/** A durable store over the `world_checkpoints` + `world_effects` tables. */
|
|
@@ -143,6 +154,7 @@ export class WorldStore {
|
|
|
143
154
|
prKey: string,
|
|
144
155
|
roundNo: number,
|
|
145
156
|
commitSha: string,
|
|
157
|
+
processKey: string | null,
|
|
146
158
|
now: string,
|
|
147
159
|
): Promise<number> {
|
|
148
160
|
const offset = await WorldStore.#nextOffsetOn(checkpoints, prKey);
|
|
@@ -152,6 +164,7 @@ export class WorldStore {
|
|
|
152
164
|
round_no: roundNo,
|
|
153
165
|
checkpoint_offset: offset,
|
|
154
166
|
commit_sha: commitSha,
|
|
167
|
+
process_key: processKey,
|
|
155
168
|
created_at: now,
|
|
156
169
|
});
|
|
157
170
|
return offset;
|
|
@@ -188,6 +201,7 @@ export class WorldStore {
|
|
|
188
201
|
*/
|
|
189
202
|
async recordCheckpoint(input: RecordCheckpointInput): Promise<number> {
|
|
190
203
|
const { prKey, roundNo, commitSha } = input;
|
|
204
|
+
const processKey = input.processKey ?? null;
|
|
191
205
|
const effects: readonly Effect[] = input.effects ?? [{ kind: "push", idempotencyKey: commitSha }];
|
|
192
206
|
const applied = input.applied ?? true;
|
|
193
207
|
const now = new Date().toISOString();
|
|
@@ -197,7 +211,7 @@ export class WorldStore {
|
|
|
197
211
|
const existing = await checkpoints.findOne({ pr_key: prKey, commit_sha: commitSha });
|
|
198
212
|
const offset = existing
|
|
199
213
|
? existing.checkpoint_offset
|
|
200
|
-
: await WorldStore.#insertCheckpointFenced(checkpoints, prKey, roundNo, commitSha, now);
|
|
214
|
+
: await WorldStore.#insertCheckpointFenced(checkpoints, prKey, roundNo, commitSha, processKey, now);
|
|
201
215
|
let seq = await WorldStore.#nextSeqOn(effectsTable, prKey, offset);
|
|
202
216
|
for (const effect of effects) {
|
|
203
217
|
await WorldStore.#appendEffect(effectsTable, prKey, offset, seq++, effect, applied, now);
|
|
@@ -258,6 +272,28 @@ export class WorldStore {
|
|
|
258
272
|
return { offset: newest.checkpoint_offset, commitSha: newest.commit_sha, roundNo: newest.round_no };
|
|
259
273
|
}
|
|
260
274
|
|
|
275
|
+
/** The newest push-checkpoint recorded by a SPECIFIC RUN of a PR at a SPECIFIC ROUND, or `null` when
|
|
276
|
+
* that run/round pushed nothing. Unlike {@link lastCheckpoint} (newest across ALL runs/rounds), this
|
|
277
|
+
* proves the checkpoint belongs to the attempt being reconciled: the no-advance self-heal (#818,
|
|
278
|
+
* #819) must NOT fast-forward the head onto a checkpoint left by a PRIOR run. `round_no` alone does
|
|
279
|
+
* not identify a run — `submitPr` resets a reopened PR's `current_round` to 1 while the prior run's
|
|
280
|
+
* checkpoints survive, so a fresh run at round 1 would select the OLD run's round-1 checkpoint and
|
|
281
|
+
* could resurrect stale work if it descends the (reset) head. Scoping to the convergence RUN's
|
|
282
|
+
* `process_key` (the instance key advanced on every reopen) AND `round_no` anchors the heal to THIS
|
|
283
|
+
* run's own pushed SHA. A `null` `processKey` (a job with no process instance) matches NOTHING and
|
|
284
|
+
* yields `null`, and a checkpoint row with a `NULL` `process_key` (pre-#819, or an un-attributable
|
|
285
|
+
* record) is likewise never returned — both degrade to a safe escalation, never a heal. Still
|
|
286
|
+
* newest-by-offset within the match, so a genuine re-push of the same round wins. */
|
|
287
|
+
async lastCheckpointForRun(prKey: string, processKey: string | null, roundNo: number): Promise<LastCheckpoint | null> {
|
|
288
|
+
if (processKey == null) return null;
|
|
289
|
+
const rows = (await this.#checkpoints().find({ pr_key: prKey })).filter(
|
|
290
|
+
(r) => r.process_key === processKey && r.round_no === roundNo,
|
|
291
|
+
);
|
|
292
|
+
if (rows.length === 0) return null;
|
|
293
|
+
const newest = rows.reduce((a, b) => (b.checkpoint_offset > a.checkpoint_offset ? b : a));
|
|
294
|
+
return { offset: newest.checkpoint_offset, commitSha: newest.commit_sha, roundNo: newest.round_no };
|
|
295
|
+
}
|
|
296
|
+
|
|
261
297
|
/** The effect tail recorded at a checkpoint offset, in `seq` order — the sequence the restore path
|
|
262
298
|
* fence-replays after checking the working tree out to that checkpoint's SHA. `seq` is the intended
|
|
263
299
|
* order, but it is allocated by a racy read-max-plus-one (`#nextSeqOn`) and the schema has no
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
-- 115_world_checkpoint_process_key.sql — issue #819 (Copilot review): bind a push-checkpoint to the
|
|
2
|
+
-- convergence RUN that produced it, so the no-advance self-heal can prove a checkpoint belongs to the
|
|
3
|
+
-- CURRENT run and never fast-forwards the PR head onto a stale checkpoint left by a PRIOR run.
|
|
4
|
+
--
|
|
5
|
+
-- WHY. The no-advance self-heal (app/world/selfHeal.ts, #818) reconciles a recorded push-checkpoint
|
|
6
|
+
-- against the PR head. #819 scoped that lookup to the current `round_no`, but `round_no` alone does
|
|
7
|
+
-- NOT identify a run: `submitPr` (app/service.ts) resets a reopened PR's `current_round` to 1 while
|
|
8
|
+
-- leaving the prior run's `world_checkpoints` rows in place, so a fresh run at round 1 would still
|
|
9
|
+
-- select the previous run's round-1 checkpoint — and if the branch was reset behind that old SHA the
|
|
10
|
+
-- fast-forward compare passes and the heal resurrects stale work. The convergence instance key
|
|
11
|
+
-- (`pull_requests.process_key`, advanced to the new run on every reopen) is the run generation, so
|
|
12
|
+
-- recording it on each checkpoint lets the self-heal scope by `(process_key, round_no)` — proving the
|
|
13
|
+
-- checkpoint is THIS run's own pushed SHA.
|
|
14
|
+
--
|
|
15
|
+
-- EXPAND (additive) phase: one nullable column, no default, nothing dropped or renamed. `NULL` on the
|
|
16
|
+
-- pre-existing rows (and on any checkpoint recorded by a job without a `processInstanceKey`) is the
|
|
17
|
+
-- SAFE value: the run-scoped lookup requires an EXACT non-null match, so a NULL-keyed checkpoint is
|
|
18
|
+
-- never attributed to a run and can never heal — it degrades to escalation, exactly as an absent
|
|
19
|
+
-- checkpoint does. Numbered after the current highest prefix on origin/main (114). The runner wraps
|
|
20
|
+
-- each file in its own transaction, so this file must NOT contain BEGIN/COMMIT.
|
|
21
|
+
|
|
22
|
+
ALTER TABLE world_checkpoints ADD COLUMN process_key TEXT;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@nanobpm/nano-workforce",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.193.0",
|
|
4
4
|
"description": "Nano Workforce — an Agent Graph Orchestration application for Agentic SDLC: durable BPMN processes that coordinate a graph of AI agents across the software delivery lifecycle.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "main.ts",
|
package/test/worldDb.ts
CHANGED
|
@@ -118,5 +118,5 @@ export function memDataFor(migrationFiles: readonly string[]): { data: DataLayer
|
|
|
118
118
|
|
|
119
119
|
/** A `DataLayer` stub over a fresh in-memory db with the world schema applied. */
|
|
120
120
|
export function memWorldData(): { data: DataLayer; db: DatabaseSync } {
|
|
121
|
-
return memDataFor(["049_world_checkpoint.sql"]);
|
|
121
|
+
return memDataFor(["049_world_checkpoint.sql", "115_world_checkpoint_process_key.sql"]);
|
|
122
122
|
}
|
|
@@ -202,6 +202,12 @@ const handler: AppJobHandler<In> = async (job, app) => {
|
|
|
202
202
|
prKey,
|
|
203
203
|
roundNo: round,
|
|
204
204
|
commitSha: marker.commitSha,
|
|
205
|
+
// Bind the checkpoint to THIS convergence run (issue #819): the no-advance self-heal scopes
|
|
206
|
+
// its lookup by `(process_key, round_no)` so it can never fast-forward the head onto a prior
|
|
207
|
+
// run's stale checkpoint. `processInstanceKey` is this run's instance key — the same value
|
|
208
|
+
// `submitPr` advances into `pull_requests.process_key` and the self-heal's ownership guard
|
|
209
|
+
// compares against.
|
|
210
|
+
processKey: processInstanceKey,
|
|
205
211
|
...(marker.effects ? { effects: marker.effects } : {}),
|
|
206
212
|
});
|
|
207
213
|
} catch (err) {
|
|
@@ -17,9 +17,11 @@
|
|
|
17
17
|
// transient GitHub hiccup can never fabricate a no-progress escalation.
|
|
18
18
|
import type { AgentInstanceSummary, AppApi, AppJobHandler } from "@nanobpm/urban";
|
|
19
19
|
import { type HeadReader, makeDefaultReadHead } from "../../app/currentHead.ts";
|
|
20
|
-
import { fetchBranchHead, fetchPrHead } from "../../app/github.ts";
|
|
20
|
+
import { compareCommits, fetchBranchHead, fetchPrHead, updateBranchRef } from "../../app/github.ts";
|
|
21
21
|
import { decideProgress, isAddressedStatus } from "../../app/roundProgress.ts";
|
|
22
22
|
import { parsePr } from "../../app/service.ts";
|
|
23
|
+
import { WorldStore } from "../../app/world/index.ts";
|
|
24
|
+
import { attemptNoAdvanceSelfHeal, type SelfHealResult } from "../../app/world/selfHeal.ts";
|
|
23
25
|
import type { WorkerInputs, WorkerOutputs } from "../../nano-generated/worker-io.d.ts";
|
|
24
26
|
|
|
25
27
|
// Input/output typed off the model data envelopes (`PrProgressCheckIn` / `PrProgressCheckOut` in
|
|
@@ -75,6 +77,45 @@ export { type HeadReader, makeDefaultReadHead };
|
|
|
75
77
|
|
|
76
78
|
const defaultReadHead: HeadReader = makeDefaultReadHead({ fetchPrHead, fetchBranchHead });
|
|
77
79
|
|
|
80
|
+
/** Attempt the no-advance self-heal (#818): reconcile a recorded push-checkpoint against the PR head
|
|
81
|
+
* and fast-forward the head onto it when safe. Injectable so tests never touch git/network; the
|
|
82
|
+
* default binds the real `WorldStore` (over `app.data`) + GitHub transport with the env credential
|
|
83
|
+
* (the same `process.env.GITHUB_TOKEN` default the head reader uses). See {@link
|
|
84
|
+
* ../../app/world/selfHeal.ts}. */
|
|
85
|
+
export type SelfHealFn = (
|
|
86
|
+
app: AppApi,
|
|
87
|
+
repo: string,
|
|
88
|
+
prNumber: number,
|
|
89
|
+
prKey: string,
|
|
90
|
+
currentHead: string,
|
|
91
|
+
roundNo: number,
|
|
92
|
+
processKey: string | null,
|
|
93
|
+
stillOwns: () => Promise<boolean>,
|
|
94
|
+
) => Promise<SelfHealResult>;
|
|
95
|
+
|
|
96
|
+
const defaultSelfHeal: SelfHealFn = (app, repo, prNumber, prKey, currentHead, roundNo, processKey, stillOwns) => {
|
|
97
|
+
const tok = process.env.GITHUB_TOKEN ?? "";
|
|
98
|
+
const store = new WorldStore(app.data);
|
|
99
|
+
return attemptNoAdvanceSelfHeal(
|
|
100
|
+
{
|
|
101
|
+
lastCheckpoint: (pk, run, rn) => store.lastCheckpointForRun(pk, run, rn),
|
|
102
|
+
prHead: async (r, n) => {
|
|
103
|
+
const h = await fetchPrHead(r, n, tok);
|
|
104
|
+
return h ? { headRef: h.headRef, headRepo: h.headRepo } : null;
|
|
105
|
+
},
|
|
106
|
+
compare: (r, base, head) => compareCommits(r, base, head, tok),
|
|
107
|
+
advanceHead: (r, branch, sha, expected) => updateBranchRef(r, branch, sha, tok, expected),
|
|
108
|
+
stillOwns,
|
|
109
|
+
},
|
|
110
|
+
repo,
|
|
111
|
+
prNumber,
|
|
112
|
+
prKey,
|
|
113
|
+
currentHead,
|
|
114
|
+
roundNo,
|
|
115
|
+
processKey,
|
|
116
|
+
);
|
|
117
|
+
};
|
|
118
|
+
|
|
78
119
|
/** An agent-instance is "durable work" for husk purposes once it has reached a terminal state — it
|
|
79
120
|
* carries a `completionDate`, or a terminal lifecycle status. A husked instance never closes (it is
|
|
80
121
|
* stuck `THINKING` with null dates — jwulf/c8ctl-plugin-nano#230), so it is NOT counted. */
|
|
@@ -203,6 +244,7 @@ function agentWorkFromEngine(engine: AppApi["engine"]): AgentWorkReader {
|
|
|
203
244
|
export function makeHandler(deps: {
|
|
204
245
|
readHead: HeadReader;
|
|
205
246
|
readAgentWork?: AgentWorkReader;
|
|
247
|
+
selfHeal?: SelfHealFn;
|
|
206
248
|
}): AppJobHandler<In, Out> {
|
|
207
249
|
return async (job, app) => {
|
|
208
250
|
const { prKey, status, repo, prNumber, round, huskRetries, roundEntryHead } = job.variables;
|
|
@@ -241,25 +283,30 @@ export function makeHandler(deps: {
|
|
|
241
283
|
// the single writer of the review-wait park (#786) — persist-round no longer sets
|
|
242
284
|
// `waiting_review`, so the row rests on its running `converging` status until exactly one write
|
|
243
285
|
// here resolves it, and the poller never sees a transient `waiting_review` for a husk-retry round.
|
|
286
|
+
// PROCESS-INSTANCE FENCE (Copilot #789). A straggler progress-check from a SUPERSEDED
|
|
287
|
+
// convergence instance can slip past the replay guard (its idempotency stamp was cleared when
|
|
288
|
+
// `submitPr` re-opened the PR) and reach here AFTER `submitPr` has reset the per-run fields and
|
|
289
|
+
// started a NEW convergence instance. Its stale baseline / status / watermark / result must not
|
|
290
|
+
// clobber the fresh run (which could be parked on `waiting_review` or replay stale output) — nor
|
|
291
|
+
// may it mutate GitHub with the old run's checkpoint (see the self-heal fence below).
|
|
292
|
+
// Re-read the row's CURRENT owner and report whether this job's own `processInstanceKey` no
|
|
293
|
+
// longer owns the row — the straggler's token lives in a terminated instance. Fail open (not
|
|
294
|
+
// superseded) when either key is absent (an older instance, or a row whose `process_key` is not
|
|
295
|
+
// yet seeded) so normal single-run behaviour is untouched. Compare as strings — `process_key` is
|
|
296
|
+
// persisted via `String(processInstanceKey)`, and a job key can arrive numeric.
|
|
297
|
+
const isSuperseded = async (): Promise<boolean> => {
|
|
298
|
+
const currentOwner = (await prs.get(prKey))?.process_key ?? null;
|
|
299
|
+
const jobOwner = job.processInstanceKey ?? null;
|
|
300
|
+
return currentOwner !== null && jobOwner !== null && String(currentOwner) !== String(jobOwner);
|
|
301
|
+
};
|
|
244
302
|
const commit = async (
|
|
245
303
|
out: Out,
|
|
246
304
|
opts: { head?: string | null; status?: "waiting_review" | "converging"; agentWatermark?: string | null },
|
|
247
305
|
): Promise<Out> => {
|
|
248
306
|
const ts = new Date().toISOString();
|
|
249
|
-
//
|
|
250
|
-
//
|
|
251
|
-
|
|
252
|
-
// started a NEW convergence instance. Its stale baseline / status / watermark / result must not
|
|
253
|
-
// clobber the fresh run (which could be parked on `waiting_review` or replay stale output).
|
|
254
|
-
// Re-read the row's CURRENT owner immediately before the write and drop the write when this
|
|
255
|
-
// job's own `processInstanceKey` no longer owns the row — the straggler's token lives in a
|
|
256
|
-
// terminated instance, so acking without persisting is correct. Fail open when either key is
|
|
257
|
-
// absent (an older instance, or a row whose `process_key` is not yet seeded) so normal
|
|
258
|
-
// single-run behaviour is untouched. Compare as strings — `process_key` is persisted via
|
|
259
|
-
// `String(processInstanceKey)`, and a job key can arrive numeric.
|
|
260
|
-
const currentOwner = (await prs.get(prKey))?.process_key ?? null;
|
|
261
|
-
const jobOwner = job.processInstanceKey ?? null;
|
|
262
|
-
if (currentOwner && jobOwner && String(currentOwner) !== String(jobOwner)) {
|
|
307
|
+
// Re-check ownership immediately before the write and drop it when this job no longer owns the
|
|
308
|
+
// row — acking without persisting is correct (the straggler's token is in a terminated instance).
|
|
309
|
+
if (await isSuperseded()) {
|
|
263
310
|
return out;
|
|
264
311
|
}
|
|
265
312
|
await prs.update(prKey, {
|
|
@@ -386,9 +433,61 @@ export function makeHandler(deps: {
|
|
|
386
433
|
// once the husk retry has been ruled out, so a husk-retry round never transits `waiting_review`.
|
|
387
434
|
return commit(out, { head: currentHead, status: "waiting_review", agentWatermark });
|
|
388
435
|
}
|
|
389
|
-
|
|
390
|
-
//
|
|
391
|
-
//
|
|
436
|
+
|
|
437
|
+
// NO-ADVANCE SELF-HEAL (#818, Layer 2). Before escalating to a human, reconcile the round's work
|
|
438
|
+
// against the remote: a producer harness that provisioned the review shape wrong pushed the fix
|
|
439
|
+
// to a throwaway fallback branch, so it IS reachable on the remote (recorded as a durable
|
|
440
|
+
// push-checkpoint via the harness's `worldMarker`) but left the PR head un-advanced. If that
|
|
441
|
+
// checkpoint SHA STRICTLY fast-forward-descends the current head, advance the head onto it and
|
|
442
|
+
// continue the loop as real progress instead of parking a human — auto-recovering the exact
|
|
443
|
+
// divergence that stranded nanobpm/nano-coder#26/#28. `currentHead` is non-null here (a null head
|
|
444
|
+
// fails OPEN upstream via decideProgress and never reaches this escalation). Every non-heal
|
|
445
|
+
// outcome (no checkpoint, not a fast-forward, a fork head, an unreadable/refused move) degrades
|
|
446
|
+
// to escalation exactly as before — the self-heal is purely additive.
|
|
447
|
+
if (currentHead) {
|
|
448
|
+
// FENCE THE REF MUTATION (Copilot review on #818 / #819). The self-heal PATCHes the PR head on
|
|
449
|
+
// GitHub — an irreversible side effect the atomic `commit` fence below can only guard for the
|
|
450
|
+
// DB write, NOT for a mutation already made. A straggler from a SUPERSEDED instance (its stale
|
|
451
|
+
// checkpoint would resurrect old work on a PR `submitPr` has since reopened under a new
|
|
452
|
+
// `process_key`) must not move the head. Two layers guard this: (1) the run-scoped checkpoint
|
|
453
|
+
// lookup — the self-heal reads `lastCheckpointForRun(prKey, jobProcessKey, roundNo)`, so a fresh
|
|
454
|
+
// run never even sees the prior run's checkpoint; and (2) a cheap early-out here PLUS a LATE
|
|
455
|
+
// ownership re-check threaded INTO the heal (`isSuperseded` inverted → `stillOwns`), evaluated
|
|
456
|
+
// immediately before the ref PATCH, closing the check-to-mutation TOCTOU where a concurrent
|
|
457
|
+
// `submitPr` supersedes this run DURING the heal's awaits. A skipped/abandoned heal falls
|
|
458
|
+
// through to the escalation `commit`, which the same fence turns into a harmless ack-without-write.
|
|
459
|
+
const jobProcessKey = job.processInstanceKey != null ? String(job.processInstanceKey) : null;
|
|
460
|
+
if (!(await isSuperseded())) {
|
|
461
|
+
const heal = await (deps.selfHeal ?? defaultSelfHeal)(
|
|
462
|
+
app,
|
|
463
|
+
ghRepo,
|
|
464
|
+
ghNumber,
|
|
465
|
+
prKey,
|
|
466
|
+
currentHead,
|
|
467
|
+
roundNo,
|
|
468
|
+
jobProcessKey,
|
|
469
|
+
async () => !(await isSuperseded()),
|
|
470
|
+
).catch((): SelfHealResult => ({ healed: false }));
|
|
471
|
+
if (heal.healed && heal.sha) {
|
|
472
|
+
app.log.info("no-advance self-heal advanced PR head to a reachable push-checkpoint", {
|
|
473
|
+
prKey,
|
|
474
|
+
round: roundNo,
|
|
475
|
+
from: currentHead,
|
|
476
|
+
to: heal.sha,
|
|
477
|
+
});
|
|
478
|
+
// Advance the baseline to the newly-pushed head and park at wait-review as genuine progress
|
|
479
|
+
// (husk counter reset), so the poller solicits the next Copilot review against the healed head.
|
|
480
|
+
return commit(
|
|
481
|
+
{ progressed: true, huskRetries: 0 },
|
|
482
|
+
{ head: heal.sha, status: "waiting_review", agentWatermark },
|
|
483
|
+
);
|
|
484
|
+
}
|
|
485
|
+
}
|
|
486
|
+
}
|
|
487
|
+
|
|
488
|
+
// The remaining outcome (progressed:false, huskRetry:false, no self-heal) escalates to a human;
|
|
489
|
+
// the persist-escalation-noprogress worker owns the status, so leave it unset — but still advance
|
|
490
|
+
// the baseline and stamp the idempotency record so a redelivered escalation replays instead of
|
|
392
491
|
// recomputing.
|
|
393
492
|
return commit(out, { head: currentHead, agentWatermark });
|
|
394
493
|
};
|