@nanobpm/nano-workforce 0.187.3 → 0.187.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/README.md +1 -1
- package/SPEC.md +71 -9
- package/app/contracts.ts +1 -0
- package/app/convergenceEscalationGuard.test.ts +3 -2
- package/app/github.test.ts +125 -1
- package/app/github.ts +43 -8
- package/app/persist-escalation.test.ts +8 -6
- package/app/persist-round.test.ts +178 -11
- package/app/pullRequestReadModel.test.ts +1 -1
- package/app/reviewWait.test.ts +7 -0
- package/app/reviewWait.ts +1 -1
- package/app/roundProgress.test.ts +755 -33
- package/app/roundProgress.ts +144 -0
- package/app/roundResultDefault.test.ts +10 -8
- package/app/service.test.ts +14 -0
- package/app/service.ts +35 -0
- package/db/migrations/102_rounds_process_instance_key.sql +28 -0
- package/db/migrations/103_pr_progress_idempotency.sql +29 -0
- package/db/migrations/104_pull_requests_read_model_progress_idempotency.sql +55 -0
- package/docs/agent-guide.md +1 -1
- package/e2e/convergence-escalation.e2e.ts +5 -4
- package/e2e/feature-run.e2e.ts +6 -1
- package/e2e/plan-fanout-sla.e2e.ts +5 -2
- package/e2e/plan-fanout.e2e.ts +6 -2
- package/e2e/support/time.ts +34 -0
- package/nano.app.json +4 -0
- package/package.json +1 -1
- package/resources/processes/convergence-loop.bpmn +211 -142
- package/test/derivation-parity/README.md +3 -3
- package/test/derivation-parity/derivation-parity.test.ts +9 -3
- package/test/derivation-parity/flows.ts +4 -4
- package/workers/capture-head/worker.test.ts +77 -0
- package/workers/capture-head/worker.ts +64 -0
- package/workers/persist-escalation/worker.ts +4 -0
- package/workers/persist-round/worker.ts +75 -9
- package/workers/progress-check/worker.ts +394 -35
package/app/roundProgress.ts
CHANGED
|
@@ -68,3 +68,147 @@ export function routeProgress(
|
|
|
68
68
|
if (!previousHead || !currentHead) return "continue";
|
|
69
69
|
return currentHead === previousHead ? "escalate" : "continue";
|
|
70
70
|
}
|
|
71
|
+
|
|
72
|
+
// ── Husk classification & bounded self-heal (issue #786) ─────────────────────
|
|
73
|
+
//
|
|
74
|
+
// A no-advance `addressed` round is not one failure mode but two, and they warrant different
|
|
75
|
+
// handling:
|
|
76
|
+
//
|
|
77
|
+
// • `husk` — the agent job completed reporting `addressed`, but the COMPLETING attempt minted NO
|
|
78
|
+
// durable work: no commit was pushed AND the round's completing `review-round` agent-instance did
|
|
79
|
+
// not reach a terminal state — either it registered a non-terminal instance, or it husked before
|
|
80
|
+
// registering any instance at all (the producer harness died mid-run, so the round is a phantom —
|
|
81
|
+
// jwulf/c8ctl-plugin-nano#230/#229). The verdict is scoped to the completing attempt via the
|
|
82
|
+
// attempt watermark (Copilot #789), so an EARLIER attempt's terminal instance neither masks nor
|
|
83
|
+
// fabricates this one. This is a transient worker/harness defect, not a real design impasse, so it
|
|
84
|
+
// is *resumable onto a healthy worker*: re-run the SAME round rather than parking a human. Bounded
|
|
85
|
+
// by {@link MAX_HUSK_RETRIES} so a persistently-husking worker still escalates instead of looping
|
|
86
|
+
// forever.
|
|
87
|
+
//
|
|
88
|
+
// • `no-advance` — the agent DID run to a terminal agent-instance but pushed no commit (it genuinely
|
|
89
|
+
// believes nothing was needed, or is wrong about the code). Re-running would loop on identical
|
|
90
|
+
// reasoning, so this escalates to a human immediately, exactly as before.
|
|
91
|
+
//
|
|
92
|
+
// The head-diff (`routeProgress`) still TRIGGERS the no-progress path; the agent-instance
|
|
93
|
+
// corroboration only SPLITS it into husk vs. no-advance so the auto-heal and the escalation message
|
|
94
|
+
// are accurate.
|
|
95
|
+
|
|
96
|
+
/** Why a no-advance `addressed` round made no progress — see the block comment above. */
|
|
97
|
+
export type NoProgressReason = "husk" | "no-advance";
|
|
98
|
+
|
|
99
|
+
/** How many times a husked round is auto-re-run onto a (hopefully healthy) worker before the loop
|
|
100
|
+
* gives up and escalates to a human. The bound is what keeps the self-heal from looping forever on a
|
|
101
|
+
* persistently-husking worker. */
|
|
102
|
+
export const MAX_HUSK_RETRIES = 2;
|
|
103
|
+
|
|
104
|
+
/** The full decision for a recorded round: whether it progressed, the running husk-retry count to
|
|
105
|
+
* carry forward, and — when it did NOT progress — whether to auto-retry the same round (`huskRetry`),
|
|
106
|
+
* the classified `reason`, and (when escalating) the human-facing `question`. This is the single
|
|
107
|
+
* source of truth the `pr.progress-check` worker returns and `gw-progress`/`gw-husk` route on. */
|
|
108
|
+
export interface ProgressDecision {
|
|
109
|
+
readonly progressed: boolean;
|
|
110
|
+
readonly huskRetries: number;
|
|
111
|
+
readonly huskRetry?: boolean;
|
|
112
|
+
readonly reason?: NoProgressReason;
|
|
113
|
+
readonly question?: string;
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
/** Coerce an externally-supplied husk-retry counter (a process variable that is null/blank on the
|
|
117
|
+
* first husk, and could be any shape after a variable regression) to a non-negative integer. */
|
|
118
|
+
function normalizeRetries(value: number | null | undefined): number {
|
|
119
|
+
if (typeof value !== "number" || !Number.isFinite(value) || value < 0) return 0;
|
|
120
|
+
return Math.floor(value);
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
/** Build the human-facing escalation question for a no-progress round, tuned to its {@link
|
|
124
|
+
* NoProgressReason} so the human sees "the agent produced no durable work (a husk)" rather than the
|
|
125
|
+
* generic "no commit was pushed" when that is what actually happened. */
|
|
126
|
+
export function noProgressQuestion(
|
|
127
|
+
round: number,
|
|
128
|
+
reason: NoProgressReason,
|
|
129
|
+
huskRetriesExhausted: boolean,
|
|
130
|
+
): string {
|
|
131
|
+
if (reason === "husk") {
|
|
132
|
+
const tail = huskRetriesExhausted
|
|
133
|
+
? ` and re-running the round ${MAX_HUSK_RETRIES} time(s) did not help (the worker keeps husking)`
|
|
134
|
+
: "";
|
|
135
|
+
return (
|
|
136
|
+
`Round ${round} reported the review comments were addressed, but the agent produced no ` +
|
|
137
|
+
`durable work — no commit was pushed and the COMPLETING review attempt recorded no terminal ` +
|
|
138
|
+
`agent-instance (a husked round; the worker likely died mid-run)${tail}. A human must decide ` +
|
|
139
|
+
`how to proceed (reply to resume the loop).`
|
|
140
|
+
);
|
|
141
|
+
}
|
|
142
|
+
return (
|
|
143
|
+
`Round ${round} reported the review comments were addressed, but the PR head did not advance ` +
|
|
144
|
+
`(no commit was pushed), so another review round would loop on identical code. A human must ` +
|
|
145
|
+
`decide how to proceed (reply to resume the loop).`
|
|
146
|
+
);
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
/** The canonical no-progress decision, mirroring {@link routeProgress} for the head-diff trigger and
|
|
150
|
+
* then splitting a no-advance round into a bounded husk auto-retry vs. an immediate escalation.
|
|
151
|
+
*
|
|
152
|
+
* • A round that progressed (head advanced, or a legitimately non-addressed status, or an
|
|
153
|
+
* unreadable head that fails OPEN, or a no-baseline addressed round) returns `{ progressed: true,
|
|
154
|
+
* huskRetries: 0 }` — real progress RESETS the husk counter so a later, unrelated husk starts
|
|
155
|
+
* fresh. A first-addressed-round husk is no longer a special case here: `pr.capture-head` records
|
|
156
|
+
* the round's entry head into `roundEntryHead` BEFORE `review-round` runs, so within-round there
|
|
157
|
+
* is always a baseline and a husk that pushed nothing takes the no-advance/husk split below.
|
|
158
|
+
* • A husked round under the retry cap returns `{ progressed: false, huskRetry: true,
|
|
159
|
+
* huskRetries: n+1 }` — `gw-husk` re-enters `review-round` (the SAME round) to try a healthy
|
|
160
|
+
* worker.
|
|
161
|
+
* • A husked round at the cap, or any `no-advance` round, returns `{ progressed: false,
|
|
162
|
+
* huskRetry: false, huskRetries: 0, question }` — `gw-husk` routes to the human escalation. The
|
|
163
|
+
* counter resets so a human-answered resume gets fresh retries.
|
|
164
|
+
*
|
|
165
|
+
* `agentWorkObserved` is the agent-instance corroboration for the COMPLETING `review-round`
|
|
166
|
+
* element-instance (see {@link decideProgress}'s reader in workers/progress-check): `true` when the
|
|
167
|
+
* newest correlated `review-round` instance is TERMINAL (→ `no-advance`); `false` — a SUCCESSFUL,
|
|
168
|
+
* positively-corroborated read whose newest correlated instance is NON-terminal (the completing
|
|
169
|
+
* attempt husked) → `husk`; and `null`/`undefined` — an UNKNOWN read (the engine channel was
|
|
170
|
+
* unavailable/absent — e.g. an empty instance list on the read-as-absence testkit — or the read
|
|
171
|
+
* threw) → `no-advance`, never an auto-retry, so a transient or absent AgentInstance read can never
|
|
172
|
+
* duplicate genuinely-completed agent work. Only a positively-corroborated non-terminal completing
|
|
173
|
+
* instance is a husk; an absent/empty read is UNKNOWN, not a husk. */
|
|
174
|
+
export function decideProgress(
|
|
175
|
+
status: string | null | undefined,
|
|
176
|
+
previousHead: string | null | undefined,
|
|
177
|
+
currentHead: string | null | undefined,
|
|
178
|
+
round: number,
|
|
179
|
+
agentWorkObserved: boolean | null | undefined,
|
|
180
|
+
currentHuskRetries: number | null | undefined,
|
|
181
|
+
maxHuskRetries: number = MAX_HUSK_RETRIES,
|
|
182
|
+
): ProgressDecision {
|
|
183
|
+
if (routeProgress(status, previousHead, currentHead) === "continue") {
|
|
184
|
+
// Fail-open: the head advanced (real progress → RESET the husk counter), a legitimately
|
|
185
|
+
// non-addressed status, an unreadable current head, OR a no-baseline addressed round. The
|
|
186
|
+
// no-baseline case is now handled structurally UPSTREAM: `pr.capture-head` records the round's
|
|
187
|
+
// entry head into `roundEntryHead` BEFORE `review-round` runs, so the baseline is always present
|
|
188
|
+
// within the round — a first-addressed-round husk leaves `currentHead === roundEntryHead` and
|
|
189
|
+
// takes the `escalate` path below (split into husk vs. no-advance), while a real push advances
|
|
190
|
+
// the head and legitimately continues. The old no-baseline husk special-case (which had to guess
|
|
191
|
+
// from `agentWorkObserved` alone, with the opposing risk of mis-escalating a straggler push) is
|
|
192
|
+
// therefore gone: with a within-round baseline there is no no-baseline case left to special-case.
|
|
193
|
+
return { progressed: true, huskRetries: 0 };
|
|
194
|
+
}
|
|
195
|
+
// Only a POSITIVELY-corroborated husk (`false` — a successful AgentInstance search whose NEWEST
|
|
196
|
+
// correlated `review-round` instance is NON-terminal, i.e. the completing attempt died mid-run) is
|
|
197
|
+
// one we may auto-retry. `true` (that newest correlated instance is TERMINAL) is a real no-advance;
|
|
198
|
+
// and an UNKNOWN read (`null`/`undefined` — an empty/absent instance list or a thrown read, i.e.
|
|
199
|
+
// the engine channel was unavailable) must NOT auto-retry, since re-running could duplicate agent
|
|
200
|
+
// work that actually did run. So an unknown read fails safe to `no-advance` (escalate to a human),
|
|
201
|
+
// exactly as the pre-#786 loop did — only a corroborated non-terminal completing instance is husk.
|
|
202
|
+
const reason: NoProgressReason = agentWorkObserved === false ? "husk" : "no-advance";
|
|
203
|
+
const retries = normalizeRetries(currentHuskRetries);
|
|
204
|
+
if (reason === "husk" && retries < maxHuskRetries) {
|
|
205
|
+
return { progressed: false, huskRetry: true, huskRetries: retries + 1, reason };
|
|
206
|
+
}
|
|
207
|
+
return {
|
|
208
|
+
progressed: false,
|
|
209
|
+
huskRetry: false,
|
|
210
|
+
huskRetries: 0,
|
|
211
|
+
reason,
|
|
212
|
+
question: noProgressQuestion(round, reason, reason === "husk" && retries >= maxHuskRetries),
|
|
213
|
+
};
|
|
214
|
+
}
|
|
@@ -74,10 +74,10 @@ test("escalation is an explicit needs_input/blocked arm gated on a non-blank que
|
|
|
74
74
|
assertStringIncludes(esc, 'trim(question) != ""');
|
|
75
75
|
});
|
|
76
76
|
|
|
77
|
-
test("the default (addressed) arm carries no condition and re-enters
|
|
77
|
+
test("the default (addressed) arm carries no condition and re-enters round processing", () => {
|
|
78
78
|
const addressed = flowElement("f_addressed");
|
|
79
79
|
assert(addressed, "f_addressed flow missing");
|
|
80
|
-
assertStringIncludes(addressed, 'targetRef="
|
|
80
|
+
assertStringIncludes(addressed, 'targetRef="persist-round"');
|
|
81
81
|
// A default flow must have NO conditionExpression.
|
|
82
82
|
assert(
|
|
83
83
|
!/conditionExpression/.test(addressed),
|
|
@@ -95,9 +95,10 @@ test("regression: an empty/unknown status no longer routes to persist-escalation
|
|
|
95
95
|
for (const f of intoEscalation) {
|
|
96
96
|
assertStringIncludes(f, "conditionExpression");
|
|
97
97
|
}
|
|
98
|
-
// The addressed default must land on
|
|
98
|
+
// The addressed default must land on round processing (persist-round → check-progress, which
|
|
99
|
+
// re-solicits the review after the round-cap gate downstream), not escalation.
|
|
99
100
|
const addressed = flowElement("f_addressed");
|
|
100
|
-
assert(addressed && /targetRef="
|
|
101
|
+
assert(addressed && /targetRef="persist-round"/.test(addressed), "default arm must re-enter round processing");
|
|
101
102
|
});
|
|
102
103
|
|
|
103
104
|
// The canonical router (app/roundResultDefault.ts) mirrors the gw-status routing above, with the
|
|
@@ -161,7 +162,7 @@ test("persist-escalation routes through the gw-escalated liveness gateway, not s
|
|
|
161
162
|
);
|
|
162
163
|
});
|
|
163
164
|
|
|
164
|
-
test("gw-escalated waits only when an escalation was opened, else re-enters
|
|
165
|
+
test("gw-escalated waits only when an escalation was opened, else re-enters round processing", () => {
|
|
165
166
|
const gw = flat.match(/<bpmn:exclusiveGateway\b[^>]*\bid="gw-escalated"[^>]*>/);
|
|
166
167
|
assert(gw, "gw-escalated gateway missing");
|
|
167
168
|
// Its default arm must be the re-enter (no-escalation) arm, never the answer-wait.
|
|
@@ -175,12 +176,13 @@ test("gw-escalated waits only when an escalation was opened, else re-enters the
|
|
|
175
176
|
assertStringIncludes(wait, 'targetRef="wait-answer"');
|
|
176
177
|
assertStringIncludes(wait, "escalated = true");
|
|
177
178
|
|
|
178
|
-
// The default (no-escalation) arm carries no condition and re-enters
|
|
179
|
-
// progress), so a non-escalation early return can
|
|
179
|
+
// The default (no-escalation) arm carries no condition and re-enters round processing
|
|
180
|
+
// (persist-round → check-progress → the round-cap gate), so a non-escalation early return can
|
|
181
|
+
// never wedge on wait-answer.
|
|
180
182
|
const reenter = flowElement("f_escReenter");
|
|
181
183
|
assert(reenter, "f_escReenter flow missing");
|
|
182
184
|
assertStringIncludes(reenter, 'sourceRef="gw-escalated"');
|
|
183
|
-
assertStringIncludes(reenter, 'targetRef="
|
|
185
|
+
assertStringIncludes(reenter, 'targetRef="persist-round"');
|
|
184
186
|
assert(
|
|
185
187
|
!/conditionExpression/.test(reenter),
|
|
186
188
|
"f_escReenter is the default arm and must not carry a conditionExpression",
|
package/app/service.test.ts
CHANGED
|
@@ -89,6 +89,10 @@ test("re-submit of a cancelled PR marks stale open escalations", async () => {
|
|
|
89
89
|
title: "old title",
|
|
90
90
|
status: "abandoned", // terminal -> re-open path
|
|
91
91
|
current_round: 3,
|
|
92
|
+
last_round_head: "stale-sha-from-prior-run",
|
|
93
|
+
last_progress_job_key: "old-job-key-from-prior-run",
|
|
94
|
+
last_progress_result: "{\"progressed\":false,\"huskRetries\":0}",
|
|
95
|
+
last_progress_agent_watermark: "999",
|
|
92
96
|
}],
|
|
93
97
|
key: "pr_key",
|
|
94
98
|
},
|
|
@@ -120,6 +124,16 @@ test("re-submit of a cancelled PR marks stale open escalations", async () => {
|
|
|
120
124
|
const pr = stores.pull_requests.rows[0] as Record<string, unknown>;
|
|
121
125
|
assertEquals(pr.status, "converging");
|
|
122
126
|
assertEquals(pr.current_round, 1);
|
|
127
|
+
// The no-progress head baseline is scoped to the prior run; a fresh run must clear it so the
|
|
128
|
+
// first addressed round is compared from a clean slate and the bounded husk retry isn't bypassed
|
|
129
|
+
// when the branch changed between runs (#786).
|
|
130
|
+
assertEquals(pr.last_round_head, null);
|
|
131
|
+
// The at-least-once replay stamp + attempt watermark are ALSO run-scoped and must be cleared so a
|
|
132
|
+
// straggler `pr.progress-check` from the prior run can't replay a stale outcome into the fresh run
|
|
133
|
+
// (Copilot PR #789).
|
|
134
|
+
assertEquals(pr.last_progress_job_key, null);
|
|
135
|
+
assertEquals(pr.last_progress_result, null);
|
|
136
|
+
assertEquals(pr.last_progress_agent_watermark, null);
|
|
123
137
|
assertEquals(pr.open_escalation_id, undefined);
|
|
124
138
|
assertEquals(pr.open_escalation_question, undefined);
|
|
125
139
|
assertEquals(pr.process_key, "PI-9");
|
package/app/service.ts
CHANGED
|
@@ -284,6 +284,23 @@ export interface PullRequest {
|
|
|
284
284
|
// review for this PR, so the nudge is throttled to one attempt per REVIEW_NUDGE_MS window.
|
|
285
285
|
// NULL means never nudged.
|
|
286
286
|
last_nudge_at: string | null;
|
|
287
|
+
// No-progress head baseline (033_pr_round_head.sql, issue #786): the PR head SHA observed at the
|
|
288
|
+
// last recorded round, written by pr.progress-check to detect an addressed round that pushed no
|
|
289
|
+
// commit. Scoped to the current convergence run — cleared on re-open so a resubmission starts from
|
|
290
|
+
// a clean slate. NULL before the first round is recorded.
|
|
291
|
+
last_round_head: string | null;
|
|
292
|
+
// At-least-once idempotency for pr.progress-check (103_pr_progress_idempotency.sql): the engine
|
|
293
|
+
// job key that produced the last committed progress decision, and that decision's serialized
|
|
294
|
+
// `PrProgressCheckOut`. On a lost-ack redelivery the guard recognizes its own job key and replays
|
|
295
|
+
// the recorded outcome instead of recomputing against the already-advanced `last_round_head`.
|
|
296
|
+
last_progress_job_key: string | null;
|
|
297
|
+
last_progress_result: string | null;
|
|
298
|
+
// Attempt watermark for pr.progress-check husk correlation (103; Copilot PR #789): the greatest
|
|
299
|
+
// `review-round` AgentInstance key an earlier progress-check already accounted for. Lets the next
|
|
300
|
+
// round tell a freshly-registered attempt from a historical one, so a current attempt that husks
|
|
301
|
+
// BEFORE registering its AgentInstance is classified as a husk instead of masked by a prior round's
|
|
302
|
+
// terminal instance. Cleared on re-open with the rest of the per-run state. NULL before any read.
|
|
303
|
+
last_progress_agent_watermark: string | null;
|
|
287
304
|
// Merge-protocol liveness (012_merge_protocol_attempt.sql): head commit last nudged by the
|
|
288
305
|
// frugal-CI fresh-head-run remedy. A rebase changes the head and therefore permits a new nudge.
|
|
289
306
|
fresh_head_run_head: string | null;
|
|
@@ -565,6 +582,24 @@ export async function submitPr(
|
|
|
565
582
|
waiting_since: null,
|
|
566
583
|
last_review_id: null,
|
|
567
584
|
last_nudge_at: null,
|
|
585
|
+
// Clear the no-progress head baseline: it is scoped to the PRIOR convergence run, and a fresh
|
|
586
|
+
// run at round 1 must compare its first addressed round against a clean slate. Leaving a stale
|
|
587
|
+
// `last_round_head` lets a resubmission whose branch changed read `currentHead !== previousHead`
|
|
588
|
+
// on its first husked round, mis-route it as progress, and bypass the bounded husk retry (#786).
|
|
589
|
+
last_round_head: null,
|
|
590
|
+
// Clear the attempt watermark too: it is scoped to the prior convergence run's `review-round`
|
|
591
|
+
// instances (Copilot #789). A fresh run mints new, higher-keyed instances so a carried-over
|
|
592
|
+
// watermark would still be below them, but clearing keeps the per-run husk-correlation state
|
|
593
|
+
// unambiguous and self-contained.
|
|
594
|
+
last_progress_agent_watermark: null,
|
|
595
|
+
// Clear the at-least-once REPLAY stamp too (Copilot PR #789). The idempotency guard replays a
|
|
596
|
+
// recorded outcome whenever a redelivered job key matches this row; if the stamp survived a
|
|
597
|
+
// re-open, an OLD `pr.progress-check` delivery redelivered after the NEW convergence instance
|
|
598
|
+
// starts would still match its job key here and replay a stale escalation/progress effect into
|
|
599
|
+
// the fresh run. Clearing it makes the new run treat any such straggler as an unknown key (a
|
|
600
|
+
// normal, freshly-computed decision) rather than replaying the prior run's outcome.
|
|
601
|
+
last_progress_job_key: null,
|
|
602
|
+
last_progress_result: null,
|
|
568
603
|
outcome: null,
|
|
569
604
|
converged_at: null,
|
|
570
605
|
merged_at: null,
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
-- Scope pr.persist-round's idempotent upsert to the process instance that wrote the row (issue #786).
|
|
2
|
+
--
|
|
3
|
+
-- The idempotent `(pr_key, round_no)` upsert in pr.persist-round exists so a husk auto-retry — which
|
|
4
|
+
-- re-enters `review-round` WITHOUT advancing the round counter, in the SAME convergence process
|
|
5
|
+
-- instance — updates its round-record row in place instead of manufacturing a duplicate history row.
|
|
6
|
+
-- Round ownership was previously INFERRED from `status` (reuse any non-`needs_input`/`blocked` row),
|
|
7
|
+
-- but status is not identity: `submitPr` re-opens a previously converged/abandoned/merged PR at
|
|
8
|
+
-- `current_round = 1` WITHOUT deleting `rounds` history, so a fresh convergence run (a NEW process
|
|
9
|
+
-- instance) at round 1 would find the prior run's `addressed`/`waiting`/`converged` round-1 row and
|
|
10
|
+
-- overwrite its summary/transcript/worker/timestamps — destroying the canonical history across
|
|
11
|
+
-- resubmissions.
|
|
12
|
+
--
|
|
13
|
+
-- Persist the writing process instance's key so the upsert can reuse ONLY a row THIS run wrote: a
|
|
14
|
+
-- husk retry (same `process_instance_key`) updates in place; a resubmission (a different key) inserts
|
|
15
|
+
-- a fresh row, leaving every prior run's history intact. Additive and nullable — pre-#786 rows and
|
|
16
|
+
-- rows written by an engine that does not surface the key read back NULL for this column.
|
|
17
|
+
--
|
|
18
|
+
-- Upgrade behaviour of those NULL rows (persist-round's reuse predicate):
|
|
19
|
+
-- * A KEYED engine job (the normal production case) reuses a row ONLY when its key equals the
|
|
20
|
+
-- current `process_instance_key`, so a NULL-key row NEVER matches and is never reused. The
|
|
21
|
+
-- status-only heuristic applies ONLY to a KEYLESS job (testkit/synthetic, no process key).
|
|
22
|
+
-- * Consequently a husk auto-retry that straddles this deploy — its first attempt wrote a NULL-key
|
|
23
|
+
-- row before the migration, its retry runs after with a concrete key — will not reuse that
|
|
24
|
+
-- earlier row and instead inserts a fresh round-record row. That lost idempotency is INTENTIONAL
|
|
25
|
+
-- and self-healing: it touches only a run whose husk-retry brackets the deploy, costs at worst
|
|
26
|
+
-- one duplicate history row for that single round, and never corrupts data or history.
|
|
27
|
+
-- Additive and nullable, so it is safe to apply forward over any earlier schema and re-runs are no-ops.
|
|
28
|
+
ALTER TABLE rounds ADD COLUMN process_instance_key TEXT;
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
-- Make the `pr.progress-check` guard (workers/progress-check/worker.ts) idempotent under the
|
|
2
|
+
-- engine's AT-LEAST-ONCE job delivery. The guard advances the `last_round_head` baseline as its
|
|
3
|
+
-- observation of the current round's head; a job whose side-effects landed but whose completion-ack
|
|
4
|
+
-- was LOST is redelivered with the SAME job key. Without an idempotency record the redelivery reads
|
|
5
|
+
-- the just-advanced baseline as `previousHead`, sees "head did not advance", and can mis-escalate an
|
|
6
|
+
-- already-progressed round (Copilot review, PR #789).
|
|
7
|
+
--
|
|
8
|
+
-- These two columns record, in the SAME atomic row update that advances the baseline and writes the
|
|
9
|
+
-- resting status, the job key that produced the decision and the serialized worker outcome. On
|
|
10
|
+
-- redelivery the guard recognizes its own job key and REPLAYS the recorded outcome instead of
|
|
11
|
+
-- recomputing against the mutated baseline. Because the stamp and the baseline advance are one
|
|
12
|
+
-- update, a redelivery either sees the whole record (replay) or none of it (recompute from the
|
|
13
|
+
-- un-advanced baseline → same decision) — never a half-state. A husk auto-retry re-enters
|
|
14
|
+
-- `review-round` as a NEW job key, so it is never mistaken for a redelivery.
|
|
15
|
+
--
|
|
16
|
+
-- • last_progress_job_key — the engine job key of the last progress-check that committed a write.
|
|
17
|
+
-- • last_progress_result — that job's serialized `PrProgressCheckOut` (JSON), replayed verbatim.
|
|
18
|
+
-- • last_progress_agent_watermark — the greatest `review-round` AgentInstance key an earlier
|
|
19
|
+
-- progress-check has ALREADY accounted for (Copilot PR #789). It lets the next round distinguish
|
|
20
|
+
-- a freshly-registered `review-round` attempt from a historical one, so a CURRENT attempt that
|
|
21
|
+
-- husks BEFORE the worker registers its AgentInstance is classified as a husk (bounded auto-
|
|
22
|
+
-- retry) instead of being masked by a prior round's terminal instance and mis-escalated.
|
|
23
|
+
--
|
|
24
|
+
-- Forward-only, additive (expand): all columns are nullable with no default. Numbered after the
|
|
25
|
+
-- current highest prefix (102); the runner wraps each file in its own transaction, so this file must
|
|
26
|
+
-- NOT contain BEGIN/COMMIT.
|
|
27
|
+
ALTER TABLE pull_requests ADD COLUMN last_progress_job_key TEXT;
|
|
28
|
+
ALTER TABLE pull_requests ADD COLUMN last_progress_result TEXT;
|
|
29
|
+
ALTER TABLE pull_requests ADD COLUMN last_progress_agent_watermark TEXT;
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
-- Re-create the `pull_requests_read_model` VIEW so it re-exports the base columns added by
|
|
2
|
+
-- 103_pr_progress_idempotency.sql (`last_progress_job_key`, `last_progress_result`,
|
|
3
|
+
-- `last_progress_agent_watermark`). The read-model VIEW must pass through EVERY base `pull_requests`
|
|
4
|
+
-- column (the static pages↔schema contract guard + app/pullRequestReadModel.test.ts DRIFT GUARD
|
|
5
|
+
-- assert it), so adding a base column obliges a fresh VIEW definition — 094 is immutable and cannot
|
|
6
|
+
-- be edited in place.
|
|
7
|
+
--
|
|
8
|
+
-- Every DERIVED column below is emitted VERBATIM from the ONE declaration in
|
|
9
|
+
-- app/pullRequestReadModel.ts (`pullRequestReadModel.sqlSelectFor(col, { baseAlias: "pr" })`) — the
|
|
10
|
+
-- same closed-DSL AST that drives the runtime TS via `fnFor`, so the two lowerings cannot diverge.
|
|
11
|
+
-- This file is a mechanical re-emission of 094 with the two new base pass-throughs added; the
|
|
12
|
+
-- derived `list_bucket`/`ack_open` expressions are unchanged. The drift guard now points at THIS
|
|
13
|
+
-- migration (the latest VIEW definition).
|
|
14
|
+
--
|
|
15
|
+
-- Forward-only VIEW definition (DROP then CREATE), sourced off the managed `pull_requests__tracking`
|
|
16
|
+
-- re-export of the base table so a terminated PR classifies on ENGINE TRUTH. SQLite does not validate
|
|
17
|
+
-- a view body at CREATE time. The runner wraps each file in its own transaction, so this file must
|
|
18
|
+
-- NOT contain BEGIN/COMMIT. Numbered after 103.
|
|
19
|
+
|
|
20
|
+
DROP VIEW IF EXISTS pull_requests_read_model;
|
|
21
|
+
|
|
22
|
+
CREATE VIEW pull_requests_read_model AS
|
|
23
|
+
SELECT
|
|
24
|
+
pr.pr_key AS pr_key,
|
|
25
|
+
pr.repo AS repo,
|
|
26
|
+
pr.number AS number,
|
|
27
|
+
pr.url AS url,
|
|
28
|
+
pr.title AS title,
|
|
29
|
+
COALESCE(pr.derived_status, pr.status) AS status,
|
|
30
|
+
pr.current_round AS current_round,
|
|
31
|
+
pr.process_key AS process_key,
|
|
32
|
+
pr.waiting_since AS waiting_since,
|
|
33
|
+
pr.last_review_id AS last_review_id,
|
|
34
|
+
pr.outcome AS outcome,
|
|
35
|
+
pr.created_at AS created_at,
|
|
36
|
+
pr.updated_at AS updated_at,
|
|
37
|
+
pr.converged_at AS converged_at,
|
|
38
|
+
pr.merged_at AS merged_at,
|
|
39
|
+
pr.active_worker AS active_worker,
|
|
40
|
+
pr.lease_until AS lease_until,
|
|
41
|
+
pr.last_nudge_at AS last_nudge_at,
|
|
42
|
+
pr.fresh_head_run_head AS fresh_head_run_head,
|
|
43
|
+
pr.abandon_token AS abandon_token,
|
|
44
|
+
pr.incident_key AS incident_key,
|
|
45
|
+
pr.incident_message AS incident_message,
|
|
46
|
+
pr.last_round_head AS last_round_head,
|
|
47
|
+
pr.last_progress_job_key AS last_progress_job_key,
|
|
48
|
+
pr.last_progress_result AS last_progress_result,
|
|
49
|
+
pr.last_progress_agent_watermark AS last_progress_agent_watermark,
|
|
50
|
+
pr.root_request_key AS root_request_key,
|
|
51
|
+
pr.epic_phase_label AS epic_phase_label,
|
|
52
|
+
pr.acknowledged_at AS acknowledged_at,
|
|
53
|
+
CASE WHEN COALESCE((COALESCE((COALESCE(("pr"."derived_status" = 'merged'), 0) OR COALESCE(("pr"."derived_status" = 'converged'), 0) OR COALESCE(("pr"."derived_status" = 'abandoned'), 0) OR COALESCE(("pr"."derived_status" = 'closed'), 0) OR COALESCE(("pr"."derived_status" = 'failed'), 0)), 0) AND COALESCE(("pr"."acknowledged_at" = "pr"."acknowledged_at"), 0)), 0) THEN 'history' ELSE 'active' END AS list_bucket,
|
|
54
|
+
CASE WHEN COALESCE((COALESCE((COALESCE(("pr"."derived_status" = 'merged'), 0) OR COALESCE(("pr"."derived_status" = 'converged'), 0) OR COALESCE(("pr"."derived_status" = 'abandoned'), 0) OR COALESCE(("pr"."derived_status" = 'closed'), 0) OR COALESCE(("pr"."derived_status" = 'failed'), 0)), 0) AND (NOT COALESCE(COALESCE(("pr"."acknowledged_at" = "pr"."acknowledged_at"), 0), 0))), 0) THEN 1 ELSE 0 END AS ack_open
|
|
55
|
+
FROM pull_requests__tracking pr;
|
package/docs/agent-guide.md
CHANGED
|
@@ -470,7 +470,7 @@ Work through this order:
|
|
|
470
470
|
3. **Check for an open escalation** (§3) — the process may simply be waiting for a
|
|
471
471
|
human answer. Answer it.
|
|
472
472
|
4. **A review that never arrives** escalates on its own after
|
|
473
|
-
`NANO_PR_REVIEW_WAIT_TIMEOUT` (default `
|
|
473
|
+
`NANO_PR_REVIEW_WAIT_TIMEOUT` (default `PT30M`); the poller also re-nudges the
|
|
474
474
|
reviewer periodically. If the reviewer bot is not provisioned on the repo, no
|
|
475
475
|
review will ever land — that is a repo-config problem, not an app bug.
|
|
476
476
|
5. **Cancel + resubmit** as a last resort. Cancel through the **app-owned** door —
|
|
@@ -181,13 +181,14 @@ describe("nano-workforce PR review-loop escalation (U4 userTask)", () => {
|
|
|
181
181
|
assert.equal(completed.body.ok, true, "the userTask was completed");
|
|
182
182
|
|
|
183
183
|
// The typed answer resumed the loop back into the review round: the token took
|
|
184
|
-
// wait-answer → record-answer (which retires the escalations row) → review-round
|
|
185
|
-
//
|
|
186
|
-
//
|
|
184
|
+
// wait-answer → record-answer (which retires the escalations row) → capture-head → review-round
|
|
185
|
+
// (the round-entry head is re-captured before each review round, #786), and the review agent saw
|
|
186
|
+
// exactly the submitted answer. An empty or wrong completion would surface a different
|
|
187
|
+
// `capturedAnswer` — this is the falsifiable core.
|
|
187
188
|
await app.settle();
|
|
188
189
|
const flows = takenFlows(app);
|
|
189
190
|
assert.ok(
|
|
190
|
-
flows.includes("wait-answer->record-answer") && flows.includes("record-answer->
|
|
191
|
+
flows.includes("wait-answer->record-answer") && flows.includes("record-answer->capture-head"),
|
|
191
192
|
`the answer resumed the loop through record-answer back to the review round (flows: ${flows.join(", ")})`,
|
|
192
193
|
);
|
|
193
194
|
assert.equal(reviewCalls, 2, "the review agent ran a second round after the answer");
|
package/e2e/feature-run.e2e.ts
CHANGED
|
@@ -23,6 +23,7 @@ import { fileURLToPath } from "node:url";
|
|
|
23
23
|
import type { EngineJob } from "@nanobpm/urban/runtime";
|
|
24
24
|
import { bootTestApp, type TestApp } from "@nanobpm/urban-testkit";
|
|
25
25
|
import { admitGithubState, installAdmitGithub } from "./support/github-admit.ts";
|
|
26
|
+
import { settleFully } from "./support/time.ts";
|
|
26
27
|
import { pollUserTasks } from "../app/service.ts";
|
|
27
28
|
|
|
28
29
|
const APP_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), "..");
|
|
@@ -103,7 +104,11 @@ describe("single-issue feature run (#172 — feature.bpmn)", () => {
|
|
|
103
104
|
const featureKey = "owner/repo#7";
|
|
104
105
|
const started = await app.api?.call("startFeature", { body: { issue: featureKey, ...startBody } });
|
|
105
106
|
assert.equal(started?.status, 202, "startFeature accepted the issue");
|
|
106
|
-
|
|
107
|
+
// Fixpoint (not a single tick): the `converge` hand-off enrolls the opened PR via `submitPr`,
|
|
108
|
+
// whose nested `createInstance` re-entrantly runs the new `capture-head` host task (#786) and
|
|
109
|
+
// leaves `pr.converge-feature`'s own completion undrained until a later settle — see
|
|
110
|
+
// `settleFully`.
|
|
111
|
+
await settleFully(app);
|
|
107
112
|
const run = await app.db
|
|
108
113
|
.table<FeatureRow>("feature_runs", "feature_key")
|
|
109
114
|
.findOne({ feature_key: featureKey });
|
|
@@ -25,7 +25,7 @@ import { fileURLToPath } from "node:url";
|
|
|
25
25
|
import type { EngineJob } from "@nanobpm/urban/runtime";
|
|
26
26
|
import { bootTestApp, type TestApp } from "@nanobpm/urban-testkit";
|
|
27
27
|
import { admitGithubState, installAdmitGithub } from "./support/github-admit.ts";
|
|
28
|
-
import { advancePastTimer } from "./support/time.ts";
|
|
28
|
+
import { advancePastTimer, settleFully } from "./support/time.ts";
|
|
29
29
|
|
|
30
30
|
const APP_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), "..");
|
|
31
31
|
|
|
@@ -95,7 +95,10 @@ describe("plan-fanout escalation SLA + assignment (U5)", () => {
|
|
|
95
95
|
const planKey = "owner/repo#1";
|
|
96
96
|
const started = await app.api?.call("startPlanFanout", { body: { issue: planKey, baseBranch: "epic/e2e" } });
|
|
97
97
|
assert.equal(started?.status, 202, "startPlanFanout accepted the issue");
|
|
98
|
-
|
|
98
|
+
// Fixpoint (not a single tick): the wave fan-out enrolls each opened PR via `submitPr`, whose
|
|
99
|
+
// nested `createInstance` re-entrantly runs the new `capture-head` host task (#786) and leaves
|
|
100
|
+
// the enroller's completion undrained until a later settle — see `settleFully`.
|
|
101
|
+
await settleFully(app);
|
|
99
102
|
const plan = await app.db
|
|
100
103
|
.table<{ plan_key: string; process_key: string | null }>("plans", "plan_key")
|
|
101
104
|
.findOne({ plan_key: planKey });
|
package/e2e/plan-fanout.e2e.ts
CHANGED
|
@@ -22,7 +22,7 @@ import { fileURLToPath } from "node:url";
|
|
|
22
22
|
import type { EngineJob } from "@nanobpm/urban/runtime";
|
|
23
23
|
import { bootTestApp, type TestApp } from "@nanobpm/urban-testkit";
|
|
24
24
|
import { admitGithubState, installAdmitGithub } from "./support/github-admit.ts";
|
|
25
|
-
import { advancePastTimer } from "./support/time.ts";
|
|
25
|
+
import { advancePastTimer, settleFully } from "./support/time.ts";
|
|
26
26
|
|
|
27
27
|
const APP_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), "..");
|
|
28
28
|
|
|
@@ -93,7 +93,11 @@ describe("plan-fanout escalations (U2 — task + plan-review + trial-merge → u
|
|
|
93
93
|
const planKey = "owner/repo#1";
|
|
94
94
|
const started = await app.api?.call("startPlanFanout", { body: { issue: planKey, baseBranch: "epic/e2e" } });
|
|
95
95
|
assert.equal(started?.status, 202, "startPlanFanout accepted the issue");
|
|
96
|
-
|
|
96
|
+
// Drive the whole fanout to quiescence: a wave whose PRs open enrolls each PR via
|
|
97
|
+
// `pr.record-wave` -> `submitPr` -> nested `createInstance`, whose re-entrant drain leaves the
|
|
98
|
+
// enroller undrained for a tick (issue #786's `pr.capture-head`). A single settle parks the
|
|
99
|
+
// plan one tick early (before the trial-merge gate opens), so settle to true quiescence.
|
|
100
|
+
await settleFully(app);
|
|
97
101
|
const plan = await app.db
|
|
98
102
|
.table<{ plan_key: string; process_key: string | null }>("plans", "plan_key")
|
|
99
103
|
.findOne({ plan_key: planKey });
|
package/e2e/support/time.ts
CHANGED
|
@@ -31,3 +31,37 @@ export async function advancePastTimer(
|
|
|
31
31
|
await app.engine.advanceTime(ms);
|
|
32
32
|
await app.settle();
|
|
33
33
|
}
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* Settle the deterministic harness to true quiescence — repeat {@link TestApp.settle} until no job
|
|
37
|
+
* remains mid-flight (`ACTIVATED`) — for any scenario where a worker ENROLLS a PR.
|
|
38
|
+
*
|
|
39
|
+
* The canonical convergence enrollment `submitPr` (used by `pr.record-wave`, `pr.converge-feature`
|
|
40
|
+
* and `pr.delivery-connector`) calls `engine.createInstance` from INSIDE a worker job handler. The
|
|
41
|
+
* urban-testkit services that nested creation with a nested `drain()`; now that `convergence-loop`
|
|
42
|
+
* opens with the host `pr.capture-head` task (issue #786), that nested drain runs real re-entrant
|
|
43
|
+
* host work and — per the testkit's documented re-entrancy — leaves the ENROLLING worker's own
|
|
44
|
+
* completion "undrained until a later settle". A single `settle()` therefore observes the enroller
|
|
45
|
+
* one tick early (e.g. a `feature_runs` row still `opened`, or a plan not yet parked on its
|
|
46
|
+
* trial-merge task). At real runtime the async job stream drains this with no extra prompting; the
|
|
47
|
+
* harness just needs to be driven to quiescence.
|
|
48
|
+
*
|
|
49
|
+
* The reliable quiescence signal is a job in state `ACTIVATED`: that is a leased, mid-flight job —
|
|
50
|
+
* the undrained enrolling-worker completion the re-entrancy leaves behind. A `settle()` drains every
|
|
51
|
+
* *registered*-worker job to completion, so once no `ACTIVATED` job remains the only jobs left are
|
|
52
|
+
* genuine external parks (`CREATED` agent jobs with no registered worker), and the harness is
|
|
53
|
+
* quiescent. Keying on `ACTIVATED` is precise, not a retry-and-hope: it is NOT sufficient to compare
|
|
54
|
+
* successive `snapshot()` signatures, because the undrained completion is invisible between certain
|
|
55
|
+
* settles (two passes look identical while work is still pending), so a naive fixpoint returns early
|
|
56
|
+
* and the re-entrant work later fires against a closing DB.
|
|
57
|
+
*/
|
|
58
|
+
export async function settleFully(
|
|
59
|
+
app: Pick<TestApp, "settle" | "engine">,
|
|
60
|
+
maxRounds = 16,
|
|
61
|
+
): Promise<void> {
|
|
62
|
+
for (let round = 0; round < maxRounds; round++) {
|
|
63
|
+
await app.settle();
|
|
64
|
+
const jobs = await app.engine.searchJobs({});
|
|
65
|
+
if (!jobs.some((job) => job.state === "ACTIVATED")) return;
|
|
66
|
+
}
|
|
67
|
+
}
|
package/nano.app.json
CHANGED
|
@@ -120,6 +120,10 @@
|
|
|
120
120
|
"taskType": "pr.persist-round",
|
|
121
121
|
"handler": "workers/persist-round/worker.ts"
|
|
122
122
|
},
|
|
123
|
+
{
|
|
124
|
+
"taskType": "pr.capture-head",
|
|
125
|
+
"handler": "workers/capture-head/worker.ts"
|
|
126
|
+
},
|
|
123
127
|
{
|
|
124
128
|
"taskType": "pr.persist-escalation",
|
|
125
129
|
"handler": "workers/persist-escalation/worker.ts"
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@nanobpm/nano-workforce",
|
|
3
|
-
"version": "0.187.
|
|
3
|
+
"version": "0.187.5",
|
|
4
4
|
"description": "Nano Workforce — an Agent Graph Orchestration application for Agentic SDLC: durable BPMN processes that coordinate a graph of AI agents across the software delivery lifecycle.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "main.ts",
|