@nanobpm/nano-workforce 0.187.4 → 0.187.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +6 -0
- package/SPEC.md +69 -7
- package/app/convergenceEscalationGuard.test.ts +3 -2
- package/app/github.test.ts +125 -1
- package/app/github.ts +43 -8
- package/app/persist-escalation.test.ts +7 -5
- package/app/persist-round.test.ts +178 -11
- package/app/pullRequestReadModel.test.ts +1 -1
- package/app/roundProgress.test.ts +755 -33
- package/app/roundProgress.ts +144 -0
- package/app/roundResultDefault.test.ts +10 -8
- package/app/service.test.ts +14 -0
- package/app/service.ts +35 -0
- package/db/migrations/102_rounds_process_instance_key.sql +28 -0
- package/db/migrations/103_pr_progress_idempotency.sql +29 -0
- package/db/migrations/104_pull_requests_read_model_progress_idempotency.sql +55 -0
- package/e2e/convergence-escalation.e2e.ts +5 -4
- package/e2e/feature-run.e2e.ts +6 -1
- package/e2e/plan-fanout-sla.e2e.ts +5 -2
- package/e2e/plan-fanout.e2e.ts +6 -2
- package/e2e/support/time.ts +34 -0
- package/nano.app.json +4 -0
- package/package.json +1 -1
- package/resources/processes/convergence-loop.bpmn +211 -142
- package/test/derivation-parity/README.md +3 -3
- package/test/derivation-parity/derivation-parity.test.ts +9 -3
- package/test/derivation-parity/flows.ts +4 -4
- package/workers/capture-head/worker.test.ts +77 -0
- package/workers/capture-head/worker.ts +64 -0
- package/workers/persist-escalation/worker.ts +4 -0
- package/workers/persist-round/worker.ts +75 -9
- package/workers/progress-check/worker.ts +394 -35
|
@@ -4,17 +4,20 @@
|
|
|
4
4
|
// compares it to the head observed at the previous recorded round. An `addressed` round whose head
|
|
5
5
|
// did NOT advance pushed no commit, so requesting another Copilot review would loop on
|
|
6
6
|
// byte-identical code — the same comments, round after round, until the round cap escalates. It
|
|
7
|
-
//
|
|
8
|
-
//
|
|
7
|
+
// then classifies WHY (issue #786): a **husk** (no commit AND no terminal `review-round`
|
|
8
|
+
// agent-instance for the round — the producer harness died mid-run, jwulf/c8ctl-plugin-nano#230) is
|
|
9
|
+
// auto-re-run onto a healthy worker up to a bound before escalating; a **no-advance** (the agent DID
|
|
10
|
+
// run to a terminal instance but pushed nothing) escalates to a human straight away. The routing
|
|
11
|
+
// decision lives in app/roundProgress.ts, the single source of truth this worker, `gw-progress`, and
|
|
12
|
+
// `gw-husk` all mirror.
|
|
9
13
|
//
|
|
10
14
|
// Every other case returns `progressed:true` (continue): a `waiting` round (round 1, awaiting the
|
|
11
15
|
// first review) legitimately has no push; an advanced head means real work landed; and a head we
|
|
12
16
|
// could not read fails OPEN — the round cap and the review-wait timeout stay the safety nets so a
|
|
13
|
-
// transient GitHub hiccup can never fabricate a no-progress escalation.
|
|
14
|
-
|
|
15
|
-
import
|
|
16
|
-
import {
|
|
17
|
-
import { isAddressedStatus, routeProgress } from "../../app/roundProgress.ts";
|
|
17
|
+
// transient GitHub hiccup can never fabricate a no-progress escalation.
|
|
18
|
+
import type { AgentInstanceSummary, AppApi, AppJobHandler } from "@nanobpm/urban";
|
|
19
|
+
import { fetchBranchHead, fetchPrHead } from "../../app/github.ts";
|
|
20
|
+
import { decideProgress, isAddressedStatus } from "../../app/roundProgress.ts";
|
|
18
21
|
import { parsePr } from "../../app/service.ts";
|
|
19
22
|
import type { WorkerInputs, WorkerOutputs } from "../../nano-generated/worker-io.d.ts";
|
|
20
23
|
|
|
@@ -25,51 +28,407 @@ type Out = WorkerOutputs["pr.progress-check"];
|
|
|
25
28
|
|
|
26
29
|
// Reads a PR's current head SHA. Injectable so unit tests never touch git/network; the default
|
|
27
30
|
// binds the real GitHub reader (the shared gh | token transport) and swallows any failure to
|
|
28
|
-
// `null` so the guard fails OPEN.
|
|
31
|
+
// `null` so the guard fails OPEN. It reads the BRANCH ref (`git/ref/heads/<branch>`) — updated
|
|
32
|
+
// atomically with the push — in preference to the PR object's asynchronously-denormalized
|
|
33
|
+
// `head.sha`, so a lagging PR projection can never fabricate a stale-but-valid no-advance
|
|
34
|
+
// escalation (#786). Once a head ref is known this trusts ONLY its atomic ref: a failed/absent
|
|
35
|
+
// ref read fails OPEN (`null`), never falling back to `head.sha`. The PR head is used only when
|
|
36
|
+
// the PR carries NO head ref at all.
|
|
29
37
|
export type HeadReader = (repo: string, prNumber: number) => Promise<string | null>;
|
|
30
38
|
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
}
|
|
39
|
+
/** The outcome of an agent-work corroboration, optionally carrying the ATTEMPT WATERMARK it
|
|
40
|
+
* consumed. `work` is the husk verdict decideProgress routes on (`true` no-advance / `false` husk /
|
|
41
|
+
* `null` unknown). `consumedKey` — when present — is the greatest `review-round` instance key this
|
|
42
|
+
* read has now ACCOUNTED FOR; the handler persists it (`last_progress_agent_watermark`) so the NEXT
|
|
43
|
+
* round can tell a freshly-registered attempt from a historical one (see {@link agentWorkFromEngine}
|
|
44
|
+
* and Copilot PR #789). A bare `boolean | null` return (the injected test readers) carries no
|
|
45
|
+
* watermark and leaves the persisted one untouched. */
|
|
46
|
+
export interface AgentWorkObservation {
|
|
47
|
+
readonly work: boolean | null;
|
|
48
|
+
readonly consumedKey?: string | null;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
// Corroborates whether a no-advance `addressed` round produced DURABLE agent work: `true` when the
|
|
52
|
+
// completing `review-round` element-instance ran to a terminal agent-instance (→ `no-advance`),
|
|
53
|
+
// `false` when the completing attempt husked — a non-terminal `review-round` instance is present, OR
|
|
54
|
+
// the completing attempt registered NO instance at all on a channel-present engine (→ `husk`),
|
|
55
|
+
// `null` when the read is UNAVAILABLE (the engine has no AgentInstance channel, or the read threw →
|
|
56
|
+
// `no-advance`, never an auto-retry). May return a bare verdict or an {@link AgentWorkObservation}
|
|
57
|
+
// that also carries the consumed attempt watermark. Injectable for tests; the default is
|
|
58
|
+
// availability- and attempt-aware over the engine's AgentInstance channel (see {@link
|
|
59
|
+
// agentWorkFromEngine}). `priorWatermark` is the greatest `review-round` instance key an earlier
|
|
60
|
+
// progress-check already accounted for, so a current pre-registration husk is not masked by a prior
|
|
61
|
+
// round's terminal instance. The `round` argument is retained for the reader contract but no longer
|
|
62
|
+
// participates in the decision — correlation is by the COMPLETING element-instance, not an aggregate
|
|
63
|
+
// round count (#786).
|
|
64
|
+
export type AgentWorkReader = (
|
|
65
|
+
processInstanceKey: string | null | undefined,
|
|
66
|
+
round: number,
|
|
67
|
+
priorWatermark?: string | null,
|
|
68
|
+
) => Promise<boolean | null | AgentWorkObservation>;
|
|
69
|
+
|
|
70
|
+
/** Normalize a reader's return (a bare verdict, or a full {@link AgentWorkObservation}) to the
|
|
71
|
+
* observation shape the handler threads: an injected `boolean | null` carries no watermark. */
|
|
72
|
+
function normalizeAgentWork(raw: boolean | null | AgentWorkObservation | undefined): AgentWorkObservation {
|
|
73
|
+
if (raw === true || raw === false || raw === null || raw === undefined) {
|
|
74
|
+
return { work: raw ?? null };
|
|
75
|
+
}
|
|
76
|
+
return raw;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/** Build the default head reader over injected GitHub fetchers. Exported (with injectable fetchers)
|
|
80
|
+
* so the branch-ref-over-stale-`head.sha` preference — the whole point of {@link fetchBranchHead}
|
|
81
|
+
* here (#786) — is covered by a handler-level regression test, not only inside the private binding:
|
|
82
|
+
* a change that stopped reading the branch ref, or fell back to `head.sha`, must turn a test red. */
|
|
83
|
+
export function makeDefaultReadHead(deps: {
|
|
84
|
+
fetchPrHead: typeof fetchPrHead;
|
|
85
|
+
fetchBranchHead: typeof fetchBranchHead;
|
|
86
|
+
}): HeadReader {
|
|
87
|
+
return async (repo, prNumber) => {
|
|
88
|
+
const token = process.env.GITHUB_TOKEN ?? "";
|
|
89
|
+
const pr = await deps.fetchPrHead(repo, prNumber, token).catch(() => null);
|
|
90
|
+
if (!pr) return null;
|
|
91
|
+
// Prefer the branch ref (atomic with the push) over the PR object's denormalized head.sha (#786).
|
|
92
|
+
// Once the head branch is known, trust ONLY its atomic ref: a failed/absent ref read fails OPEN
|
|
93
|
+
// (`null`) rather than falling back to the PR object's asynchronously-denormalized head.sha, which
|
|
94
|
+
// can still report a stale-but-valid SHA after a push and fabricate a no-advance escalation — the
|
|
95
|
+
// very projection this branch-ref read exists to avoid. The ref is read in the repository the
|
|
96
|
+
// head branch actually lives in (the fork for a cross-repo PR — see below), so a fork PR fails
|
|
97
|
+
// open safely instead of comparing an unrelated base-repo SHA. Fall back to the PR head only when
|
|
98
|
+
// there is NO head ref.
|
|
99
|
+
if (pr.headRef) {
|
|
100
|
+
// Resolve the head ref in the repository the head branch actually lives in — the FORK for a
|
|
101
|
+
// cross-repo PR (`pr.headRepo`), else the base `repo`. Querying the base repo unconditionally
|
|
102
|
+
// would, for a fork PR whose head branch shares a name with a base-repo branch, read the
|
|
103
|
+
// unrelated base-branch SHA and fabricate progress/no-progress (#786). When the head repo
|
|
104
|
+
// cannot be resolved (a deleted fork ⇒ `headRepo:null`) fail OPEN to `null` rather than fall
|
|
105
|
+
// back to the base repo and risk that collision.
|
|
106
|
+
const headRepo = pr.headRepo;
|
|
107
|
+
if (!headRepo) return null;
|
|
108
|
+
return await deps.fetchBranchHead(headRepo, pr.headRef, token).catch(() => null);
|
|
109
|
+
}
|
|
110
|
+
return pr.headSha ?? null;
|
|
111
|
+
};
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
const defaultReadHead: HeadReader = makeDefaultReadHead({ fetchPrHead, fetchBranchHead });
|
|
36
115
|
|
|
37
|
-
/**
|
|
38
|
-
*
|
|
39
|
-
|
|
116
|
+
/** An agent-instance is "durable work" for husk purposes once it has reached a terminal state — it
|
|
117
|
+
* carries a `completionDate`, or a terminal lifecycle status. A husked instance never closes (it is
|
|
118
|
+
* stuck `THINKING` with null dates — jwulf/c8ctl-plugin-nano#230), so it is NOT counted. */
|
|
119
|
+
function isTerminalInstance(s: AgentInstanceSummary): boolean {
|
|
120
|
+
if (typeof s.completionDate === "string" && s.completionDate.trim() !== "") return true;
|
|
121
|
+
return /^(completed|complete|done|finished|failed|terminated)$/i.test(s.status ?? "");
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
/** Parse a 64-bit engine key string to a `BigInt` for monotonic ordering; an absent/blank/malformed
|
|
125
|
+
* key sorts oldest (`0n`). Engine keys are 64-bit, so a numeric `parseInt`/`Number` compare would
|
|
126
|
+
* lose precision — `BigInt` compares them exactly. */
|
|
127
|
+
function engineKey(s: string | undefined | null): bigint {
|
|
128
|
+
if (typeof s !== "string" || s.trim() === "") return 0n;
|
|
129
|
+
try {
|
|
130
|
+
return BigInt(s.trim());
|
|
131
|
+
} catch {
|
|
132
|
+
return 0n;
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
/** The monotonic recency key that orders an AgentInstance by creation: the GREATEST of its occupancy
|
|
137
|
+
* `elementInstanceKeys` (the `review-round` element-instance keys the engine mints per attempt),
|
|
138
|
+
* falling back to its `agentInstanceKey`. Engine keys increase with creation, so the instance with
|
|
139
|
+
* the greatest recency key is the most-recently-created — the COMPLETING attempt in the
|
|
140
|
+
* single-threaded review-round loop. */
|
|
141
|
+
function recencyKey(s: AgentInstanceSummary): bigint {
|
|
142
|
+
let max = engineKey(s.agentInstanceKey);
|
|
143
|
+
for (const k of s.elementInstanceKeys ?? []) {
|
|
144
|
+
const v = engineKey(k);
|
|
145
|
+
if (v > max) max = v;
|
|
146
|
+
}
|
|
147
|
+
return max;
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
/** Default agent-work corroboration — AVAILABILITY-AWARE and correlated to the COMPLETING
|
|
151
|
+
* `review-round` element-instance (issue #786, Option 1):
|
|
152
|
+
*
|
|
153
|
+
* • AVAILABILITY PROBE (ADR 0056 fail-safe) — TWO-TIER, because an empty `review-round` search is
|
|
154
|
+
* AMBIGUOUS. `review-round` is an external-agent `serviceTask`: the engine creates an ORDINARY
|
|
155
|
+
* job and the WORKER registers its AgentInstance via `CreateAgentInstance` (nanobpmn engine-core
|
|
156
|
+
* `bpmn.rs`; a husked round is "a COMPLETED job that minted no AgentInstance", `event.rs`). So a
|
|
157
|
+
* completing review-round job that HUSKED *before the worker registered* mints NO instance — an
|
|
158
|
+
* empty scoped search is then a genuine husk, NOT proof of an absent channel. Distinguish the two
|
|
159
|
+
* with a process-instance-WIDE probe: (a) scoped `review-round` search non-empty → classify it
|
|
160
|
+
* (below); (b) scoped empty BUT the process instance has ANY agent instance at all (e.g.
|
|
161
|
+
* `classify-scope`, or an earlier round) → the channel is provably PRESENT, so a review-round
|
|
162
|
+
* that minted nothing is a genuine HUSK (`false`, auto-retry under the cap); (c) scoped empty AND
|
|
163
|
+
* the whole process instance has NO agent instance → the channel is UNKNOWN/ABSENT (the testkit
|
|
164
|
+
* WASM double, or a non-agentic engine) → `null` → `no-advance`, never an auto-retry, so a
|
|
165
|
+
* channel-absent engine can't loop. (The irreducible residual: the VERY FIRST agent task in a
|
|
166
|
+
* process husking before registering, with zero prior instances anywhere, is indistinguishable
|
|
167
|
+
* from an absent channel and fails safe to `no-advance` — a human resume, not a wedge.)
|
|
168
|
+
*
|
|
169
|
+
* • CORRELATION to the COMPLETING attempt via an ATTEMPT WATERMARK (Copilot PR #789). Once the
|
|
170
|
+
* channel is known present, the newest `review-round` instance is NOT necessarily the completing
|
|
171
|
+
* attempt: because a pre-registration husk mints NOTHING, a current attempt that husks before
|
|
172
|
+
* registering leaves the newest instance pointing at an EARLIER, terminal attempt — which, read
|
|
173
|
+
* naively, would classify as `no-advance` and bypass the bounded husk retry. So the completing
|
|
174
|
+
* attempt is correlated against `priorWatermark`: the greatest `review-round` instance key an
|
|
175
|
+
* earlier progress-check already accounted for. (a) A `review-round` instance NEWER than the
|
|
176
|
+
* watermark exists ⇒ the current attempt DID register ⇒ classify that newest instance — terminal
|
|
177
|
+
* ⇒ `no-advance` (`true`), non-terminal ⇒ the completing attempt husked (`false`) — and consume
|
|
178
|
+
* it (advance the watermark to its key). (b) NO instance newer than the watermark ⇒ the current
|
|
179
|
+
* attempt registered nothing new ⇒ it husked pre-registration ⇒ `husk` (`false`, auto-retry),
|
|
180
|
+
* leaving the watermark where it is. Each attempt (initial, review-loop, answer-loop resume, husk
|
|
181
|
+
* retry) re-enters `review-round` as a FRESH element-instance with a strictly greater monotonic
|
|
182
|
+
* key (its `elementInstanceKeys`, else its `agentInstanceKey`; see {@link recencyKey}), so "newer
|
|
183
|
+
* than the watermark" is exactly "a not-yet-accounted-for attempt". A stale terminal instance
|
|
184
|
+
* from an earlier attempt can therefore neither fabricate nor mask a current husk. The watermark
|
|
185
|
+
* is maintained on EVERY addressed round with a readable head (including progressing rounds), so a
|
|
186
|
+
* progressing round's fresh instance is consumed and can't be mistaken for the next round's
|
|
187
|
+
* completing attempt.
|
|
188
|
+
*
|
|
189
|
+
* Any read FAILURE degrades to `null` (unknown → `no-advance`, never an auto-retry), so a transient
|
|
190
|
+
* read outage can never duplicate genuinely-completed agent work. */
|
|
191
|
+
function agentWorkFromEngine(engine: AppApi["engine"]): AgentWorkReader {
|
|
192
|
+
return async (processInstanceKey, _round, priorWatermark): Promise<AgentWorkObservation> => {
|
|
193
|
+
if (!processInstanceKey) return { work: null };
|
|
194
|
+
try {
|
|
195
|
+
const instances = await engine.searchAgentInstances({
|
|
196
|
+
processInstanceKey: String(processInstanceKey),
|
|
197
|
+
elementId: "review-round",
|
|
198
|
+
});
|
|
199
|
+
if (instances.length === 0) {
|
|
200
|
+
// Two-tier availability probe (Copilot #789): the scoped search is empty, which is
|
|
201
|
+
// AMBIGUOUS for an external-agent task (a pre-registration husk mints no instance). Probe
|
|
202
|
+
// the process instance WIDE: if it holds ANY agent instance the channel is provably PRESENT,
|
|
203
|
+
// so a review-round that minted nothing genuinely HUSKED (`false`); if it holds NONE the
|
|
204
|
+
// channel is UNKNOWN/ABSENT → `null` (no-advance, never an auto-retry).
|
|
205
|
+
const anyInProcess = await engine.searchAgentInstances({
|
|
206
|
+
processInstanceKey: String(processInstanceKey),
|
|
207
|
+
});
|
|
208
|
+
return { work: anyInProcess.length === 0 ? null : false };
|
|
209
|
+
}
|
|
210
|
+
// Find the newest `review-round` instance (greatest monotonic engine key).
|
|
211
|
+
let newest = instances[0];
|
|
212
|
+
let newestKey = recencyKey(newest);
|
|
213
|
+
for (const inst of instances) {
|
|
214
|
+
const k = recencyKey(inst);
|
|
215
|
+
if (k > newestKey) {
|
|
216
|
+
newest = inst;
|
|
217
|
+
newestKey = k;
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
// ATTEMPT-WATERMARK correlation (Copilot #789): only an instance NEWER than the watermark is
|
|
221
|
+
// the current, not-yet-accounted-for attempt. If none is newer, the completing attempt
|
|
222
|
+
// registered nothing new — it husked before registering — even though a stale terminal instance
|
|
223
|
+
// from an earlier attempt is still present. That is a genuine husk (auto-retry), NOT a
|
|
224
|
+
// no-advance; classifying `newest` naively here would wrongly escalate it.
|
|
225
|
+
const prior = engineKey(priorWatermark);
|
|
226
|
+
if (newestKey <= prior) {
|
|
227
|
+
return { work: false, consumedKey: priorWatermark ?? null };
|
|
228
|
+
}
|
|
229
|
+
// A fresh attempt registered: classify it (terminal ⇒ no-advance, non-terminal ⇒ husk) and
|
|
230
|
+
// consume it so the next round can distinguish the attempt AFTER it from this one.
|
|
231
|
+
return { work: isTerminalInstance(newest), consumedKey: newestKey.toString() };
|
|
232
|
+
} catch {
|
|
233
|
+
return { work: null };
|
|
234
|
+
}
|
|
235
|
+
};
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
/** Build the handler with injectable readers (see {@link HeadReader} / {@link AgentWorkReader}). The
|
|
239
|
+
* default export binds the real GitHub reader; the agent-work reader defaults to the engine's
|
|
240
|
+
* AgentInstance channel when not injected. Tests inject stubs. */
|
|
241
|
+
export function makeHandler(deps: {
|
|
242
|
+
readHead: HeadReader;
|
|
243
|
+
readAgentWork?: AgentWorkReader;
|
|
244
|
+
}): AppJobHandler<In, Out> {
|
|
40
245
|
return async (job, app) => {
|
|
41
|
-
const { prKey, status, repo, prNumber } = job.variables;
|
|
246
|
+
const { prKey, status, repo, prNumber, round, huskRetries, roundEntryHead } = job.variables;
|
|
247
|
+
const jobKey = job.jobKey;
|
|
42
248
|
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
249
|
+
const prs = app.data.table<{
|
|
250
|
+
pr_key: string;
|
|
251
|
+
process_key: string | null;
|
|
252
|
+
last_round_head: string | null;
|
|
253
|
+
status: string | null;
|
|
254
|
+
updated_at: string | null;
|
|
255
|
+
last_progress_job_key: string | null;
|
|
256
|
+
last_progress_result: string | null;
|
|
257
|
+
last_progress_agent_watermark: string | null;
|
|
258
|
+
}>("pull_requests", "pr_key");
|
|
259
|
+
const row = await prs.get(prKey);
|
|
260
|
+
|
|
261
|
+
// REDELIVERY REPLAY GUARD (Copilot #789 / engine at-least-once delivery). A job whose
|
|
262
|
+
// side-effects landed but whose completion-ack was lost is redelivered with the SAME job key.
|
|
263
|
+
// Replaying the recorded outcome — instead of re-deriving against the now-advanced
|
|
264
|
+
// `last_round_head` baseline — stops a redelivered PROGRESSED round from reading its own
|
|
265
|
+
// just-written head as "no advance" and mis-escalating an already-progressed round. A husk
|
|
266
|
+
// auto-retry re-enters `review-round` as a NEW job key, so it is never mistaken for a redelivery.
|
|
267
|
+
// The stamp and the baseline advance are written in ONE atomic row update (see `commit` below),
|
|
268
|
+
// so this guard can never disagree with the baseline it replays against.
|
|
269
|
+
if (jobKey && row?.last_progress_job_key === jobKey && typeof row.last_progress_result === "string") {
|
|
270
|
+
// biome-ignore lint/plugin: replay a worker outcome persisted as JSON in the idempotency stamp.
|
|
271
|
+
return JSON.parse(row.last_progress_result) as Out;
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
// The SINGLE atomic terminal write for this delivery. It advances the head baseline (when a head
|
|
275
|
+
// was read), applies the resting status effect, and stamps the idempotency key + serialized
|
|
276
|
+
// outcome. Folding all three into one row update is what makes the observation and the decision
|
|
277
|
+
// atomic (Copilot #789): a redelivery either sees the whole record (→ replay above) or none of
|
|
278
|
+
// it (→ recompute from the un-advanced baseline → same decision), never a half-state. It is also
|
|
279
|
+
// the single writer of the review-wait park (#786) — persist-round no longer sets
|
|
280
|
+
// `waiting_review`, so the row rests on its running `converging` status until exactly one write
|
|
281
|
+
// here resolves it, and the poller never sees a transient `waiting_review` for a husk-retry round.
|
|
282
|
+
const commit = async (
|
|
283
|
+
out: Out,
|
|
284
|
+
opts: { head?: string | null; status?: "waiting_review" | "converging"; agentWatermark?: string | null },
|
|
285
|
+
): Promise<Out> => {
|
|
286
|
+
const ts = new Date().toISOString();
|
|
287
|
+
// PROCESS-INSTANCE FENCE (Copilot #789). A straggler progress-check from a SUPERSEDED
|
|
288
|
+
// convergence instance can slip past the replay guard (its idempotency stamp was cleared when
|
|
289
|
+
// `submitPr` re-opened the PR) and reach here AFTER `submitPr` has reset the per-run fields and
|
|
290
|
+
// started a NEW convergence instance. Its stale baseline / status / watermark / result must not
|
|
291
|
+
// clobber the fresh run (which could be parked on `waiting_review` or replay stale output).
|
|
292
|
+
// Re-read the row's CURRENT owner immediately before the write and drop the write when this
|
|
293
|
+
// job's own `processInstanceKey` no longer owns the row — the straggler's token lives in a
|
|
294
|
+
// terminated instance, so acking without persisting is correct. Fail open when either key is
|
|
295
|
+
// absent (an older instance, or a row whose `process_key` is not yet seeded) so normal
|
|
296
|
+
// single-run behaviour is untouched. Compare as strings — `process_key` is persisted via
|
|
297
|
+
// `String(processInstanceKey)`, and a job key can arrive numeric.
|
|
298
|
+
const currentOwner = (await prs.get(prKey))?.process_key ?? null;
|
|
299
|
+
const jobOwner = job.processInstanceKey ?? null;
|
|
300
|
+
if (currentOwner && jobOwner && String(currentOwner) !== String(jobOwner)) {
|
|
301
|
+
return out;
|
|
302
|
+
}
|
|
303
|
+
await prs.update(prKey, {
|
|
304
|
+
// Only overwrite the baseline when we actually read a head, so a null (unreadable) head
|
|
305
|
+
// never clobbers a good `last_round_head`.
|
|
306
|
+
...(opts.head ? { last_round_head: opts.head } : {}),
|
|
307
|
+
...(opts.status === "waiting_review" ? { status: "waiting_review", waiting_since: ts } : {}),
|
|
308
|
+
...(opts.status === "converging" ? { status: "converging" } : {}),
|
|
309
|
+
// Advance the attempt watermark ONLY when an agent-work read consumed a `review-round`
|
|
310
|
+
// instance (Copilot #789); `undefined` leaves the persisted watermark untouched (an
|
|
311
|
+
// unreadable head, a non-addressed round, or an injected bare-verdict reader), so the next
|
|
312
|
+
// round still correlates against the last real attempt.
|
|
313
|
+
...(opts.agentWatermark !== undefined ? { last_progress_agent_watermark: opts.agentWatermark } : {}),
|
|
314
|
+
...(jobKey ? { last_progress_job_key: jobKey, last_progress_result: JSON.stringify(out) } : {}),
|
|
315
|
+
updated_at: ts,
|
|
316
|
+
});
|
|
317
|
+
return out;
|
|
318
|
+
};
|
|
49
319
|
|
|
50
320
|
// Prefer the carried repo/prNumber; fall back to parsing the canonical `owner/repo#N` prKey so
|
|
51
321
|
// an older in-flight instance (or a process-variable regression) still resolves a target. If
|
|
52
|
-
// neither yields one, fail open rather than guess
|
|
322
|
+
// neither yields one, fail open rather than guess — but still park for review, since the loop
|
|
323
|
+
// proceeds to wait-review and the poller must watch this PR.
|
|
53
324
|
const parsed = parsePr(prKey);
|
|
54
325
|
const ghRepo = repo ?? parsed?.repo;
|
|
55
326
|
const ghNumber = typeof prNumber === "number" ? prNumber : parsed?.number;
|
|
56
|
-
if (!ghRepo || typeof ghNumber !== "number")
|
|
327
|
+
if (!ghRepo || typeof ghNumber !== "number") {
|
|
328
|
+
return commit({ progressed: true, huskRetries: 0 }, { status: "waiting_review" });
|
|
329
|
+
}
|
|
57
330
|
|
|
331
|
+
// Read the head and record the baseline on EVERY round — including a non-addressed `waiting`
|
|
332
|
+
// round — BEFORE the addressed-only escalation logic. The within-round baseline is now
|
|
333
|
+
// `roundEntryHead` (captured by `pr.capture-head` before `review-round`); persisting each round's
|
|
334
|
+
// exit head into `last_round_head` still matters as the FALLBACK baseline for an older in-flight
|
|
335
|
+
// instance whose fixed flow predates capture-head, or when a capture read failed open to null.
|
|
58
336
|
const currentHead = await deps.readHead(ghRepo, ghNumber).catch(() => null);
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
337
|
+
// Baseline = the head captured by `pr.capture-head` at THIS round's entry, BEFORE `review-round`
|
|
338
|
+
// ran (the `roundEntryHead` process variable). It is a within-round baseline, so it structurally
|
|
339
|
+
// closes the no-baseline gap: a real push advances `currentHead` past it (→ progress, never a
|
|
340
|
+
// false husk), and a husk that pushed nothing leaves `currentHead === roundEntryHead` (→ a
|
|
341
|
+
// genuine no-advance, split into husk vs. no-advance by the agent-instance corroboration).
|
|
342
|
+
// capture-head publishes the EMPTY string when it could not read the entry head (its "unknown"
|
|
343
|
+
// sentinel) — treat that as no round-entry baseline and fall back to the persisted prior-round
|
|
344
|
+
// head (an OLDER in-flight instance whose fixed flow lacks capture-head resolves the var to
|
|
345
|
+
// undefined and lands here too). Absent both, the no-advance path fails open (see decideProgress).
|
|
346
|
+
const previousHead =
|
|
347
|
+
typeof roundEntryHead === "string" && roundEntryHead !== ""
|
|
348
|
+
? roundEntryHead
|
|
349
|
+
: (row?.last_round_head ?? null);
|
|
350
|
+
|
|
351
|
+
// Round numbers are 1-based; coerce a missing/invalid `round` to a positive 1. The round no
|
|
352
|
+
// longer gates the agent-work corroboration (correlation is by the completing element-instance,
|
|
353
|
+
// not an aggregate round count — #786); it only tunes the human-facing escalation question.
|
|
354
|
+
const roundNo = typeof round === "number" && round > 0 ? Math.floor(round) : 1;
|
|
355
|
+
// Corroborate durable agent work on EVERY round — the addressed rounds AND the non-addressed
|
|
356
|
+
// `waiting` round, AND even a round whose head could not be read. The husk verdict itself is only
|
|
357
|
+
// consulted when the head did not advance past the round-entry baseline, but the read must ALSO
|
|
358
|
+
// run to MAINTAIN THE ATTEMPT WATERMARK (Copilot
|
|
359
|
+
// #789): every round runs `review-round` BEFORE this progress-check
|
|
360
|
+
// (`…→review-round→gw-status→…→check-progress`), registering a fresh `review-round` instance that
|
|
361
|
+
// must be CONSUMED into the watermark, else the NEXT round's pre-registration husk would see that
|
|
362
|
+
// stale terminal instance as "newer than the watermark" and mis-escalate as no-advance. This
|
|
363
|
+
// includes the `waiting` round (Copilot #789 worker.ts:329): a `waiting` round's own review runs
|
|
364
|
+
// and can leave a terminal instance, so if its progress-check returned early WITHOUT consuming it,
|
|
365
|
+
// the first addressed round's pre-registration husk would inherit that unconsumed terminal
|
|
366
|
+
// instance and bypass the bounded husk retry. It ALSO includes a round whose CURRENT HEAD could
|
|
367
|
+
// not be read (Copilot #789): the head-diff fails open regardless, but if that GitHub outage
|
|
368
|
+
// coincided with a round whose review DID register a terminal instance, skipping the agent read
|
|
369
|
+
// would leave that instance UNCONSUMED — a later pre-registration husk would then see it as newer
|
|
370
|
+
// than the stale watermark and mis-classify itself as no-advance. So read the channel here too and
|
|
371
|
+
// let a successful read advance the watermark even when the head is unreadable.
|
|
372
|
+
const readAgentWork = deps.readAgentWork ?? agentWorkFromEngine(app.engine);
|
|
373
|
+
const priorWatermark = row?.last_progress_agent_watermark ?? null;
|
|
374
|
+
const observation = normalizeAgentWork(
|
|
375
|
+
await readAgentWork(job.processInstanceKey, roundNo, priorWatermark).catch(() => null),
|
|
62
376
|
);
|
|
63
|
-
const
|
|
64
|
-
|
|
377
|
+
const agentWorkObserved = observation.work;
|
|
378
|
+
// The watermark to persist: the key this read consumed (a fresh attempt), else undefined so the
|
|
379
|
+
// stored one is left untouched (an unreadable head or an injected bare-verdict reader).
|
|
380
|
+
const agentWatermark = observation.consumedKey;
|
|
65
381
|
|
|
66
|
-
//
|
|
67
|
-
//
|
|
68
|
-
|
|
69
|
-
|
|
382
|
+
// Only an `addressed` round claims a push, so only it can be a no-progress round — and a
|
|
383
|
+
// blank/unknown status counts as `addressed` here (gw-status defaults it down the addressed arm
|
|
384
|
+
// and pr.persist-round records a missing status as `addressed`), so it is the safe-default trap
|
|
385
|
+
// this guard exists for. An explicitly recognized non-addressed status (`waiting`, etc.)
|
|
386
|
+
// legitimately has no push and always continues to the review wait — after the baseline write.
|
|
387
|
+
// Park it for review (persist-round no longer does), but still advance the attempt watermark so a
|
|
388
|
+
// following addressed husk isn't masked by this round's own review instance (Copilot #789).
|
|
389
|
+
if (!isAddressedStatus(status)) {
|
|
390
|
+
return commit(
|
|
391
|
+
{ progressed: true, huskRetries: 0 },
|
|
392
|
+
{ head: currentHead, status: "waiting_review", agentWatermark },
|
|
393
|
+
);
|
|
70
394
|
}
|
|
71
395
|
|
|
72
|
-
|
|
396
|
+
const decision = decideProgress(
|
|
397
|
+
status,
|
|
398
|
+
previousHead,
|
|
399
|
+
currentHead,
|
|
400
|
+
roundNo,
|
|
401
|
+
agentWorkObserved,
|
|
402
|
+
typeof huskRetries === "number" ? huskRetries : null,
|
|
403
|
+
);
|
|
404
|
+
|
|
405
|
+
const out: Out = {
|
|
406
|
+
progressed: decision.progressed,
|
|
407
|
+
huskRetries: decision.huskRetries,
|
|
408
|
+
...(decision.huskRetry !== undefined ? { huskRetry: decision.huskRetry } : {}),
|
|
409
|
+
...(decision.reason !== undefined ? { noProgressReason: decision.reason } : {}),
|
|
410
|
+
...(decision.question !== undefined ? { noProgressQuestion: decision.question } : {}),
|
|
411
|
+
};
|
|
412
|
+
|
|
413
|
+
// Resolve the row's resting status with the single atomic `commit`, now that the husk decision
|
|
414
|
+
// is known — closing the persist-round→progress-check race (#786).
|
|
415
|
+
if (decision.huskRetry === true) {
|
|
416
|
+
// Husk auto-retry re-enters `review-round` immediately (it does NOT wait for a new review), so
|
|
417
|
+
// keep the PR on the running `converging` aggregate: the poller must see an in-flight round,
|
|
418
|
+
// not a `waiting_review` it would solicit a spurious Copilot review for (and pollJobActivation
|
|
419
|
+
// treats only `converging` as live).
|
|
420
|
+
return commit(out, { head: currentHead, status: "converging", agentWatermark });
|
|
421
|
+
}
|
|
422
|
+
if (decision.progressed) {
|
|
423
|
+
// Genuine progress → the loop parks at wait-review. THIS is the review-wait park, written only
|
|
424
|
+
// once the husk retry has been ruled out, so a husk-retry round never transits `waiting_review`.
|
|
425
|
+
return commit(out, { head: currentHead, status: "waiting_review", agentWatermark });
|
|
426
|
+
}
|
|
427
|
+
// The remaining outcome (progressed:false, huskRetry:false) escalates to a human; the
|
|
428
|
+
// persist-escalation-noprogress worker owns the status, so leave it unset — but still advance the
|
|
429
|
+
// baseline and stamp the idempotency record so a redelivered escalation replays instead of
|
|
430
|
+
// recomputing.
|
|
431
|
+
return commit(out, { head: currentHead, agentWatermark });
|
|
73
432
|
};
|
|
74
433
|
}
|
|
75
434
|
|