@nanobpm/nano-workforce 0.193.0 → 0.193.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/SPEC.md +21 -0
- package/app/convergenceStaleRoundReset.test.ts +92 -0
- package/app/persist-escalation.test.ts +56 -15
- package/app/persistEscalationElements.test.ts +40 -0
- package/app/pollUserTasks.test.ts +572 -3
- package/app/service.ts +158 -0
- package/app/userTasks.ts +34 -0
- package/package.json +2 -2
- package/resources/processes/convergence-loop.bpmn +6 -0
- package/workers/capture-head/worker.ts +10 -0
- package/workers/persist-escalation/worker.ts +29 -14
package/CHANGELOG.md
CHANGED
|
@@ -1,3 +1,15 @@
|
|
|
1
|
+
## [0.193.2](https://github.com/nanobpm/nano-workforce/compare/v0.193.1...v0.193.2) (2026-09-29)
|
|
2
|
+
|
|
3
|
+
### Bug Fixes
|
|
4
|
+
|
|
5
|
+
* **convergence:** heal PR rows stuck at status=escalated after escalation answered ([#829](https://github.com/nanobpm/nano-workforce/issues/829)) ([235a004](https://github.com/nanobpm/nano-workforce/commit/235a004f73ec1b27f52b5dfe88e2f2a28c1d99e8)), closes [#642](https://github.com/nanobpm/nano-workforce/issues/642) [#828](https://github.com/nanobpm/nano-workforce/issues/828) [#828](https://github.com/nanobpm/nano-workforce/issues/828) [#828](https://github.com/nanobpm/nano-workforce/issues/828) [#828](https://github.com/nanobpm/nano-workforce/issues/828)
|
|
6
|
+
|
|
7
|
+
## [0.193.1](https://github.com/nanobpm/nano-workforce/compare/v0.193.0...v0.193.1) (2026-09-28)
|
|
8
|
+
|
|
9
|
+
### Bug Fixes
|
|
10
|
+
|
|
11
|
+
* **convergence:** reset round output vars on entry so a no-result round can't re-escalate ([#823](https://github.com/nanobpm/nano-workforce/issues/823)) ([3154918](https://github.com/nanobpm/nano-workforce/commit/3154918f3aeea477733347277ec067c1e0359e84)), closes [nano-coder#35](https://github.com/nanobpm/nano-coder/issues/35) [#806](https://github.com/nanobpm/nano-workforce/issues/806) [#822](https://github.com/nanobpm/nano-workforce/issues/822) [#822](https://github.com/nanobpm/nano-workforce/issues/822)
|
|
12
|
+
|
|
1
13
|
## [0.193.0](https://github.com/nanobpm/nano-workforce/compare/v0.192.0...v0.193.0) (2026-09-28)
|
|
2
14
|
|
|
3
15
|
### Features
|
package/SPEC.md
CHANGED
|
@@ -300,6 +300,27 @@ Notes:
|
|
|
300
300
|
poller does not solicit a spurious review against the still-running round. The
|
|
301
301
|
round cap and the review-wait timeout remain the outer safety nets.
|
|
302
302
|
|
|
303
|
+
- **Stale-output reset on round entry (issue #822).** `pr.capture-head` — the
|
|
304
|
+
single task every entry into `review-round` routes through first (from `Start`,
|
|
305
|
+
the review-loop re-enter, the human-answer resume, the husk auto-retry, and the
|
|
306
|
+
ack-only retry) — additionally CLEARS the round's output variables on entry:
|
|
307
|
+
`status`, `summary`, and `question` (→ `null`) and `escalated` (→ `false`),
|
|
308
|
+
mirroring the merge loop's `arm-merge` (which already clears `status` on entry
|
|
309
|
+
before its agent runs). A `review-round` job taken by a misconfigured worker can
|
|
310
|
+
exit 0 with no result variables, empty output, and no push; because nothing
|
|
311
|
+
overwrote them, the PREVIOUS round's `status`/`summary`/`question` would
|
|
312
|
+
otherwise persist and `gw-status` would re-route on that stale decision —
|
|
313
|
+
re-opening a word-for-word duplicate of an escalation a human had just answered
|
|
314
|
+
(the answer never reaching an agent). Resetting on entry makes a no-result round
|
|
315
|
+
enter on a blank `status`/`question`, so it can no longer inherit the prior
|
|
316
|
+
round's decision: it falls through `gw-status`'s `f_addressed` default into
|
|
317
|
+
`persist-round` → `check-progress`, where the unchanged head is caught by the
|
|
318
|
+
no-progress guard above and classified (husk vs. no-advance) on its own merits.
|
|
319
|
+
These vars are reset on ENTRY here rather than on `review-round`'s own output
|
|
320
|
+
because a same-task output-clear would clobber the agent's real verdict;
|
|
321
|
+
`answer`/`scopePending` (inputs the agent consumes) stay cleared on the
|
|
322
|
+
review-round output.
|
|
323
|
+
|
|
303
324
|
|
|
304
325
|
## 5. Agent job contract (`senior:pr-review`)
|
|
305
326
|
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
// Regression guard for issue #822 — the convergence loop must not REUSE a previous round's
|
|
2
|
+
// status/question/summary/escalated when an agent round returns NO result.
|
|
3
|
+
//
|
|
4
|
+
// The defect (proc 86936 / nano-coder#35): a `review-round` job taken by a misconfigured worker
|
|
5
|
+
// exited 0 in ~11s with NO result vars, no output, no commits, no push. Because nothing overwrote
|
|
6
|
+
// them, the PREVIOUS round's `status = "needs_input"`, `summary` and `question` stayed in scope.
|
|
7
|
+
// `gw-status` then routed on that stale `status`/`question` to `persist-escalation`, opening a
|
|
8
|
+
// word-for-word DUPLICATE of the escalation a human had just answered — the human's answer never
|
|
9
|
+
// reached an agent.
|
|
10
|
+
//
|
|
11
|
+
// The fix (categorical, not a special-case): clear the round's OUTPUT variables ON ENTRY to the
|
|
12
|
+
// round, on `capture-head` — the single task every entry into `review-round` routes through first
|
|
13
|
+
// (from Start, the review-loop re-enter, the human-answer resume, the husk auto-retry, and the
|
|
14
|
+
// ack-only retry). This mirrors the merge loop's `arm-merge`, which clears `status` on entry before
|
|
15
|
+
// its agent runs. With the round entering on a blank `status`/`question`, a no-result round can no
|
|
16
|
+
// longer inherit the prior round's decision: it falls through `gw-status`'s `f_addressed` default
|
|
17
|
+
// into `persist-round` → `check-progress`, where the unchanged PR head is caught as a no-progress
|
|
18
|
+
// round. `decideProgress` then classifies it on positive evidence (`app/roundProgress.ts`): only a
|
|
19
|
+
// corroborated non-terminal completing `review-round` instance is a husk (auto-retried within the
|
|
20
|
+
// budget); a terminal instance OR an absent/unknown read fails safe to `no-advance` and is escalated
|
|
21
|
+
// immediately as "produced nothing" (never re-running, to avoid duplicating agent work that may have
|
|
22
|
+
// run). Either way the round is routed as no-progress and NOT re-asked as an already-answered
|
|
23
|
+
// question.
|
|
24
|
+
//
|
|
25
|
+
// `answer`/`scopePending` (INPUTS the agent CONSUMES) are cleared on the review-round OUTPUT so the
|
|
26
|
+
// NEXT round can't re-consume them; `status`/`summary`/`question`/`escalated` (OUTPUTS the agent
|
|
27
|
+
// PRODUCES) must instead be cleared on ENTRY, because a same-task output-clear would clobber the
|
|
28
|
+
// agent's real verdict. Hence they live on `capture-head`, not on `review-round`.
|
|
29
|
+
//
|
|
30
|
+
// Pure text assertions over the committed BPMN (no engine), matching the repo's lightweight
|
|
31
|
+
// model-guard style (see convergenceEscalationGuard.test.ts, roundResultDefault.test.ts).
|
|
32
|
+
|
|
33
|
+
import { test } from "node:test";
|
|
34
|
+
import { assert, assertStringIncludes } from "#test-assert";
|
|
35
|
+
import { readFileSync } from "node:fs";
|
|
36
|
+
|
|
37
|
+
const bpmn = readFileSync("resources/processes/convergence-loop.bpmn", "utf8");
|
|
38
|
+
// Collapse whitespace so attribute-order / line-wrapping churn doesn't make the assertions brittle.
|
|
39
|
+
const flat = bpmn.replace(/\s+/g, " ");
|
|
40
|
+
|
|
41
|
+
/** The <serviceTask> element for the given id (whole element, up to its close), or null. */
|
|
42
|
+
function serviceTask(id: string): string | null {
|
|
43
|
+
const m = flat.match(new RegExp(`<bpmn:serviceTask\\b[^>]*\\bid="${id}"[\\s\\S]*?</bpmn:serviceTask>`));
|
|
44
|
+
return m ? m[0] : null;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
// The round-output variables that must be reset on entry so a no-result round can't inherit them.
|
|
48
|
+
// `escalated` is reset to `false` (a boolean flag); the rest to `null` (absent).
|
|
49
|
+
const RESET_ON_ENTRY: ReadonlyArray<{ target: string; source: string }> = [
|
|
50
|
+
{ target: "status", source: "=null" },
|
|
51
|
+
{ target: "summary", source: "=null" },
|
|
52
|
+
{ target: "question", source: "=null" },
|
|
53
|
+
{ target: "escalated", source: "=false" },
|
|
54
|
+
];
|
|
55
|
+
|
|
56
|
+
test("capture-head clears the round's output vars on entry to every review round", () => {
|
|
57
|
+
const el = serviceTask("capture-head");
|
|
58
|
+
assert(el, "capture-head service task must exist");
|
|
59
|
+
for (const { target, source } of RESET_ON_ENTRY) {
|
|
60
|
+
const re = new RegExp(`<zeebe:output\\b[^>]*\\bsource="${source}"[^>]*\\btarget="${target}"[^>]*/>`);
|
|
61
|
+
const alt = new RegExp(`<zeebe:output\\b[^>]*\\btarget="${target}"[^>]*\\bsource="${source}"[^>]*/>`);
|
|
62
|
+
assert(
|
|
63
|
+
re.test(el!) || alt.test(el!),
|
|
64
|
+
`capture-head must clear "${target}" (source ${source}) on entry so a no-result round can't reuse the prior round's ${target} (issue #822)`,
|
|
65
|
+
);
|
|
66
|
+
}
|
|
67
|
+
});
|
|
68
|
+
|
|
69
|
+
test("capture-head still precedes review-round on every loop entry (the reset is truly on entry)", () => {
|
|
70
|
+
// The reset is only "on entry to the round" if capture-head runs immediately before review-round
|
|
71
|
+
// for EVERY way the loop re-enters. capture-head's single outgoing must target review-round, and
|
|
72
|
+
// it must be the join for all the re-entry flows.
|
|
73
|
+
const el = serviceTask("capture-head");
|
|
74
|
+
assert(el, "capture-head service task must exist");
|
|
75
|
+
const incoming = [...el!.matchAll(/<bpmn:incoming>([^<]+)<\/bpmn:incoming>/g)].map((m) => m[1]);
|
|
76
|
+
for (const flow of ["f_start", "f_reviewLoop", "f_answerLoop", "f_huskRetry", "f_ackRetry"]) {
|
|
77
|
+
assert(incoming.includes(flow), `capture-head must be the entry join for ${flow} so the reset covers that re-entry (issue #822)`);
|
|
78
|
+
}
|
|
79
|
+
const capture = flat.match(/<bpmn:sequenceFlow[^>]*\bid="f_capture"[^>]*\/>/);
|
|
80
|
+
assert(capture, "f_capture flow missing");
|
|
81
|
+
assertStringIncludes(capture![0], 'sourceRef="capture-head"');
|
|
82
|
+
assertStringIncludes(capture![0], 'targetRef="review-round"');
|
|
83
|
+
});
|
|
84
|
+
|
|
85
|
+
test("gw-status still defaults a blank status to the addressed (re-enter) arm, not escalation", () => {
|
|
86
|
+
// The reset only helps if a blank status routes to the no-progress/husk path rather than
|
|
87
|
+
// escalation. This pins the interaction: with status/question cleared, gw-status falls to
|
|
88
|
+
// f_addressed (persist-round → check-progress), where an unchanged head is caught as a husk.
|
|
89
|
+
const gw = flat.match(/<bpmn:exclusiveGateway\b[^>]*\bid="gw-status"[^>]*>/);
|
|
90
|
+
assert(gw, "gw-status gateway missing");
|
|
91
|
+
assertStringIncludes(gw![0], 'default="f_addressed"');
|
|
92
|
+
});
|
|
@@ -8,29 +8,56 @@
|
|
|
8
8
|
// guard moved downstream of progress classification) and likewise passes `recordRound=false`; an
|
|
9
9
|
// agent-raised arm with no prior round row omits the flag and must still record the round.
|
|
10
10
|
import { test } from "node:test";
|
|
11
|
-
import { assertEquals } from "#test-assert";
|
|
11
|
+
import { assertEquals, assertRejects } from "#test-assert";
|
|
12
12
|
import handler from "../workers/persist-escalation/worker.ts";
|
|
13
13
|
|
|
14
|
-
|
|
14
|
+
// `failOn` forces the named table's `update` to throw, so a test can assert the escalation-insert +
|
|
15
|
+
// PR-stamp transaction rolls back as a unit (issue #829 — the insert-before-timestamp window).
|
|
16
|
+
function fakeApp(opts: { failOn?: string } = {}) {
|
|
15
17
|
const inserts: Record<string, unknown[]> = { rounds: [], escalations: [] };
|
|
16
18
|
const updates: Record<string, unknown[]> = { pull_requests: [] };
|
|
17
19
|
const rows: Record<string, Map<string, unknown>> = {};
|
|
20
|
+
// `apply` decides WHEN a staged write lands: outside a transaction it applies immediately; inside
|
|
21
|
+
// `open().tx()` it is buffered and flushed only on a clean return, so a throw discards it (rollback).
|
|
22
|
+
const makeTable = (name: string, apply: (fn: () => void) => void) => {
|
|
23
|
+
const store = (rows[name] ??= new Map());
|
|
24
|
+
return {
|
|
25
|
+
async get(key: string) {
|
|
26
|
+
return store.get(key);
|
|
27
|
+
},
|
|
28
|
+
async insert(row: unknown) {
|
|
29
|
+
const pk = name === "escalations" ? "id" : name === "rounds" ? "id" : "pr_key";
|
|
30
|
+
const id = name === "escalations" ? 42 : name === "rounds" ? 1 : (row as any).pr_key;
|
|
31
|
+
const stored = pk === "id" ? { id, ...(row as Record<string, unknown>) } : row;
|
|
32
|
+
apply(() => {
|
|
33
|
+
(inserts[name] ??= []).push(row);
|
|
34
|
+
store.set((stored as any)[pk], stored);
|
|
35
|
+
});
|
|
36
|
+
return id;
|
|
37
|
+
},
|
|
38
|
+
async update(key: string, patch: unknown) {
|
|
39
|
+
if (opts.failOn === name) throw new Error(`simulated ${name} update failure`);
|
|
40
|
+
apply(() => {
|
|
41
|
+
(updates[name] ??= []).push({ key, patch });
|
|
42
|
+
});
|
|
43
|
+
},
|
|
44
|
+
};
|
|
45
|
+
};
|
|
46
|
+
const applyNow = (fn: () => void) => fn();
|
|
18
47
|
const app = {
|
|
19
48
|
data: {
|
|
20
|
-
table(name: string, _key
|
|
21
|
-
|
|
49
|
+
table(name: string, _key?: string) {
|
|
50
|
+
return makeTable(name, applyNow);
|
|
51
|
+
},
|
|
52
|
+
open() {
|
|
22
53
|
return {
|
|
23
|
-
async
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
const
|
|
29
|
-
|
|
30
|
-
return name === "escalations" ? 42 : 1;
|
|
31
|
-
},
|
|
32
|
-
async update(key: string, patch: unknown) {
|
|
33
|
-
(updates[name] ??= []).push({ key, patch });
|
|
54
|
+
async tx<T>(fn: (t: unknown) => Promise<T>): Promise<T> {
|
|
55
|
+
const staged: Array<() => void> = [];
|
|
56
|
+
const stage = (fn2: () => void) => staged.push(fn2);
|
|
57
|
+
const handle = { table: (name: string, _key?: string) => makeTable(name, stage) };
|
|
58
|
+
const result = await fn(handle); // a throw here skips the flush below — the staged writes roll back
|
|
59
|
+
for (const s of staged) s();
|
|
60
|
+
return result;
|
|
34
61
|
},
|
|
35
62
|
};
|
|
36
63
|
},
|
|
@@ -50,6 +77,20 @@ test("stalled arm (recordRound=false) does not insert a duplicate rounds row", a
|
|
|
50
77
|
assertEquals((out as any).escalationId, 42);
|
|
51
78
|
});
|
|
52
79
|
|
|
80
|
+
// Red/green regression for the insert-before-timestamp race (issue #829, Copilot review). The
|
|
81
|
+
// escalated-PR self-heal snapshots the PR generation + open-escalation ids and fences its repair on
|
|
82
|
+
// them. If this worker inserted the fresh `open` escalation BEFORE stamping the PR's `updated_at`
|
|
83
|
+
// (two separate writes), a heal snapshot landing between them would see the new escalation under the
|
|
84
|
+
// OLD generation and could retire it, orphaning a live user task. The insert and the PR stamp are now
|
|
85
|
+
// one transaction, so they are all-or-nothing: a failed PR stamp must roll the escalation insert back
|
|
86
|
+
// — proving no `open` row is ever visible while the PR still carries the pre-escalation generation.
|
|
87
|
+
test("persist-escalation opens the escalation and stamps the PR atomically — a failed PR stamp rolls back the escalation insert (issue #829)", async () => {
|
|
88
|
+
const { app, inserts } = fakeApp({ failOn: "pull_requests" });
|
|
89
|
+
const job = { variables: { prKey: "o/r#1", round: 3, status: "blocked", question: "needs input" } };
|
|
90
|
+
await assertRejects(() => handler(job as any, app as any));
|
|
91
|
+
assertEquals(inserts.escalations.length, 0, "the escalation insert rolls back with the failed PR stamp — no orphan open row under the old generation");
|
|
92
|
+
});
|
|
93
|
+
|
|
53
94
|
test("an agent-raised arm without the flag still records the round", async () => {
|
|
54
95
|
const { app, inserts } = fakeApp();
|
|
55
96
|
const job = { variables: { prKey: "o/r#1", round: 3, status: "blocked", question: "needs input" } };
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
// Drift guard for `PR_ESCALATION_PRODUCER_ELEMENTS` (issue #829).
|
|
2
|
+
//
|
|
3
|
+
// The escalated-PR self-heal (`pollUserTasks` in app/service.ts) treats an ACTIVE `pr.persist-escalation`
|
|
4
|
+
// PRODUCER element as positive evidence an escalation is being raised, so it must NOT retire the live
|
|
5
|
+
// escalation. That guard is only as correct as the element-id set it checks against — a NEW escalation
|
|
6
|
+
// arm added to either loop's BPMN, or a renamed one, would silently fall outside the set and re-open the
|
|
7
|
+
// race the fix closes. This test derives the truth from the deployed BPMN: every service task whose
|
|
8
|
+
// `zeebe:taskDefinition type="pr.persist-escalation"` in convergence-loop.bpmn / merge-loop.bpmn must be
|
|
9
|
+
// listed in the constant, and the constant must list nothing that isn't such a task. It fails loudly on
|
|
10
|
+
// either drift so the set stays the single source of truth.
|
|
11
|
+
import { readFileSync } from "node:fs";
|
|
12
|
+
import { test } from "node:test";
|
|
13
|
+
import { assertEquals } from "#test-assert";
|
|
14
|
+
import { PR_ESCALATION_PRODUCER_ELEMENTS } from "./userTasks.ts";
|
|
15
|
+
|
|
16
|
+
/** Extract the `id`s of every `<bpmn:serviceTask>` whose `<zeebe:taskDefinition type="pr.persist-escalation"/>`
|
|
17
|
+
* from one BPMN file. Matches each service task element and keeps the ones carrying the producer job type. */
|
|
18
|
+
function persistEscalationTaskIds(bpmnPath: string): string[] {
|
|
19
|
+
const xml = readFileSync(bpmnPath, "utf8");
|
|
20
|
+
const ids: string[] = [];
|
|
21
|
+
const taskRe = /<bpmn:serviceTask\b[^>]*\bid="([^"]+)"[\s\S]*?<\/bpmn:serviceTask>/g;
|
|
22
|
+
for (let m = taskRe.exec(xml); m !== null; m = taskRe.exec(xml)) {
|
|
23
|
+
if (/<zeebe:taskDefinition\b[^>]*\btype="pr\.persist-escalation"/.test(m[0])) ids.push(m[1]);
|
|
24
|
+
}
|
|
25
|
+
return ids;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
test("PR_ESCALATION_PRODUCER_ELEMENTS matches every pr.persist-escalation task in both PR loops (issue #829)", () => {
|
|
29
|
+
const modelled = [
|
|
30
|
+
...persistEscalationTaskIds("resources/processes/convergence-loop.bpmn"),
|
|
31
|
+
...persistEscalationTaskIds("resources/processes/merge-loop.bpmn"),
|
|
32
|
+
].sort();
|
|
33
|
+
// Sanity: the BPMN really does carry producer tasks — a zero here would make the match vacuously pass.
|
|
34
|
+
assertEquals(modelled.length > 0, true, "expected at least one pr.persist-escalation task in the BPMN");
|
|
35
|
+
assertEquals(
|
|
36
|
+
[...PR_ESCALATION_PRODUCER_ELEMENTS].sort(),
|
|
37
|
+
modelled,
|
|
38
|
+
"PR_ESCALATION_PRODUCER_ELEMENTS drifted from the pr.persist-escalation tasks modelled in the BPMN — update the constant in app/userTasks.ts",
|
|
39
|
+
);
|
|
40
|
+
});
|
|
@@ -15,6 +15,34 @@ function memData(seed: Record<string, any[]> = {}): { data: DataLayer; stores: R
|
|
|
15
15
|
const stores: Record<string, any[]> = {};
|
|
16
16
|
for (const [k, v] of Object.entries(seed)) stores[k] = v.map((r) => ({ ...r }));
|
|
17
17
|
function tbl(name: string, pk = "id") {
|
|
18
|
+
// The ADR-0065 derived tracking VIEW (`<base>__tracking`) is modelled as a read-only projection
|
|
19
|
+
// over its base store, augmenting each row with `derived_status` (Copilot review of #829 — the PR
|
|
20
|
+
// self-heal now reads candidates through `prsTracking` and filters on `derived_status="escalated"`).
|
|
21
|
+
// Default `derived_status` to the base `status`, so a fixture that does NOT model out-of-band
|
|
22
|
+
// termination reads live (base status passes through). A test that wants a row the reconciler has
|
|
23
|
+
// already folded terminal seeds an explicit `derived_status` (e.g. `"abandoned"`) on the base row.
|
|
24
|
+
const trackingMatch = /^(.*)__tracking$/.exec(name);
|
|
25
|
+
if (trackingMatch) {
|
|
26
|
+
const base = (stores[trackingMatch[1]] ??= [] as any[]);
|
|
27
|
+
// biome-ignore lint/suspicious/noExplicitAny: see above
|
|
28
|
+
const project = (r: any) => ({ ...r, derived_status: r.derived_status ?? r.status });
|
|
29
|
+
// biome-ignore lint/suspicious/noExplicitAny: see above
|
|
30
|
+
const match = (r: any, where: any) => Object.entries(where).every(([k, v]) => r[k] === v);
|
|
31
|
+
return {
|
|
32
|
+
async all() {
|
|
33
|
+
return base.map(project);
|
|
34
|
+
},
|
|
35
|
+
// biome-ignore lint/suspicious/noExplicitAny: see above
|
|
36
|
+
async get(id: any) {
|
|
37
|
+
const r = base.find((row) => row[pk] === id);
|
|
38
|
+
return r ? project(r) : undefined;
|
|
39
|
+
},
|
|
40
|
+
// biome-ignore lint/suspicious/noExplicitAny: see above
|
|
41
|
+
async find(where: any = {}) {
|
|
42
|
+
return base.map(project).filter((r) => match(r, where));
|
|
43
|
+
},
|
|
44
|
+
};
|
|
45
|
+
}
|
|
18
46
|
// biome-ignore lint/suspicious/noExplicitAny: see above
|
|
19
47
|
const rows = (stores[name] ??= [] as any[]);
|
|
20
48
|
// biome-ignore lint/suspicious/noExplicitAny: see above
|
|
@@ -48,26 +76,107 @@ function memData(seed: Record<string, any[]> = {}): { data: DataLayer; stores: R
|
|
|
48
76
|
},
|
|
49
77
|
};
|
|
50
78
|
}
|
|
51
|
-
const data =
|
|
79
|
+
const data = openableData(stores, tbl);
|
|
52
80
|
return { data, stores };
|
|
53
81
|
}
|
|
54
82
|
|
|
83
|
+
/** Wrap the in-memory `stores` as a `DataLayer` that ALSO exposes `open()` → a `DataSource` with the
|
|
84
|
+
* `exec` (guarded UPDATE) + `tx` (snapshot/rollback) surface the escalated-PR self-heal now uses for its
|
|
85
|
+
* conditional, atomic repair (Copilot review of #829). `exec` interprets ONLY the two guarded statements
|
|
86
|
+
* the heal issues — the snapshot-fenced `pull_requests` CAS and the by-id `escalations` retirement —
|
|
87
|
+
* mutating the same store objects `table()` reads, so both surfaces stay consistent within a test. The
|
|
88
|
+
* SQL shape itself is exercised against real SQLite by the app's integration/e2e suites; here the fake
|
|
89
|
+
* need only honour the guards so a race assertion can observe the CAS refusing a moved snapshot. */
|
|
90
|
+
// biome-ignore lint/suspicious/noExplicitAny: in-memory rows are untyped fixtures
|
|
91
|
+
function openableData(stores: Record<string, any[]>, tbl: (name: string, pk?: string) => any): DataLayer {
|
|
92
|
+
const norm = (v: unknown) => v ?? null;
|
|
93
|
+
const exec = async (sql: string, params: unknown[] = []) => {
|
|
94
|
+
const flip =
|
|
95
|
+
/UPDATE "pull_requests" SET "status" = 'converging', "updated_at" = \? WHERE "pr_key" = \? AND "status" = 'escalated' AND "process_key" IS \? AND "updated_at" IS \?/.exec(
|
|
96
|
+
sql,
|
|
97
|
+
);
|
|
98
|
+
if (flip) {
|
|
99
|
+
const [at, prKey, processKey, updatedAt] = params;
|
|
100
|
+
const row = (stores.pull_requests ?? []).find((r) => r.pr_key === prKey);
|
|
101
|
+
if (row && row.status === "escalated" && norm(row.process_key) === norm(processKey) && norm(row.updated_at) === norm(updatedAt)) {
|
|
102
|
+
row.status = "converging";
|
|
103
|
+
row.updated_at = at;
|
|
104
|
+
return { changed: 1 };
|
|
105
|
+
}
|
|
106
|
+
return { changed: 0 };
|
|
107
|
+
}
|
|
108
|
+
const retire = /UPDATE "escalations" SET "status" = 'stale' WHERE "id" = \? AND "status" = 'open'/.exec(sql);
|
|
109
|
+
if (retire) {
|
|
110
|
+
const row = (stores.escalations ?? []).find((r) => r.id === params[0]);
|
|
111
|
+
if (row && row.status === "open") {
|
|
112
|
+
row.status = "stale";
|
|
113
|
+
return { changed: 1 };
|
|
114
|
+
}
|
|
115
|
+
return { changed: 0 };
|
|
116
|
+
}
|
|
117
|
+
throw new Error(`unexpected exec sql: ${sql}`);
|
|
118
|
+
};
|
|
119
|
+
const source: any = {
|
|
120
|
+
exec,
|
|
121
|
+
table: (n: string, pk?: string) => tbl(n, pk),
|
|
122
|
+
tx: async (fn: (t: any) => Promise<unknown>) => {
|
|
123
|
+
const snap = JSON.parse(JSON.stringify(stores));
|
|
124
|
+
try {
|
|
125
|
+
return await fn(source);
|
|
126
|
+
} catch (e) {
|
|
127
|
+
for (const [n, rows] of Object.entries(stores)) {
|
|
128
|
+
rows.length = 0;
|
|
129
|
+
rows.push(...(snap[n] ?? []));
|
|
130
|
+
}
|
|
131
|
+
throw e;
|
|
132
|
+
}
|
|
133
|
+
},
|
|
134
|
+
};
|
|
135
|
+
return { table: (n: string, pk?: string) => tbl(n, pk), open: () => source } as unknown as DataLayer;
|
|
136
|
+
}
|
|
137
|
+
|
|
55
138
|
/** A single engine-reported user task in the fixture. `state` mirrors the engine lifecycle; it
|
|
56
139
|
* defaults to `"CREATED"` (the only open/answerable state) so existing fixtures read as live tasks.
|
|
57
140
|
* A looping instance holds multiple tasks for one element (COMPLETED from prior rounds + the live one). */
|
|
58
141
|
type FakeTask = { userTaskKey: string; elementId?: string; state?: "CREATED" | "COMPLETED" | "CANCELED"; formKey?: string };
|
|
59
142
|
|
|
143
|
+
/** A single engine-reported element instance in the fixture — the answer-recording service task the
|
|
144
|
+
* escalation self-heal probes for (issue #829). `state` mirrors the engine lifecycle; `"ACTIVE"` is the
|
|
145
|
+
* in-flight marker that an answer is being recorded. */
|
|
146
|
+
type FakeElement = { elementId: string; state: string };
|
|
147
|
+
|
|
60
148
|
/** A fake engine whose user tasks are keyed by processInstanceKey (the only field the poller queries on
|
|
61
149
|
* for plan / PR instances). It models the real engine's two accessors from ONE fixture so a test
|
|
62
150
|
* genuinely exercises the lifecycle-state filtering: `searchUserTasks` returns tasks in ANY state
|
|
63
|
-
* (COMPLETED first, as the live API does — issue #294), while `openUserTasks` pins `state:"CREATED"`.
|
|
64
|
-
|
|
151
|
+
* (COMPLETED first, as the live API does — issue #294), while `openUserTasks` pins `state:"CREATED"`.
|
|
152
|
+
* `searchElementInstances` is fed from an OPTIONAL second fixture (default empty) so the PR escalation
|
|
153
|
+
* self-heal can see an ACTIVE answer-recording task (issue #829). `searchProcessInstances` is fed from an
|
|
154
|
+
* OPTIONAL third fixture keyed by processInstanceKey: any key NOT listed defaults to reporting the
|
|
155
|
+
* instance `ACTIVE` (so existing heal fixtures, which model a genuinely-resumed live loop, stay green); a
|
|
156
|
+
* value of `null` models an instance ABSENT from the read model; a string overrides its lifecycle state
|
|
157
|
+
* (e.g. `"TERMINATED"`) — the positive-ACTIVE liveness gate the self-heal requires (issue #829). */
|
|
158
|
+
function fakeEngine(
|
|
159
|
+
byInstance: Record<string, FakeTask[]>,
|
|
160
|
+
elementsByInstance: Record<string, FakeElement[]> = {},
|
|
161
|
+
instanceStateByInstance: Record<string, string | null> = {},
|
|
162
|
+
): EngineClient {
|
|
65
163
|
const all = (filter?: { processInstanceKey?: string }) =>
|
|
66
164
|
filter?.processInstanceKey ? (byInstance[filter.processInstanceKey] ?? []) : [];
|
|
67
165
|
return {
|
|
68
166
|
searchUserTasks: (filter?: { processInstanceKey?: string }) => Promise.resolve(all(filter)),
|
|
69
167
|
openUserTasks: (filter?: { processInstanceKey?: string }) =>
|
|
70
168
|
Promise.resolve(all(filter).filter((t) => (t.state ?? "CREATED") === "CREATED")),
|
|
169
|
+
searchElementInstances: (filter?: { processInstanceKey?: string }) =>
|
|
170
|
+
Promise.resolve(filter?.processInstanceKey ? (elementsByInstance[filter.processInstanceKey] ?? []) : []),
|
|
171
|
+
searchProcessInstances: (filter?: { processInstanceKeys?: string[] }) =>
|
|
172
|
+
Promise.resolve(
|
|
173
|
+
(filter?.processInstanceKeys ?? [])
|
|
174
|
+
.map((k) => {
|
|
175
|
+
const state = k in instanceStateByInstance ? instanceStateByInstance[k] : "ACTIVE";
|
|
176
|
+
return state === null ? null : { processInstanceKey: k, state };
|
|
177
|
+
})
|
|
178
|
+
.filter((it): it is { processInstanceKey: string; state: string } => it != null),
|
|
179
|
+
),
|
|
71
180
|
} as unknown as EngineClient;
|
|
72
181
|
}
|
|
73
182
|
|
|
@@ -1072,3 +1181,463 @@ test("pollUserTasks (typed-seam fallback): self-heals an escalated run with no o
|
|
|
1072
1181
|
const byKey = Object.fromEntries((stores.feature_runs ?? []).map((r) => [r.feature_key, r]));
|
|
1073
1182
|
assertEquals(byKey["o/r#632"].status, "running", "the stranded escalated run is healed to running");
|
|
1074
1183
|
});
|
|
1184
|
+
|
|
1185
|
+
test("pollUserTasks (engine-first): self-heals an escalated PR stranded off its parked task, sparing a genuinely parked one (issue #828)", async () => {
|
|
1186
|
+
// The convergence-path twin of the #642 feature heal. A PR's `status="escalated"` must hold ONLY while
|
|
1187
|
+
// a `wait-answer` / `wait-merge-answer` task is open. A PR whose instance the engine no longer reports
|
|
1188
|
+
// parked (its escalation was answered, but the write-side `answer-escalation` flip was lost around an
|
|
1189
|
+
// app restart — the #828 tear) is reconciled to `converging`; a PR whose task IS still open is left
|
|
1190
|
+
// escalated. The engine's open set is the authority, not the raw `pull_requests.status` column. The
|
|
1191
|
+
// orphaned `open` escalation rows are retired to `stale` so the audit trail matches.
|
|
1192
|
+
const { data, stores } = memData({
|
|
1193
|
+
pull_requests: [
|
|
1194
|
+
{ pr_key: "o/r#828", status: "escalated", process_key: "rp-828", url: "https://github.com/o/r/pull/828", title: "stranded" },
|
|
1195
|
+
{ pr_key: "o/r#77", status: "escalated", process_key: "rp-77", url: "https://github.com/o/r/pull/77", title: "still parked" },
|
|
1196
|
+
],
|
|
1197
|
+
escalations: [
|
|
1198
|
+
{ id: 1, pr_key: "o/r#828", status: "open", question: "orphaned — never answered", answer: null, answered_at: null },
|
|
1199
|
+
{ id: 2, pr_key: "o/r#77", status: "open", question: "genuinely open", answer: null, answered_at: null },
|
|
1200
|
+
],
|
|
1201
|
+
});
|
|
1202
|
+
const restore = stubUserTaskSearch([
|
|
1203
|
+
// Only rp-77 is genuinely parked; the engine reports NO open task on rp-828.
|
|
1204
|
+
{ userTaskKey: "ut-parked", elementId: "wait-answer", processInstanceKey: "rp-77", state: "CREATED" },
|
|
1205
|
+
]);
|
|
1206
|
+
try {
|
|
1207
|
+
await pollUserTasks(data, fakeEngine({ "rp-77": [{ userTaskKey: "ut-parked", elementId: "wait-answer" }] }), REST);
|
|
1208
|
+
} finally {
|
|
1209
|
+
restore();
|
|
1210
|
+
}
|
|
1211
|
+
|
|
1212
|
+
const byKey = Object.fromEntries((stores.pull_requests ?? []).map((r) => [r.pr_key, r]));
|
|
1213
|
+
assertEquals(byKey["o/r#828"].status, "converging", "the stranded escalated PR is healed to converging");
|
|
1214
|
+
assertEquals(byKey["o/r#77"].status, "escalated", "the genuinely parked PR stays escalated");
|
|
1215
|
+
const escById = Object.fromEntries((stores.escalations ?? []).map((r) => [r.id, r]));
|
|
1216
|
+
assertEquals(escById[1].status, "stale", "the healed PR's orphaned open escalation row is retired to stale");
|
|
1217
|
+
assertEquals(escById[2].status, "open", "the genuinely parked PR's escalation row stays open");
|
|
1218
|
+
});
|
|
1219
|
+
|
|
1220
|
+
test("pollUserTasks (engine-first): self-heals an escalated MERGE-loop PR off its parked wait-merge-answer task (issue #828)", async () => {
|
|
1221
|
+
// Both loops write `pr.persist-escalation` (`status="escalated"`) and are answered by the same
|
|
1222
|
+
// `answer-escalation` flip, so the heal must confirm against BOTH PR escalation elements — a merge
|
|
1223
|
+
// escalation parks on `wait-merge-answer`.
|
|
1224
|
+
const { data, stores } = memData({
|
|
1225
|
+
pull_requests: [
|
|
1226
|
+
{ pr_key: "o/r#828m", status: "escalated", process_key: "mp-828", url: "https://github.com/o/r/pull/828", title: "stranded merge" },
|
|
1227
|
+
],
|
|
1228
|
+
escalations: [{ id: 1, pr_key: "o/r#828m", status: "open", question: "not mergeable", answer: null, answered_at: null }],
|
|
1229
|
+
});
|
|
1230
|
+
const restore = stubUserTaskSearch([]); // engine reports NO open task on mp-828
|
|
1231
|
+
const engine = fakeEngine({ "mp-828": [] });
|
|
1232
|
+
try {
|
|
1233
|
+
await pollUserTasks(data, engine, REST);
|
|
1234
|
+
} finally {
|
|
1235
|
+
restore();
|
|
1236
|
+
}
|
|
1237
|
+
const byKey = Object.fromEntries((stores.pull_requests ?? []).map((r) => [r.pr_key, r]));
|
|
1238
|
+
assertEquals(byKey["o/r#828m"].status, "converging", "the stranded escalated merge-loop PR is healed to converging");
|
|
1239
|
+
});
|
|
1240
|
+
|
|
1241
|
+
test("pollUserTasks (engine-first): does NOT heal an escalated PR while its answer-recording job is in flight (issue #829)", async () => {
|
|
1242
|
+
// The dangerous race: both BPMN models flow the completed `wait-answer` / `wait-merge-answer` user task
|
|
1243
|
+
// DIRECTLY to the `record-answer` / `record-merge-answer` service task (`pr.answer-escalation`). Between
|
|
1244
|
+
// the operator completing the task and that job running, the user task is already gone (openUserTasks
|
|
1245
|
+
// returns none) yet the answer is being recorded RIGHT NOW. The escalation can be arbitrarily old (past
|
|
1246
|
+
// the raise-time grace), so the grace window does not cover this. Healing here would flip the row to
|
|
1247
|
+
// `converging` and retire the still-`open` escalation to `stale`, and the in-flight worker would then
|
|
1248
|
+
// find no target and silently drop the operator's answer. An ACTIVE answer-recording element instance is
|
|
1249
|
+
// positive evidence the answer is in flight, so the heal must skip it — exactly like an open task.
|
|
1250
|
+
const stale = new Date(Date.now() - 60 * 60_000).toISOString();
|
|
1251
|
+
const { data, stores } = memData({
|
|
1252
|
+
pull_requests: [
|
|
1253
|
+
{ pr_key: "o/r#829r", status: "escalated", process_key: "rp-829r", updated_at: stale, url: "https://github.com/o/r/pull/829", title: "answer in flight (review)" },
|
|
1254
|
+
{ pr_key: "o/r#829m", status: "escalated", process_key: "rp-829m", updated_at: stale, url: "https://github.com/o/r/pull/8290", title: "answer in flight (merge)" },
|
|
1255
|
+
],
|
|
1256
|
+
escalations: [
|
|
1257
|
+
{ id: 1, pr_key: "o/r#829r", status: "open", question: "answered, recording", answer: "do it", answered_at: stale },
|
|
1258
|
+
{ id: 2, pr_key: "o/r#829m", status: "open", question: "answered, recording", answer: "rebase", answered_at: stale },
|
|
1259
|
+
],
|
|
1260
|
+
});
|
|
1261
|
+
const restore = stubUserTaskSearch([]); // the user task has already completed on both instances
|
|
1262
|
+
// Each instance's answer-recording service task is ACTIVE — the operator's answer is being written.
|
|
1263
|
+
const engine = fakeEngine(
|
|
1264
|
+
{ "rp-829r": [], "rp-829m": [] },
|
|
1265
|
+
{
|
|
1266
|
+
"rp-829r": [{ elementId: "record-answer", state: "ACTIVE" }],
|
|
1267
|
+
"rp-829m": [{ elementId: "record-merge-answer", state: "ACTIVE" }],
|
|
1268
|
+
},
|
|
1269
|
+
);
|
|
1270
|
+
try {
|
|
1271
|
+
await pollUserTasks(data, engine, REST);
|
|
1272
|
+
} finally {
|
|
1273
|
+
restore();
|
|
1274
|
+
}
|
|
1275
|
+
const byKey = Object.fromEntries((stores.pull_requests ?? []).map((r) => [r.pr_key, r]));
|
|
1276
|
+
assertEquals(byKey["o/r#829r"].status, "escalated", "a PR with an in-flight review answer-recorder is spared the heal");
|
|
1277
|
+
assertEquals(byKey["o/r#829m"].status, "escalated", "a PR with an in-flight merge answer-recorder is spared the heal");
|
|
1278
|
+
const escById = Object.fromEntries((stores.escalations ?? []).map((r) => [r.id, r]));
|
|
1279
|
+
assertEquals(escById[1].status, "open", "the in-flight review escalation row is left for the worker to answer");
|
|
1280
|
+
assertEquals(escById[2].status, "open", "the in-flight merge escalation row is left for the worker to answer");
|
|
1281
|
+
});
|
|
1282
|
+
|
|
1283
|
+
test("pollUserTasks (engine-first): DOES heal an escalated PR once its answer-recorder has completed (issue #829)", async () => {
|
|
1284
|
+
// The genuine #828 lost-write tear: the answer-recording job COMPLETED (its element instance is no
|
|
1285
|
+
// longer ACTIVE) but the durable `status="converging"` flip was lost around an app restart. With no open
|
|
1286
|
+
// task AND no ACTIVE answer-recorder, the PR is genuinely stranded and must heal — the in-flight guard
|
|
1287
|
+
// must not over-fire and wedge it permanently.
|
|
1288
|
+
const stale = new Date(Date.now() - 60 * 60_000).toISOString();
|
|
1289
|
+
const { data, stores } = memData({
|
|
1290
|
+
pull_requests: [{ pr_key: "o/r#829c", status: "escalated", process_key: "rp-829c", updated_at: stale, url: "https://github.com/o/r/pull/8291", title: "recorder completed, write lost" }],
|
|
1291
|
+
escalations: [{ id: 1, pr_key: "o/r#829c", status: "open", question: "orphaned", answer: null, answered_at: null }],
|
|
1292
|
+
});
|
|
1293
|
+
const restore = stubUserTaskSearch([]);
|
|
1294
|
+
const engine = fakeEngine({ "rp-829c": [] }, { "rp-829c": [{ elementId: "record-answer", state: "COMPLETED" }] });
|
|
1295
|
+
try {
|
|
1296
|
+
await pollUserTasks(data, engine, REST);
|
|
1297
|
+
} finally {
|
|
1298
|
+
restore();
|
|
1299
|
+
}
|
|
1300
|
+
const byKey = Object.fromEntries((stores.pull_requests ?? []).map((r) => [r.pr_key, r]));
|
|
1301
|
+
assertEquals(byKey["o/r#829c"].status, "converging", "a stranded PR whose answer-recorder already completed is healed");
|
|
1302
|
+
const escById = Object.fromEntries((stores.escalations ?? []).map((r) => [r.id, r]));
|
|
1303
|
+
assertEquals(escById[1].status, "stale", "the orphaned open escalation row is retired to stale");
|
|
1304
|
+
});
|
|
1305
|
+
|
|
1306
|
+
test("pollUserTasks (engine-first): does NOT heal an escalated PR whose process instance is TERMINAL (issue #829)", async () => {
|
|
1307
|
+
// Copilot review of the #828 heal: "no open task and no ACTIVE escalation element" is ALSO true for a
|
|
1308
|
+
// TERMINAL instance (cancelled / completed / failed), whose loop never resumed. Flipping such a frozen
|
|
1309
|
+
// `escalated` row to `converging` and retiring its audit rows fabricates a live loop the engine will
|
|
1310
|
+
// never advance; terminal PRs are owned by tracking/reconciliation (they read `abandoned`/settled on
|
|
1311
|
+
// `derived_status`). The heal now requires POSITIVE engine-truth that the instance is `ACTIVE`, so a
|
|
1312
|
+
// TERMINATED (and, below, an absent) instance is spared. One case per known terminal state.
|
|
1313
|
+
const stale = new Date(Date.now() - 60 * 60_000).toISOString();
|
|
1314
|
+
const { data, stores } = memData({
|
|
1315
|
+
pull_requests: [
|
|
1316
|
+
{ pr_key: "o/r#829tt", status: "escalated", process_key: "rp-829tt", updated_at: stale, url: "https://github.com/o/r/pull/8297", title: "terminated instance" },
|
|
1317
|
+
{ pr_key: "o/r#829tc", status: "escalated", process_key: "rp-829tc", updated_at: stale, url: "https://github.com/o/r/pull/8298", title: "cancelled instance" },
|
|
1318
|
+
],
|
|
1319
|
+
escalations: [
|
|
1320
|
+
{ id: 1, pr_key: "o/r#829tt", status: "open", question: "loop gone", answer: null, answered_at: null },
|
|
1321
|
+
{ id: 2, pr_key: "o/r#829tc", status: "open", question: "loop gone", answer: null, answered_at: null },
|
|
1322
|
+
],
|
|
1323
|
+
});
|
|
1324
|
+
const restore = stubUserTaskSearch([]); // no open task — but the loop is dead, not merely mid-transition
|
|
1325
|
+
// No open task, no ACTIVE escalation element, but the instance itself is a known terminal state.
|
|
1326
|
+
const engine = fakeEngine(
|
|
1327
|
+
{ "rp-829tt": [], "rp-829tc": [] },
|
|
1328
|
+
{},
|
|
1329
|
+
{ "rp-829tt": "TERMINATED", "rp-829tc": "CANCELED" },
|
|
1330
|
+
);
|
|
1331
|
+
try {
|
|
1332
|
+
await pollUserTasks(data, engine, REST);
|
|
1333
|
+
} finally {
|
|
1334
|
+
restore();
|
|
1335
|
+
}
|
|
1336
|
+
const byKey = Object.fromEntries((stores.pull_requests ?? []).map((r) => [r.pr_key, r]));
|
|
1337
|
+
assertEquals(byKey["o/r#829tt"].status, "escalated", "a TERMINATED instance is left to tracking/reconciliation, not healed");
|
|
1338
|
+
assertEquals(byKey["o/r#829tc"].status, "escalated", "a CANCELED instance is left to tracking/reconciliation, not healed");
|
|
1339
|
+
const escById = Object.fromEntries((stores.escalations ?? []).map((r) => [r.id, r]));
|
|
1340
|
+
assertEquals(escById[1].status, "open", "the terminal instance's escalation row is not retired");
|
|
1341
|
+
assertEquals(escById[2].status, "open", "the cancelled instance's escalation row is not retired");
|
|
1342
|
+
});
|
|
1343
|
+
|
|
1344
|
+
test("pollUserTasks (engine-first): does NOT heal an escalated PR whose process instance is ABSENT / unknown-state (issue #829)", async () => {
|
|
1345
|
+
// The other halves of the tri-state (Copilot review of #828): an instance genuinely ABSENT from the
|
|
1346
|
+
// read model (engine answered, no match) is treated like a terminal — its loop is gone, so leave it to
|
|
1347
|
+
// tracking rather than fabricate a resumed loop. An instance present with an UNKNOWN/empty `state` is a
|
|
1348
|
+
// partial read this app cannot interpret and is SPARED (never healed off a wire shape we misread). Both
|
|
1349
|
+
// stay `escalated`. One case each.
|
|
1350
|
+
const stale = new Date(Date.now() - 60 * 60_000).toISOString();
|
|
1351
|
+
const { data, stores } = memData({
|
|
1352
|
+
pull_requests: [
|
|
1353
|
+
{ pr_key: "o/r#829ab", status: "escalated", process_key: "rp-829ab", updated_at: stale, url: "https://github.com/o/r/pull/8299", title: "absent instance" },
|
|
1354
|
+
{ pr_key: "o/r#829uk", status: "escalated", process_key: "rp-829uk", updated_at: stale, url: "https://github.com/o/r/pull/82990", title: "unknown-state instance" },
|
|
1355
|
+
],
|
|
1356
|
+
escalations: [
|
|
1357
|
+
{ id: 1, pr_key: "o/r#829ab", status: "open", question: "loop gone", answer: null, answered_at: null },
|
|
1358
|
+
{ id: 2, pr_key: "o/r#829uk", status: "open", question: "partial read", answer: null, answered_at: null },
|
|
1359
|
+
],
|
|
1360
|
+
});
|
|
1361
|
+
const restore = stubUserTaskSearch([]);
|
|
1362
|
+
// `null` models an instance absent from the read model; a blank string models an unknown/partial state.
|
|
1363
|
+
const engine = fakeEngine({ "rp-829ab": [], "rp-829uk": [] }, {}, { "rp-829ab": null, "rp-829uk": "" });
|
|
1364
|
+
try {
|
|
1365
|
+
await pollUserTasks(data, engine, REST);
|
|
1366
|
+
} finally {
|
|
1367
|
+
restore();
|
|
1368
|
+
}
|
|
1369
|
+
const byKey = Object.fromEntries((stores.pull_requests ?? []).map((r) => [r.pr_key, r]));
|
|
1370
|
+
assertEquals(byKey["o/r#829ab"].status, "escalated", "an ABSENT instance is left to tracking/reconciliation, not healed");
|
|
1371
|
+
assertEquals(byKey["o/r#829uk"].status, "escalated", "an UNKNOWN-state instance is spared, not healed off a misread wire shape");
|
|
1372
|
+
const escById = Object.fromEntries((stores.escalations ?? []).map((r) => [r.id, r]));
|
|
1373
|
+
assertEquals(escById[1].status, "open", "the absent instance's escalation row is not retired");
|
|
1374
|
+
assertEquals(escById[2].status, "open", "the unknown-state instance's escalation row is not retired");
|
|
1375
|
+
});
|
|
1376
|
+
|
|
1377
|
+
test("pollUserTasks (engine-first): a terminated-while-escalated PR is EXCLUDED by the derived tracking VIEW — never probed (issue #829)", async () => {
|
|
1378
|
+
// Copilot review of #829: the candidate scan reads through `prsTracking` and requires
|
|
1379
|
+
// `derived_status="escalated"`, so a PR TERMINATED while its base status was `escalated` — whose VIEW
|
|
1380
|
+
// folds `derived_status` to `abandoned` — is excluded BEFORE any per-instance probe. Without the VIEW
|
|
1381
|
+
// filter this historical row would be reselected every poll and burn engine RPCs forever. We seed an
|
|
1382
|
+
// explicit `derived_status: "abandoned"` (the reconciler's terminal fold) and assert the heal issues NO
|
|
1383
|
+
// engine RPC for it at all, and its frozen base row / escalation are left untouched.
|
|
1384
|
+
const stale = new Date(Date.now() - 60 * 60_000).toISOString();
|
|
1385
|
+
const { data, stores } = memData({
|
|
1386
|
+
pull_requests: [
|
|
1387
|
+
{ pr_key: "o/r#829term", status: "escalated", derived_status: "abandoned", process_key: "rp-829term", updated_at: stale, url: "https://github.com/o/r/pull/82991", title: "terminated-while-escalated (VIEW-folded)" },
|
|
1388
|
+
],
|
|
1389
|
+
escalations: [{ id: 1, pr_key: "o/r#829term", status: "open", question: "loop gone", answer: null, answered_at: null }],
|
|
1390
|
+
});
|
|
1391
|
+
const restore = stubUserTaskSearch([]);
|
|
1392
|
+
const probed: string[] = [];
|
|
1393
|
+
const engine = {
|
|
1394
|
+
searchUserTasks: () => Promise.resolve([]),
|
|
1395
|
+
openUserTasks: (f?: { processInstanceKey?: string }) => {
|
|
1396
|
+
if (f?.processInstanceKey) probed.push(f.processInstanceKey);
|
|
1397
|
+
return Promise.resolve([]);
|
|
1398
|
+
},
|
|
1399
|
+
searchElementInstances: (f?: { processInstanceKey?: string }) => {
|
|
1400
|
+
if (f?.processInstanceKey) probed.push(f.processInstanceKey);
|
|
1401
|
+
return Promise.resolve([]);
|
|
1402
|
+
},
|
|
1403
|
+
searchProcessInstances: (f?: { processInstanceKeys?: string[] }) => {
|
|
1404
|
+
for (const k of f?.processInstanceKeys ?? []) probed.push(k);
|
|
1405
|
+
return Promise.resolve([]);
|
|
1406
|
+
},
|
|
1407
|
+
} as unknown as EngineClient;
|
|
1408
|
+
try {
|
|
1409
|
+
await pollUserTasks(data, engine, REST);
|
|
1410
|
+
} finally {
|
|
1411
|
+
restore();
|
|
1412
|
+
}
|
|
1413
|
+
assertEquals(probed.includes("rp-829term"), false, "a VIEW-folded terminal row is excluded from the scan — no engine RPC is issued for it");
|
|
1414
|
+
const byKey = Object.fromEntries((stores.pull_requests ?? []).map((r) => [r.pr_key, r]));
|
|
1415
|
+
assertEquals(byKey["o/r#829term"].status, "escalated", "the terminated row's frozen base status is left to tracking, not rewritten");
|
|
1416
|
+
const escById = Object.fromEntries((stores.escalations ?? []).map((r) => [r.id, r]));
|
|
1417
|
+
assertEquals(escById[1].status, "open", "the terminated row's escalation is not retired");
|
|
1418
|
+
});
|
|
1419
|
+
|
|
1420
|
+
test("pollUserTasks (engine-first): does NOT heal an escalated PR while its persist-escalation producer is still ACTIVE (issue #829)", async () => {
|
|
1421
|
+
// The producer-side twin of the answer-recorder race. `pr.persist-escalation` COMMITS the `open` row and
|
|
1422
|
+
// `status="escalated"` BEFORE the engine creates the `wait-answer` user task, so there is a window with an
|
|
1423
|
+
// `escalated` PR, an `open` escalation, and NO user task yet. The raise-time grace normally covers it, but
|
|
1424
|
+
// if the producer service task stays ACTIVE past the grace (an app restart / lease delay between the DB
|
|
1425
|
+
// commit and the job completing) the row is old enough to heal while the escalation is being raised RIGHT
|
|
1426
|
+
// NOW. Healing here would retire the just-raised escalation to `stale` and the question would never surface.
|
|
1427
|
+
// An ACTIVE persist-escalation element is positive evidence, so the heal must skip it — one case per loop.
|
|
1428
|
+
const stale = new Date(Date.now() - 60 * 60_000).toISOString();
|
|
1429
|
+
const { data, stores } = memData({
|
|
1430
|
+
pull_requests: [
|
|
1431
|
+
{ pr_key: "o/r#829pr", status: "escalated", process_key: "rp-829pr", updated_at: stale, url: "https://github.com/o/r/pull/8293", title: "producer active (review)" },
|
|
1432
|
+
{ pr_key: "o/r#829pm", status: "escalated", process_key: "rp-829pm", updated_at: stale, url: "https://github.com/o/r/pull/8294", title: "producer active (merge)" },
|
|
1433
|
+
],
|
|
1434
|
+
escalations: [
|
|
1435
|
+
{ id: 1, pr_key: "o/r#829pr", status: "open", question: "being raised", answer: null, answered_at: null },
|
|
1436
|
+
{ id: 2, pr_key: "o/r#829pm", status: "open", question: "being raised", answer: null, answered_at: null },
|
|
1437
|
+
],
|
|
1438
|
+
});
|
|
1439
|
+
const restore = stubUserTaskSearch([]); // the user task does not exist yet — the producer hasn't completed
|
|
1440
|
+
const engine = fakeEngine(
|
|
1441
|
+
{ "rp-829pr": [], "rp-829pm": [] },
|
|
1442
|
+
{
|
|
1443
|
+
"rp-829pr": [{ elementId: "persist-escalation-blockedcomments", state: "ACTIVE" }],
|
|
1444
|
+
"rp-829pm": [{ elementId: "merge-esc-conflict", state: "ACTIVE" }],
|
|
1445
|
+
},
|
|
1446
|
+
);
|
|
1447
|
+
try {
|
|
1448
|
+
await pollUserTasks(data, engine, REST);
|
|
1449
|
+
} finally {
|
|
1450
|
+
restore();
|
|
1451
|
+
}
|
|
1452
|
+
const byKey = Object.fromEntries((stores.pull_requests ?? []).map((r) => [r.pr_key, r]));
|
|
1453
|
+
assertEquals(byKey["o/r#829pr"].status, "escalated", "a PR with an ACTIVE review persist-escalation producer is spared the heal");
|
|
1454
|
+
assertEquals(byKey["o/r#829pm"].status, "escalated", "a PR with an ACTIVE merge persist-escalation producer is spared the heal");
|
|
1455
|
+
const escById = Object.fromEntries((stores.escalations ?? []).map((r) => [r.id, r]));
|
|
1456
|
+
assertEquals(escById[1].status, "open", "the just-raised review escalation row is left open");
|
|
1457
|
+
assertEquals(escById[2].status, "open", "the just-raised merge escalation row is left open");
|
|
1458
|
+
});
|
|
1459
|
+
|
|
1460
|
+
test("pollUserTasks (engine-first): does NOT heal an escalated PR when the producer→user-task transition lands BETWEEN the two engine reads (issue #829)", async () => {
|
|
1461
|
+
// The producer→user-task TOCTOU the prior two guards miss. `openUserTasks()` (read 1) returns empty while
|
|
1462
|
+
// a long-running persist producer is still ACTIVE; that producer then COMPLETES and the engine creates the
|
|
1463
|
+
// `wait-answer` / `wait-merge-answer` user task BEFORE `searchElementInstances()` (read 2). Read 2 therefore
|
|
1464
|
+
// sees an ACTIVE wait element — NOT the producer. Because user-task creation does not re-stamp the PR row's
|
|
1465
|
+
// `updated_at` (only `pr.persist-escalation` does, before the task exists), the row is past the raise-time
|
|
1466
|
+
// grace, so neither the grace nor the producer/answer-recorder predicate covers it: a predicate that ignores
|
|
1467
|
+
// the wait elements lets the CAS win and stale a live escalation. Treat an ACTIVE `wait-answer` /
|
|
1468
|
+
// `wait-merge-answer` element as positive evidence in the second snapshot too — one case per loop.
|
|
1469
|
+
const stale = new Date(Date.now() - 60 * 60_000).toISOString();
|
|
1470
|
+
const { data, stores } = memData({
|
|
1471
|
+
pull_requests: [
|
|
1472
|
+
{ pr_key: "o/r#829wr", status: "escalated", process_key: "rp-829wr", updated_at: stale, url: "https://github.com/o/r/pull/8295", title: "producer→wait (review)" },
|
|
1473
|
+
{ pr_key: "o/r#829wm", status: "escalated", process_key: "rp-829wm", updated_at: stale, url: "https://github.com/o/r/pull/8296", title: "producer→wait (merge)" },
|
|
1474
|
+
],
|
|
1475
|
+
escalations: [
|
|
1476
|
+
{ id: 1, pr_key: "o/r#829wr", status: "open", question: "raised mid-read", answer: null, answered_at: null },
|
|
1477
|
+
{ id: 2, pr_key: "o/r#829wm", status: "open", question: "raised mid-read", answer: null, answered_at: null },
|
|
1478
|
+
],
|
|
1479
|
+
});
|
|
1480
|
+
// The typed `openUserTasks` seam reports NO open task (the producer was still ACTIVE at read 1), while the
|
|
1481
|
+
// element search (read 2) now sees the freshly-created wait element ACTIVE — the transition landed mid-read.
|
|
1482
|
+
const restore = stubUserTaskSearch([]);
|
|
1483
|
+
const engine = fakeEngine(
|
|
1484
|
+
{ "rp-829wr": [], "rp-829wm": [] },
|
|
1485
|
+
{
|
|
1486
|
+
"rp-829wr": [{ elementId: "wait-answer", state: "ACTIVE" }],
|
|
1487
|
+
"rp-829wm": [{ elementId: "wait-merge-answer", state: "ACTIVE" }],
|
|
1488
|
+
},
|
|
1489
|
+
);
|
|
1490
|
+
try {
|
|
1491
|
+
await pollUserTasks(data, engine, REST);
|
|
1492
|
+
} finally {
|
|
1493
|
+
restore();
|
|
1494
|
+
}
|
|
1495
|
+
const byKey = Object.fromEntries((stores.pull_requests ?? []).map((r) => [r.pr_key, r]));
|
|
1496
|
+
assertEquals(byKey["o/r#829wr"].status, "escalated", "a PR whose wait-answer appeared mid-read is spared the heal");
|
|
1497
|
+
assertEquals(byKey["o/r#829wm"].status, "escalated", "a PR whose wait-merge-answer appeared mid-read is spared the heal");
|
|
1498
|
+
const escById = Object.fromEntries((stores.escalations ?? []).map((r) => [r.id, r]));
|
|
1499
|
+
assertEquals(escById[1].status, "open", "the live review escalation row is left open");
|
|
1500
|
+
assertEquals(escById[2].status, "open", "the live merge escalation row is left open");
|
|
1501
|
+
});
|
|
1502
|
+
|
|
1503
|
+
test("pollUserTasks (engine-first): a re-escalation in the read→write window is NOT clobbered — the guarded CAS refuses the moved snapshot (issue #829)", async () => {
|
|
1504
|
+
// The TOCTOU race Copilot flagged: the heal reads the PR, then does two remote engine RPCs, THEN writes.
|
|
1505
|
+
// If the SAME instance re-escalates in that window (`pr.persist-escalation` re-stamps `updated_at` and
|
|
1506
|
+
// INSERTs a fresh open escalation), a blind write would clobber the fresh `escalated` status and stale
|
|
1507
|
+
// the fresh escalation — dropping the operator's next answer. We reproduce the concurrent re-escalation
|
|
1508
|
+
// by mutating the store from inside `searchElementInstances` (the last read before the write), then
|
|
1509
|
+
// assert the snapshot-fenced CAS makes NO change: the PR stays `escalated` and the fresh escalation open.
|
|
1510
|
+
// NB the mutation advances `updated_at` AND inserts the fresh escalation TOGETHER, modelling the now-atomic
|
|
1511
|
+
// `pr.persist-escalation` producer (its insert + PR re-stamp share one transaction, issue #829) — so a
|
|
1512
|
+
// captured snapshot can never see the new escalation under the OLD generation; the covering red/green for
|
|
1513
|
+
// that producer atomicity lives in app/persist-escalation.test.ts.
|
|
1514
|
+
const stale = new Date(Date.now() - 60 * 60_000).toISOString();
|
|
1515
|
+
const { data, stores } = memData({
|
|
1516
|
+
pull_requests: [{ pr_key: "o/r#829race", status: "escalated", process_key: "rp-829race", updated_at: stale, url: "https://github.com/o/r/pull/8292", title: "re-escalates mid-heal" }],
|
|
1517
|
+
escalations: [{ id: 1, pr_key: "o/r#829race", status: "open", question: "first", answer: null, answered_at: null }],
|
|
1518
|
+
});
|
|
1519
|
+
const restore = stubUserTaskSearch([]);
|
|
1520
|
+
let reEscalated = false;
|
|
1521
|
+
const engine = {
|
|
1522
|
+
searchUserTasks: () => Promise.resolve([]),
|
|
1523
|
+
openUserTasks: () => Promise.resolve([]),
|
|
1524
|
+
// Report the instance ACTIVE so the heal reaches its guarded CAS (Copilot review of #829): without a
|
|
1525
|
+
// `searchProcessInstances` response the positive-liveness probe throws, sets `queryErrored`, and the
|
|
1526
|
+
// row would be spared for the WRONG reason — never exercising the snapshot-fenced CAS this test asserts.
|
|
1527
|
+
searchProcessInstances: () => Promise.resolve([{ processInstanceKey: "rp-829race", state: "ACTIVE" }]),
|
|
1528
|
+
searchElementInstances: (filter?: { processInstanceKey?: string }) => {
|
|
1529
|
+
// Simulate the concurrent re-escalation landing AFTER the heal captured its snapshot but BEFORE its
|
|
1530
|
+
// write: the row's generation advances (`updated_at`) and a brand-new open escalation is inserted.
|
|
1531
|
+
if (filter?.processInstanceKey === "rp-829race" && !reEscalated) {
|
|
1532
|
+
reEscalated = true;
|
|
1533
|
+
const row = (stores.pull_requests ?? []).find((r) => r.pr_key === "o/r#829race");
|
|
1534
|
+
if (row) row.updated_at = new Date().toISOString();
|
|
1535
|
+
(stores.escalations ?? []).push({ id: 2, pr_key: "o/r#829race", status: "open", question: "second", answer: null, answered_at: null });
|
|
1536
|
+
}
|
|
1537
|
+
return Promise.resolve([{ elementId: "record-answer", state: "COMPLETED" }]);
|
|
1538
|
+
},
|
|
1539
|
+
} as unknown as EngineClient;
|
|
1540
|
+
try {
|
|
1541
|
+
await pollUserTasks(data, engine, REST);
|
|
1542
|
+
} finally {
|
|
1543
|
+
restore();
|
|
1544
|
+
}
|
|
1545
|
+
const byKey = Object.fromEntries((stores.pull_requests ?? []).map((r) => [r.pr_key, r]));
|
|
1546
|
+
assertEquals(byKey["o/r#829race"].status, "escalated", "the re-escalated PR is left escalated — the CAS refused the moved snapshot");
|
|
1547
|
+
const escById = Object.fromEntries((stores.escalations ?? []).map((r) => [r.id, r]));
|
|
1548
|
+
assertEquals(escById[2].status, "open", "the fresh escalation opened in the window is NOT retired");
|
|
1549
|
+
assertEquals(escById[1].status, "open", "the snapshot's own escalation is left untouched too — the whole tx rolled to a no-op");
|
|
1550
|
+
});
|
|
1551
|
+
|
|
1552
|
+
test("pollUserTasks (engine-first): skips the per-instance open-task RPC for an escalated PR already seen parked in this pass's sweep (issue #828)", async () => {
|
|
1553
|
+
// best-effort sweep may truncate (drop tasks) but never invents one. Re-confirming with a per-instance
|
|
1554
|
+
// `openUserTasks` RPC is a redundant N+1 query; the self-heal must skip it. Only a PR NOT confirmed
|
|
1555
|
+
// parked by the sweep still needs the per-instance check.
|
|
1556
|
+
const { data, stores } = memData({
|
|
1557
|
+
pull_requests: [
|
|
1558
|
+
{ pr_key: "o/r#parked", status: "escalated", process_key: "rp-parked", url: "https://github.com/o/r/pull/1", title: "parked" },
|
|
1559
|
+
{ pr_key: "o/r#stranded", status: "escalated", process_key: "rp-stranded", url: "https://github.com/o/r/pull/2", title: "stranded" },
|
|
1560
|
+
],
|
|
1561
|
+
escalations: [{ id: 1, pr_key: "o/r#parked", status: "open", question: "open", answer: null, answered_at: null }],
|
|
1562
|
+
});
|
|
1563
|
+
const restore = stubUserTaskSearch([
|
|
1564
|
+
{ userTaskKey: "ut-parked", elementId: "wait-answer", processInstanceKey: "rp-parked", state: "CREATED" },
|
|
1565
|
+
]);
|
|
1566
|
+
const openUserTasksCalls: string[] = [];
|
|
1567
|
+
const engine = {
|
|
1568
|
+
searchUserTasks: () => Promise.resolve([]),
|
|
1569
|
+
openUserTasks: (filter?: { processInstanceKey?: string }) => {
|
|
1570
|
+
if (filter?.processInstanceKey) openUserTasksCalls.push(filter.processInstanceKey);
|
|
1571
|
+
return Promise.resolve([]);
|
|
1572
|
+
},
|
|
1573
|
+
searchElementInstances: () => Promise.resolve([]),
|
|
1574
|
+
// The stranded instance genuinely resumed (ACTIVE) — so the sweep-miss falls through to the
|
|
1575
|
+
// per-instance confirm AND the positive-liveness gate, and heals (issue #829).
|
|
1576
|
+
searchProcessInstances: (filter?: { processInstanceKeys?: string[] }) =>
|
|
1577
|
+
Promise.resolve((filter?.processInstanceKeys ?? []).map((k) => ({ processInstanceKey: k, state: "ACTIVE" }))),
|
|
1578
|
+
} as unknown as EngineClient;
|
|
1579
|
+
try {
|
|
1580
|
+
await pollUserTasks(data, engine, REST);
|
|
1581
|
+
} finally {
|
|
1582
|
+
restore();
|
|
1583
|
+
}
|
|
1584
|
+
const byKey = Object.fromEntries((stores.pull_requests ?? []).map((r) => [r.pr_key, r]));
|
|
1585
|
+
assertEquals(byKey["o/r#parked"].status, "escalated", "the swept-parked PR stays escalated without a per-instance query");
|
|
1586
|
+
assertEquals(byKey["o/r#stranded"].status, "converging", "the PR absent from the sweep is still confirmed per-instance and healed");
|
|
1587
|
+
assertEquals(openUserTasksCalls.includes("rp-parked"), false, "no redundant per-instance RPC for the already-parked PR");
|
|
1588
|
+
assertEquals(openUserTasksCalls.includes("rp-stranded"), true, "the unconfirmed PR still needs the per-instance RPC");
|
|
1589
|
+
});
|
|
1590
|
+
|
|
1591
|
+
test("pollUserTasks (engine-first): does NOT heal an escalated PR when the per-instance open-task query errors (issue #828)", async () => {
|
|
1592
|
+
// A per-instance query error is not proof the PR is unparked — mutating on that negative evidence would
|
|
1593
|
+
// steal a parked escalation. On query error the PR must be left `escalated` for a later pass.
|
|
1594
|
+
const { data, stores } = memData({
|
|
1595
|
+
pull_requests: [{ pr_key: "o/r#err", status: "escalated", process_key: "rp-err", url: "https://github.com/o/r/pull/9", title: "query errors" }],
|
|
1596
|
+
escalations: [{ id: 1, pr_key: "o/r#err", status: "open", question: "open", answer: null, answered_at: null }],
|
|
1597
|
+
});
|
|
1598
|
+
const restore = stubUserTaskSearch([]);
|
|
1599
|
+
const engine = {
|
|
1600
|
+
searchUserTasks: () => Promise.resolve([]),
|
|
1601
|
+
openUserTasks: (filter?: { processInstanceKey?: string }) =>
|
|
1602
|
+
filter?.processInstanceKey === "rp-err" ? Promise.reject(new Error("engine down")) : Promise.resolve([]),
|
|
1603
|
+
searchElementInstances: () => Promise.resolve([]),
|
|
1604
|
+
} as unknown as EngineClient;
|
|
1605
|
+
try {
|
|
1606
|
+
await pollUserTasks(data, engine, REST);
|
|
1607
|
+
} finally {
|
|
1608
|
+
restore();
|
|
1609
|
+
}
|
|
1610
|
+
const byKey = Object.fromEntries((stores.pull_requests ?? []).map((r) => [r.pr_key, r]));
|
|
1611
|
+
assertEquals(byKey["o/r#err"].status, "escalated", "a failed per-instance query leaves the PR escalated");
|
|
1612
|
+
const escById = Object.fromEntries((stores.escalations ?? []).map((r) => [r.id, r]));
|
|
1613
|
+
assertEquals(escById[1].status, "open", "the escalation row is untouched when the heal is skipped");
|
|
1614
|
+
});
|
|
1615
|
+
|
|
1616
|
+
test("pollUserTasks (engine-first): does NOT heal a JUST-escalated PR inside the grace window before its user task exists (issue #828)", async () => {
|
|
1617
|
+
// `pr.persist-escalation` stamps `status="escalated"` (and `updated_at`) IMMEDIATELY BEFORE the engine
|
|
1618
|
+
// creates the `wait-answer` user task. A poll landing in that window sees `openUserTasks` return none
|
|
1619
|
+
// and would wrongly flip the fresh escalation back to `converging`. A short grace window on `updated_at`
|
|
1620
|
+
// spares a just-written escalation while still healing genuinely-stranded (old) rows.
|
|
1621
|
+
const fresh = new Date().toISOString();
|
|
1622
|
+
const stale = new Date(Date.now() - 60 * 60_000).toISOString();
|
|
1623
|
+
const { data, stores } = memData({
|
|
1624
|
+
pull_requests: [
|
|
1625
|
+
{ pr_key: "o/r#fresh", status: "escalated", process_key: "rp-fresh", updated_at: fresh, url: "https://github.com/o/r/pull/3", title: "just escalated" },
|
|
1626
|
+
{ pr_key: "o/r#old", status: "escalated", process_key: "rp-old", updated_at: stale, url: "https://github.com/o/r/pull/4", title: "genuinely stranded" },
|
|
1627
|
+
],
|
|
1628
|
+
escalations: [
|
|
1629
|
+
{ id: 1, pr_key: "o/r#fresh", status: "open", question: "fresh", answer: null, answered_at: null },
|
|
1630
|
+
{ id: 2, pr_key: "o/r#old", status: "open", question: "old", answer: null, answered_at: null },
|
|
1631
|
+
],
|
|
1632
|
+
});
|
|
1633
|
+
const restore = stubUserTaskSearch([]);
|
|
1634
|
+
const engine = fakeEngine({});
|
|
1635
|
+
try {
|
|
1636
|
+
await pollUserTasks(data, engine, REST);
|
|
1637
|
+
} finally {
|
|
1638
|
+
restore();
|
|
1639
|
+
}
|
|
1640
|
+
const byKey = Object.fromEntries((stores.pull_requests ?? []).map((r) => [r.pr_key, r]));
|
|
1641
|
+
assertEquals(byKey["o/r#fresh"].status, "escalated", "a just-escalated PR inside the grace window is spared the heal");
|
|
1642
|
+
assertEquals(byKey["o/r#old"].status, "converging", "a genuinely-stranded (old) PR is still healed");
|
|
1643
|
+
});
|
package/app/service.ts
CHANGED
|
@@ -106,6 +106,9 @@ import {
|
|
|
106
106
|
latestTrialMergeQuestion,
|
|
107
107
|
type OpenEscalation,
|
|
108
108
|
PLAN_REVIEW_ELEMENT,
|
|
109
|
+
PR_ESCALATION_PRODUCER_ELEMENTS,
|
|
110
|
+
PR_RECORD_ANSWER_ELEMENT,
|
|
111
|
+
PR_RECORD_MERGE_ANSWER_ELEMENT,
|
|
109
112
|
PR_WAIT_ANSWER_ELEMENT,
|
|
110
113
|
PR_WAIT_MERGE_ANSWER_ELEMENT,
|
|
111
114
|
prEscalations,
|
|
@@ -251,6 +254,15 @@ export const REVIEW_NUDGE_MS = clampNudgeMinutes(process.env.NANO_PR_REVIEW_NUDG
|
|
|
251
254
|
* still reconciling genuinely-stranded rows. */
|
|
252
255
|
export const FEATURE_ESCALATION_HEAL_GRACE_MS = 60_000;
|
|
253
256
|
|
|
257
|
+
/** Grace window (ms) before the `escalated`→`converging` PR self-heal (below) may act on a row — the
|
|
258
|
+
* convergence-path twin of `FEATURE_ESCALATION_HEAL_GRACE_MS` (issue #828). The sole writer of a PR's
|
|
259
|
+
* `status="escalated"` — `pr.persist-escalation` — stamps the row IMMEDIATELY BEFORE the engine creates
|
|
260
|
+
* the `wait-answer` / `wait-merge-answer` user task. A poll landing in that window would see
|
|
261
|
+
* `openUserTasks` report none and wrongly heal the just-raised escalation back to `converging`, hiding
|
|
262
|
+
* it before the operator can answer. Only heal rows whose escalation is older than this window, by when
|
|
263
|
+
* the user task must already be observable. */
|
|
264
|
+
export const PR_ESCALATION_HEAL_GRACE_MS = 60_000;
|
|
265
|
+
|
|
254
266
|
/** Whether a converged PR is automatically driven to merge (the merge-loop). Default on; set
|
|
255
267
|
* `NANO_PR_AUTO_MERGE=0` to stop at `converged` (review-only mode). */
|
|
256
268
|
export const AUTO_MERGE = !["0", "false", "off", "no"].includes(
|
|
@@ -3325,6 +3337,152 @@ export async function pollUserTasks(
|
|
|
3325
3337
|
if (queryErrored || stillParked) continue;
|
|
3326
3338
|
await featureRuns(data).update(run.feature_key, { status: "running", updated_at: at });
|
|
3327
3339
|
}
|
|
3340
|
+
|
|
3341
|
+
// ── Self-heal: a convergence PR row holds `escalated` ONLY while parked (issue #828) ─────────────
|
|
3342
|
+
// The convergence-path twin of the feature self-heal above (issue #642). `pr.persist-escalation` is
|
|
3343
|
+
// the sole writer of a PR's `status="escalated"`, and the sole writer that moves it back off is the
|
|
3344
|
+
// `pr.answer-escalation` worker (the guarded `UPDATE … SET status='converging'` on user-task
|
|
3345
|
+
// completion). When that write is lost — e.g. the app restarts while a `wait-answer` completion /
|
|
3346
|
+
// `answer-escalation` job is in flight — the engine resumes the loop (token back on `review-round`,
|
|
3347
|
+
// no open user task) but the PR row stays `escalated` forever, since NOTHING on the PR path repairs
|
|
3348
|
+
// it (unlike the feature path). `/status` then shows `status="escalated"` with a null derived
|
|
3349
|
+
// `openEscalation` indefinitely, inviting a human to answer an already-answered question.
|
|
3350
|
+
//
|
|
3351
|
+
// Heal it engine-first, on the SAME positive-evidence contract as the feature sweep: a PR at
|
|
3352
|
+
// `status="escalated"` is reconciled back to `converging` only when the engine confirms NO open PR
|
|
3353
|
+
// escalation task (`wait-answer` / `wait-merge-answer`) for its instance. As above, presence in THIS
|
|
3354
|
+
// pass's `desired` set is itself positive evidence of parking (truncation only ever DROPS tasks), so a
|
|
3355
|
+
// PR whose escalation task was already swept is genuinely parked — skip its per-instance RPC. Only a
|
|
3356
|
+
// PR not confirmed parked by the sweep falls through to the per-instance `openUserTasks` check, and a
|
|
3357
|
+
// query error is negative evidence (skip the heal), so durable state flips only on positive proof.
|
|
3358
|
+
//
|
|
3359
|
+
// Candidate set: read through the ADR-0065 derived tracking VIEW and require `derived_status="escalated"`
|
|
3360
|
+
// (Copilot review of #829). A PR TERMINATED while its base status was `escalated` keeps that frozen base
|
|
3361
|
+
// status forever — the reconciler never rewrites it (app/instanceTracking.ts) — so a raw `prs.find({
|
|
3362
|
+
// status:"escalated" })` scan would reselect every such HISTORICAL row on every poll and burn up to three
|
|
3363
|
+
// sequential engine RPCs on each, indefinitely. The VIEW folds the terminal edge to `derived_status`
|
|
3364
|
+
// (out-of-band terminate → `abandoned`), so filtering on `derived_status="escalated"` bounds the pass to
|
|
3365
|
+
// LIVE/UNKNOWN candidates and leaves terminal rows to tracking as intended; the ACTIVE-instance gate below
|
|
3366
|
+
// is the correctness backstop for a candidate the VIEW hasn't yet reconciled. Writes still target `prs`.
|
|
3367
|
+
const sweptParkedPrEscalations = new Set<string>();
|
|
3368
|
+
for (const r of desired) {
|
|
3369
|
+
if (r.element_id !== PR_WAIT_ANSWER_ELEMENT && r.element_id !== PR_WAIT_MERGE_ANSWER_ELEMENT) continue;
|
|
3370
|
+
if (r.subject_type !== "pr") continue;
|
|
3371
|
+
if (r.process_key) sweptParkedPrEscalations.add(r.process_key);
|
|
3372
|
+
if (r.subject_key) sweptParkedPrEscalations.add(r.subject_key);
|
|
3373
|
+
}
|
|
3374
|
+
for (const pr of await prsTracking(data).find({ derived_status: "escalated" })) {
|
|
3375
|
+
if (!pr.process_key) continue;
|
|
3376
|
+
if (sweptParkedPrEscalations.has(pr.process_key) || sweptParkedPrEscalations.has(pr.pr_key)) continue; // parked this pass — no RPC, no heal
|
|
3377
|
+
// Don't race a just-raised escalation: `pr.persist-escalation` stamps `updated_at` immediately
|
|
3378
|
+
// before the engine creates the user task, so heal only rows past the grace window.
|
|
3379
|
+
const escalatedAt = Date.parse(pr.updated_at ?? "");
|
|
3380
|
+
if (Number.isFinite(escalatedAt) && Date.now() - escalatedAt < PR_ESCALATION_HEAL_GRACE_MS) continue;
|
|
3381
|
+
// Snapshot the PR generation (`process_key` + `updated_at`) and the `open` escalation rows we observe
|
|
3382
|
+
// NOW — BEFORE the remote engine reads below. The repair writes are fenced on this snapshot, so a
|
|
3383
|
+
// FRESH escalation the SAME instance opens in the read→write window is neither clobbered nor retired.
|
|
3384
|
+
// The convergence loop REUSES one process instance across rounds, so a re-escalation keeps the same
|
|
3385
|
+
// `process_key`; the discriminator is `updated_at`, which `pr.persist-escalation` re-stamps every time
|
|
3386
|
+
// it writes `status="escalated"`. Capturing the open-escalation ids here (not re-reading `status=open`
|
|
3387
|
+
// after the write) means a fresh escalation row — a NEW id inserted after this point — is never retired
|
|
3388
|
+
// to `stale` even though it is `open`, since it was not part of the snapshot we healed (issue #829).
|
|
3389
|
+
const snapshotProcessKey = pr.process_key;
|
|
3390
|
+
const snapshotUpdatedAt = pr.updated_at ?? null;
|
|
3391
|
+
const snapshotOpenEscalationIds = (await escs(data).find({ pr_key: pr.pr_key, status: "open" })).map((e) => e.id);
|
|
3392
|
+
let stillParked = false;
|
|
3393
|
+
let escalationInFlight = false;
|
|
3394
|
+
let instanceActive = false;
|
|
3395
|
+
let queryErrored = false;
|
|
3396
|
+
try {
|
|
3397
|
+
const openTasks = await engine.openUserTasks({ processInstanceKey: pr.process_key });
|
|
3398
|
+
stillParked = openTasks.some((t) => t.elementId === PR_WAIT_ANSWER_ELEMENT || t.elementId === PR_WAIT_MERGE_ANSWER_ELEMENT);
|
|
3399
|
+
// An empty open-task set is NOT yet proof the escalation is dead. Two distinct DB/engine boundaries
|
|
3400
|
+
// each leave the row `escalated` with NO open user task while an escalation is genuinely live:
|
|
3401
|
+
//
|
|
3402
|
+
// 1. The ANSWER side. `pr.answer-escalation` (`record-answer` / `record-merge-answer`) is the sole
|
|
3403
|
+
// writer that moves a PR OFF `escalated`, and both BPMN models flow the completed `wait-answer` /
|
|
3404
|
+
// `wait-merge-answer` user task DIRECTLY to that service task. So a normal answer has a window
|
|
3405
|
+
// where the user task is already gone (this query returns none) while the answer-recording job is
|
|
3406
|
+
// still queued or running.
|
|
3407
|
+
// 2. The PRODUCER side. `pr.persist-escalation` (the `PR_ESCALATION_PRODUCER_ELEMENTS`) COMMITS the
|
|
3408
|
+
// `open` row and `status="escalated"` IMMEDIATELY BEFORE the engine creates the `wait-answer` /
|
|
3409
|
+
// `wait-merge-answer` user task. The raise-time grace window covers the normal case, but if that
|
|
3410
|
+
// producer service task stays ACTIVE past the grace (e.g. an app restart / lease delay between
|
|
3411
|
+
// the DB commit and the job completing) the row is old enough to heal yet no user task exists.
|
|
3412
|
+
// 3. The PRODUCER→USER-TASK TOCTOU BETWEEN THE TWO READS. `openUserTasks` (read 1) can return empty
|
|
3413
|
+
// while a persist producer is still ACTIVE; that producer then COMPLETES and the engine creates
|
|
3414
|
+
// the `wait-answer` / `wait-merge-answer` user task BEFORE `searchElementInstances` (read 2). Read 2
|
|
3415
|
+
// then sees an ACTIVE WAIT element — not the producer. Because user-task creation does NOT re-stamp
|
|
3416
|
+
// the PR row's `updated_at` (only `pr.persist-escalation` does, before the task exists), the row is
|
|
3417
|
+
// past the grace, so neither the grace nor the producer/answer-recorder check covers it. Treat an
|
|
3418
|
+
// ACTIVE wait element as positive evidence here too, or the CAS wins and stales a live escalation.
|
|
3419
|
+
//
|
|
3420
|
+
// In EITHER window the escalation is arbitrarily old (past the raise-time grace), so the grace above
|
|
3421
|
+
// does not cover it. Healing here flips the row to `converging` and retires the still-`open`
|
|
3422
|
+
// escalation to `stale`, and the live escalation is lost — the answer-recorder finds no target and
|
|
3423
|
+
// silently drops the operator's answer, or the just-raised question never surfaces (issue #829). An
|
|
3424
|
+
// ACTIVE answer-recording, persist-escalation, OR wait-answer/wait-merge-answer element instance is
|
|
3425
|
+
// positive evidence an escalation is in flight, so treat it exactly like an open task and skip the
|
|
3426
|
+
// heal. But even a PR with NEITHER an open task NOR any ACTIVE escalation element is NOT yet
|
|
3427
|
+
// heal-eligible — it must ALSO be a confirmed-ACTIVE instance (the positive-liveness gate below);
|
|
3428
|
+
// a terminal/absent instance matches this negative shape too but never resumed (the #828 tear).
|
|
3429
|
+
if (!stillParked) {
|
|
3430
|
+
const elements = await engine.searchElementInstances({ processInstanceKey: pr.process_key });
|
|
3431
|
+
escalationInFlight = elements.some(
|
|
3432
|
+
(el) =>
|
|
3433
|
+
el.state === "ACTIVE" &&
|
|
3434
|
+
(el.elementId === PR_RECORD_ANSWER_ELEMENT ||
|
|
3435
|
+
el.elementId === PR_RECORD_MERGE_ANSWER_ELEMENT ||
|
|
3436
|
+
el.elementId === PR_WAIT_ANSWER_ELEMENT ||
|
|
3437
|
+
el.elementId === PR_WAIT_MERGE_ANSWER_ELEMENT ||
|
|
3438
|
+
(el.elementId != null && PR_ESCALATION_PRODUCER_ELEMENTS.includes(el.elementId))),
|
|
3439
|
+
);
|
|
3440
|
+
}
|
|
3441
|
+
// "No open task and no in-flight escalation element" is necessary but NOT sufficient to heal
|
|
3442
|
+
// (Copilot review of #829). A TERMINAL or ABSENT process instance — one cancelled, completed, or
|
|
3443
|
+
// dropped from the read model — ALSO reports no open task and no ACTIVE escalation element, yet its
|
|
3444
|
+
// loop never resumed: flipping such a frozen `escalated` row to `converging` and retiring its audit
|
|
3445
|
+
// rows fabricates a live loop the engine will never advance. Terminal/absent PRs are owned by the
|
|
3446
|
+
// ADR-0065 derived tracking + reconciliation (they read `abandoned`/settled on `derived_status`), so
|
|
3447
|
+
// leave them be. Require POSITIVE engine-truth that the instance is `ACTIVE` before healing — the
|
|
3448
|
+
// same tri-state contract as the feature-handoff probe (search ~L2562) and `makeEngineActiveProbe`
|
|
3449
|
+
// (app/reconcile.ts): ONLY a confirmed `ACTIVE` state heals; a known-terminal or genuinely-absent
|
|
3450
|
+
// instance is left to tracking; a missing/empty/unknown `state` (or a query error) is spared, never
|
|
3451
|
+
// healed off a wire shape we misread. So a healed row is provably the #828 lost-write tear: an
|
|
3452
|
+
// ACTIVE loop that resumed but whose `status="converging"` flip was lost.
|
|
3453
|
+
const snapshots = await engine.searchProcessInstances({ processInstanceKeys: [pr.process_key] });
|
|
3454
|
+
const match = snapshots.find((s) => String(s.processInstanceKey) === pr.process_key);
|
|
3455
|
+
instanceActive = match != null && String(match.state ?? "").trim().toUpperCase() === "ACTIVE";
|
|
3456
|
+
} catch (err) {
|
|
3457
|
+
console.error(`[poller] escalated-pr self-heal (${pr.pr_key} @ ${pr.process_key}): ${err}`);
|
|
3458
|
+
queryErrored = true;
|
|
3459
|
+
}
|
|
3460
|
+
if (queryErrored || stillParked || escalationInFlight || !instanceActive) continue;
|
|
3461
|
+
// Repair conditionally and atomically (Copilot review of #829). The read→write window between the
|
|
3462
|
+
// engine reads above and here is wide (two remote RPCs), so the PR may have re-escalated on the same
|
|
3463
|
+
// instance meanwhile; a blind `update` would then overwrite that fresh `escalated` status and retire
|
|
3464
|
+
// its new `open` escalation. The PR flip is therefore a SINGLE guarded UPDATE fenced on the inspected
|
|
3465
|
+
// snapshot — still `escalated`, same `process_key`, same `updated_at` — an atomic compare-and-swap
|
|
3466
|
+
// (SQLite serialises it against the worker's guarded writes in the other process). Only when that CAS
|
|
3467
|
+
// WON (`changed === 1`) do we retire the snapshot's own orphan escalation rows, in the SAME
|
|
3468
|
+
// transaction: a crash can never leave a healed PR with a dangling `open` row, nor an un-healed PR
|
|
3469
|
+
// with retired rows — a rolled-back heal is simply retried by a later pass. Retiring by the captured
|
|
3470
|
+
// snapshot ids (never a fresh `status=open` re-read) means a just-opened escalation is left untouched.
|
|
3471
|
+
const db = data.open();
|
|
3472
|
+
await db.tx(async (t) => {
|
|
3473
|
+
const flipped = await t.exec(
|
|
3474
|
+
`UPDATE "pull_requests" SET "status" = 'converging', "updated_at" = ? WHERE "pr_key" = ? AND "status" = 'escalated' AND "process_key" IS ? AND "updated_at" IS ?`,
|
|
3475
|
+
[at, pr.pr_key, snapshotProcessKey, snapshotUpdatedAt],
|
|
3476
|
+
);
|
|
3477
|
+
if (flipped.changed !== 1) return; // snapshot moved (a fresh escalation re-stamped the row) — leave it for a later pass
|
|
3478
|
+
// Retire the orphaned `open` escalation rows so the audit trail matches the healed PR row — the
|
|
3479
|
+
// `answer-escalation` write that would have answered/retired them was the very write we lost. We
|
|
3480
|
+
// mark them `stale` (not `answered`) because no operator answer was recorded for them.
|
|
3481
|
+
for (const escId of snapshotOpenEscalationIds) {
|
|
3482
|
+
await t.exec(`UPDATE "escalations" SET "status" = 'stale' WHERE "id" = ? AND "status" = 'open'`, [escId]);
|
|
3483
|
+
}
|
|
3484
|
+
});
|
|
3485
|
+
}
|
|
3328
3486
|
}
|
|
3329
3487
|
|
|
3330
3488
|
/** One full poll pass: advance the review stage, the merge stage, the wave-merge barrier, and
|
package/app/userTasks.ts
CHANGED
|
@@ -52,6 +52,40 @@ export const PR_WAIT_ANSWER_ELEMENT = "wait-answer";
|
|
|
52
52
|
* canonical `completeUserTask` door and surfaced in this same Tasks inbox. */
|
|
53
53
|
export const PR_WAIT_MERGE_ANSWER_ELEMENT = "wait-merge-answer";
|
|
54
54
|
|
|
55
|
+
/** The PR review-loop answer-recording service task (convergence-loop.bpmn) — the `pr.answer-escalation`
|
|
56
|
+
* job the token moves to IMMEDIATELY when `wait-answer` completes. Its ACTIVE element instance is the
|
|
57
|
+
* positive-evidence marker that an operator's answer is in-flight (recorded but the loop hasn't looped
|
|
58
|
+
* yet), so the escalation self-heal must NOT steal the row while it runs (issue #829). */
|
|
59
|
+
export const PR_RECORD_ANSWER_ELEMENT = "record-answer";
|
|
60
|
+
|
|
61
|
+
/** The PR merge-loop answer-recording service task (merge-loop.bpmn) — the merge-path twin of
|
|
62
|
+
* `PR_RECORD_ANSWER_ELEMENT`, reached when `wait-merge-answer` completes (issue #829). */
|
|
63
|
+
export const PR_RECORD_MERGE_ANSWER_ELEMENT = "record-merge-answer";
|
|
64
|
+
|
|
65
|
+
/** Every `pr.persist-escalation` PRODUCER element across the two PR loops — the service tasks that write
|
|
66
|
+
* `status="escalated"` (and INSERT the `open` escalation row) IMMEDIATELY BEFORE the engine creates the
|
|
67
|
+
* `wait-answer` / `wait-merge-answer` user task. An ACTIVE instance of any of these is positive evidence
|
|
68
|
+
* an escalation is being RAISED right now: the row is already `escalated` but no user task exists yet, so
|
|
69
|
+
* neither the open-task probe nor the answer-recorder probe covers it. If such a producer stays ACTIVE
|
|
70
|
+
* past the raise-time grace window (e.g. an app restart / lease delay between the DB commit and the job
|
|
71
|
+
* completing), the escalated-PR self-heal would otherwise see no parked task and wrongly retire the live
|
|
72
|
+
* escalation. So the heal must treat an ACTIVE producer as in-flight too (issue #829). This is the single
|
|
73
|
+
* source of truth for that set; `app/persistEscalationElements.test.ts` fails if it drifts from the
|
|
74
|
+
* `pr.persist-escalation` tasks actually modelled in the BPMN. */
|
|
75
|
+
export const PR_ESCALATION_PRODUCER_ELEMENTS: readonly string[] = [
|
|
76
|
+
// convergence-loop.bpmn
|
|
77
|
+
"persist-escalation",
|
|
78
|
+
"persist-escalation-noprogress",
|
|
79
|
+
"persist-escalation-maxrounds",
|
|
80
|
+
"persist-escalation-blockedcomments",
|
|
81
|
+
"persist-escalation-scope",
|
|
82
|
+
"persist-review-stalled",
|
|
83
|
+
// merge-loop.bpmn
|
|
84
|
+
"merge-esc-conflict",
|
|
85
|
+
"merge-esc-attempt",
|
|
86
|
+
"merge-esc-landed",
|
|
87
|
+
];
|
|
88
|
+
|
|
55
89
|
/** The ACP permission-prompt escalation (issue #559, ADR 0056) — the Tasks-inbox kind a bridged
|
|
56
90
|
* `session/request_permission` surfaces under when an escalate-policy agent asks a human to Allow/Deny
|
|
57
91
|
* a proposed action. Unlike the other escalation elements this is NOT a BPMN user-task element; it is
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@nanobpm/nano-workforce",
|
|
3
|
-
"version": "0.193.
|
|
3
|
+
"version": "0.193.2",
|
|
4
4
|
"description": "Nano Workforce — an Agent Graph Orchestration application for Agentic SDLC: durable BPMN processes that coordinate a graph of AI agents across the software delivery lifecycle.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "main.ts",
|
|
@@ -83,7 +83,7 @@
|
|
|
83
83
|
"yaml": "^2.9.0"
|
|
84
84
|
},
|
|
85
85
|
"overrides": {
|
|
86
|
-
"@nanobpm/engine-wasm": "^0.
|
|
86
|
+
"@nanobpm/engine-wasm": "^0.10.0",
|
|
87
87
|
"@semantic-release/npm": "^13.1.5"
|
|
88
88
|
}
|
|
89
89
|
}
|
|
@@ -145,6 +145,12 @@
|
|
|
145
145
|
<zeebe:property name="io.nanobpm.dataEnvelope.in" value="PrCaptureHeadIn" />
|
|
146
146
|
<zeebe:property name="io.nanobpm.dataEnvelope.out" value="PrCaptureHeadOut" />
|
|
147
147
|
</zeebe:properties>
|
|
148
|
+
<zeebe:ioMapping>
|
|
149
|
+
<zeebe:output source="=null" target="status" />
|
|
150
|
+
<zeebe:output source="=null" target="summary" />
|
|
151
|
+
<zeebe:output source="=null" target="question" />
|
|
152
|
+
<zeebe:output source="=false" target="escalated" />
|
|
153
|
+
</zeebe:ioMapping>
|
|
148
154
|
</bpmn:extensionElements>
|
|
149
155
|
<bpmn:incoming>f_start</bpmn:incoming>
|
|
150
156
|
<bpmn:incoming>f_reviewLoop</bpmn:incoming>
|
|
@@ -23,6 +23,16 @@
|
|
|
23
23
|
// fresh on every round entry (including husk retries), so it never carries a stale prior-round value.
|
|
24
24
|
// The read fails OPEN to `null` (a transient GitHub hiccup must never fabricate a no-progress verdict);
|
|
25
25
|
// progress-check then falls back to the persisted `last_round_head`.
|
|
26
|
+
//
|
|
27
|
+
// Because this task is the single on-entry join before `review-round`, the model also uses it to RESET
|
|
28
|
+
// the round's OUTPUT variables on entry (issue #822): its `ioMapping` clears `status`, `summary`,
|
|
29
|
+
// `question` (→ null) and `escalated` (→ false) in convergence-loop.bpmn. A `review-round` that returns
|
|
30
|
+
// NO result (a misconfigured worker that exits 0 with empty output, no commits, no push) then can't
|
|
31
|
+
// inherit the PREVIOUS round's `status`/`question` and re-open a word-for-word duplicate of an
|
|
32
|
+
// already-answered escalation; instead it falls through `gw-status`'s addressed default into the
|
|
33
|
+
// no-progress (husk) path. These OUTPUT vars are reset on ENTRY (here) rather than on the review-round's
|
|
34
|
+
// own output — a same-task output-clear would clobber the agent's real verdict — mirroring the merge
|
|
35
|
+
// loop's `arm-merge`, which clears `status` on entry before its agent runs.
|
|
26
36
|
import type { AppJobHandler } from "@nanobpm/urban";
|
|
27
37
|
import { fetchBranchHead, fetchPrHead } from "../../app/github.ts";
|
|
28
38
|
import { parsePr } from "../../app/service.ts";
|
|
@@ -111,20 +111,35 @@ const handler: AppJobHandler<In> = async (job, app) => {
|
|
|
111
111
|
process_instance_key: job.processInstanceKey != null ? String(job.processInstanceKey) : null,
|
|
112
112
|
});
|
|
113
113
|
}
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
114
|
+
// Open the escalation row and stamp the PR to `escalated` ATOMICALLY, in ONE transaction (Copilot
|
|
115
|
+
// review of #829). `pr.persist-escalation` is the SOLE producer of a PR's escalated generation, and the
|
|
116
|
+
// escalated-PR self-heal (`pollUserTasks` in app/service.ts) fences its repair on a snapshot of the PR
|
|
117
|
+
// generation (`process_key` + `updated_at`) plus the `open` escalation ids it observes. If the fresh
|
|
118
|
+
// `open` row were INSERTED before the PR's `updated_at` re-stamp — two separate writes, as this worker
|
|
119
|
+
// used to do — a heal snapshot landing between them would capture the new escalation while the PR still
|
|
120
|
+
// matched the OLD generation; its compare-and-swap, still matching the un-bumped `updated_at`, could then
|
|
121
|
+
// WIN and retire the just-opened escalation to `stale`, orphaning a live user task and dropping the
|
|
122
|
+
// operator's next answer. Committing the insert and the generation re-stamp together closes that window
|
|
123
|
+
// categorically: the heal can only ever observe the new escalation AND the bumped `updated_at` atomically,
|
|
124
|
+
// so its snapshot-fenced CAS misses (the discriminator moved) and never clobbers. The same transaction
|
|
125
|
+
// also means a crash mid-write rolls back cleanly — never an `open` escalation row with no `escalated` PR.
|
|
126
|
+
let escalationId: number | bigint = 0;
|
|
127
|
+
await app.data.open().tx(async (t) => {
|
|
128
|
+
escalationId = await t.table("escalations", "id").insert({
|
|
129
|
+
pr_key: prKey,
|
|
130
|
+
round_no: round,
|
|
131
|
+
kind,
|
|
132
|
+
question,
|
|
133
|
+
transcript,
|
|
134
|
+
worker,
|
|
135
|
+
status: "open",
|
|
136
|
+
asked_at: now,
|
|
137
|
+
});
|
|
138
|
+
await t.table("pull_requests", "pr_key").update(prKey, {
|
|
139
|
+
status: "escalated",
|
|
140
|
+
current_round: round,
|
|
141
|
+
updated_at: now,
|
|
142
|
+
});
|
|
128
143
|
});
|
|
129
144
|
|
|
130
145
|
return { escalationId: Number(escalationId), escalated: true, question };
|