@nanobpm/nano-workforce 0.201.1 → 0.201.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,169 @@
1
+ // Agent-task SLA-escalation reason builder (issue #879) — the single canonical, pure text builder for
2
+ // the human-facing escalation raised when an agent service task's interrupting SLA timer boundary
3
+ // fires. It distinguishes the two outcomes the old wording conflated:
4
+ //
5
+ // • NEVER OBSERVED ACTIVATED — no worker was ever observed to pick the job up (almost certainly
6
+ // queue starvation / no capacity for the job type). The job sat CREATED for ~the whole SLA budget
7
+ // and was cancelled by the boundary. Telling the human it is "hung or looping" and to "check its
8
+ // progress" is a false diagnosis — the likely fix is CAPACITY, not intervention. The wording is
9
+ // hedged (a worker could lease it in the final moment before the boundary fires, between activation
10
+ // polls, unrecorded), so it reports the likely cause without over-claiming certainty.
11
+ // • STARTED BUT EXCEEDED — a worker leased the job (an agent ran) but did not complete within the
12
+ // budget — it may genuinely be hung or looping, and intervening / checking its progress is apt.
13
+ //
14
+ // The distinction is read from the durable activation ledger (`app/agentActivationLedger.ts`), which
15
+ // the poller populates while the job is live. Every escalation names the JOB TYPE and HOW LONG it
16
+ // waited, per the issue's acceptance criteria.
17
+ //
18
+ // Derivation over duplication: this is the ONE builder every agent-SLA recorder (implement-cell,
19
+ // merge-cell, retro) routes through, so the never-started/hung wording can never drift between sites.
20
+ // The pure `buildAgentSlaEscalationReason` is exhaustively testable; `agentSlaEscalationQuestion` is
21
+ // the thin async wrapper that reads the ledger and calls it.
22
+ import type { DataLayer } from "@nanobpm/urban";
23
+ import { type AgentJobActivation, getAgentJobActivation } from "./agentActivationLedger.ts";
24
+ import { isoDurationToMs } from "./reviewWait.ts";
25
+
26
+ /** The activation facts the builder needs (a slice of the ledger row), or `null` when the poller never
27
+ * observed the job (activation unknown). */
28
+ export interface AgentSlaActivation {
29
+ /** ISO ts a leasing worker was first observed; `null`/absent ⇒ never activated. */
30
+ activatedAt?: string | null;
31
+ /** ISO ts the job was first observed queued. */
32
+ firstSeenAt?: string | null;
33
+ /** The leasing worker's name, when known. */
34
+ worker?: string | null;
35
+ }
36
+
37
+ export interface AgentSlaEscalationInput {
38
+ /** The agent job type that timed out, e.g. `senior:feature`. Named in the reason. */
39
+ jobType: string;
40
+ /** The SLA budget (ISO-8601 duration) the boundary armed, e.g. `PT2H`. */
41
+ sla: string;
42
+ /** The activation record for this job, or `null` when the poller never observed it. */
43
+ activation: AgentSlaActivation | null;
44
+ /** The escalation moment (ISO ts) — the elapsed wait is measured against it. */
45
+ now: string;
46
+ /** How the human resumes after answering — appended verbatim so each caller keeps its own
47
+ * routing hint (e.g. "answer to resume … or abandon it"). */
48
+ recovery: string;
49
+ }
50
+
51
+ /** Humanise a millisecond span to a compact, human-facing string (e.g. `2h 5m`, `45m`, `30s`). A
52
+ * non-positive/NaN span renders `less than a minute`. */
53
+ export function humanizeMs(ms: number): string {
54
+ if (!Number.isFinite(ms) || ms <= 0) return "less than a minute";
55
+ const totalMinutes = Math.floor(ms / 60_000);
56
+ if (totalMinutes < 1) return `${Math.max(1, Math.round(ms / 1000))}s`;
57
+ const days = Math.floor(totalMinutes / 1440);
58
+ const hours = Math.floor((totalMinutes % 1440) / 60);
59
+ const minutes = totalMinutes % 60;
60
+ const parts: string[] = [];
61
+ if (days > 0) parts.push(`${days}d`);
62
+ if (hours > 0) parts.push(`${hours}h`);
63
+ if (minutes > 0) parts.push(`${minutes}m`);
64
+ return parts.join(" ") || "less than a minute";
65
+ }
66
+
67
+ /** Humanise an ISO-8601 duration (the SLA budget) to the same compact form, via the canonical parser. */
68
+ export function humanizeIsoDuration(iso: string): string {
69
+ return humanizeMs(isoDurationToMs(iso, iso));
70
+ }
71
+
72
+ const elapsedMs = (fromIso: string | null | undefined, toIso: string): number | null => {
73
+ if (!fromIso) return null;
74
+ const from = Date.parse(fromIso);
75
+ const to = Date.parse(toIso);
76
+ if (!Number.isFinite(from) || !Number.isFinite(to)) return null;
77
+ return to - from;
78
+ };
79
+
80
+ /** Build the accurate, self-contained agent-SLA escalation reason. Pure — distinguishes never-started
81
+ * (queue starvation) from started-but-hung, names the job type and how long it waited, then appends
82
+ * the caller's recovery hint. */
83
+ export function buildAgentSlaEscalationReason(input: AgentSlaEscalationInput): string {
84
+ const { jobType, sla, activation, now, recovery } = input;
85
+ const slaHuman = humanizeIsoDuration(sla);
86
+ const activatedAt = activation?.activatedAt ?? null;
87
+
88
+ if (activatedAt) {
89
+ // STARTED BUT EXCEEDED: a worker leased the job, so it may genuinely be hung or looping.
90
+ const queueMs = elapsedMs(activation?.firstSeenAt, activatedAt);
91
+ const queuedClause = queueMs !== null && queueMs > 0
92
+ ? ` (after waiting ~${humanizeMs(queueMs)} in the queue)`
93
+ : "";
94
+ const ranMs = elapsedMs(activatedAt, now);
95
+ const ranClause = ranMs !== null && ranMs > 0 ? ` and then ran for ~${humanizeMs(ranMs)}` : "";
96
+ const workerClause = activation?.worker ? ` (worker "${activation.worker}")` : "";
97
+ return (
98
+ `The ${jobType} agent started${queuedClause}${workerClause}${ranClause} but exceeded its time budget ` +
99
+ `(SLA ${sla} ≈ ${slaHuman}) without completing — it may be hung or looping. ${recovery}`
100
+ );
101
+ }
102
+
103
+ // Measure the wait: from first-seen to now when the poller observed the queued job, else the full
104
+ // SLA budget (the boundary fired exactly one budget after the task was entered).
105
+ const waitedMs = elapsedMs(activation?.firstSeenAt, now);
106
+ const waited = waitedMs !== null && waitedMs > 0 ? humanizeMs(waitedMs) : slaHuman;
107
+
108
+ if (activation === null) {
109
+ // ACTIVATION UNKNOWN: the poller never observed the job (e.g. the activation poll was unavailable).
110
+ // Report the ambiguity honestly rather than guessing "hung" or "never started".
111
+ return (
112
+ `The ${jobType} agent did not complete within its time budget (SLA ${sla} ≈ ${slaHuman}), and no ` +
113
+ `worker-activation was ever recorded for it, so we cannot confirm whether a worker ever picked it ` +
114
+ `up — it may have been starved of capacity for ${jobType}, or started and hung. ${recovery}`
115
+ );
116
+ }
117
+
118
+ // NEVER OBSERVED ACTIVATED: seen queued, never seen leased. This is the best-effort observation of
119
+ // queue starvation — hedged, because a worker could in principle lease the job in the final moment
120
+ // before the boundary cancels it, between activation polls, and that would not be recorded. We report
121
+ // the likely diagnosis (and its capacity fix) without over-claiming certainty.
122
+ return (
123
+ `No worker was ever observed to pick up the ${jobType} agent's job for its entire SLA budget ` +
124
+ `(SLA ${sla} ≈ ${slaHuman}): it was seen queued for ~${waited}, then the SLA boundary cancelled it. ` +
125
+ `This is almost certainly queue starvation (no capacity for ${jobType}) rather than a hung or ` +
126
+ `looping agent, so the likely fix is to add capacity for ${jobType} (enrol a worker that serves it, ` +
127
+ `or reduce concurrent demand) rather than to check a non-existent agent's progress. (If a worker ` +
128
+ `leased it in the final moment before the boundary fired, between activation polls, that would not ` +
129
+ `have been recorded here.) ${recovery}`
130
+ );
131
+ }
132
+
133
+ /** Read the activation ledger for this job and build the escalation reason — the thin async wrapper the
134
+ * recorder workers call. `processInstanceKey` is the instance the agent job AND its SLA recorder share,
135
+ * so the lookup is by (instance, jobType). An absent ledger row ⇒ activation unknown. */
136
+ export async function agentSlaEscalationQuestion(
137
+ data: DataLayer,
138
+ input: {
139
+ processInstanceKey: string;
140
+ jobType: string;
141
+ sla: string;
142
+ recovery: string;
143
+ now?: string;
144
+ },
145
+ ): Promise<string> {
146
+ const now = input.now ?? new Date().toISOString();
147
+ let activation: AgentSlaActivation | null = null;
148
+ if (input.processInstanceKey && input.processInstanceKey.length > 0) {
149
+ const row: AgentJobActivation | undefined = await getAgentJobActivation(
150
+ data,
151
+ input.processInstanceKey,
152
+ input.jobType,
153
+ );
154
+ if (row) {
155
+ activation = {
156
+ activatedAt: row.activated_at,
157
+ firstSeenAt: row.first_seen_at,
158
+ worker: row.worker,
159
+ };
160
+ }
161
+ }
162
+ return buildAgentSlaEscalationReason({
163
+ jobType: input.jobType,
164
+ sla: input.sla,
165
+ activation,
166
+ now,
167
+ recovery: input.recovery,
168
+ });
169
+ }
@@ -0,0 +1,172 @@
1
+ // DEFECT-CLASS GUARD (PR #881 review, thread r4214170285): `agentSlaJobType` is the marker that
2
+ // selects the agent-SLA escalation builder in the four recorder workers (`pr.persist-escalation`,
3
+ // `pr.record-feature-escalation`, `pr.record-trial-merge`, `pr.conformance-record`). It is set
4
+ // ONLY by the BPMN SLA arm (an interrupting timer boundary's route), never by the agent — but the
5
+ // c8ctl harness HOISTS the agent's result-JSON keys into the SAME process-variable namespace the
6
+ // recorders read it from. So a NORMAL (non-SLA) agent result that happens to carry an
7
+ // `agentSlaJobType` key would wrongly select the SLA builder and replace a genuine question or
8
+ // audit summary with a timeout diagnosis.
9
+ //
10
+ // This is the exact defect class PR #864 (review r4181010853) already closed for the boolean
11
+ // SLA-control flags `agentSlaElapsed` / `preserveConformance` — by pinning each to a LITERAL on
12
+ // every recorder task's ioMapping so hoisted agent output can never select the internal control
13
+ // path. `agentSlaJobType` was left unpinned. This guard closes that gap: every NORMAL-path recorder
14
+ // (one that does NOT set the marker to a real job-type literal) must pin `agentSlaJobType` to the
15
+ // `=null` literal, so a hoisted agent-supplied value is overridden to null on the normal path.
16
+ // SLA-arm recorders legitimately pin it to a job-type literal (e.g. `="senior:feature"`); those
17
+ // are the trusted source and are NOT required to pin null.
18
+ //
19
+ // The guard scans the deployed corpus and fails if any recorder task reads `agentSlaJobType`
20
+ // without pinning it to a literal (null on the normal path, a job-type on an SLA arm) — so a
21
+ // newly-added or edited recorder can never re-open the hole.
22
+ //
23
+ // EXCEPTION — merge-loop's SHARED SLA-sink recorders (PR #881 review, adversarial round 3): unlike
24
+ // the other processes, merge-loop has NO dedicated `*-sla` recorder task. Its fix-ci / rebase
25
+ // interrupting-SLA arms (`end_ci_sla` → "senior:fix-ci", `end_reb_sla` → "senior:rebase") and its
26
+ // NON-SLA escalation arms CONVERGE on the SAME two recorders (`merge-esc-attempt`,
27
+ // `merge-esc-conflict`). So those two recorders MUST READ `agentSlaJobType` from process scope
28
+ // (set by the trusted SLA end event) — a blanket `=null` input pin there would clobber the
29
+ // legitimate SLA job type and silently regress the issue-#879 never-started-vs-hung diagnosis. The
30
+ // hoisted-agent-output hole is closed on these shared sinks a DIFFERENT way: every path that can
31
+ // carry a hoisted (spoofable) value CLEARS `agentSlaJobType` to null via an OUTPUT mapping before
32
+ // it reaches the recorder — at the loop head (`arm-merge`, scrubbing any stale/cross-iteration
33
+ // value), on the two non-SLA escalate end events that bypass the loop head (`end_ci_esc`,
34
+ // `end_reb_blocked`), and on the CI reconcile re-check exit (`end_ci_recheck`, which re-enters the
35
+ // merge wait via `gw-merge-entry` without passing through `arm-merge`). Only the trusted SLA arms
36
+ // leave it set. This guard therefore (a) requires the shared sinks to NOT pin, and (b) requires
37
+ // those clears + the SLA-arm literals to exist.
38
+ import { readFileSync } from "node:fs";
39
+ import { test } from "node:test";
40
+ import { dirname, join } from "node:path";
41
+ import { fileURLToPath } from "node:url";
42
+ import { assert } from "#test-assert";
43
+
44
+ const PROCESSES_DIR = join(dirname(fileURLToPath(import.meta.url)), "../resources/processes");
45
+ const PROCESS_FILES = [
46
+ "convergence-loop.bpmn",
47
+ "merge-loop.bpmn",
48
+ "implement-cell.bpmn",
49
+ "merge-cell.bpmn",
50
+ "plan-fanout.bpmn",
51
+ "retro.bpmn",
52
+ ];
53
+
54
+ // The recorder job types that read `agentSlaJobType` to select the SLA builder.
55
+ const RECORDER_TYPES = [
56
+ "pr.persist-escalation",
57
+ "pr.record-feature-escalation",
58
+ "pr.record-trial-merge",
59
+ "pr.conformance-record",
60
+ ];
61
+
62
+ const SERVICE_TASK = /<bpmn:serviceTask\b[\s\S]*?<\/bpmn:serviceTask>/g;
63
+ // A trusted pin is a LITERAL input: `=null` (normal path) or `="..."`/`=&#34;...&#34;` (SLA arm's
64
+ // job-type). A process-variable REFERENCE (e.g. `=agentSlaJobType`) is NOT a pin — it would read
65
+ // hoisted agent output, which is exactly the hole being closed.
66
+ const LITERAL_PIN =
67
+ /<zeebe:input\s+source="=(?:null|&#[0-9]+;[^"]*&#[0-9]+;|"[^"]*")"\s+target="agentSlaJobType"\s*\/>/;
68
+
69
+ // Any INPUT mapping on `agentSlaJobType` (literal OR reference) — used to assert the shared SLA-sink
70
+ // recorders carry NONE (they must read the SLA arm's value from process scope, not shadow it).
71
+ const ANY_INPUT_PIN = /<zeebe:input\s+source="=[^"]*"\s+target="agentSlaJobType"\s*\/>/;
72
+ // A `=null` OUTPUT clear (writes process scope) — the fail-closed scrub on merge-loop's spoofable
73
+ // non-SLA paths.
74
+ const NULL_OUTPUT_CLEAR = /<zeebe:output\s+source="=null"\s+target="agentSlaJobType"\s*\/>/;
75
+
76
+ // merge-loop's two SHARED SLA-sink recorders: both SLA and non-SLA arms converge on them, so they
77
+ // MUST read `agentSlaJobType` from scope and MUST NOT pin it (see header).
78
+ const SHARED_SLA_SINKS = new Set(["merge-esc-attempt", "merge-esc-conflict"]);
79
+
80
+ /** Extract the XML block for a BPMN element by id (any element tag). */
81
+ function elementBlock(xml: string, id: string): string | undefined {
82
+ const re = new RegExp(`<bpmn:(\\w+)\\b[^>]*\\bid="${id}"[\\s\\S]*?</bpmn:\\1>`);
83
+ return xml.match(re)?.[0];
84
+ }
85
+
86
+ test("DEFECT-CLASS GUARD: every SLA-builder recorder pins agentSlaJobType to a literal (agent output cannot select the SLA path)", () => {
87
+ let recorderCount = 0;
88
+ for (const file of PROCESS_FILES) {
89
+ const xml = readFileSync(join(PROCESSES_DIR, file), "utf8");
90
+ for (const [block] of xml.matchAll(SERVICE_TASK)) {
91
+ if (!RECORDER_TYPES.some((t) => block.includes(`type="${t}"`))) continue;
92
+ const id = block.match(/<bpmn:serviceTask\b[^>]*\bid="([^"]*)"/)?.[1] ?? "(unknown)";
93
+ // The shared SLA-sink recorders read the marker from scope — they must NOT pin it, else a
94
+ // `=null` pin would clobber the trusted SLA job type (issue #879 regression). They are
95
+ // protected by the OUTPUT clears asserted below, not by an input pin.
96
+ if (SHARED_SLA_SINKS.has(id)) {
97
+ assert(
98
+ !ANY_INPUT_PIN.test(block),
99
+ `${file}: shared SLA-sink recorder "${id}" must NOT pin agentSlaJobType — both the SLA ` +
100
+ `and non-SLA arms converge on it, so it must READ the SLA end event's value from ` +
101
+ `process scope. A =null pin here clobbers the trusted SLA job type and regresses the ` +
102
+ `never-started-vs-hung diagnosis (PR #881 adversarial review). Remove the pin; the ` +
103
+ `hole is closed by the =null OUTPUT clears on arm-merge / end_ci_esc / end_reb_blocked.`,
104
+ );
105
+ continue;
106
+ }
107
+ recorderCount++;
108
+ assert(
109
+ LITERAL_PIN.test(block),
110
+ `${file}: recorder task "${id}" reads agentSlaJobType but does not pin it to a literal ` +
111
+ `(=null on the normal path, a job-type literal on an SLA arm). A hoisted agent result ` +
112
+ `carrying agentSlaJobType would wrongly select the SLA escalation builder (PR #881 review ` +
113
+ `r4214170285). Pin it: <zeebe:input source="=null" target="agentSlaJobType" /> on the ` +
114
+ `normal path.`,
115
+ );
116
+ }
117
+ }
118
+ // Sanity: the guard is not vacuously green — the corpus really has these recorders (the
119
+ // normal-path recorders + the SLA arms across the six processes).
120
+ assert(recorderCount >= 11, `expected >=11 SLA-builder recorder tasks, found ${recorderCount}`);
121
+ });
122
+
123
+ test("DEFECT-CLASS GUARD: merge-loop's shared SLA sinks are fed a trusted agentSlaJobType (SLA arms set it; spoofable non-SLA paths clear it)", () => {
124
+ const xml = readFileSync(join(PROCESSES_DIR, "merge-loop.bpmn"), "utf8");
125
+
126
+ // The two shared sinks really exist and read from scope (no input pin) — guards against the file
127
+ // being renamed/removed and silently passing.
128
+ for (const id of SHARED_SLA_SINKS) {
129
+ const block = elementBlock(xml, id);
130
+ assert(block, `merge-loop.bpmn: expected shared SLA-sink recorder "${id}" to exist`);
131
+ assert(
132
+ !ANY_INPUT_PIN.test(block),
133
+ `merge-loop.bpmn: "${id}" must read agentSlaJobType from scope, not pin it`,
134
+ );
135
+ }
136
+
137
+ // The trusted SLA arms SET the job-type literal (this is the value the shared sinks must receive).
138
+ for (const [id, literal] of [
139
+ ["end_ci_sla", "senior:fix-ci"],
140
+ ["end_reb_sla", "senior:rebase"],
141
+ ] as const) {
142
+ const block = elementBlock(xml, id);
143
+ assert(block, `merge-loop.bpmn: expected SLA arm "${id}" to exist`);
144
+ const re = new RegExp(
145
+ `<zeebe:output\\s+source="=&#34;${literal.replace(/[-/\\^$*+?.()|[\]{}]/g, "\\$&")}&#34;"\\s+target="agentSlaJobType"\\s*/>`,
146
+ );
147
+ assert(
148
+ re.test(block),
149
+ `merge-loop.bpmn: SLA arm "${id}" must set agentSlaJobType to the "${literal}" literal via ` +
150
+ `an output mapping (the trusted source the shared sink reads).`,
151
+ );
152
+ }
153
+
154
+ // Every spoofable path that can carry a hoisted agentSlaJobType into a shared sink CLEARS it to
155
+ // null via output BEFORE the recorder: the loop head (stale/cross-iteration), the two non-SLA
156
+ // escalate end events that bypass the loop head (same-iteration agent spoof), and the CI
157
+ // reconcile re-check exit (`end_ci_recheck`), which re-enters the merge wait via `gw-merge-entry`
158
+ // — bypassing `arm-merge` — so a fix-ci result with status:"blocked"/pushed:false carrying a
159
+ // hoisted marker would otherwise reach `merge-esc-conflict` on a later draft/budget-exhausted
160
+ // merge with no SLA timer having fired (PR #881 review, "Previously missed" r5451284362).
161
+ for (const id of ["arm-merge", "end_ci_esc", "end_reb_blocked", "end_ci_recheck"]) {
162
+ const block = elementBlock(xml, id);
163
+ assert(block, `merge-loop.bpmn: expected "${id}" to exist`);
164
+ assert(
165
+ NULL_OUTPUT_CLEAR.test(block),
166
+ `merge-loop.bpmn: "${id}" must clear agentSlaJobType to null via ` +
167
+ `<zeebe:output source="=null" target="agentSlaJobType" /> so a hoisted (spoofable) agent ` +
168
+ `value can never reach the shared SLA-sink recorder and fabricate a never-started/hung ` +
169
+ `SLA diagnosis (PR #881 adversarial review).`,
170
+ );
171
+ }
172
+ });
@@ -28,7 +28,7 @@
28
28
  // Plus the terminate semantics the refactor had to preserve: `MergeAbandoned`
29
29
  // terminates the whole instance (it stays at the root, not inside a sub-process).
30
30
  import { after, test } from "node:test";
31
- import { assert, assertStringIncludes } from "#test-assert";
31
+ import { assert, assertEquals, assertStringIncludes } from "#test-assert";
32
32
  import { readFileSync } from "node:fs";
33
33
  import {
34
34
  assertThatInstance,
@@ -562,6 +562,10 @@ test("the fix-ci agent SLA interrupts the sub-process and escalates", async () =
562
562
  await assertThatUserTask(engine, { instance: byProcessId("merge-loop"), elementId: "wait-merge-answer" }).isCreated();
563
563
  assertThatInstance(engine, byProcessId("merge-loop")).hasCompletedElements("merge-esc-attempt");
564
564
  assertStringIncludes(String(escalation(engine).question ?? ""), "time budget (SLA)", "the SLA escalation must name the SLA trigger");
565
+ // #879 regression: the trusted fix-ci SLA arm must deliver its job-type marker to the shared
566
+ // persist-escalation sink so the worker builds the never-started-vs-hung diagnosis — a =null
567
+ // input pin on the shared recorder silently clobbered this (PR #881 adversarial review).
568
+ assertEquals(escalation(engine).agentSlaJobType, "senior:fix-ci", "the fix-ci SLA arm must reach the recorder with its job-type marker");
565
569
  });
566
570
 
567
571
  // ---------------------------------------------------------------------------
@@ -613,8 +617,49 @@ test("the rebase agent SLA interrupts the sub-process and escalates as not-merge
613
617
  await engine.advanceTime(AGENT_SLA_MS + 1);
614
618
  await assertThatUserTask(engine, { instance: byProcessId("merge-loop"), elementId: "wait-merge-answer" }).isCreated();
615
619
  assertThatInstance(engine, byProcessId("merge-loop")).hasCompletedElements("merge-esc-conflict");
620
+ // #879 regression: the trusted rebase SLA arm must deliver its job-type marker to the shared sink.
621
+ assertEquals(escalation(engine).agentSlaJobType, "senior:rebase", "the rebase SLA arm must reach the recorder with its job-type marker");
616
622
  });
617
623
 
624
+ test("a non-SLA escalation carrying a spoofed agentSlaJobType is scrubbed before the recorder (#879 fail-closed)", async () => {
625
+ // The c8ctl harness hoists an agent's result-JSON keys into process scope, so a NON-SLA rebase
626
+ // agent that returns a bogus `agentSlaJobType` must NOT be able to fabricate an SLA diagnosis on
627
+ // the shared persist-escalation sink. The loop head / non-SLA escalate end events clear it.
628
+ const engine = await boot({ responses: { "senior:rebase": { status: "blocked", agentSlaJobType: "senior:evil" } } });
629
+ await driveToRebase(engine);
630
+ await assertThatUserTask(engine, { instance: byProcessId("merge-loop"), elementId: "wait-merge-answer" }).isCreated();
631
+ assertThatInstance(engine, byProcessId("merge-loop")).hasCompletedElements("merge-esc-attempt");
632
+ const spoofed = escalation(engine).agentSlaJobType;
633
+ assert(spoofed === null || spoofed === undefined, `a spoofed agentSlaJobType must be scrubbed, got ${JSON.stringify(spoofed)}`);
634
+ });
635
+
636
+ test("a spoofed agentSlaJobType surviving the CI-reconcile re-check route is scrubbed before a later merge-esc-conflict (#881 Previously-missed)", async () => {
637
+ // The CI reconcile route (fix-ci blocked/pushed:false → ci-reconcile → end_ci_recheck →
638
+ // gw-merge-entry) re-enters the merge wait WITHOUT passing through the loop head `arm-merge` or
639
+ // the non-SLA escalate clears. A hoisted (spoofable) agentSlaJobType from that fix-ci result
640
+ // would otherwise survive into a later draft / budget-exhausted merge and fabricate an SLA
641
+ // never-started/hung diagnosis on the shared merge-esc-conflict sink with NO SLA timer having
642
+ // fired. `end_ci_recheck` must clear it (PR #881 review, "Previously missed" r5451284362).
643
+ const engine = await boot({
644
+ responses: { "senior:fix-ci": { status: "blocked", pushed: false, agentSlaJobType: "senior:evil" } },
645
+ });
646
+ await driveToCiFix(engine); // blocked/pushed:false → ci-reconcile → end_ci_recheck → re-check mergeable
647
+ assertThatInstance(engine, byProcessId("merge-loop"))
648
+ .isActive()
649
+ .hasActiveElement("wait-mergeable")
650
+ .hasCompletedElements("ci-reconcile")
651
+ .hasVariable("ciBlockedReconciled", true);
652
+ assert(!completedElementIds(engine).has("merge-esc-conflict"), "the reconcile pass must not escalate yet");
653
+ // A later merge-ready reports the PR as draft → f_m_mDraft → merge-esc-conflict reads the marker
654
+ // directly (the fix-ci agent does NOT run again on this route).
655
+ await engine.publishMessage({ name: "merge-ready", correlationKey: "pr-1", variables: { mergeState: "draft" } });
656
+ await assertThatUserTask(engine, { instance: byProcessId("merge-loop"), elementId: "wait-merge-answer" }).isCreated();
657
+ assertThatInstance(engine, byProcessId("merge-loop")).hasCompletedElements("merge-esc-conflict");
658
+ const spoofed = escalation(engine).agentSlaJobType;
659
+ assert(spoofed === null || spoofed === undefined, `the reconcile-route spoof must be scrubbed, got ${JSON.stringify(spoofed)}`);
660
+ // The human gets the genuine draft question, not a fabricated SLA diagnosis.
661
+ assertStringIncludes(String(escalation(engine).question ?? ""), "draft", "the escalation must be the genuine draft question, not an SLA diagnosis");
662
+ });
618
663
  // ---------------------------------------------------------------------------
619
664
  // Escalation user task (mergeEscalationUserTask #256, mergeEscalationQuestion #329/#454)
620
665
  // ---------------------------------------------------------------------------
package/app/service.ts CHANGED
@@ -10,6 +10,7 @@
10
10
  import type { DataLayer, EngineClient } from "@nanobpm/urban";
11
11
  import { ABANDONED_STATUS, abandonUrl, mintAbandonToken, renderAbandonBrief } from "./abandon.ts";
12
12
  import { matchAdjudication, prAdjudications, resetAdjudications } from "./adjudications.ts";
13
+ import { AGENT_SLA_JOB_TYPES, recordAgentJobObservation } from "./agentActivationLedger.ts";
13
14
  import { completeEscalationAutoApplied, escalationFormId } from "./agentCompletion.ts";
14
15
  import { AGENT_SLA_TIMEOUT } from "./agentSla.ts";
15
16
  import {
@@ -1733,6 +1734,98 @@ interface JobSearchItem {
1733
1734
  /** The job's `zeebe:taskDefinition` type — read to defensively re-filter the wire `$in` type
1734
1735
  * filter so a non-agent job can never masquerade as a leasing convergence agent. */
1735
1736
  type?: string;
1737
+ /** The owning process instance — keyed into the activation ledger (issue #879). */
1738
+ processInstanceKey?: string;
1739
+ /** The engine job key — the unique id of THIS job attempt. The activation ledger uses it to tell one
1740
+ * attempt of a retrying agent task from the next (issue #879 review). */
1741
+ jobKey?: string;
1742
+ }
1743
+
1744
+ /** One agent-activation ledger poll pass (issue #879). Unlike {@link pollJobActivationImpl} (which
1745
+ * tracks only the review-round for the pages grid), this records — for EVERY bounded agent job type —
1746
+ * whether a worker ever picked the job up, into the durable `agent_job_activations` ledger. When an
1747
+ * agent-task SLA boundary later cancels a job, the job's engine state is gone, so the escalation
1748
+ * recorder cannot ask the engine whether it was ever activated; this ledger, captured WHILE the job is
1749
+ * live, is what lets the escalation distinguish a NEVER-ACTIVATED (queue-starved) job from an
1750
+ * ACTIVATED-but-hung one.
1751
+ *
1752
+ * It reads the same activation signal as the review-round pass: the Camunda-8 `/v2/jobs/search` wire
1753
+ * API collapses Activated -> CREATED (no ACTIVATED enum), so a leased job is recognised by its
1754
+ * `worker`/`deadline` fields. One paged search across all bounded agent types per pass upserts the
1755
+ * ledger by (processInstanceKey, type), keyed to the current attempt by `jobKey`. Best-effort: a
1756
+ * transport failure leaves the ledger untouched and the next pass retries. */
1757
+ async function pollAgentActivations(
1758
+ data: DataLayer,
1759
+ restAddress: string,
1760
+ engineToken: string | undefined,
1761
+ ) {
1762
+ const base = restAddress.replace(/\/+$/, "");
1763
+ const headers: Record<string, string> = { "content-type": "application/json" };
1764
+ if (engineToken) headers.authorization = `Bearer ${engineToken}`;
1765
+ await pollAgentActivationsImpl(data, base, headers);
1766
+ }
1767
+
1768
+ /** Testable core of {@link pollAgentActivations}: query the engine for every CREATED bounded-agent job
1769
+ * and upsert the activation ledger. Split out so tests can drive it with a stubbed `fetch`. */
1770
+ export async function pollAgentActivationsImpl(
1771
+ data: DataLayer,
1772
+ base: string,
1773
+ headers: Record<string, string>,
1774
+ ) {
1775
+ // Page through EVERY matching open job: a single first-page read would, under fleet saturation, let
1776
+ // long-lived jobs occupy the page on every pass while later jobs time out with their activation never
1777
+ // recorded — exactly when the never-started/hung distinction matters most. Mirror the paging contract
1778
+ // the other engine sweeps use (`page: { from, limit }`, stop on a short page). (#881 review)
1779
+ const ts = now();
1780
+ const limit = 200;
1781
+ let from = 0;
1782
+ for (let guard = 0; guard < 1000; guard++) {
1783
+ let items: JobSearchItem[] = [];
1784
+ try {
1785
+ const res = await fetch(`${base}/jobs/search`, {
1786
+ method: "POST",
1787
+ headers,
1788
+ body: JSON.stringify({
1789
+ filter: {
1790
+ type: { $in: [...AGENT_SLA_JOB_TYPES] },
1791
+ // A merely-created (queued) AND an activated-but-not-yet-completed job both project to
1792
+ // CREATED on the wire (Camunda has no ACTIVATED enum) — so this one filter captures both,
1793
+ // and the leasing `worker` field discriminates them at upsert time.
1794
+ state: "CREATED",
1795
+ },
1796
+ page: { from, limit },
1797
+ }),
1798
+ });
1799
+ if (!res.ok) return; // engine unhappy → keep the ledger as-is, retry next pass
1800
+ // biome-ignore lint/plugin: runtime/framework contract boundary for external data shape
1801
+ const body = (await res.json()) as { items?: JobSearchItem[] };
1802
+ items = body.items ?? [];
1803
+ } catch (err) {
1804
+ console.error(`[poller] agent-activation search: ${err}`);
1805
+ return;
1806
+ }
1807
+ for (const j of items) {
1808
+ const jobType = j.type ?? "";
1809
+ const pik = j.processInstanceKey ?? "";
1810
+ // Defensively re-filter the wire `$in` type filter and require a process instance to key on.
1811
+ if (pik.length === 0 || !AGENT_SLA_JOB_TYPES.includes(jobType)) continue;
1812
+ const worker = typeof j.worker === "string" && j.worker.length > 0 ? j.worker : null;
1813
+ const jobKey = typeof j.jobKey === "string" && j.jobKey.length > 0 ? j.jobKey : null;
1814
+ try {
1815
+ await recordAgentJobObservation(data, {
1816
+ processInstanceKey: pik,
1817
+ jobType,
1818
+ worker,
1819
+ jobKey,
1820
+ now: ts,
1821
+ });
1822
+ } catch (err) {
1823
+ console.error(`[poller] agent-activation upsert ${pik}:${jobType}: ${err}`);
1824
+ }
1825
+ }
1826
+ if (items.length < limit) break; // last page
1827
+ from += items.length;
1828
+ }
1736
1829
  }
1737
1830
 
1738
1831
  /** One job-activation poll pass. The `converging` status means the process is parked at a
@@ -3717,6 +3810,7 @@ export async function pollOnce(
3717
3810
  await pollWaveGatesImpl(data, engine, token, base, headers);
3718
3811
  await pollCapabilityGatesImpl(data, engine, base, headers);
3719
3812
  await pollJobActivation(data, engineRest.restAddress, engineRest.token);
3813
+ await pollAgentActivations(data, engineRest.restAddress, engineRest.token);
3720
3814
  await pollIncidents(data, engineRest.restAddress, engineRest.token);
3721
3815
  }
3722
3816
  }
@@ -0,0 +1,50 @@
1
+ -- 119_agent_job_activations.sql — issue #879: distinguish a NEVER-ACTIVATED (queue-starved) agent
2
+ -- job from an ACTIVATED-but-hung one when an agent-task SLA boundary fires.
3
+ --
4
+ -- The agent-task SLA timer (`<bpmn:timeDuration>=agentSlaTimeout`) starts when the service task is
5
+ -- ENTERED, so it measures QUEUE time + RUN time. When no worker can take the job type (fleet
6
+ -- saturation / no capacity), the job sits CREATED (never leased) for the whole budget and the
7
+ -- boundary fires — yet the escalation reported it as "hung or looping", telling the human to "check
8
+ -- its progress" when there is none. (The reported case: a `senior:feature` job queued 2h, never
9
+ -- activated, then cancelled by its SLA boundary.)
10
+ --
11
+ -- The engine tracks JobState::Created vs ::Activated, but the Camunda-8 `/v2/jobs/search` wire API
12
+ -- collapses Activated -> CREATED (no ACTIVATED enum value); activation is read off the `worker` +
13
+ -- `deadline` fields — an activated job carries a leasing worker's name and a lock deadline, a
14
+ -- merely-queued one carries neither (same signal `pull_requests.active_worker`/`lease_until` use for
15
+ -- the review-round, migration 005). This ledger generalises that observation to EVERY bounded agent
16
+ -- job type (not just the review-round) and PERSISTS it, so when the SLA boundary cancels the job the
17
+ -- escalation recorder can still tell whether a worker ever picked it up.
18
+ --
19
+ -- The poller (`pollAgentActivations`, app/service.ts) upserts one row per (process_instance_key,
20
+ -- job_type) each pass: `first_seen_at` on first observation (job CREATED), and `activated_at` +
21
+ -- `worker` the first time it observes a leasing worker. The SLA-escalation recorders read it by the
22
+ -- same key (the agent job and its escalation recorder share one process instance) to word the
23
+ -- escalation accurately — "never started (queue starvation)" vs "started but exceeded its budget" —
24
+ -- including the job type and how long it waited.
25
+ --
26
+ -- `job_key` holds the engine job key of the CURRENT attempt so the row tracks that attempt, not a
27
+ -- stale earlier one: the same (process_instance_key, job_type) runs MULTIPLE times within one instance
28
+ -- when its cell retries (implement-cell `ic_reImplement`, merge-cell `mc_rerun`, merge-loop fix-ci /
29
+ -- rebase reloops). Keyed only by (instance, job_type) a sticky first-activation would make a later
30
+ -- queued retry read the EARLIER attempt's worker and misdiagnose it as started/hung. When the poller
31
+ -- observes a different `job_key` for the pair it RESETS the row to the new attempt (first_seen_at /
32
+ -- activated_at / worker), so the escalation always reflects the attempt the SLA boundary cancelled.
33
+ --
34
+ -- Keyed by a surrogate TEXT id (`<process_instance_key>:<job_type>`) so the single-pk `data.table`
35
+ -- accessor can upsert it; the (process_instance_key, job_type) pair is unique per id. FK-free by
36
+ -- design — the owning instance lives in the durable ENGINE store, not app.db, so there is no app-tier
37
+ -- parent row to reference (mirrors `worker_durable_resume`/`worker_harness_protocol`). EXPAND
38
+ -- (additive) phase: one new table; nothing is dropped or renamed. The runner wraps each file in its
39
+ -- own transaction, so this file must NOT contain BEGIN/COMMIT.
40
+ CREATE TABLE IF NOT EXISTS agent_job_activations (
41
+ id TEXT PRIMARY KEY, -- surrogate: `<process_instance_key>:<job_type>`
42
+ process_instance_key TEXT NOT NULL, -- the engine instance the agent job (and its SLA recorder) runs in
43
+ job_type TEXT NOT NULL, -- the agent job's `<zeebe:taskDefinition>` type (e.g. senior:feature)
44
+ first_seen_at TEXT NOT NULL, -- ISO ts the poller first observed the job (CREATED/queued)
45
+ activated_at TEXT, -- ISO ts first observed ACTIVATED (leasing worker seen); NULL = never started
46
+ worker TEXT, -- the leasing worker's name at activation; NULL until activated
47
+ job_key TEXT, -- the engine job key of the CURRENT attempt; a new key RESETS the row (retry isolation)
48
+ updated_at TEXT NOT NULL
49
+ );
50
+ CREATE INDEX IF NOT EXISTS ix_agent_job_activations_instance ON agent_job_activations (process_instance_key);