@nanobpm/nano-workforce 0.201.1 → 0.201.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +6 -0
- package/app/agentActivationLedger.test.ts +109 -0
- package/app/agentActivationLedger.ts +144 -0
- package/app/agentActivationsPoll.test.ts +180 -0
- package/app/agentSlaEscalation.test.ts +137 -0
- package/app/agentSlaEscalation.ts +169 -0
- package/app/agentSlaMarkerGuard.test.ts +172 -0
- package/app/mergeLoopBehaviour.test.ts +46 -1
- package/app/service.ts +94 -0
- package/db/migrations/119_agent_job_activations.sql +50 -0
- package/e2e/agent-sla-boundary.e2e.ts +51 -8
- package/package.json +1 -1
- package/resources/processes/convergence-loop.bpmn +10 -0
- package/resources/processes/implement-cell.bpmn +4 -1
- package/resources/processes/merge-cell.bpmn +4 -1
- package/resources/processes/merge-loop.bpmn +7 -0
- package/resources/processes/plan-fanout.bpmn +5 -0
- package/resources/processes/retro.bpmn +5 -2
- package/test/derivation-parity/flows.ts +5 -3
- package/workers/conformance-record/worker.ts +15 -1
- package/workers/persist-escalation/worker.ts +22 -3
- package/workers/record-feature-escalation/worker.ts +28 -12
- package/workers/record-trial-merge/worker.ts +16 -1
|
@@ -0,0 +1,169 @@
|
|
|
1
|
+
// Agent-task SLA-escalation reason builder (issue #879) — the single canonical, pure text builder for
|
|
2
|
+
// the human-facing escalation raised when an agent service task's interrupting SLA timer boundary
|
|
3
|
+
// fires. It distinguishes the two outcomes the old wording conflated:
|
|
4
|
+
//
|
|
5
|
+
// • NEVER OBSERVED ACTIVATED — no worker was ever observed to pick the job up (almost certainly
|
|
6
|
+
// queue starvation / no capacity for the job type). The job sat CREATED for ~the whole SLA budget
|
|
7
|
+
// and was cancelled by the boundary. Telling the human it is "hung or looping" and to "check its
|
|
8
|
+
// progress" is a false diagnosis — the likely fix is CAPACITY, not intervention. The wording is
|
|
9
|
+
// hedged (a worker could lease it in the final moment before the boundary fires, between activation
|
|
10
|
+
// polls, unrecorded), so it reports the likely cause without over-claiming certainty.
|
|
11
|
+
// • STARTED BUT EXCEEDED — a worker leased the job (an agent ran) but did not complete within the
|
|
12
|
+
// budget — it may genuinely be hung or looping, and intervening / checking its progress is apt.
|
|
13
|
+
//
|
|
14
|
+
// The distinction is read from the durable activation ledger (`app/agentActivationLedger.ts`), which
|
|
15
|
+
// the poller populates while the job is live. Every escalation names the JOB TYPE and HOW LONG it
|
|
16
|
+
// waited, per the issue's acceptance criteria.
|
|
17
|
+
//
|
|
18
|
+
// Derivation over duplication: this is the ONE builder every agent-SLA recorder (implement-cell,
|
|
19
|
+
// merge-cell, retro) routes through, so the never-started/hung wording can never drift between sites.
|
|
20
|
+
// The pure `buildAgentSlaEscalationReason` is exhaustively testable; `agentSlaEscalationQuestion` is
|
|
21
|
+
// the thin async wrapper that reads the ledger and calls it.
|
|
22
|
+
import type { DataLayer } from "@nanobpm/urban";
|
|
23
|
+
import { type AgentJobActivation, getAgentJobActivation } from "./agentActivationLedger.ts";
|
|
24
|
+
import { isoDurationToMs } from "./reviewWait.ts";
|
|
25
|
+
|
|
26
|
+
/** The activation facts the builder needs (a slice of the ledger row), or `null` when the poller never
|
|
27
|
+
* observed the job (activation unknown). */
|
|
28
|
+
export interface AgentSlaActivation {
|
|
29
|
+
/** ISO ts a leasing worker was first observed; `null`/absent ⇒ never activated. */
|
|
30
|
+
activatedAt?: string | null;
|
|
31
|
+
/** ISO ts the job was first observed queued. */
|
|
32
|
+
firstSeenAt?: string | null;
|
|
33
|
+
/** The leasing worker's name, when known. */
|
|
34
|
+
worker?: string | null;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
export interface AgentSlaEscalationInput {
|
|
38
|
+
/** The agent job type that timed out, e.g. `senior:feature`. Named in the reason. */
|
|
39
|
+
jobType: string;
|
|
40
|
+
/** The SLA budget (ISO-8601 duration) the boundary armed, e.g. `PT2H`. */
|
|
41
|
+
sla: string;
|
|
42
|
+
/** The activation record for this job, or `null` when the poller never observed it. */
|
|
43
|
+
activation: AgentSlaActivation | null;
|
|
44
|
+
/** The escalation moment (ISO ts) — the elapsed wait is measured against it. */
|
|
45
|
+
now: string;
|
|
46
|
+
/** How the human resumes after answering — appended verbatim so each caller keeps its own
|
|
47
|
+
* routing hint (e.g. "answer to resume … or abandon it"). */
|
|
48
|
+
recovery: string;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/** Humanise a millisecond span to a compact, human-facing string (e.g. `2h 5m`, `45m`, `30s`). A
|
|
52
|
+
* non-positive/NaN span renders `less than a minute`. */
|
|
53
|
+
export function humanizeMs(ms: number): string {
|
|
54
|
+
if (!Number.isFinite(ms) || ms <= 0) return "less than a minute";
|
|
55
|
+
const totalMinutes = Math.floor(ms / 60_000);
|
|
56
|
+
if (totalMinutes < 1) return `${Math.max(1, Math.round(ms / 1000))}s`;
|
|
57
|
+
const days = Math.floor(totalMinutes / 1440);
|
|
58
|
+
const hours = Math.floor((totalMinutes % 1440) / 60);
|
|
59
|
+
const minutes = totalMinutes % 60;
|
|
60
|
+
const parts: string[] = [];
|
|
61
|
+
if (days > 0) parts.push(`${days}d`);
|
|
62
|
+
if (hours > 0) parts.push(`${hours}h`);
|
|
63
|
+
if (minutes > 0) parts.push(`${minutes}m`);
|
|
64
|
+
return parts.join(" ") || "less than a minute";
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/** Humanise an ISO-8601 duration (the SLA budget) to the same compact form, via the canonical parser. */
|
|
68
|
+
export function humanizeIsoDuration(iso: string): string {
|
|
69
|
+
return humanizeMs(isoDurationToMs(iso, iso));
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
const elapsedMs = (fromIso: string | null | undefined, toIso: string): number | null => {
|
|
73
|
+
if (!fromIso) return null;
|
|
74
|
+
const from = Date.parse(fromIso);
|
|
75
|
+
const to = Date.parse(toIso);
|
|
76
|
+
if (!Number.isFinite(from) || !Number.isFinite(to)) return null;
|
|
77
|
+
return to - from;
|
|
78
|
+
};
|
|
79
|
+
|
|
80
|
+
/** Build the accurate, self-contained agent-SLA escalation reason. Pure — distinguishes never-started
|
|
81
|
+
* (queue starvation) from started-but-hung, names the job type and how long it waited, then appends
|
|
82
|
+
* the caller's recovery hint. */
|
|
83
|
+
export function buildAgentSlaEscalationReason(input: AgentSlaEscalationInput): string {
|
|
84
|
+
const { jobType, sla, activation, now, recovery } = input;
|
|
85
|
+
const slaHuman = humanizeIsoDuration(sla);
|
|
86
|
+
const activatedAt = activation?.activatedAt ?? null;
|
|
87
|
+
|
|
88
|
+
if (activatedAt) {
|
|
89
|
+
// STARTED BUT EXCEEDED: a worker leased the job, so it may genuinely be hung or looping.
|
|
90
|
+
const queueMs = elapsedMs(activation?.firstSeenAt, activatedAt);
|
|
91
|
+
const queuedClause = queueMs !== null && queueMs > 0
|
|
92
|
+
? ` (after waiting ~${humanizeMs(queueMs)} in the queue)`
|
|
93
|
+
: "";
|
|
94
|
+
const ranMs = elapsedMs(activatedAt, now);
|
|
95
|
+
const ranClause = ranMs !== null && ranMs > 0 ? ` and then ran for ~${humanizeMs(ranMs)}` : "";
|
|
96
|
+
const workerClause = activation?.worker ? ` (worker "${activation.worker}")` : "";
|
|
97
|
+
return (
|
|
98
|
+
`The ${jobType} agent started${queuedClause}${workerClause}${ranClause} but exceeded its time budget ` +
|
|
99
|
+
`(SLA ${sla} ≈ ${slaHuman}) without completing — it may be hung or looping. ${recovery}`
|
|
100
|
+
);
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
// Measure the wait: from first-seen to now when the poller observed the queued job, else the full
|
|
104
|
+
// SLA budget (the boundary fired exactly one budget after the task was entered).
|
|
105
|
+
const waitedMs = elapsedMs(activation?.firstSeenAt, now);
|
|
106
|
+
const waited = waitedMs !== null && waitedMs > 0 ? humanizeMs(waitedMs) : slaHuman;
|
|
107
|
+
|
|
108
|
+
if (activation === null) {
|
|
109
|
+
// ACTIVATION UNKNOWN: the poller never observed the job (e.g. the activation poll was unavailable).
|
|
110
|
+
// Report the ambiguity honestly rather than guessing "hung" or "never started".
|
|
111
|
+
return (
|
|
112
|
+
`The ${jobType} agent did not complete within its time budget (SLA ${sla} ≈ ${slaHuman}), and no ` +
|
|
113
|
+
`worker-activation was ever recorded for it, so we cannot confirm whether a worker ever picked it ` +
|
|
114
|
+
`up — it may have been starved of capacity for ${jobType}, or started and hung. ${recovery}`
|
|
115
|
+
);
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
// NEVER OBSERVED ACTIVATED: seen queued, never seen leased. This is the best-effort observation of
|
|
119
|
+
// queue starvation — hedged, because a worker could in principle lease the job in the final moment
|
|
120
|
+
// before the boundary cancels it, between activation polls, and that would not be recorded. We report
|
|
121
|
+
// the likely diagnosis (and its capacity fix) without over-claiming certainty.
|
|
122
|
+
return (
|
|
123
|
+
`No worker was ever observed to pick up the ${jobType} agent's job for its entire SLA budget ` +
|
|
124
|
+
`(SLA ${sla} ≈ ${slaHuman}): it was seen queued for ~${waited}, then the SLA boundary cancelled it. ` +
|
|
125
|
+
`This is almost certainly queue starvation (no capacity for ${jobType}) rather than a hung or ` +
|
|
126
|
+
`looping agent, so the likely fix is to add capacity for ${jobType} (enrol a worker that serves it, ` +
|
|
127
|
+
`or reduce concurrent demand) rather than to check a non-existent agent's progress. (If a worker ` +
|
|
128
|
+
`leased it in the final moment before the boundary fired, between activation polls, that would not ` +
|
|
129
|
+
`have been recorded here.) ${recovery}`
|
|
130
|
+
);
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
/** Read the activation ledger for this job and build the escalation reason — the thin async wrapper the
|
|
134
|
+
* recorder workers call. `processInstanceKey` is the instance the agent job AND its SLA recorder share,
|
|
135
|
+
* so the lookup is by (instance, jobType). An absent ledger row ⇒ activation unknown. */
|
|
136
|
+
export async function agentSlaEscalationQuestion(
|
|
137
|
+
data: DataLayer,
|
|
138
|
+
input: {
|
|
139
|
+
processInstanceKey: string;
|
|
140
|
+
jobType: string;
|
|
141
|
+
sla: string;
|
|
142
|
+
recovery: string;
|
|
143
|
+
now?: string;
|
|
144
|
+
},
|
|
145
|
+
): Promise<string> {
|
|
146
|
+
const now = input.now ?? new Date().toISOString();
|
|
147
|
+
let activation: AgentSlaActivation | null = null;
|
|
148
|
+
if (input.processInstanceKey && input.processInstanceKey.length > 0) {
|
|
149
|
+
const row: AgentJobActivation | undefined = await getAgentJobActivation(
|
|
150
|
+
data,
|
|
151
|
+
input.processInstanceKey,
|
|
152
|
+
input.jobType,
|
|
153
|
+
);
|
|
154
|
+
if (row) {
|
|
155
|
+
activation = {
|
|
156
|
+
activatedAt: row.activated_at,
|
|
157
|
+
firstSeenAt: row.first_seen_at,
|
|
158
|
+
worker: row.worker,
|
|
159
|
+
};
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
return buildAgentSlaEscalationReason({
|
|
163
|
+
jobType: input.jobType,
|
|
164
|
+
sla: input.sla,
|
|
165
|
+
activation,
|
|
166
|
+
now,
|
|
167
|
+
recovery: input.recovery,
|
|
168
|
+
});
|
|
169
|
+
}
|
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
// DEFECT-CLASS GUARD (PR #881 review, thread r4214170285): `agentSlaJobType` is the marker that
|
|
2
|
+
// selects the agent-SLA escalation builder in the four recorder workers (`pr.persist-escalation`,
|
|
3
|
+
// `pr.record-feature-escalation`, `pr.record-trial-merge`, `pr.conformance-record`). It is set
|
|
4
|
+
// ONLY by the BPMN SLA arm (an interrupting timer boundary's route), never by the agent — but the
|
|
5
|
+
// c8ctl harness HOISTS the agent's result-JSON keys into the SAME process-variable namespace the
|
|
6
|
+
// recorders read it from. So a NORMAL (non-SLA) agent result that happens to carry an
|
|
7
|
+
// `agentSlaJobType` key would wrongly select the SLA builder and replace a genuine question or
|
|
8
|
+
// audit summary with a timeout diagnosis.
|
|
9
|
+
//
|
|
10
|
+
// This is the exact defect class PR #864 (review r4181010853) already closed for the boolean
|
|
11
|
+
// SLA-control flags `agentSlaElapsed` / `preserveConformance` — by pinning each to a LITERAL on
|
|
12
|
+
// every recorder task's ioMapping so hoisted agent output can never select the internal control
|
|
13
|
+
// path. `agentSlaJobType` was left unpinned. This guard closes that gap: every NORMAL-path recorder
|
|
14
|
+
// (one that does NOT set the marker to a real job-type literal) must pin `agentSlaJobType` to the
|
|
15
|
+
// `=null` literal, so a hoisted agent-supplied value is overridden to null on the normal path.
|
|
16
|
+
// SLA-arm recorders legitimately pin it to a job-type literal (e.g. `="senior:feature"`); those
|
|
17
|
+
// are the trusted source and are NOT required to pin null.
|
|
18
|
+
//
|
|
19
|
+
// The guard scans the deployed corpus and fails if any recorder task reads `agentSlaJobType`
|
|
20
|
+
// without pinning it to a literal (null on the normal path, a job-type on an SLA arm) — so a
|
|
21
|
+
// newly-added or edited recorder can never re-open the hole.
|
|
22
|
+
//
|
|
23
|
+
// EXCEPTION — merge-loop's SHARED SLA-sink recorders (PR #881 review, adversarial round 3): unlike
|
|
24
|
+
// the other processes, merge-loop has NO dedicated `*-sla` recorder task. Its fix-ci / rebase
|
|
25
|
+
// interrupting-SLA arms (`end_ci_sla` → "senior:fix-ci", `end_reb_sla` → "senior:rebase") and its
|
|
26
|
+
// NON-SLA escalation arms CONVERGE on the SAME two recorders (`merge-esc-attempt`,
|
|
27
|
+
// `merge-esc-conflict`). So those two recorders MUST READ `agentSlaJobType` from process scope
|
|
28
|
+
// (set by the trusted SLA end event) — a blanket `=null` input pin there would clobber the
|
|
29
|
+
// legitimate SLA job type and silently regress the issue-#879 never-started-vs-hung diagnosis. The
|
|
30
|
+
// hoisted-agent-output hole is closed on these shared sinks a DIFFERENT way: every path that can
|
|
31
|
+
// carry a hoisted (spoofable) value CLEARS `agentSlaJobType` to null via an OUTPUT mapping before
|
|
32
|
+
// it reaches the recorder — at the loop head (`arm-merge`, scrubbing any stale/cross-iteration
|
|
33
|
+
// value), on the two non-SLA escalate end events that bypass the loop head (`end_ci_esc`,
|
|
34
|
+
// `end_reb_blocked`), and on the CI reconcile re-check exit (`end_ci_recheck`, which re-enters the
|
|
35
|
+
// merge wait via `gw-merge-entry` without passing through `arm-merge`). Only the trusted SLA arms
|
|
36
|
+
// leave it set. This guard therefore (a) requires the shared sinks to NOT pin, and (b) requires
|
|
37
|
+
// those clears + the SLA-arm literals to exist.
|
|
38
|
+
import { readFileSync } from "node:fs";
|
|
39
|
+
import { test } from "node:test";
|
|
40
|
+
import { dirname, join } from "node:path";
|
|
41
|
+
import { fileURLToPath } from "node:url";
|
|
42
|
+
import { assert } from "#test-assert";
|
|
43
|
+
|
|
44
|
+
const PROCESSES_DIR = join(dirname(fileURLToPath(import.meta.url)), "../resources/processes");
|
|
45
|
+
const PROCESS_FILES = [
|
|
46
|
+
"convergence-loop.bpmn",
|
|
47
|
+
"merge-loop.bpmn",
|
|
48
|
+
"implement-cell.bpmn",
|
|
49
|
+
"merge-cell.bpmn",
|
|
50
|
+
"plan-fanout.bpmn",
|
|
51
|
+
"retro.bpmn",
|
|
52
|
+
];
|
|
53
|
+
|
|
54
|
+
// The recorder job types that read `agentSlaJobType` to select the SLA builder.
|
|
55
|
+
const RECORDER_TYPES = [
|
|
56
|
+
"pr.persist-escalation",
|
|
57
|
+
"pr.record-feature-escalation",
|
|
58
|
+
"pr.record-trial-merge",
|
|
59
|
+
"pr.conformance-record",
|
|
60
|
+
];
|
|
61
|
+
|
|
62
|
+
const SERVICE_TASK = /<bpmn:serviceTask\b[\s\S]*?<\/bpmn:serviceTask>/g;
|
|
63
|
+
// A trusted pin is a LITERAL input: `=null` (normal path) or `="..."`/`="..."` (SLA arm's
|
|
64
|
+
// job-type). A process-variable REFERENCE (e.g. `=agentSlaJobType`) is NOT a pin — it would read
|
|
65
|
+
// hoisted agent output, which is exactly the hole being closed.
|
|
66
|
+
const LITERAL_PIN =
|
|
67
|
+
/<zeebe:input\s+source="=(?:null|&#[0-9]+;[^"]*&#[0-9]+;|"[^"]*")"\s+target="agentSlaJobType"\s*\/>/;
|
|
68
|
+
|
|
69
|
+
// Any INPUT mapping on `agentSlaJobType` (literal OR reference) — used to assert the shared SLA-sink
|
|
70
|
+
// recorders carry NONE (they must read the SLA arm's value from process scope, not shadow it).
|
|
71
|
+
const ANY_INPUT_PIN = /<zeebe:input\s+source="=[^"]*"\s+target="agentSlaJobType"\s*\/>/;
|
|
72
|
+
// A `=null` OUTPUT clear (writes process scope) — the fail-closed scrub on merge-loop's spoofable
|
|
73
|
+
// non-SLA paths.
|
|
74
|
+
const NULL_OUTPUT_CLEAR = /<zeebe:output\s+source="=null"\s+target="agentSlaJobType"\s*\/>/;
|
|
75
|
+
|
|
76
|
+
// merge-loop's two SHARED SLA-sink recorders: both SLA and non-SLA arms converge on them, so they
|
|
77
|
+
// MUST read `agentSlaJobType` from scope and MUST NOT pin it (see header).
|
|
78
|
+
const SHARED_SLA_SINKS = new Set(["merge-esc-attempt", "merge-esc-conflict"]);
|
|
79
|
+
|
|
80
|
+
/** Extract the XML block for a BPMN element by id (any element tag). */
|
|
81
|
+
function elementBlock(xml: string, id: string): string | undefined {
|
|
82
|
+
const re = new RegExp(`<bpmn:(\\w+)\\b[^>]*\\bid="${id}"[\\s\\S]*?</bpmn:\\1>`);
|
|
83
|
+
return xml.match(re)?.[0];
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
test("DEFECT-CLASS GUARD: every SLA-builder recorder pins agentSlaJobType to a literal (agent output cannot select the SLA path)", () => {
|
|
87
|
+
let recorderCount = 0;
|
|
88
|
+
for (const file of PROCESS_FILES) {
|
|
89
|
+
const xml = readFileSync(join(PROCESSES_DIR, file), "utf8");
|
|
90
|
+
for (const [block] of xml.matchAll(SERVICE_TASK)) {
|
|
91
|
+
if (!RECORDER_TYPES.some((t) => block.includes(`type="${t}"`))) continue;
|
|
92
|
+
const id = block.match(/<bpmn:serviceTask\b[^>]*\bid="([^"]*)"/)?.[1] ?? "(unknown)";
|
|
93
|
+
// The shared SLA-sink recorders read the marker from scope — they must NOT pin it, else a
|
|
94
|
+
// `=null` pin would clobber the trusted SLA job type (issue #879 regression). They are
|
|
95
|
+
// protected by the OUTPUT clears asserted below, not by an input pin.
|
|
96
|
+
if (SHARED_SLA_SINKS.has(id)) {
|
|
97
|
+
assert(
|
|
98
|
+
!ANY_INPUT_PIN.test(block),
|
|
99
|
+
`${file}: shared SLA-sink recorder "${id}" must NOT pin agentSlaJobType — both the SLA ` +
|
|
100
|
+
`and non-SLA arms converge on it, so it must READ the SLA end event's value from ` +
|
|
101
|
+
`process scope. A =null pin here clobbers the trusted SLA job type and regresses the ` +
|
|
102
|
+
`never-started-vs-hung diagnosis (PR #881 adversarial review). Remove the pin; the ` +
|
|
103
|
+
`hole is closed by the =null OUTPUT clears on arm-merge / end_ci_esc / end_reb_blocked.`,
|
|
104
|
+
);
|
|
105
|
+
continue;
|
|
106
|
+
}
|
|
107
|
+
recorderCount++;
|
|
108
|
+
assert(
|
|
109
|
+
LITERAL_PIN.test(block),
|
|
110
|
+
`${file}: recorder task "${id}" reads agentSlaJobType but does not pin it to a literal ` +
|
|
111
|
+
`(=null on the normal path, a job-type literal on an SLA arm). A hoisted agent result ` +
|
|
112
|
+
`carrying agentSlaJobType would wrongly select the SLA escalation builder (PR #881 review ` +
|
|
113
|
+
`r4214170285). Pin it: <zeebe:input source="=null" target="agentSlaJobType" /> on the ` +
|
|
114
|
+
`normal path.`,
|
|
115
|
+
);
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
// Sanity: the guard is not vacuously green — the corpus really has these recorders (the
|
|
119
|
+
// normal-path recorders + the SLA arms across the six processes).
|
|
120
|
+
assert(recorderCount >= 11, `expected >=11 SLA-builder recorder tasks, found ${recorderCount}`);
|
|
121
|
+
});
|
|
122
|
+
|
|
123
|
+
test("DEFECT-CLASS GUARD: merge-loop's shared SLA sinks are fed a trusted agentSlaJobType (SLA arms set it; spoofable non-SLA paths clear it)", () => {
|
|
124
|
+
const xml = readFileSync(join(PROCESSES_DIR, "merge-loop.bpmn"), "utf8");
|
|
125
|
+
|
|
126
|
+
// The two shared sinks really exist and read from scope (no input pin) — guards against the file
|
|
127
|
+
// being renamed/removed and silently passing.
|
|
128
|
+
for (const id of SHARED_SLA_SINKS) {
|
|
129
|
+
const block = elementBlock(xml, id);
|
|
130
|
+
assert(block, `merge-loop.bpmn: expected shared SLA-sink recorder "${id}" to exist`);
|
|
131
|
+
assert(
|
|
132
|
+
!ANY_INPUT_PIN.test(block),
|
|
133
|
+
`merge-loop.bpmn: "${id}" must read agentSlaJobType from scope, not pin it`,
|
|
134
|
+
);
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
// The trusted SLA arms SET the job-type literal (this is the value the shared sinks must receive).
|
|
138
|
+
for (const [id, literal] of [
|
|
139
|
+
["end_ci_sla", "senior:fix-ci"],
|
|
140
|
+
["end_reb_sla", "senior:rebase"],
|
|
141
|
+
] as const) {
|
|
142
|
+
const block = elementBlock(xml, id);
|
|
143
|
+
assert(block, `merge-loop.bpmn: expected SLA arm "${id}" to exist`);
|
|
144
|
+
const re = new RegExp(
|
|
145
|
+
`<zeebe:output\\s+source="="${literal.replace(/[-/\\^$*+?.()|[\]{}]/g, "\\$&")}""\\s+target="agentSlaJobType"\\s*/>`,
|
|
146
|
+
);
|
|
147
|
+
assert(
|
|
148
|
+
re.test(block),
|
|
149
|
+
`merge-loop.bpmn: SLA arm "${id}" must set agentSlaJobType to the "${literal}" literal via ` +
|
|
150
|
+
`an output mapping (the trusted source the shared sink reads).`,
|
|
151
|
+
);
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
// Every spoofable path that can carry a hoisted agentSlaJobType into a shared sink CLEARS it to
|
|
155
|
+
// null via output BEFORE the recorder: the loop head (stale/cross-iteration), the two non-SLA
|
|
156
|
+
// escalate end events that bypass the loop head (same-iteration agent spoof), and the CI
|
|
157
|
+
// reconcile re-check exit (`end_ci_recheck`), which re-enters the merge wait via `gw-merge-entry`
|
|
158
|
+
// — bypassing `arm-merge` — so a fix-ci result with status:"blocked"/pushed:false carrying a
|
|
159
|
+
// hoisted marker would otherwise reach `merge-esc-conflict` on a later draft/budget-exhausted
|
|
160
|
+
// merge with no SLA timer having fired (PR #881 review, "Previously missed" r5451284362).
|
|
161
|
+
for (const id of ["arm-merge", "end_ci_esc", "end_reb_blocked", "end_ci_recheck"]) {
|
|
162
|
+
const block = elementBlock(xml, id);
|
|
163
|
+
assert(block, `merge-loop.bpmn: expected "${id}" to exist`);
|
|
164
|
+
assert(
|
|
165
|
+
NULL_OUTPUT_CLEAR.test(block),
|
|
166
|
+
`merge-loop.bpmn: "${id}" must clear agentSlaJobType to null via ` +
|
|
167
|
+
`<zeebe:output source="=null" target="agentSlaJobType" /> so a hoisted (spoofable) agent ` +
|
|
168
|
+
`value can never reach the shared SLA-sink recorder and fabricate a never-started/hung ` +
|
|
169
|
+
`SLA diagnosis (PR #881 adversarial review).`,
|
|
170
|
+
);
|
|
171
|
+
}
|
|
172
|
+
});
|
|
@@ -28,7 +28,7 @@
|
|
|
28
28
|
// Plus the terminate semantics the refactor had to preserve: `MergeAbandoned`
|
|
29
29
|
// terminates the whole instance (it stays at the root, not inside a sub-process).
|
|
30
30
|
import { after, test } from "node:test";
|
|
31
|
-
import { assert, assertStringIncludes } from "#test-assert";
|
|
31
|
+
import { assert, assertEquals, assertStringIncludes } from "#test-assert";
|
|
32
32
|
import { readFileSync } from "node:fs";
|
|
33
33
|
import {
|
|
34
34
|
assertThatInstance,
|
|
@@ -562,6 +562,10 @@ test("the fix-ci agent SLA interrupts the sub-process and escalates", async () =
|
|
|
562
562
|
await assertThatUserTask(engine, { instance: byProcessId("merge-loop"), elementId: "wait-merge-answer" }).isCreated();
|
|
563
563
|
assertThatInstance(engine, byProcessId("merge-loop")).hasCompletedElements("merge-esc-attempt");
|
|
564
564
|
assertStringIncludes(String(escalation(engine).question ?? ""), "time budget (SLA)", "the SLA escalation must name the SLA trigger");
|
|
565
|
+
// #879 regression: the trusted fix-ci SLA arm must deliver its job-type marker to the shared
|
|
566
|
+
// persist-escalation sink so the worker builds the never-started-vs-hung diagnosis — a =null
|
|
567
|
+
// input pin on the shared recorder silently clobbered this (PR #881 adversarial review).
|
|
568
|
+
assertEquals(escalation(engine).agentSlaJobType, "senior:fix-ci", "the fix-ci SLA arm must reach the recorder with its job-type marker");
|
|
565
569
|
});
|
|
566
570
|
|
|
567
571
|
// ---------------------------------------------------------------------------
|
|
@@ -613,8 +617,49 @@ test("the rebase agent SLA interrupts the sub-process and escalates as not-merge
|
|
|
613
617
|
await engine.advanceTime(AGENT_SLA_MS + 1);
|
|
614
618
|
await assertThatUserTask(engine, { instance: byProcessId("merge-loop"), elementId: "wait-merge-answer" }).isCreated();
|
|
615
619
|
assertThatInstance(engine, byProcessId("merge-loop")).hasCompletedElements("merge-esc-conflict");
|
|
620
|
+
// #879 regression: the trusted rebase SLA arm must deliver its job-type marker to the shared sink.
|
|
621
|
+
assertEquals(escalation(engine).agentSlaJobType, "senior:rebase", "the rebase SLA arm must reach the recorder with its job-type marker");
|
|
616
622
|
});
|
|
617
623
|
|
|
624
|
+
test("a non-SLA escalation carrying a spoofed agentSlaJobType is scrubbed before the recorder (#879 fail-closed)", async () => {
|
|
625
|
+
// The c8ctl harness hoists an agent's result-JSON keys into process scope, so a NON-SLA rebase
|
|
626
|
+
// agent that returns a bogus `agentSlaJobType` must NOT be able to fabricate an SLA diagnosis on
|
|
627
|
+
// the shared persist-escalation sink. The loop head / non-SLA escalate end events clear it.
|
|
628
|
+
const engine = await boot({ responses: { "senior:rebase": { status: "blocked", agentSlaJobType: "senior:evil" } } });
|
|
629
|
+
await driveToRebase(engine);
|
|
630
|
+
await assertThatUserTask(engine, { instance: byProcessId("merge-loop"), elementId: "wait-merge-answer" }).isCreated();
|
|
631
|
+
assertThatInstance(engine, byProcessId("merge-loop")).hasCompletedElements("merge-esc-attempt");
|
|
632
|
+
const spoofed = escalation(engine).agentSlaJobType;
|
|
633
|
+
assert(spoofed === null || spoofed === undefined, `a spoofed agentSlaJobType must be scrubbed, got ${JSON.stringify(spoofed)}`);
|
|
634
|
+
});
|
|
635
|
+
|
|
636
|
+
test("a spoofed agentSlaJobType surviving the CI-reconcile re-check route is scrubbed before a later merge-esc-conflict (#881 Previously-missed)", async () => {
|
|
637
|
+
// The CI reconcile route (fix-ci blocked/pushed:false → ci-reconcile → end_ci_recheck →
|
|
638
|
+
// gw-merge-entry) re-enters the merge wait WITHOUT passing through the loop head `arm-merge` or
|
|
639
|
+
// the non-SLA escalate clears. A hoisted (spoofable) agentSlaJobType from that fix-ci result
|
|
640
|
+
// would otherwise survive into a later draft / budget-exhausted merge and fabricate an SLA
|
|
641
|
+
// never-started/hung diagnosis on the shared merge-esc-conflict sink with NO SLA timer having
|
|
642
|
+
// fired. `end_ci_recheck` must clear it (PR #881 review, "Previously missed" r5451284362).
|
|
643
|
+
const engine = await boot({
|
|
644
|
+
responses: { "senior:fix-ci": { status: "blocked", pushed: false, agentSlaJobType: "senior:evil" } },
|
|
645
|
+
});
|
|
646
|
+
await driveToCiFix(engine); // blocked/pushed:false → ci-reconcile → end_ci_recheck → re-check mergeable
|
|
647
|
+
assertThatInstance(engine, byProcessId("merge-loop"))
|
|
648
|
+
.isActive()
|
|
649
|
+
.hasActiveElement("wait-mergeable")
|
|
650
|
+
.hasCompletedElements("ci-reconcile")
|
|
651
|
+
.hasVariable("ciBlockedReconciled", true);
|
|
652
|
+
assert(!completedElementIds(engine).has("merge-esc-conflict"), "the reconcile pass must not escalate yet");
|
|
653
|
+
// A later merge-ready reports the PR as draft → f_m_mDraft → merge-esc-conflict reads the marker
|
|
654
|
+
// directly (the fix-ci agent does NOT run again on this route).
|
|
655
|
+
await engine.publishMessage({ name: "merge-ready", correlationKey: "pr-1", variables: { mergeState: "draft" } });
|
|
656
|
+
await assertThatUserTask(engine, { instance: byProcessId("merge-loop"), elementId: "wait-merge-answer" }).isCreated();
|
|
657
|
+
assertThatInstance(engine, byProcessId("merge-loop")).hasCompletedElements("merge-esc-conflict");
|
|
658
|
+
const spoofed = escalation(engine).agentSlaJobType;
|
|
659
|
+
assert(spoofed === null || spoofed === undefined, `the reconcile-route spoof must be scrubbed, got ${JSON.stringify(spoofed)}`);
|
|
660
|
+
// The human gets the genuine draft question, not a fabricated SLA diagnosis.
|
|
661
|
+
assertStringIncludes(String(escalation(engine).question ?? ""), "draft", "the escalation must be the genuine draft question, not an SLA diagnosis");
|
|
662
|
+
});
|
|
618
663
|
// ---------------------------------------------------------------------------
|
|
619
664
|
// Escalation user task (mergeEscalationUserTask #256, mergeEscalationQuestion #329/#454)
|
|
620
665
|
// ---------------------------------------------------------------------------
|
package/app/service.ts
CHANGED
|
@@ -10,6 +10,7 @@
|
|
|
10
10
|
import type { DataLayer, EngineClient } from "@nanobpm/urban";
|
|
11
11
|
import { ABANDONED_STATUS, abandonUrl, mintAbandonToken, renderAbandonBrief } from "./abandon.ts";
|
|
12
12
|
import { matchAdjudication, prAdjudications, resetAdjudications } from "./adjudications.ts";
|
|
13
|
+
import { AGENT_SLA_JOB_TYPES, recordAgentJobObservation } from "./agentActivationLedger.ts";
|
|
13
14
|
import { completeEscalationAutoApplied, escalationFormId } from "./agentCompletion.ts";
|
|
14
15
|
import { AGENT_SLA_TIMEOUT } from "./agentSla.ts";
|
|
15
16
|
import {
|
|
@@ -1733,6 +1734,98 @@ interface JobSearchItem {
|
|
|
1733
1734
|
/** The job's `zeebe:taskDefinition` type — read to defensively re-filter the wire `$in` type
|
|
1734
1735
|
* filter so a non-agent job can never masquerade as a leasing convergence agent. */
|
|
1735
1736
|
type?: string;
|
|
1737
|
+
/** The owning process instance — keyed into the activation ledger (issue #879). */
|
|
1738
|
+
processInstanceKey?: string;
|
|
1739
|
+
/** The engine job key — the unique id of THIS job attempt. The activation ledger uses it to tell one
|
|
1740
|
+
* attempt of a retrying agent task from the next (issue #879 review). */
|
|
1741
|
+
jobKey?: string;
|
|
1742
|
+
}
|
|
1743
|
+
|
|
1744
|
+
/** One agent-activation ledger poll pass (issue #879). Unlike {@link pollJobActivationImpl} (which
|
|
1745
|
+
* tracks only the review-round for the pages grid), this records — for EVERY bounded agent job type —
|
|
1746
|
+
* whether a worker ever picked the job up, into the durable `agent_job_activations` ledger. When an
|
|
1747
|
+
* agent-task SLA boundary later cancels a job, the job's engine state is gone, so the escalation
|
|
1748
|
+
* recorder cannot ask the engine whether it was ever activated; this ledger, captured WHILE the job is
|
|
1749
|
+
* live, is what lets the escalation distinguish a NEVER-ACTIVATED (queue-starved) job from an
|
|
1750
|
+
* ACTIVATED-but-hung one.
|
|
1751
|
+
*
|
|
1752
|
+
* It reads the same activation signal as the review-round pass: the Camunda-8 `/v2/jobs/search` wire
|
|
1753
|
+
* API collapses Activated -> CREATED (no ACTIVATED enum), so a leased job is recognised by its
|
|
1754
|
+
* `worker`/`deadline` fields. One paged search across all bounded agent types per pass upserts the
|
|
1755
|
+
* ledger by (processInstanceKey, type), keyed to the current attempt by `jobKey`. Best-effort: a
|
|
1756
|
+
* transport failure leaves the ledger untouched and the next pass retries. */
|
|
1757
|
+
async function pollAgentActivations(
|
|
1758
|
+
data: DataLayer,
|
|
1759
|
+
restAddress: string,
|
|
1760
|
+
engineToken: string | undefined,
|
|
1761
|
+
) {
|
|
1762
|
+
const base = restAddress.replace(/\/+$/, "");
|
|
1763
|
+
const headers: Record<string, string> = { "content-type": "application/json" };
|
|
1764
|
+
if (engineToken) headers.authorization = `Bearer ${engineToken}`;
|
|
1765
|
+
await pollAgentActivationsImpl(data, base, headers);
|
|
1766
|
+
}
|
|
1767
|
+
|
|
1768
|
+
/** Testable core of {@link pollAgentActivations}: query the engine for every CREATED bounded-agent job
|
|
1769
|
+
* and upsert the activation ledger. Split out so tests can drive it with a stubbed `fetch`. */
|
|
1770
|
+
export async function pollAgentActivationsImpl(
|
|
1771
|
+
data: DataLayer,
|
|
1772
|
+
base: string,
|
|
1773
|
+
headers: Record<string, string>,
|
|
1774
|
+
) {
|
|
1775
|
+
// Page through EVERY matching open job: a single first-page read would, under fleet saturation, let
|
|
1776
|
+
// long-lived jobs occupy the page on every pass while later jobs time out with their activation never
|
|
1777
|
+
// recorded — exactly when the never-started/hung distinction matters most. Mirror the paging contract
|
|
1778
|
+
// the other engine sweeps use (`page: { from, limit }`, stop on a short page). (#881 review)
|
|
1779
|
+
const ts = now();
|
|
1780
|
+
const limit = 200;
|
|
1781
|
+
let from = 0;
|
|
1782
|
+
for (let guard = 0; guard < 1000; guard++) {
|
|
1783
|
+
let items: JobSearchItem[] = [];
|
|
1784
|
+
try {
|
|
1785
|
+
const res = await fetch(`${base}/jobs/search`, {
|
|
1786
|
+
method: "POST",
|
|
1787
|
+
headers,
|
|
1788
|
+
body: JSON.stringify({
|
|
1789
|
+
filter: {
|
|
1790
|
+
type: { $in: [...AGENT_SLA_JOB_TYPES] },
|
|
1791
|
+
// A merely-created (queued) AND an activated-but-not-yet-completed job both project to
|
|
1792
|
+
// CREATED on the wire (Camunda has no ACTIVATED enum) — so this one filter captures both,
|
|
1793
|
+
// and the leasing `worker` field discriminates them at upsert time.
|
|
1794
|
+
state: "CREATED",
|
|
1795
|
+
},
|
|
1796
|
+
page: { from, limit },
|
|
1797
|
+
}),
|
|
1798
|
+
});
|
|
1799
|
+
if (!res.ok) return; // engine unhappy → keep the ledger as-is, retry next pass
|
|
1800
|
+
// biome-ignore lint/plugin: runtime/framework contract boundary for external data shape
|
|
1801
|
+
const body = (await res.json()) as { items?: JobSearchItem[] };
|
|
1802
|
+
items = body.items ?? [];
|
|
1803
|
+
} catch (err) {
|
|
1804
|
+
console.error(`[poller] agent-activation search: ${err}`);
|
|
1805
|
+
return;
|
|
1806
|
+
}
|
|
1807
|
+
for (const j of items) {
|
|
1808
|
+
const jobType = j.type ?? "";
|
|
1809
|
+
const pik = j.processInstanceKey ?? "";
|
|
1810
|
+
// Defensively re-filter the wire `$in` type filter and require a process instance to key on.
|
|
1811
|
+
if (pik.length === 0 || !AGENT_SLA_JOB_TYPES.includes(jobType)) continue;
|
|
1812
|
+
const worker = typeof j.worker === "string" && j.worker.length > 0 ? j.worker : null;
|
|
1813
|
+
const jobKey = typeof j.jobKey === "string" && j.jobKey.length > 0 ? j.jobKey : null;
|
|
1814
|
+
try {
|
|
1815
|
+
await recordAgentJobObservation(data, {
|
|
1816
|
+
processInstanceKey: pik,
|
|
1817
|
+
jobType,
|
|
1818
|
+
worker,
|
|
1819
|
+
jobKey,
|
|
1820
|
+
now: ts,
|
|
1821
|
+
});
|
|
1822
|
+
} catch (err) {
|
|
1823
|
+
console.error(`[poller] agent-activation upsert ${pik}:${jobType}: ${err}`);
|
|
1824
|
+
}
|
|
1825
|
+
}
|
|
1826
|
+
if (items.length < limit) break; // last page
|
|
1827
|
+
from += items.length;
|
|
1828
|
+
}
|
|
1736
1829
|
}
|
|
1737
1830
|
|
|
1738
1831
|
/** One job-activation poll pass. The `converging` status means the process is parked at a
|
|
@@ -3717,6 +3810,7 @@ export async function pollOnce(
|
|
|
3717
3810
|
await pollWaveGatesImpl(data, engine, token, base, headers);
|
|
3718
3811
|
await pollCapabilityGatesImpl(data, engine, base, headers);
|
|
3719
3812
|
await pollJobActivation(data, engineRest.restAddress, engineRest.token);
|
|
3813
|
+
await pollAgentActivations(data, engineRest.restAddress, engineRest.token);
|
|
3720
3814
|
await pollIncidents(data, engineRest.restAddress, engineRest.token);
|
|
3721
3815
|
}
|
|
3722
3816
|
}
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
-- 119_agent_job_activations.sql — issue #879: distinguish a NEVER-ACTIVATED (queue-starved) agent
|
|
2
|
+
-- job from an ACTIVATED-but-hung one when an agent-task SLA boundary fires.
|
|
3
|
+
--
|
|
4
|
+
-- The agent-task SLA timer (`<bpmn:timeDuration>=agentSlaTimeout`) starts when the service task is
|
|
5
|
+
-- ENTERED, so it measures QUEUE time + RUN time. When no worker can take the job type (fleet
|
|
6
|
+
-- saturation / no capacity), the job sits CREATED (never leased) for the whole budget and the
|
|
7
|
+
-- boundary fires — yet the escalation reported it as "hung or looping", telling the human to "check
|
|
8
|
+
-- its progress" when there is none. (The reported case: a `senior:feature` job queued 2h, never
|
|
9
|
+
-- activated, then cancelled by its SLA boundary.)
|
|
10
|
+
--
|
|
11
|
+
-- The engine tracks JobState::Created vs ::Activated, but the Camunda-8 `/v2/jobs/search` wire API
|
|
12
|
+
-- collapses Activated -> CREATED (no ACTIVATED enum value); activation is read off the `worker` +
|
|
13
|
+
-- `deadline` fields — an activated job carries a leasing worker's name and a lock deadline, a
|
|
14
|
+
-- merely-queued one carries neither (same signal `pull_requests.active_worker`/`lease_until` use for
|
|
15
|
+
-- the review-round, migration 005). This ledger generalises that observation to EVERY bounded agent
|
|
16
|
+
-- job type (not just the review-round) and PERSISTS it, so when the SLA boundary cancels the job the
|
|
17
|
+
-- escalation recorder can still tell whether a worker ever picked it up.
|
|
18
|
+
--
|
|
19
|
+
-- The poller (`pollAgentActivations`, app/service.ts) upserts one row per (process_instance_key,
|
|
20
|
+
-- job_type) each pass: `first_seen_at` on first observation (job CREATED), and `activated_at` +
|
|
21
|
+
-- `worker` the first time it observes a leasing worker. The SLA-escalation recorders read it by the
|
|
22
|
+
-- same key (the agent job and its escalation recorder share one process instance) to word the
|
|
23
|
+
-- escalation accurately — "never started (queue starvation)" vs "started but exceeded its budget" —
|
|
24
|
+
-- including the job type and how long it waited.
|
|
25
|
+
--
|
|
26
|
+
-- `job_key` holds the engine job key of the CURRENT attempt so the row tracks that attempt, not a
|
|
27
|
+
-- stale earlier one: the same (process_instance_key, job_type) runs MULTIPLE times within one instance
|
|
28
|
+
-- when its cell retries (implement-cell `ic_reImplement`, merge-cell `mc_rerun`, merge-loop fix-ci /
|
|
29
|
+
-- rebase reloops). Keyed only by (instance, job_type) a sticky first-activation would make a later
|
|
30
|
+
-- queued retry read the EARLIER attempt's worker and misdiagnose it as started/hung. When the poller
|
|
31
|
+
-- observes a different `job_key` for the pair it RESETS the row to the new attempt (first_seen_at /
|
|
32
|
+
-- activated_at / worker), so the escalation always reflects the attempt the SLA boundary cancelled.
|
|
33
|
+
--
|
|
34
|
+
-- Keyed by a surrogate TEXT id (`<process_instance_key>:<job_type>`) so the single-pk `data.table`
|
|
35
|
+
-- accessor can upsert it; the (process_instance_key, job_type) pair is unique per id. FK-free by
|
|
36
|
+
-- design — the owning instance lives in the durable ENGINE store, not app.db, so there is no app-tier
|
|
37
|
+
-- parent row to reference (mirrors `worker_durable_resume`/`worker_harness_protocol`). EXPAND
|
|
38
|
+
-- (additive) phase: one new table; nothing is dropped or renamed. The runner wraps each file in its
|
|
39
|
+
-- own transaction, so this file must NOT contain BEGIN/COMMIT.
|
|
40
|
+
CREATE TABLE IF NOT EXISTS agent_job_activations (
|
|
41
|
+
id TEXT PRIMARY KEY, -- surrogate: `<process_instance_key>:<job_type>`
|
|
42
|
+
process_instance_key TEXT NOT NULL, -- the engine instance the agent job (and its SLA recorder) runs in
|
|
43
|
+
job_type TEXT NOT NULL, -- the agent job's `<zeebe:taskDefinition>` type (e.g. senior:feature)
|
|
44
|
+
first_seen_at TEXT NOT NULL, -- ISO ts the poller first observed the job (CREATED/queued)
|
|
45
|
+
activated_at TEXT, -- ISO ts first observed ACTIVATED (leasing worker seen); NULL = never started
|
|
46
|
+
worker TEXT, -- the leasing worker's name at activation; NULL until activated
|
|
47
|
+
job_key TEXT, -- the engine job key of the CURRENT attempt; a new key RESETS the row (retry isolation)
|
|
48
|
+
updated_at TEXT NOT NULL
|
|
49
|
+
);
|
|
50
|
+
CREATE INDEX IF NOT EXISTS ix_agent_job_activations_instance ON agent_job_activations (process_instance_key);
|