@nanobpm/nano-workforce 0.202.0 → 0.203.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +6 -0
- package/SPEC.md +48 -0
- package/app/admittedStaging.test.ts +71 -0
- package/app/agentActivationLedger.ts +5 -0
- package/app/agentCompletion.test.ts +33 -0
- package/app/agentCompletion.ts +108 -2
- package/app/agentic/vocab/agent-sla-boundary.test.ts +6 -1
- package/app/contracts.ts +23 -0
- package/app/deliveryGraphDispatch.ts +6 -0
- package/app/deliveryGraphReadModel.test.ts +2 -2
- package/app/deliveryGraphRun.ts +18 -0
- package/app/deliveryRunner.ts +9 -1
- package/app/escalationSteward.test.ts +452 -0
- package/app/escalationSteward.ts +537 -0
- package/app/escalationStewardPoll.test.ts +123 -0
- package/app/feature.test.ts +16 -0
- package/app/feature.ts +11 -0
- package/app/featureDelivery.test.ts +23 -0
- package/app/mcpToolSurface.ts +17 -1
- package/app/plan.ts +49 -6
- package/app/planLowering.test.ts +26 -1
- package/app/planLowering.ts +5 -0
- package/app/promotionPoll.test.ts +19 -0
- package/app/pullRequestReadModel.test.ts +1 -1
- package/app/service.test.ts +36 -0
- package/app/service.ts +139 -5
- package/db/migrations/120_escalation_auto_resolutions.sql +229 -0
- package/db/migrations/121_delivery_graph_read_model_auto_resolve.sql +54 -0
- package/db/migrations/122_pull_requests_read_model_auto_resolve.sql +56 -0
- package/db/migrations/123_escalation_auto_resolutions_selected_option.sql +47 -0
- package/db/migrations/124_admitted_epics_auto_resolve.sql +21 -0
- package/docs/agent-guide.md +69 -30
- package/docs/mcp-runbook.md +2 -0
- package/e2e/escalation-steward.e2e.ts +211 -0
- package/nano.app.json +10 -1
- package/openapi.yaml +365 -0
- package/operations/dispatchDeliveryGraph.ts +6 -1
- package/operations/listActivePrs.test.ts +60 -1
- package/operations/listActivePrs.ts +3 -3
- package/operations/listAutoResolutions.ts +83 -0
- package/operations/reviewAutoResolution.test.ts +137 -0
- package/operations/reviewAutoResolution.ts +42 -0
- package/operations/startConvergenceLoop.ts +5 -1
- package/operations/startEpicSet.admission.integration.test.ts +2 -2
- package/operations/startEpicSet.ts +8 -2
- package/operations/startFeature.ts +5 -0
- package/operations/startPlanFanout.ts +4 -1
- package/package.json +1 -1
- package/pages/_nav.json +12 -0
- package/pages/auto-resolutions.page.json +191 -0
- package/pages/board.page.json +17 -0
- package/pages/cockpit.page.json +17 -0
- package/pages/delivery-graph-detail.page.json +17 -0
- package/pages/delivery-graphs/staged.mount.js +17 -5
- package/pages/delivery-graphs.page.json +17 -0
- package/pages/epic-detail.page.json +17 -0
- package/pages/epic.page.json +19 -1
- package/pages/feature.page.json +19 -1
- package/pages/home.page.json +22 -0
- package/pages/lineage.page.json +17 -0
- package/pages/mcp.page.json +17 -0
- package/pages/overview.page.json +17 -0
- package/pages/tasks.page.json +17 -0
- package/pages/velocity.page.json +17 -0
- package/resources/processes/escalation-steward.bpmn +162 -0
- package/resources/prompts/escalation-steward.md +87 -0
- package/test/delivery-graphs-staged-embed.test.ts +13 -0
- package/test/derivation-parity/derivation-parity.test.ts +1 -0
- package/test/derivation-parity/flows.ts +12 -0
- package/workers/converge-feature/worker.ts +6 -1
- package/workers/delivery-connector/worker.test.ts +72 -1
- package/workers/delivery-connector/worker.ts +53 -2
- package/workers/escalation-steward-collect/worker.test.ts +180 -0
- package/workers/escalation-steward-collect/worker.ts +99 -0
- package/workers/escalation-steward-record/worker.test.ts +740 -0
- package/workers/escalation-steward-record/worker.ts +428 -0
- package/workers/record-wave/worker.ts +5 -1
|
@@ -0,0 +1,537 @@
|
|
|
1
|
+
// Opt-in escalation steward (issue #889) — the domain core for the timer-driven loop that reviews
|
|
2
|
+
// OPEN escalations on OPTED-IN work and submits an agent's recommended choice for each, keeping the
|
|
3
|
+
// fleet moving while every automatic decision stays reviewable by a human after the fact.
|
|
4
|
+
//
|
|
5
|
+
// This module owns the PURE/host-side pieces so they are unit-testable without the engine:
|
|
6
|
+
//
|
|
7
|
+
// • Eligibility — which open escalation user tasks the steward may touch: the task's owning work
|
|
8
|
+
// must have OPTED IN (`auto_resolve_escalations` on its subject row), the escalation must be past
|
|
9
|
+
// a short minimum age (so a human already answering wins), it must not be claimed by an in-flight
|
|
10
|
+
// steward run (a claim/lease so overlapping ticks never double-answer), and it must be a kind the
|
|
11
|
+
// steward is permitted to answer (never a human-only or irreversible-by-construction kind).
|
|
12
|
+
// • The claim/lease fence — a durable compare-and-swap on `escalation_auto_resolutions` so two
|
|
13
|
+
// overlapping steward ticks (or a steward tick racing a human) can never both act on one task.
|
|
14
|
+
// • The record — one `escalation_auto_resolutions` row per escalation the steward resolved or
|
|
15
|
+
// abstained from, so the Auto-resolutions view can render it and an operator can dismiss or
|
|
16
|
+
// intervene.
|
|
17
|
+
// • The loop guard + destructive-option abstention — pure functions the steward's collect step and
|
|
18
|
+
// the agent's record step share, so the "never auto-answer an irreversible/credential choice" and
|
|
19
|
+
// "stop re-answering a subject that keeps re-escalating" rules are enforced in ONE place.
|
|
20
|
+
//
|
|
21
|
+
// The engine-facing orchestration (the timer-start process, the collect/record service tasks, and the
|
|
22
|
+
// `senior:escalation-steward` agent job) lives in `resources/processes/escalation-steward.bpmn` and
|
|
23
|
+
// `workers/escalation-steward-*`; the actual ANSWER is submitted through the ONE canonical attributed
|
|
24
|
+
// door (`completeUserTaskAttributed`, app/agentCompletion.ts) with AGENT attribution — never recorded
|
|
25
|
+
// as a human decision (ADR 0046).
|
|
26
|
+
import type { DataLayer } from "@nanobpm/urban";
|
|
27
|
+
import { ESCALATION_TASK_ELEMENTS, snapshotUserTaskFormOptions } from "./agentCompletion.ts";
|
|
28
|
+
import { isUniqueConstraintFence } from "./dbFence.ts";
|
|
29
|
+
import { ESCALATION_FORM, GENERIC_HUMAN_FORM, isDeliveryHumanElement } from "./deliveryHuman.ts";
|
|
30
|
+
import type { UserTaskRow } from "./userTasks.ts";
|
|
31
|
+
|
|
32
|
+
/** The minimal shape of a subject row that carries the opt-in flag. The index signature lets the
|
|
33
|
+
* steward read each table's distinct PRIMARY-KEY column generically (see {@link optInBySubjectKey}). */
|
|
34
|
+
interface OptInSubjectRow {
|
|
35
|
+
process_key: string | null;
|
|
36
|
+
auto_resolve_escalations: number;
|
|
37
|
+
[column: string]: string | number | null;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
const now = () => new Date().toISOString();
|
|
41
|
+
|
|
42
|
+
/** Read a string field off a subject row by dynamic column name, returning `null` when it is absent
|
|
43
|
+
* or not a string. The row is widened to a string-indexed record via its own type (no `as` cast), so
|
|
44
|
+
* the generic per-table key read in {@link optInBySubjectKey} stays type-safe. */
|
|
45
|
+
function keyOf(row: OptInSubjectRow, column: string): string | null {
|
|
46
|
+
const record: Record<string, unknown> = row;
|
|
47
|
+
const v: unknown = record[column];
|
|
48
|
+
return typeof v === "string" && v !== "" ? v : null;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/** The `senior:escalation-steward` agent's job type (the dispatch verb the steward process emits). */
|
|
52
|
+
export const ESCALATION_STEWARD_JOB_TYPE = "senior:escalation-steward";
|
|
53
|
+
/** The steward process definition id (`resources/processes/escalation-steward.bpmn`). */
|
|
54
|
+
export const ESCALATION_STEWARD_PROCESS_ID = "escalation-steward";
|
|
55
|
+
|
|
56
|
+
/** How long an escalation must be open before the steward considers it — a grace window so a human
|
|
57
|
+
* who is ALREADY answering wins and the steward never snatches a task out from under an operator.
|
|
58
|
+
* Configurable (`NANO_ESCALATION_STEWARD_MIN_AGE_MS`); default 5 minutes per the issue. */
|
|
59
|
+
export const DEFAULT_MIN_AGE_MS = 5 * 60 * 1000;
|
|
60
|
+
|
|
61
|
+
/** How many times a single subject may be auto-resolved before the steward abstains and leaves the
|
|
62
|
+
* re-escalation for a human — repeated escalation means the automatic answer isn't working. */
|
|
63
|
+
export const LOOP_GUARD_MAX_RESOLUTIONS = 2;
|
|
64
|
+
|
|
65
|
+
/** The escalation user-task `elementId`s the steward must NEVER auto-answer, because the decision is
|
|
66
|
+
* irreversible-by-construction, a credential/permission grant, or a gate whose whole point is a human
|
|
67
|
+
* judgement. These are refused BEFORE the agent is even dispatched (defence-in-depth under the
|
|
68
|
+
* prompt's own abstain rule), so the steward can never be the path that abandons work, approves a
|
|
69
|
+
* merge, grants a permission, or overrides a readiness gate.
|
|
70
|
+
*
|
|
71
|
+
* This guard is INTERSECTED with the canonical agent-answerable allowlist (`ESCALATION_TASK_ELEMENTS`
|
|
72
|
+
* + the dynamic delivery-human predicate): an element is steward-eligible only if it is BOTH
|
|
73
|
+
* agent-answerable AND not in this guard set (see {@link isStewardAnswerableElement}). */
|
|
74
|
+
export const STEWARD_NEVER_RESOLVE_ELEMENTS: ReadonlySet<string> = new Set([
|
|
75
|
+
// Human-only merge approval (issue #826) — the fleet must never auto-approve a merge.
|
|
76
|
+
"merge-approval",
|
|
77
|
+
// ACP permission prompt (issue #559) — a credential/permission grant; always a human decision.
|
|
78
|
+
"acp-permission",
|
|
79
|
+
// Readiness/preflight gates (issue #674) — an agent must never auto-answer "is upstream ready?".
|
|
80
|
+
"readiness-escalation",
|
|
81
|
+
"readiness-escalation-pf",
|
|
82
|
+
// Human-only terminal acknowledgements — a blocked run / a conformance shortfall / an empty plan are
|
|
83
|
+
// exactly the "the automatic path produced nothing usable" cases a human must adjudicate.
|
|
84
|
+
"feature-blocked",
|
|
85
|
+
"conformance-escalation",
|
|
86
|
+
"empty-plan-escalation",
|
|
87
|
+
]);
|
|
88
|
+
|
|
89
|
+
/** An `escalation_auto_resolutions` row — the durable, reviewable record of one steward action. */
|
|
90
|
+
export interface EscalationAutoResolution {
|
|
91
|
+
id: number;
|
|
92
|
+
user_task_key: string;
|
|
93
|
+
escalation_kind: string | null;
|
|
94
|
+
element_id: string | null;
|
|
95
|
+
subject_type: string | null;
|
|
96
|
+
subject_key: string | null;
|
|
97
|
+
subject_url: string | null;
|
|
98
|
+
process_instance_key: string | null;
|
|
99
|
+
question: string | null;
|
|
100
|
+
/** JSON array snapshot of the form's option values/labels at resolution time. */
|
|
101
|
+
options: string | null;
|
|
102
|
+
/** `resolved` (the steward submitted an answer) | `abstained` (it deliberately left it for a human).
|
|
103
|
+
* `claimed` is the transient, non-final sentinel for an open claim/lease placeholder (never shown in
|
|
104
|
+
* the review surface); the finalise step overwrites it to `resolved`/`abstained`. */
|
|
105
|
+
outcome: "resolved" | "abstained" | "claimed";
|
|
106
|
+
/** JSON of the variables submitted; NULL when abstained. */
|
|
107
|
+
submitted_variables: string | null;
|
|
108
|
+
/** The option value the steward actually selected (derived host-side from the decision's own form
|
|
109
|
+
* variables at record time), so the review surface shows WHAT was chosen without decoding
|
|
110
|
+
* `submitted_variables`. NULL when abstained / not derivable. */
|
|
111
|
+
selected_option: string | null;
|
|
112
|
+
rationale: string | null;
|
|
113
|
+
/** JSON/markdown: what the agent checked, with links. */
|
|
114
|
+
evidence: string | null;
|
|
115
|
+
steward_instance_key: string | null;
|
|
116
|
+
agent_job_key: string | null;
|
|
117
|
+
worker: string | null;
|
|
118
|
+
model: string | null;
|
|
119
|
+
created_at: string;
|
|
120
|
+
review_state: "pending" | "dismissed" | "intervened";
|
|
121
|
+
reviewed_by: string | null;
|
|
122
|
+
reviewed_at: string | null;
|
|
123
|
+
review_note: string | null;
|
|
124
|
+
/** Claim/lease bookkeeping (not part of the review surface): which steward instance holds the claim
|
|
125
|
+
* and until when. NULL once the row is finalised (resolved/abstained recorded). */
|
|
126
|
+
claimed_by: string | null;
|
|
127
|
+
claimed_until: string | null;
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
export const escalationAutoResolutions = (data: DataLayer) =>
|
|
131
|
+
data.table<EscalationAutoResolution>("escalation_auto_resolutions", "id");
|
|
132
|
+
|
|
133
|
+
/** Whether an escalation element kind is one the steward is permitted to answer at all. The agent
|
|
134
|
+
* prompt ALSO carries an abstain rule; this is the host-side fail-closed guard so a mis-prompted or
|
|
135
|
+
* misrouted steward can never touch a human-only / irreversible / credential kind.
|
|
136
|
+
*
|
|
137
|
+
* This is an ALLOWLIST gated on the canonical agent-answerable surface, not a denylist: an element is
|
|
138
|
+
* steward-eligible only if it is BOTH agent-answerable (in `ESCALATION_TASK_ELEMENTS`, or a dynamic
|
|
139
|
+
* delivery-human node matched by the single-source `isDeliveryHumanElement` predicate — the same
|
|
140
|
+
* surface the completer resolves against) AND not in the explicit {@link STEWARD_NEVER_RESOLVE_ELEMENTS}
|
|
141
|
+
* safety guard. Gating on the allowlist (issue #889 review) means an arbitrary/unknown user task that
|
|
142
|
+
* merely escapes the NEVER_RESOLVE set is refused up front, instead of being claimed and dispatched
|
|
143
|
+
* only for the completer to refuse it — which would record a spurious abstention and permanently
|
|
144
|
+
* exclude that task from future steward ticks. */
|
|
145
|
+
export function isStewardAnswerableElement(elementId: string): boolean {
|
|
146
|
+
const agentAnswerable = ESCALATION_TASK_ELEMENTS.has(elementId) || isDeliveryHumanElement(elementId);
|
|
147
|
+
return agentAnswerable && !STEWARD_NEVER_RESOLVE_ELEMENTS.has(elementId);
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
/** The ONLY delivery-graph `human` forms a steward may auto-answer. A delivery-graph `human` node
|
|
151
|
+
* renders a DIFFERENT form per node (explicit → category → generic → agent-router,
|
|
152
|
+
* `app/deliveryHuman.ts` `resolveHumanForm`), and — unlike the fixed-form escalations — the agent
|
|
153
|
+
* completer (`completeEscalationAsAgent`) intentionally does NOT validate these dynamic forms (their
|
|
154
|
+
* typed-emit contract is bound, not field-validated). So ELIGIBILITY, not the completer, is the
|
|
155
|
+
* fail-closed fence for them. Only the typed-value/decision forms are something an agent can
|
|
156
|
+
* legitimately produce: the GENERIC typed-emit capture and the continue/retry service-node
|
|
157
|
+
* ESCALATION form. The bespoke CATEGORY forms are manual/out-of-band human work the steward cannot
|
|
158
|
+
* perform — `delivery-human-publish` is a manual OTP/OIDC credential publish, `delivery-human-ack` is
|
|
159
|
+
* a "do X out of band, then click done" acknowledgement — so admitting a delivery-human task on ANY
|
|
160
|
+
* other (or an absent/unknown) form would let a mis-prompted steward blind-complete a
|
|
161
|
+
* credential/manual task (issue #889 review). Hence an ALLOWLIST: fail closed for everything else. */
|
|
162
|
+
export const STEWARD_SAFE_DELIVERY_HUMAN_FORMS: ReadonlySet<string> = new Set([
|
|
163
|
+
GENERIC_HUMAN_FORM,
|
|
164
|
+
ESCALATION_FORM,
|
|
165
|
+
]);
|
|
166
|
+
|
|
167
|
+
/** Pure, form-aware fail-closed guard for the dynamic delivery-graph `human` task. A FIXED-form
|
|
168
|
+
* escalation (plan-review-decision, wait-answer, …) is governed by its element id and validated by
|
|
169
|
+
* the completer, so its form is NOT gated here (returns true). A DYNAMIC delivery-human element,
|
|
170
|
+
* whose form is only known at runtime and is unvalidated by the completer, is steward-answerable
|
|
171
|
+
* ONLY when its resolved `formKey` is a known-safe form ({@link STEWARD_SAFE_DELIVERY_HUMAN_FORMS});
|
|
172
|
+
* an absent/unknown/category form fails closed. */
|
|
173
|
+
export function isStewardAnswerableForm(elementId: string, formKey: string | null | undefined): boolean {
|
|
174
|
+
if (!isDeliveryHumanElement(elementId)) return true;
|
|
175
|
+
return formKey != null && STEWARD_SAFE_DELIVERY_HUMAN_FORMS.has(formKey);
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
/** Pure: is one open escalation user task ELIGIBLE for the steward right now? Requires:
|
|
179
|
+
* • the owning work opted in (`optedIn` — derived by the caller from the subject row);
|
|
180
|
+
* • the kind is steward-answerable ({@link isStewardAnswerableElement});
|
|
181
|
+
* • for a dynamic delivery-human task, its resolved form is steward-safe ({@link isStewardAnswerableForm});
|
|
182
|
+
* • the task is past the minimum age (`minAgeMs`) so a human already answering wins;
|
|
183
|
+
* • the task is not currently claimed by a live steward run (`claimedUserTaskKeys`).
|
|
184
|
+
* `at` is the current time (ms epoch); the task's `created_at` is its open-since timestamp. */
|
|
185
|
+
export function isEligibleEscalation(
|
|
186
|
+
row: Pick<UserTaskRow, "user_task_key" | "element_id" | "created_at" | "form_key">,
|
|
187
|
+
opts: { optedIn: boolean; minAgeMs: number; claimedUserTaskKeys: ReadonlySet<string>; at: number },
|
|
188
|
+
): boolean {
|
|
189
|
+
if (!opts.optedIn) return false;
|
|
190
|
+
if (!isStewardAnswerableElement(row.element_id)) return false;
|
|
191
|
+
if (!isStewardAnswerableForm(row.element_id, row.form_key)) return false;
|
|
192
|
+
if (opts.claimedUserTaskKeys.has(row.user_task_key)) return false;
|
|
193
|
+
const openedAt = Date.parse(row.created_at);
|
|
194
|
+
if (Number.isNaN(openedAt)) return false;
|
|
195
|
+
return opts.at - openedAt >= opts.minAgeMs;
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
/** Pure: the loop guard. Given the count of prior steward RESOLUTIONS for a subject, whether the
|
|
199
|
+
* steward must abstain on a fresh escalation for that same subject (it has already auto-answered
|
|
200
|
+
* `maxResolutions` times and the work escalated again — the automatic answer isn't working). */
|
|
201
|
+
export function loopGuardTripped(priorResolutions: number, maxResolutions: number = LOOP_GUARD_MAX_RESOLUTIONS): boolean {
|
|
202
|
+
return priorResolutions >= maxResolutions;
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
/** Pure: does choosing this option abandon/destroy work or merge past a check — an IRREVERSIBLE or
|
|
206
|
+
* destructive choice the steward must abstain from even when the kind is otherwise answerable? The
|
|
207
|
+
* agent is instructed to abstain on these; this guard lets the host ALSO refuse to record a
|
|
208
|
+
* destructive resolution an agent erroneously submitted (fail-closed). Matched case-insensitively
|
|
209
|
+
* against the option value. */
|
|
210
|
+
const DESTRUCTIVE_OPTION_VALUES = /^(abandon|abandon-run|delete|destroy|discard|force-merge|merge-admin|override-checks?)$/i;
|
|
211
|
+
export function isDestructiveOption(optionValue: string): boolean {
|
|
212
|
+
return DESTRUCTIVE_OPTION_VALUES.test(optionValue.trim());
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
/** The `trial-merge-decision` element id (plan-fanout.bpmn) — the D3 trial-merge integration gate. */
|
|
216
|
+
export const TRIAL_MERGE_DECISION_ELEMENT = "trial-merge-decision";
|
|
217
|
+
|
|
218
|
+
/** Pure: is choosing this option on THIS element unsafe for the steward to apply automatically —
|
|
219
|
+
* either generically destructive ({@link isDestructiveOption}) OR an element-specific override of a
|
|
220
|
+
* red gate (issue #889 review)? On `trial-merge-decision`, `action: "proceed"` is NOT universally
|
|
221
|
+
* safe: it explicitly overrides a `suite-failed` D3 trial merge and lets the wave continue past a red
|
|
222
|
+
* integration gate (SPEC.md §13.2). Because `isDestructiveOption("proceed")` is false, a
|
|
223
|
+
* value-only guard would let the steward auto-bypass that red gate — contrary to the fail-closed
|
|
224
|
+
* rule for merging past failures. For that element only the NON-override path (`rebase` — rerun the
|
|
225
|
+
* trial after a fix) is steward-safe; `proceed` (override) and `abandon` (give up / destroy the wave)
|
|
226
|
+
* are left for a human. Element-aware so a benign `proceed` on any OTHER element is unaffected. */
|
|
227
|
+
export function isUnsafeAutoOption(elementId: string | null, optionValue: string): boolean {
|
|
228
|
+
if (isDestructiveOption(optionValue)) return true;
|
|
229
|
+
if (elementId === TRIAL_MERGE_DECISION_ELEMENT) {
|
|
230
|
+
// Only the non-override "rerun the trial" path is steward-safe on the red integration gate.
|
|
231
|
+
return optionValue.trim().toLowerCase() !== "rebase";
|
|
232
|
+
}
|
|
233
|
+
return false;
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
/** The claim/lease TTL: a steward instance that crashes mid-run releases its claims after this long,
|
|
237
|
+
* so a later tick can pick the escalation up. Generous relative to the agent SLA. */
|
|
238
|
+
export const CLAIM_TTL_MS = 15 * 60 * 1000;
|
|
239
|
+
|
|
240
|
+
/** Atomically CLAIM a set of eligible escalation tasks for one steward run, so overlapping ticks never
|
|
241
|
+
* double-answer. For each task we insert a `pending`-outcome placeholder row keyed on `user_task_key`
|
|
242
|
+
* carrying the claim (`claimed_by`/`claimed_until`); the UNIQUE index on `user_task_key` among
|
|
243
|
+
* unresolved placeholder rows is the durable fence. A task already claimed (live lease) or already
|
|
244
|
+
* recorded is skipped. Returns the claimed rows (the subset this run may act on).
|
|
245
|
+
*
|
|
246
|
+
* The fence is enforced by a partial UNIQUE index created in migration 120
|
|
247
|
+
* (`ux_auto_resolutions_open_claim` on `user_task_key` WHERE `outcome` IS the claim sentinel). The
|
|
248
|
+
* placeholder uses `outcome = 'claimed'` (a non-final sentinel distinct from `resolved`/`abstained`),
|
|
249
|
+
* which the finalise step overwrites. */
|
|
250
|
+
export async function claimEscalations(
|
|
251
|
+
data: DataLayer,
|
|
252
|
+
rows: UserTaskRow[],
|
|
253
|
+
stewardInstanceKey: string,
|
|
254
|
+
ttlMs: number = CLAIM_TTL_MS,
|
|
255
|
+
): Promise<EscalationAutoResolution[]> {
|
|
256
|
+
const claimed: EscalationAutoResolution[] = [];
|
|
257
|
+
const until = new Date(Date.now() + ttlMs).toISOString();
|
|
258
|
+
// Drop any expired claims first so a crashed steward's leases don't wedge a task forever.
|
|
259
|
+
await releaseExpiredClaims(data);
|
|
260
|
+
for (const row of rows) {
|
|
261
|
+
try {
|
|
262
|
+
// Snapshot the task's REAL form choices at claim time, resolving the form from the task's
|
|
263
|
+
// engine-reported `form_key` first (a delivery-graph human task's element id is dynamic and has
|
|
264
|
+
// no static form linkage, so an element-id-only snapshot would record NULL options for it —
|
|
265
|
+
// issue #889 review). Safe: a `form_key` that names no bundled form yields null, never a throw.
|
|
266
|
+
const optionSnapshot = snapshotUserTaskFormOptions(row.form_key, row.element_id);
|
|
267
|
+
const id = await escalationAutoResolutions(data).insert({
|
|
268
|
+
user_task_key: row.user_task_key,
|
|
269
|
+
escalation_kind: row.kind_label,
|
|
270
|
+
element_id: row.element_id,
|
|
271
|
+
subject_type: row.subject_type,
|
|
272
|
+
subject_key: row.subject_key,
|
|
273
|
+
subject_url: row.subject_url,
|
|
274
|
+
process_instance_key: row.process_key,
|
|
275
|
+
question: row.question,
|
|
276
|
+
options: optionSnapshot != null ? JSON.stringify(optionSnapshot) : null,
|
|
277
|
+
outcome: "claimed",
|
|
278
|
+
submitted_variables: null,
|
|
279
|
+
selected_option: null,
|
|
280
|
+
rationale: null,
|
|
281
|
+
evidence: null,
|
|
282
|
+
steward_instance_key: stewardInstanceKey,
|
|
283
|
+
agent_job_key: null,
|
|
284
|
+
worker: null,
|
|
285
|
+
model: null,
|
|
286
|
+
created_at: now(),
|
|
287
|
+
review_state: "pending",
|
|
288
|
+
reviewed_by: null,
|
|
289
|
+
reviewed_at: null,
|
|
290
|
+
review_note: null,
|
|
291
|
+
claimed_by: stewardInstanceKey,
|
|
292
|
+
claimed_until: until,
|
|
293
|
+
});
|
|
294
|
+
const inserted = await escalationAutoResolutions(data).get(Number(id));
|
|
295
|
+
if (inserted) claimed.push(inserted);
|
|
296
|
+
} catch (err) {
|
|
297
|
+
// A UNIQUE collision means another steward tick (or a racing claim) already holds this task —
|
|
298
|
+
// skip it (the fence fired; that is the intended at-most-once outcome, not an error).
|
|
299
|
+
if (isUniqueConstraintFence(err)) continue;
|
|
300
|
+
throw err;
|
|
301
|
+
}
|
|
302
|
+
}
|
|
303
|
+
return claimed;
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
/** Release claims whose lease has expired (a steward instance that crashed before finalising). */
|
|
307
|
+
export async function releaseExpiredClaims(data: DataLayer): Promise<number> {
|
|
308
|
+
const res = await data.open().exec(
|
|
309
|
+
`DELETE FROM "escalation_auto_resolutions" WHERE "outcome" = 'claimed' AND "claimed_until" IS NOT NULL AND "claimed_until" < ?`,
|
|
310
|
+
[now()],
|
|
311
|
+
);
|
|
312
|
+
return res.changed;
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
/** Finalise a claim into a `resolved` or `abstained` record. Overwrites the placeholder row the claim
|
|
316
|
+
* created (matched by `user_task_key` + the `claimed` sentinel), so there is exactly ONE durable row
|
|
317
|
+
* per steward-acted escalation.
|
|
318
|
+
*
|
|
319
|
+
* ATOMIC OWNER+LEASE CAS (issue #889 review): when `owner` is supplied the UPDATE is fenced on
|
|
320
|
+
* `claimed_by = owner AND claimed_until >= now`, and the function returns whether that guarded UPDATE
|
|
321
|
+
* actually flipped the row. The completion→finalise pair is NOT atomic — the engine completion lands
|
|
322
|
+
* first, then this row write — so between the worker's claim read and this finalise an overlapping
|
|
323
|
+
* tick can release a lapsed lease and RECLAIM the task. An unfenced `WHERE user_task_key + outcome =
|
|
324
|
+
* 'claimed'` would then finalise the OTHER run's live claim (violating the no-double-answer fence).
|
|
325
|
+
* Fencing on the current owner + an unexpired lease makes the finalise a compare-and-swap: SQLite
|
|
326
|
+
* serialises the writes, so a stale/foreign owner changes ZERO rows and reports `false`, leaving the
|
|
327
|
+
* real claim holder's row untouched. (When `owner` is omitted the legacy unfenced behaviour is kept
|
|
328
|
+
* for back-compat callers and `true` is returned.) */
|
|
329
|
+
export async function recordResolution(
|
|
330
|
+
data: DataLayer,
|
|
331
|
+
userTaskKey: string,
|
|
332
|
+
patch: {
|
|
333
|
+
outcome: "resolved" | "abstained";
|
|
334
|
+
submittedVariables?: Record<string, unknown> | null;
|
|
335
|
+
selectedOption?: string | null;
|
|
336
|
+
rationale?: string | null;
|
|
337
|
+
evidence?: string | null;
|
|
338
|
+
options?: unknown[] | null;
|
|
339
|
+
agentJobKey?: string | null;
|
|
340
|
+
worker?: string | null;
|
|
341
|
+
model?: string | null;
|
|
342
|
+
},
|
|
343
|
+
owner?: string,
|
|
344
|
+
): Promise<boolean> {
|
|
345
|
+
const params = [
|
|
346
|
+
patch.outcome,
|
|
347
|
+
patch.submittedVariables != null ? JSON.stringify(patch.submittedVariables) : null,
|
|
348
|
+
patch.selectedOption ?? null,
|
|
349
|
+
patch.rationale ?? null,
|
|
350
|
+
patch.evidence ?? null,
|
|
351
|
+
patch.options != null ? JSON.stringify(patch.options) : null,
|
|
352
|
+
patch.agentJobKey ?? null,
|
|
353
|
+
patch.worker ?? null,
|
|
354
|
+
patch.model ?? null,
|
|
355
|
+
userTaskKey,
|
|
356
|
+
];
|
|
357
|
+
const res =
|
|
358
|
+
owner == null
|
|
359
|
+
? await data.open().exec(
|
|
360
|
+
`UPDATE "escalation_auto_resolutions"
|
|
361
|
+
SET "outcome" = ?, "submitted_variables" = ?, "selected_option" = ?, "rationale" = ?, "evidence" = ?, "options" = ?,
|
|
362
|
+
"agent_job_key" = ?, "worker" = ?, "model" = ?, "claimed_by" = NULL, "claimed_until" = NULL
|
|
363
|
+
WHERE "user_task_key" = ? AND "outcome" = 'claimed'`,
|
|
364
|
+
params,
|
|
365
|
+
)
|
|
366
|
+
: await data.open().exec(
|
|
367
|
+
`UPDATE "escalation_auto_resolutions"
|
|
368
|
+
SET "outcome" = ?, "submitted_variables" = ?, "selected_option" = ?, "rationale" = ?, "evidence" = ?, "options" = ?,
|
|
369
|
+
"agent_job_key" = ?, "worker" = ?, "model" = ?, "claimed_by" = NULL, "claimed_until" = NULL
|
|
370
|
+
WHERE "user_task_key" = ? AND "outcome" = 'claimed' AND "claimed_by" = ? AND "claimed_until" IS NOT NULL AND "claimed_until" >= ?`,
|
|
371
|
+
[...params, owner, now()],
|
|
372
|
+
);
|
|
373
|
+
return res.changed > 0;
|
|
374
|
+
}
|
|
375
|
+
|
|
376
|
+
/** Re-acquire (renew) THIS run's own claim lease before finalising it (issue #889 review).
|
|
377
|
+
*
|
|
378
|
+
* Why it exists: the claim lease ({@link CLAIM_TTL_MS}) is a short CRASH-RECOVERY TTL, but the
|
|
379
|
+
* `senior:escalation-steward` agent task it must survive has NO bounded runtime (no interrupting
|
|
380
|
+
* boundary timer). On a long agent run the lease lapses before the record step runs, and
|
|
381
|
+
* {@link recordResolution}'s owner+lease CAS would then REJECT the expired claim and discard the
|
|
382
|
+
* agent's valid decision — the next tick would simply repeat the work. Renewing re-asserts the lease
|
|
383
|
+
* so the finalise CAS can win.
|
|
384
|
+
*
|
|
385
|
+
* Why owner-fenced but NOT expiry-gated: the whole point is to recover a LAPSED lease, so gating on
|
|
386
|
+
* an unexpired lease would defeat it. Scoping to `claimed_by = owner` is a sufficient exclusivity
|
|
387
|
+
* fence here because the steward's overlap guard (collect worker) blocks any OTHER tick from touching
|
|
388
|
+
* claims while this run is ACTIVE — so no overlapping tick can have released+reclaimed the row out
|
|
389
|
+
* from under this still-active run. A foreign reclaim (`claimed_by != owner`) or a released/finalised
|
|
390
|
+
* row (no open `claimed` row) therefore changes ZERO rows and reports `false`, so a stale/foreign run
|
|
391
|
+
* can never steal or resurrect a claim. Returns whether this run's claim was renewed. */
|
|
392
|
+
export async function renewClaim(
|
|
393
|
+
data: DataLayer,
|
|
394
|
+
userTaskKey: string,
|
|
395
|
+
owner: string,
|
|
396
|
+
ttlMs: number = CLAIM_TTL_MS,
|
|
397
|
+
): Promise<boolean> {
|
|
398
|
+
const until = new Date(Date.now() + ttlMs).toISOString();
|
|
399
|
+
const res = await data.open().exec(
|
|
400
|
+
`UPDATE "escalation_auto_resolutions" SET "claimed_until" = ?
|
|
401
|
+
WHERE "user_task_key" = ? AND "outcome" = 'claimed' AND "claimed_by" = ?`,
|
|
402
|
+
[until, userTaskKey, owner],
|
|
403
|
+
);
|
|
404
|
+
return res.changed > 0;
|
|
405
|
+
}
|
|
406
|
+
|
|
407
|
+
/** Count a subject's prior steward RESOLUTIONS (for the loop guard). Only `resolved` rows count — an
|
|
408
|
+
* abstention did not apply an answer, so it must not trip the guard. Scoped by BOTH `subject_type`
|
|
409
|
+
* AND `subject_key` (issue #889 review): a bare `subject_key` collides across subject types (a plan
|
|
410
|
+
* `o/r#1` and a PR `o/r#1` share the key), so an unscoped count would let one subject's resolutions
|
|
411
|
+
* trip an UNRELATED subject's guard. */
|
|
412
|
+
export async function countSubjectResolutions(data: DataLayer, subjectType: string | null, subjectKey: string): Promise<number> {
|
|
413
|
+
const rows = await escalationAutoResolutions(data).find({ subject_key: subjectKey, outcome: "resolved" });
|
|
414
|
+
if (subjectType == null) return rows.length;
|
|
415
|
+
return rows.filter((r) => r.subject_type === subjectType).length;
|
|
416
|
+
}
|
|
417
|
+
|
|
418
|
+
/** The set of `user_task_key`s currently claimed by a live steward run (unexpired, unfinalised). */
|
|
419
|
+
export async function liveClaimedTaskKeys(data: DataLayer): Promise<Set<string>> {
|
|
420
|
+
const rows = await escalationAutoResolutions(data).find({ outcome: "claimed" });
|
|
421
|
+
const out = new Set<string>();
|
|
422
|
+
const at = now();
|
|
423
|
+
for (const r of rows) {
|
|
424
|
+
if (r.claimed_until != null && r.claimed_until >= at) out.add(r.user_task_key);
|
|
425
|
+
}
|
|
426
|
+
return out;
|
|
427
|
+
}
|
|
428
|
+
|
|
429
|
+
/** The set of `user_task_key`s the steward has ALREADY finally recorded (`resolved`/`abstained`).
|
|
430
|
+
* The claim fence's partial UNIQUE index only fences an OPEN (`claimed`) claim, so once a row is
|
|
431
|
+
* finalised it leaves that index — and because an abstention does NOT complete the user task, the
|
|
432
|
+
* task stays open and every later tick would otherwise re-claim the SAME `user_task_key` and record
|
|
433
|
+
* another abstention indefinitely (issue #889 review). Exclude already-final keys from eligibility so
|
|
434
|
+
* a task the steward already answered/abstained is never re-claimed. (A genuinely NEW escalation of
|
|
435
|
+
* the same work gets a NEW engine `user_task_key`, so this never blocks a fresh escalation.) */
|
|
436
|
+
export async function finalRecordedTaskKeys(data: DataLayer): Promise<Set<string>> {
|
|
437
|
+
const rows = await escalationAutoResolutions(data).all();
|
|
438
|
+
const out = new Set<string>();
|
|
439
|
+
for (const r of rows) {
|
|
440
|
+
if (r.outcome === "resolved" || r.outcome === "abstained") out.add(r.user_task_key);
|
|
441
|
+
}
|
|
442
|
+
return out;
|
|
443
|
+
}
|
|
444
|
+
|
|
445
|
+
/** Mark a record dismissed (reviewed, no action) or intervened (operator stepped in), with an optional
|
|
446
|
+
* note. Only a FINAL (`resolved`/`abstained`), still-`pending` record transitions — an in-flight
|
|
447
|
+
* `claimed` placeholder starts life `review_state='pending'` too, so without the outcome guard a
|
|
448
|
+
* guessed/old claim id could be "reviewed" before it finalises, letting its eventual resolution slip
|
|
449
|
+
* past the pending-review queue (matching the list/read-model, which only ever surfaces final rows).
|
|
450
|
+
* An already-reviewed record is left alone (idempotent). */
|
|
451
|
+
export async function reviewResolution(
|
|
452
|
+
data: DataLayer,
|
|
453
|
+
id: number,
|
|
454
|
+
action: "dismissed" | "intervened",
|
|
455
|
+
reviewedBy: string,
|
|
456
|
+
note: string | null,
|
|
457
|
+
): Promise<boolean> {
|
|
458
|
+
const res = await data.open().exec(
|
|
459
|
+
`UPDATE "escalation_auto_resolutions"
|
|
460
|
+
SET "review_state" = ?, "reviewed_by" = ?, "reviewed_at" = ?, "review_note" = ?
|
|
461
|
+
WHERE "id" = ? AND "review_state" = 'pending' AND "outcome" IN ('resolved', 'abstained')`,
|
|
462
|
+
[action, reviewedBy, now(), note, id],
|
|
463
|
+
);
|
|
464
|
+
return res.changed === 1;
|
|
465
|
+
}
|
|
466
|
+
|
|
467
|
+
/** List auto-resolution records, newest first, optionally filtered by review state / subject / kind. */
|
|
468
|
+
export async function listAutoResolutions(
|
|
469
|
+
data: DataLayer,
|
|
470
|
+
filter: { reviewState?: string; subjectKey?: string; kind?: string } = {},
|
|
471
|
+
): Promise<EscalationAutoResolution[]> {
|
|
472
|
+
let rows = await escalationAutoResolutions(data).all();
|
|
473
|
+
// Only FINAL records are reviewable — an unfinalised `claimed` placeholder is an in-flight steward
|
|
474
|
+
// action, not a reviewable decision, so it never reaches the view.
|
|
475
|
+
rows = rows.filter((r) => r.outcome === "resolved" || r.outcome === "abstained");
|
|
476
|
+
if (filter.reviewState) rows = rows.filter((r) => r.review_state === filter.reviewState);
|
|
477
|
+
if (filter.subjectKey) rows = rows.filter((r) => r.subject_key === filter.subjectKey);
|
|
478
|
+
if (filter.kind) rows = rows.filter((r) => r.escalation_kind === filter.kind || r.element_id === filter.kind);
|
|
479
|
+
return rows.sort((a, b) => (a.created_at < b.created_at ? 1 : a.created_at > b.created_at ? -1 : 0));
|
|
480
|
+
}
|
|
481
|
+
|
|
482
|
+
/** Count of PENDING (not-yet-reviewed) final records — the nav badge / list header. */
|
|
483
|
+
export async function pendingAutoResolutionCount(data: DataLayer): Promise<number> {
|
|
484
|
+
return (await listAutoResolutions(data, { reviewState: "pending" })).length;
|
|
485
|
+
}
|
|
486
|
+
|
|
487
|
+
/** Build the `processInstanceKey` → opted-in map across EVERY escalation-raising subject table
|
|
488
|
+
* (`feature_runs`, `plans`, `pull_requests`, `delivery_graph_runs`). The steward's eligibility reads
|
|
489
|
+
* the flag off the SUBJECT row keyed by the escalation's owning instance — the durable source of
|
|
490
|
+
* truth, immune to an agent-completion clobber of the process variable (#887/#888). A subject whose
|
|
491
|
+
* row is absent, has no `process_key`, or carries `auto_resolve_escalations != 1` maps to NOT opted
|
|
492
|
+
* in (fail-closed: work that didn't opt in is never touched). */
|
|
493
|
+
export async function optInByProcessKey(data: DataLayer): Promise<Map<string, boolean>> {
|
|
494
|
+
const out = new Map<string, boolean>();
|
|
495
|
+
const collect = async (table: string, key: string) => {
|
|
496
|
+
const rows: OptInSubjectRow[] = await data.table<OptInSubjectRow>(table, key).all();
|
|
497
|
+
for (const r of rows) {
|
|
498
|
+
if (r.process_key) out.set(String(r.process_key), r.auto_resolve_escalations === 1);
|
|
499
|
+
}
|
|
500
|
+
};
|
|
501
|
+
await collect("feature_runs", "feature_key");
|
|
502
|
+
await collect("plans", "plan_key");
|
|
503
|
+
await collect("pull_requests", "pr_key");
|
|
504
|
+
await collect("delivery_graph_runs", "run_key");
|
|
505
|
+
return out;
|
|
506
|
+
}
|
|
507
|
+
|
|
508
|
+
/** The same opt-in derivation keyed by the SUBJECT's primary key (`feature_key`/`plan_key`/`pr_key`/
|
|
509
|
+
* `run_key`) rather than its engine `process_key`. The collect step needs BOTH: an escalation that
|
|
510
|
+
* parks INSIDE a callActivity child cell (the shared `human-escalation`/`implement-cell`, ADR 0006 S4)
|
|
511
|
+
* is denormalised onto `user_tasks` with the CHILD's `process_key` (which no subject row tracks) but
|
|
512
|
+
* the PARENT run's `subject_key` (which `pollUserTasks` correlates via the root instance, #603/#633).
|
|
513
|
+
* Resolving eligibility off the subject key lets those child-cell escalations opt in exactly as their
|
|
514
|
+
* owning run did, where a `process_key`-only lookup would always read them as non-opted-in. */
|
|
515
|
+
export async function optInBySubjectKey(data: DataLayer): Promise<Map<string, boolean>> {
|
|
516
|
+
const out = new Map<string, boolean>();
|
|
517
|
+
// Each subject table's PRIMARY-KEY column differs, so read it generically: the row's key field is
|
|
518
|
+
// the one named `key`. The map is keyed by a `<subjectType>:<subjectKey>` COMPOUND key, never the
|
|
519
|
+
// bare subject key — a `feature_key`, `plan_key`, `pr_key` and `run_key` share one string space, so
|
|
520
|
+
// an un-scoped key would let one subject's opt-in (or opt-OUT) silently clobber a DIFFERENT subject
|
|
521
|
+
// type's row with a colliding key, making a non-opted-in task eligible (issue #889 review). A
|
|
522
|
+
// missing/blank key is dropped so it never matches a real escalation subject.
|
|
523
|
+
const collect = async (table: string, key: string, subjectType: string) => {
|
|
524
|
+
const rows: OptInSubjectRow[] = await data.table<OptInSubjectRow>(table, key).all();
|
|
525
|
+
for (const r of rows) {
|
|
526
|
+
// Read the subject's primary-key column generically (it differs per table). `keyOf` is a type
|
|
527
|
+
// guard, not an `as` cast, so the dynamic read stays type-safe.
|
|
528
|
+
const keyValue = keyOf(r, key);
|
|
529
|
+
if (keyValue) out.set(`${subjectType}:${keyValue}`, r.auto_resolve_escalations === 1);
|
|
530
|
+
}
|
|
531
|
+
};
|
|
532
|
+
await collect("feature_runs", "feature_key", "feature");
|
|
533
|
+
await collect("plans", "plan_key", "plan");
|
|
534
|
+
await collect("pull_requests", "pr_key", "pr");
|
|
535
|
+
await collect("delivery_graph_runs", "run_key", "delivery");
|
|
536
|
+
return out;
|
|
537
|
+
}
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
// Deterministic unit coverage for `pollEscalationSteward` (issue #889 review) — the steward tick's
|
|
2
|
+
// pacing + master-switch path, previously untested. Drives the exported poller against a stub
|
|
3
|
+
// EngineClient and asserts the four branches that decide whether the steward runs at all:
|
|
4
|
+
// • the master switch (`NANO_ESCALATION_STEWARD` off) — never starts an instance;
|
|
5
|
+
// • interval pacing (`NANO_ESCALATION_STEWARD_INTERVAL`) — a tick inside the interval is suppressed;
|
|
6
|
+
// • active-instance suppression — a still-ACTIVE prior run makes the tick a no-op;
|
|
7
|
+
// • failed-start retry — a `createInstance` throw leaves the tick un-advanced so the next pass retries.
|
|
8
|
+
//
|
|
9
|
+
// `pollEscalationSteward` paces itself via a MODULE-LEVEL `lastStewardTick`, so each test imports a
|
|
10
|
+
// FRESH (cache-busted) copy of `service.ts` to isolate that state — otherwise one test's tick would
|
|
11
|
+
// suppress the next test's first tick through the shared interval clock.
|
|
12
|
+
import { test } from "node:test";
|
|
13
|
+
import { assertEquals, assertRejects } from "#test-assert";
|
|
14
|
+
import type { EngineClient } from "@nanobpm/urban";
|
|
15
|
+
|
|
16
|
+
/** A stub EngineClient capturing createInstance calls and reporting a configurable ACTIVE set. */
|
|
17
|
+
function stubEngine(opts: { active?: string[]; failStart?: boolean } = {}) {
|
|
18
|
+
const created: { processDefinitionId: string; variables: Record<string, unknown> }[] = [];
|
|
19
|
+
const engine = {
|
|
20
|
+
searchProcessInstances: (filter?: { processDefinitionId?: string; state?: string }) =>
|
|
21
|
+
Promise.resolve(
|
|
22
|
+
filter?.processDefinitionId === "escalation-steward" && filter?.state === "ACTIVE"
|
|
23
|
+
? (opts.active ?? []).map((k) => ({ processInstanceKey: k, state: "ACTIVE" }))
|
|
24
|
+
: [],
|
|
25
|
+
),
|
|
26
|
+
createInstance: (input: { processDefinitionId: string; variables: Record<string, unknown> }) => {
|
|
27
|
+
if (opts.failStart) return Promise.reject(new Error("engine unavailable"));
|
|
28
|
+
created.push(input);
|
|
29
|
+
return Promise.resolve({ processInstanceKey: `PI-${created.length}` });
|
|
30
|
+
},
|
|
31
|
+
};
|
|
32
|
+
return { engine: engine as unknown as EngineClient, created };
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/** Import a FRESH `service.ts` (isolated `lastStewardTick`) and return its poller. */
|
|
36
|
+
async function freshPoller(): Promise<(engine: EngineClient) => Promise<void>> {
|
|
37
|
+
const mod = await import(`./service.ts?iso=${Date.now()}-${Math.random()}`);
|
|
38
|
+
return mod.pollEscalationSteward;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/** Run `fn` with the given env vars set, restoring process.env afterwards. */
|
|
42
|
+
async function withEnv<T>(env: Record<string, string>, fn: () => Promise<T>): Promise<T> {
|
|
43
|
+
const prev: Record<string, string | undefined> = {};
|
|
44
|
+
for (const k of Object.keys(env)) prev[k] = process.env[k];
|
|
45
|
+
for (const [k, v] of Object.entries(env)) process.env[k] = v;
|
|
46
|
+
try {
|
|
47
|
+
return await fn();
|
|
48
|
+
} finally {
|
|
49
|
+
for (const k of Object.keys(env)) {
|
|
50
|
+
if (prev[k] === undefined) delete process.env[k];
|
|
51
|
+
else process.env[k] = prev[k];
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
const STEWARD_ENV = { NANO_ESCALATION_STEWARD: "1", NANO_ESCALATION_STEWARD_INTERVAL: "PT30M" };
|
|
57
|
+
|
|
58
|
+
test("pollEscalationSteward: the master switch OFF never starts an instance", async () => {
|
|
59
|
+
const poll = await freshPoller();
|
|
60
|
+
const { engine, created } = stubEngine();
|
|
61
|
+
await withEnv({ NANO_ESCALATION_STEWARD: "off" }, () => poll(engine));
|
|
62
|
+
assertEquals(created.length, 0, "disabled → no instance started");
|
|
63
|
+
});
|
|
64
|
+
|
|
65
|
+
test("pollEscalationSteward: a still-ACTIVE prior run suppresses the tick (overlap guard)", async () => {
|
|
66
|
+
const poll = await freshPoller();
|
|
67
|
+
const { engine, created } = stubEngine({ active: ["PI-PRIOR"] });
|
|
68
|
+
await withEnv(STEWARD_ENV, () => poll(engine));
|
|
69
|
+
assertEquals(created.length, 0, "an active prior run suppresses the start");
|
|
70
|
+
});
|
|
71
|
+
|
|
72
|
+
test("pollEscalationSteward: a failed start leaves the tick un-advanced so the next pass retries immediately", async () => {
|
|
73
|
+
const poll = await freshPoller();
|
|
74
|
+
// First pass: the engine start THROWS. The tick must NOT advance (lastStewardTick stays 0), so a
|
|
75
|
+
// second pass within the interval still attempts a start rather than suppressing retries.
|
|
76
|
+
const failing = stubEngine({ failStart: true });
|
|
77
|
+
await assertRejects(() => withEnv(STEWARD_ENV, () => poll(failing.engine)), Error, "engine unavailable");
|
|
78
|
+
assertEquals(failing.created.length, 0, "the failed start recorded nothing");
|
|
79
|
+
|
|
80
|
+
// Immediately retry (same module clock, well inside the interval) with a now-healthy engine: the
|
|
81
|
+
// un-advanced tick lets the start straight through.
|
|
82
|
+
const healthy = stubEngine();
|
|
83
|
+
await withEnv(STEWARD_ENV, () => poll(healthy.engine));
|
|
84
|
+
assertEquals(healthy.created.length, 1, "the retry is not suppressed by the failed attempt");
|
|
85
|
+
assertEquals(healthy.created[0].processDefinitionId, "escalation-steward");
|
|
86
|
+
});
|
|
87
|
+
|
|
88
|
+
test("pollEscalationSteward: interval pacing suppresses a tick that lands inside the interval", async () => {
|
|
89
|
+
const poll = await freshPoller();
|
|
90
|
+
const { engine, created } = stubEngine();
|
|
91
|
+
// First tick starts an instance and advances the module clock.
|
|
92
|
+
await withEnv(STEWARD_ENV, () => poll(engine));
|
|
93
|
+
assertEquals(created.length, 1, "the first tick starts an instance");
|
|
94
|
+
// A second tick immediately after (well inside the 30-min interval) is suppressed by pacing.
|
|
95
|
+
await withEnv(STEWARD_ENV, () => poll(engine));
|
|
96
|
+
assertEquals(created.length, 1, "a tick inside the interval is suppressed");
|
|
97
|
+
});
|
|
98
|
+
|
|
99
|
+
test("pollEscalationSteward: a zero/negative interval is clamped so the steward never free-runs every poll", async () => {
|
|
100
|
+
const poll = await freshPoller();
|
|
101
|
+
const { engine, created } = stubEngine();
|
|
102
|
+
// PT0S parses to 0 → clamped to the 30-min default. The first tick starts; the immediate second is
|
|
103
|
+
// still suppressed by the clamped interval (a 0-interval must never start every poll).
|
|
104
|
+
await withEnv({ NANO_ESCALATION_STEWARD: "1", NANO_ESCALATION_STEWARD_INTERVAL: "PT0S" }, () => poll(engine));
|
|
105
|
+
assertEquals(created.length, 1, "the first tick starts even with a zero interval");
|
|
106
|
+
await withEnv({ NANO_ESCALATION_STEWARD: "1", NANO_ESCALATION_STEWARD_INTERVAL: "PT0S" }, () => poll(engine));
|
|
107
|
+
assertEquals(created.length, 1, "a zero interval is clamped — the immediate re-tick is suppressed");
|
|
108
|
+
});
|
|
109
|
+
|
|
110
|
+
test("pollEscalationSteward: the steward start seeds the canonical agentSlaTimeout so its agent boundary can fire", async () => {
|
|
111
|
+
const poll = await freshPoller();
|
|
112
|
+
const { engine, created } = stubEngine();
|
|
113
|
+
await withEnv(STEWARD_ENV, () => poll(engine));
|
|
114
|
+
assertEquals(created.length, 1, "the tick starts an instance");
|
|
115
|
+
// The `be_steward_sla` interrupting timer boundary (escalation-steward.bpmn) evaluates
|
|
116
|
+
// `<bpmn:timeDuration>=agentSlaTimeout</…>` at timer creation (FEEL). If the start does not seed
|
|
117
|
+
// `agentSlaTimeout` the boundary can never arm and a hung steward agent parks the instance ACTIVE
|
|
118
|
+
// forever — the overlap guard then suppresses every later tick (issue #889 review). Assert the
|
|
119
|
+
// seeded variable is a non-blank ISO-8601 duration (the canonical AGENT_SLA_TIMEOUT default).
|
|
120
|
+
const seeded = created[0].variables.agentSlaTimeout;
|
|
121
|
+
assertEquals(typeof seeded, "string", "agentSlaTimeout is seeded as a string");
|
|
122
|
+
assertEquals(/^P/.test(seeded), true, `agentSlaTimeout is an ISO-8601 duration (got ${JSON.stringify(seeded)})`);
|
|
123
|
+
});
|