@nanobpm/nano-workforce 0.201.5 → 0.203.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. package/CHANGELOG.md +12 -0
  2. package/SPEC.md +48 -0
  3. package/app/admittedStaging.test.ts +71 -0
  4. package/app/agentActivationLedger.ts +5 -0
  5. package/app/agentCompletion.test.ts +33 -0
  6. package/app/agentCompletion.ts +108 -2
  7. package/app/agentic/README.md +48 -0
  8. package/app/agentic/claim-registry.test.ts +61 -0
  9. package/app/agentic/claim-registry.ts +35 -1
  10. package/app/agentic/families/claim.family.test.ts +216 -0
  11. package/app/agentic/families/claim.family.ts +100 -4
  12. package/app/agentic/vocab/agent-sla-boundary.test.ts +6 -1
  13. package/app/contracts.ts +31 -0
  14. package/app/deliveryGraphDispatch.ts +6 -0
  15. package/app/deliveryGraphReadModel.test.ts +2 -2
  16. package/app/deliveryGraphRun.ts +18 -0
  17. package/app/deliveryRunner.ts +9 -1
  18. package/app/escalationSteward.test.ts +452 -0
  19. package/app/escalationSteward.ts +537 -0
  20. package/app/escalationStewardPoll.test.ts +123 -0
  21. package/app/feature.test.ts +16 -0
  22. package/app/feature.ts +11 -0
  23. package/app/featureDelivery.test.ts +23 -0
  24. package/app/mcpToolSurface.ts +17 -1
  25. package/app/plan.ts +49 -6
  26. package/app/planLowering.test.ts +26 -1
  27. package/app/planLowering.ts +5 -0
  28. package/app/promotionPoll.test.ts +19 -0
  29. package/app/pullRequestReadModel.test.ts +1 -1
  30. package/app/service.test.ts +36 -0
  31. package/app/service.ts +139 -5
  32. package/db/migrations/120_escalation_auto_resolutions.sql +229 -0
  33. package/db/migrations/121_delivery_graph_read_model_auto_resolve.sql +54 -0
  34. package/db/migrations/122_pull_requests_read_model_auto_resolve.sql +56 -0
  35. package/db/migrations/123_escalation_auto_resolutions_selected_option.sql +47 -0
  36. package/db/migrations/124_admitted_epics_auto_resolve.sql +21 -0
  37. package/docs/agent-guide.md +69 -30
  38. package/docs/agentic-cockpit.md +34 -0
  39. package/docs/mcp-runbook.md +2 -0
  40. package/e2e/escalation-steward.e2e.ts +211 -0
  41. package/nano.app.json +10 -1
  42. package/openapi.yaml +365 -0
  43. package/operations/dispatchDeliveryGraph.ts +6 -1
  44. package/operations/listActivePrs.test.ts +60 -1
  45. package/operations/listActivePrs.ts +3 -3
  46. package/operations/listAutoResolutions.ts +83 -0
  47. package/operations/reviewAutoResolution.test.ts +137 -0
  48. package/operations/reviewAutoResolution.ts +42 -0
  49. package/operations/startConvergenceLoop.ts +5 -1
  50. package/operations/startEpicSet.admission.integration.test.ts +2 -2
  51. package/operations/startEpicSet.ts +8 -2
  52. package/operations/startFeature.ts +5 -0
  53. package/operations/startPlanFanout.ts +4 -1
  54. package/package.json +1 -1
  55. package/pages/_nav.json +12 -0
  56. package/pages/auto-resolutions.page.json +191 -0
  57. package/pages/board.page.json +17 -0
  58. package/pages/cockpit.page.json +17 -0
  59. package/pages/delivery-graph-detail.page.json +17 -0
  60. package/pages/delivery-graphs/staged.mount.js +17 -5
  61. package/pages/delivery-graphs.page.json +17 -0
  62. package/pages/epic-detail.page.json +17 -0
  63. package/pages/epic.page.json +19 -1
  64. package/pages/feature.page.json +19 -1
  65. package/pages/home.page.json +22 -0
  66. package/pages/lineage.page.json +17 -0
  67. package/pages/mcp.page.json +17 -0
  68. package/pages/overview.page.json +17 -0
  69. package/pages/tasks.page.json +17 -0
  70. package/pages/velocity.page.json +17 -0
  71. package/resources/processes/escalation-steward.bpmn +162 -0
  72. package/resources/prompts/escalation-steward.md +87 -0
  73. package/test/delivery-graphs-staged-embed.test.ts +13 -0
  74. package/test/derivation-parity/derivation-parity.test.ts +1 -0
  75. package/test/derivation-parity/flows.ts +12 -0
  76. package/workers/converge-feature/worker.ts +6 -1
  77. package/workers/delivery-connector/worker.test.ts +72 -1
  78. package/workers/delivery-connector/worker.ts +53 -2
  79. package/workers/escalation-steward-collect/worker.test.ts +180 -0
  80. package/workers/escalation-steward-collect/worker.ts +99 -0
  81. package/workers/escalation-steward-record/worker.test.ts +740 -0
  82. package/workers/escalation-steward-record/worker.ts +428 -0
  83. package/workers/record-wave/worker.ts +5 -1
@@ -0,0 +1,537 @@
1
+ // Opt-in escalation steward (issue #889) — the domain core for the timer-driven loop that reviews
2
+ // OPEN escalations on OPTED-IN work and submits an agent's recommended choice for each, keeping the
3
+ // fleet moving while every automatic decision stays reviewable by a human after the fact.
4
+ //
5
+ // This module owns the PURE/host-side pieces so they are unit-testable without the engine:
6
+ //
7
+ // • Eligibility — which open escalation user tasks the steward may touch: the task's owning work
8
+ // must have OPTED IN (`auto_resolve_escalations` on its subject row), the escalation must be past
9
+ // a short minimum age (so a human already answering wins), it must not be claimed by an in-flight
10
+ // steward run (a claim/lease so overlapping ticks never double-answer), and it must be a kind the
11
+ // steward is permitted to answer (never a human-only or irreversible-by-construction kind).
12
+ // • The claim/lease fence — a durable compare-and-swap on `escalation_auto_resolutions` so two
13
+ // overlapping steward ticks (or a steward tick racing a human) can never both act on one task.
14
+ // • The record — one `escalation_auto_resolutions` row per escalation the steward resolved or
15
+ // abstained from, so the Auto-resolutions view can render it and an operator can dismiss or
16
+ // intervene.
17
+ // • The loop guard + destructive-option abstention — pure functions the steward's collect step and
18
+ // the agent's record step share, so the "never auto-answer an irreversible/credential choice" and
19
+ // "stop re-answering a subject that keeps re-escalating" rules are enforced in ONE place.
20
+ //
21
+ // The engine-facing orchestration (the timer-start process, the collect/record service tasks, and the
22
+ // `senior:escalation-steward` agent job) lives in `resources/processes/escalation-steward.bpmn` and
23
+ // `workers/escalation-steward-*`; the actual ANSWER is submitted through the ONE canonical attributed
24
+ // door (`completeUserTaskAttributed`, app/agentCompletion.ts) with AGENT attribution — never recorded
25
+ // as a human decision (ADR 0046).
26
+ import type { DataLayer } from "@nanobpm/urban";
27
+ import { ESCALATION_TASK_ELEMENTS, snapshotUserTaskFormOptions } from "./agentCompletion.ts";
28
+ import { isUniqueConstraintFence } from "./dbFence.ts";
29
+ import { ESCALATION_FORM, GENERIC_HUMAN_FORM, isDeliveryHumanElement } from "./deliveryHuman.ts";
30
+ import type { UserTaskRow } from "./userTasks.ts";
31
+
32
+ /** The minimal shape of a subject row that carries the opt-in flag. The index signature lets the
33
+ * steward read each table's distinct PRIMARY-KEY column generically (see {@link optInBySubjectKey}). */
34
+ interface OptInSubjectRow {
35
+ process_key: string | null;
36
+ auto_resolve_escalations: number;
37
+ [column: string]: string | number | null;
38
+ }
39
+
40
+ const now = () => new Date().toISOString();
41
+
42
+ /** Read a string field off a subject row by dynamic column name, returning `null` when it is absent
43
+ * or not a string. The row is widened to a string-indexed record via its own type (no `as` cast), so
44
+ * the generic per-table key read in {@link optInBySubjectKey} stays type-safe. */
45
+ function keyOf(row: OptInSubjectRow, column: string): string | null {
46
+ const record: Record<string, unknown> = row;
47
+ const v: unknown = record[column];
48
+ return typeof v === "string" && v !== "" ? v : null;
49
+ }
50
+
51
+ /** The `senior:escalation-steward` agent's job type (the dispatch verb the steward process emits). */
52
+ export const ESCALATION_STEWARD_JOB_TYPE = "senior:escalation-steward";
53
+ /** The steward process definition id (`resources/processes/escalation-steward.bpmn`). */
54
+ export const ESCALATION_STEWARD_PROCESS_ID = "escalation-steward";
55
+
56
+ /** How long an escalation must be open before the steward considers it — a grace window so a human
57
+ * who is ALREADY answering wins and the steward never snatches a task out from under an operator.
58
+ * Configurable (`NANO_ESCALATION_STEWARD_MIN_AGE_MS`); default 5 minutes per the issue. */
59
+ export const DEFAULT_MIN_AGE_MS = 5 * 60 * 1000;
60
+
61
+ /** How many times a single subject may be auto-resolved before the steward abstains and leaves the
62
+ * re-escalation for a human — repeated escalation means the automatic answer isn't working. */
63
+ export const LOOP_GUARD_MAX_RESOLUTIONS = 2;
64
+
65
+ /** The escalation user-task `elementId`s the steward must NEVER auto-answer, because the decision is
66
+ * irreversible-by-construction, a credential/permission grant, or a gate whose whole point is a human
67
+ * judgement. These are refused BEFORE the agent is even dispatched (defence-in-depth under the
68
+ * prompt's own abstain rule), so the steward can never be the path that abandons work, approves a
69
+ * merge, grants a permission, or overrides a readiness gate.
70
+ *
71
+ * This guard is INTERSECTED with the canonical agent-answerable allowlist (`ESCALATION_TASK_ELEMENTS`
72
+ * + the dynamic delivery-human predicate): an element is steward-eligible only if it is BOTH
73
+ * agent-answerable AND not in this guard set (see {@link isStewardAnswerableElement}). */
74
+ export const STEWARD_NEVER_RESOLVE_ELEMENTS: ReadonlySet<string> = new Set([
75
+ // Human-only merge approval (issue #826) — the fleet must never auto-approve a merge.
76
+ "merge-approval",
77
+ // ACP permission prompt (issue #559) — a credential/permission grant; always a human decision.
78
+ "acp-permission",
79
+ // Readiness/preflight gates (issue #674) — an agent must never auto-answer "is upstream ready?".
80
+ "readiness-escalation",
81
+ "readiness-escalation-pf",
82
+ // Human-only terminal acknowledgements — a blocked run / a conformance shortfall / an empty plan are
83
+ // exactly the "the automatic path produced nothing usable" cases a human must adjudicate.
84
+ "feature-blocked",
85
+ "conformance-escalation",
86
+ "empty-plan-escalation",
87
+ ]);
88
+
89
+ /** An `escalation_auto_resolutions` row — the durable, reviewable record of one steward action. */
90
+ export interface EscalationAutoResolution {
91
+ id: number;
92
+ user_task_key: string;
93
+ escalation_kind: string | null;
94
+ element_id: string | null;
95
+ subject_type: string | null;
96
+ subject_key: string | null;
97
+ subject_url: string | null;
98
+ process_instance_key: string | null;
99
+ question: string | null;
100
+ /** JSON array snapshot of the form's option values/labels at resolution time. */
101
+ options: string | null;
102
+ /** `resolved` (the steward submitted an answer) | `abstained` (it deliberately left it for a human).
103
+ * `claimed` is the transient, non-final sentinel for an open claim/lease placeholder (never shown in
104
+ * the review surface); the finalise step overwrites it to `resolved`/`abstained`. */
105
+ outcome: "resolved" | "abstained" | "claimed";
106
+ /** JSON of the variables submitted; NULL when abstained. */
107
+ submitted_variables: string | null;
108
+ /** The option value the steward actually selected (derived host-side from the decision's own form
109
+ * variables at record time), so the review surface shows WHAT was chosen without decoding
110
+ * `submitted_variables`. NULL when abstained / not derivable. */
111
+ selected_option: string | null;
112
+ rationale: string | null;
113
+ /** JSON/markdown: what the agent checked, with links. */
114
+ evidence: string | null;
115
+ steward_instance_key: string | null;
116
+ agent_job_key: string | null;
117
+ worker: string | null;
118
+ model: string | null;
119
+ created_at: string;
120
+ review_state: "pending" | "dismissed" | "intervened";
121
+ reviewed_by: string | null;
122
+ reviewed_at: string | null;
123
+ review_note: string | null;
124
+ /** Claim/lease bookkeeping (not part of the review surface): which steward instance holds the claim
125
+ * and until when. NULL once the row is finalised (resolved/abstained recorded). */
126
+ claimed_by: string | null;
127
+ claimed_until: string | null;
128
+ }
129
+
130
+ export const escalationAutoResolutions = (data: DataLayer) =>
131
+ data.table<EscalationAutoResolution>("escalation_auto_resolutions", "id");
132
+
133
+ /** Whether an escalation element kind is one the steward is permitted to answer at all. The agent
134
+ * prompt ALSO carries an abstain rule; this is the host-side fail-closed guard so a mis-prompted or
135
+ * misrouted steward can never touch a human-only / irreversible / credential kind.
136
+ *
137
+ * This is an ALLOWLIST gated on the canonical agent-answerable surface, not a denylist: an element is
138
+ * steward-eligible only if it is BOTH agent-answerable (in `ESCALATION_TASK_ELEMENTS`, or a dynamic
139
+ * delivery-human node matched by the single-source `isDeliveryHumanElement` predicate — the same
140
+ * surface the completer resolves against) AND not in the explicit {@link STEWARD_NEVER_RESOLVE_ELEMENTS}
141
+ * safety guard. Gating on the allowlist (issue #889 review) means an arbitrary/unknown user task that
142
+ * merely escapes the NEVER_RESOLVE set is refused up front, instead of being claimed and dispatched
143
+ * only for the completer to refuse it — which would record a spurious abstention and permanently
144
+ * exclude that task from future steward ticks. */
145
+ export function isStewardAnswerableElement(elementId: string): boolean {
146
+ const agentAnswerable = ESCALATION_TASK_ELEMENTS.has(elementId) || isDeliveryHumanElement(elementId);
147
+ return agentAnswerable && !STEWARD_NEVER_RESOLVE_ELEMENTS.has(elementId);
148
+ }
149
+
150
+ /** The ONLY delivery-graph `human` forms a steward may auto-answer. A delivery-graph `human` node
151
+ * renders a DIFFERENT form per node (explicit → category → generic → agent-router,
152
+ * `app/deliveryHuman.ts` `resolveHumanForm`), and — unlike the fixed-form escalations — the agent
153
+ * completer (`completeEscalationAsAgent`) intentionally does NOT validate these dynamic forms (their
154
+ * typed-emit contract is bound, not field-validated). So ELIGIBILITY, not the completer, is the
155
+ * fail-closed fence for them. Only the typed-value/decision forms are something an agent can
156
+ * legitimately produce: the GENERIC typed-emit capture and the continue/retry service-node
157
+ * ESCALATION form. The bespoke CATEGORY forms are manual/out-of-band human work the steward cannot
158
+ * perform — `delivery-human-publish` is a manual OTP/OIDC credential publish, `delivery-human-ack` is
159
+ * a "do X out of band, then click done" acknowledgement — so admitting a delivery-human task on ANY
160
+ * other (or an absent/unknown) form would let a mis-prompted steward blind-complete a
161
+ * credential/manual task (issue #889 review). Hence an ALLOWLIST: fail closed for everything else. */
162
+ export const STEWARD_SAFE_DELIVERY_HUMAN_FORMS: ReadonlySet<string> = new Set([
163
+ GENERIC_HUMAN_FORM,
164
+ ESCALATION_FORM,
165
+ ]);
166
+
167
+ /** Pure, form-aware fail-closed guard for the dynamic delivery-graph `human` task. A FIXED-form
168
+ * escalation (plan-review-decision, wait-answer, …) is governed by its element id and validated by
169
+ * the completer, so its form is NOT gated here (returns true). A DYNAMIC delivery-human element,
170
+ * whose form is only known at runtime and is unvalidated by the completer, is steward-answerable
171
+ * ONLY when its resolved `formKey` is a known-safe form ({@link STEWARD_SAFE_DELIVERY_HUMAN_FORMS});
172
+ * an absent/unknown/category form fails closed. */
173
+ export function isStewardAnswerableForm(elementId: string, formKey: string | null | undefined): boolean {
174
+ if (!isDeliveryHumanElement(elementId)) return true;
175
+ return formKey != null && STEWARD_SAFE_DELIVERY_HUMAN_FORMS.has(formKey);
176
+ }
177
+
178
+ /** Pure: is one open escalation user task ELIGIBLE for the steward right now? Requires:
179
+ * • the owning work opted in (`optedIn` — derived by the caller from the subject row);
180
+ * • the kind is steward-answerable ({@link isStewardAnswerableElement});
181
+ * • for a dynamic delivery-human task, its resolved form is steward-safe ({@link isStewardAnswerableForm});
182
+ * • the task is past the minimum age (`minAgeMs`) so a human already answering wins;
183
+ * • the task is not currently claimed by a live steward run (`claimedUserTaskKeys`).
184
+ * `at` is the current time (ms epoch); the task's `created_at` is its open-since timestamp. */
185
+ export function isEligibleEscalation(
186
+ row: Pick<UserTaskRow, "user_task_key" | "element_id" | "created_at" | "form_key">,
187
+ opts: { optedIn: boolean; minAgeMs: number; claimedUserTaskKeys: ReadonlySet<string>; at: number },
188
+ ): boolean {
189
+ if (!opts.optedIn) return false;
190
+ if (!isStewardAnswerableElement(row.element_id)) return false;
191
+ if (!isStewardAnswerableForm(row.element_id, row.form_key)) return false;
192
+ if (opts.claimedUserTaskKeys.has(row.user_task_key)) return false;
193
+ const openedAt = Date.parse(row.created_at);
194
+ if (Number.isNaN(openedAt)) return false;
195
+ return opts.at - openedAt >= opts.minAgeMs;
196
+ }
197
+
198
+ /** Pure: the loop guard. Given the count of prior steward RESOLUTIONS for a subject, whether the
199
+ * steward must abstain on a fresh escalation for that same subject (it has already auto-answered
200
+ * `maxResolutions` times and the work escalated again — the automatic answer isn't working). */
201
+ export function loopGuardTripped(priorResolutions: number, maxResolutions: number = LOOP_GUARD_MAX_RESOLUTIONS): boolean {
202
+ return priorResolutions >= maxResolutions;
203
+ }
204
+
205
+ /** Pure: does choosing this option abandon/destroy work or merge past a check — an IRREVERSIBLE or
206
+ * destructive choice the steward must abstain from even when the kind is otherwise answerable? The
207
+ * agent is instructed to abstain on these; this guard lets the host ALSO refuse to record a
208
+ * destructive resolution an agent erroneously submitted (fail-closed). Matched case-insensitively
209
+ * against the option value. */
210
+ const DESTRUCTIVE_OPTION_VALUES = /^(abandon|abandon-run|delete|destroy|discard|force-merge|merge-admin|override-checks?)$/i;
211
+ export function isDestructiveOption(optionValue: string): boolean {
212
+ return DESTRUCTIVE_OPTION_VALUES.test(optionValue.trim());
213
+ }
214
+
215
+ /** The `trial-merge-decision` element id (plan-fanout.bpmn) — the D3 trial-merge integration gate. */
216
+ export const TRIAL_MERGE_DECISION_ELEMENT = "trial-merge-decision";
217
+
218
+ /** Pure: is choosing this option on THIS element unsafe for the steward to apply automatically —
219
+ * either generically destructive ({@link isDestructiveOption}) OR an element-specific override of a
220
+ * red gate (issue #889 review)? On `trial-merge-decision`, `action: "proceed"` is NOT universally
221
+ * safe: it explicitly overrides a `suite-failed` D3 trial merge and lets the wave continue past a red
222
+ * integration gate (SPEC.md §13.2). Because `isDestructiveOption("proceed")` is false, a
223
+ * value-only guard would let the steward auto-bypass that red gate — contrary to the fail-closed
224
+ * rule for merging past failures. For that element only the NON-override path (`rebase` — rerun the
225
+ * trial after a fix) is steward-safe; `proceed` (override) and `abandon` (give up / destroy the wave)
226
+ * are left for a human. Element-aware so a benign `proceed` on any OTHER element is unaffected. */
227
+ export function isUnsafeAutoOption(elementId: string | null, optionValue: string): boolean {
228
+ if (isDestructiveOption(optionValue)) return true;
229
+ if (elementId === TRIAL_MERGE_DECISION_ELEMENT) {
230
+ // Only the non-override "rerun the trial" path is steward-safe on the red integration gate.
231
+ return optionValue.trim().toLowerCase() !== "rebase";
232
+ }
233
+ return false;
234
+ }
235
+
236
+ /** The claim/lease TTL: a steward instance that crashes mid-run releases its claims after this long,
237
+ * so a later tick can pick the escalation up. Generous relative to the agent SLA. */
238
+ export const CLAIM_TTL_MS = 15 * 60 * 1000;
239
+
240
+ /** Atomically CLAIM a set of eligible escalation tasks for one steward run, so overlapping ticks never
241
+ * double-answer. For each task we insert a `pending`-outcome placeholder row keyed on `user_task_key`
242
+ * carrying the claim (`claimed_by`/`claimed_until`); the UNIQUE index on `user_task_key` among
243
+ * unresolved placeholder rows is the durable fence. A task already claimed (live lease) or already
244
+ * recorded is skipped. Returns the claimed rows (the subset this run may act on).
245
+ *
246
+ * The fence is enforced by a partial UNIQUE index created in migration 120
247
+ * (`ux_auto_resolutions_open_claim` on `user_task_key` WHERE `outcome` IS the claim sentinel). The
248
+ * placeholder uses `outcome = 'claimed'` (a non-final sentinel distinct from `resolved`/`abstained`),
249
+ * which the finalise step overwrites. */
250
+ export async function claimEscalations(
251
+ data: DataLayer,
252
+ rows: UserTaskRow[],
253
+ stewardInstanceKey: string,
254
+ ttlMs: number = CLAIM_TTL_MS,
255
+ ): Promise<EscalationAutoResolution[]> {
256
+ const claimed: EscalationAutoResolution[] = [];
257
+ const until = new Date(Date.now() + ttlMs).toISOString();
258
+ // Drop any expired claims first so a crashed steward's leases don't wedge a task forever.
259
+ await releaseExpiredClaims(data);
260
+ for (const row of rows) {
261
+ try {
262
+ // Snapshot the task's REAL form choices at claim time, resolving the form from the task's
263
+ // engine-reported `form_key` first (a delivery-graph human task's element id is dynamic and has
264
+ // no static form linkage, so an element-id-only snapshot would record NULL options for it —
265
+ // issue #889 review). Safe: a `form_key` that names no bundled form yields null, never a throw.
266
+ const optionSnapshot = snapshotUserTaskFormOptions(row.form_key, row.element_id);
267
+ const id = await escalationAutoResolutions(data).insert({
268
+ user_task_key: row.user_task_key,
269
+ escalation_kind: row.kind_label,
270
+ element_id: row.element_id,
271
+ subject_type: row.subject_type,
272
+ subject_key: row.subject_key,
273
+ subject_url: row.subject_url,
274
+ process_instance_key: row.process_key,
275
+ question: row.question,
276
+ options: optionSnapshot != null ? JSON.stringify(optionSnapshot) : null,
277
+ outcome: "claimed",
278
+ submitted_variables: null,
279
+ selected_option: null,
280
+ rationale: null,
281
+ evidence: null,
282
+ steward_instance_key: stewardInstanceKey,
283
+ agent_job_key: null,
284
+ worker: null,
285
+ model: null,
286
+ created_at: now(),
287
+ review_state: "pending",
288
+ reviewed_by: null,
289
+ reviewed_at: null,
290
+ review_note: null,
291
+ claimed_by: stewardInstanceKey,
292
+ claimed_until: until,
293
+ });
294
+ const inserted = await escalationAutoResolutions(data).get(Number(id));
295
+ if (inserted) claimed.push(inserted);
296
+ } catch (err) {
297
+ // A UNIQUE collision means another steward tick (or a racing claim) already holds this task —
298
+ // skip it (the fence fired; that is the intended at-most-once outcome, not an error).
299
+ if (isUniqueConstraintFence(err)) continue;
300
+ throw err;
301
+ }
302
+ }
303
+ return claimed;
304
+ }
305
+
306
+ /** Release claims whose lease has expired (a steward instance that crashed before finalising). */
307
+ export async function releaseExpiredClaims(data: DataLayer): Promise<number> {
308
+ const res = await data.open().exec(
309
+ `DELETE FROM "escalation_auto_resolutions" WHERE "outcome" = 'claimed' AND "claimed_until" IS NOT NULL AND "claimed_until" < ?`,
310
+ [now()],
311
+ );
312
+ return res.changed;
313
+ }
314
+
315
+ /** Finalise a claim into a `resolved` or `abstained` record. Overwrites the placeholder row the claim
316
+ * created (matched by `user_task_key` + the `claimed` sentinel), so there is exactly ONE durable row
317
+ * per steward-acted escalation.
318
+ *
319
+ * ATOMIC OWNER+LEASE CAS (issue #889 review): when `owner` is supplied the UPDATE is fenced on
320
+ * `claimed_by = owner AND claimed_until >= now`, and the function returns whether that guarded UPDATE
321
+ * actually flipped the row. The completion→finalise pair is NOT atomic — the engine completion lands
322
+ * first, then this row write — so between the worker's claim read and this finalise an overlapping
323
+ * tick can release a lapsed lease and RECLAIM the task. An unfenced `WHERE user_task_key + outcome =
324
+ * 'claimed'` would then finalise the OTHER run's live claim (violating the no-double-answer fence).
325
+ * Fencing on the current owner + an unexpired lease makes the finalise a compare-and-swap: SQLite
326
+ * serialises the writes, so a stale/foreign owner changes ZERO rows and reports `false`, leaving the
327
+ * real claim holder's row untouched. (When `owner` is omitted the legacy unfenced behaviour is kept
328
+ * for back-compat callers and `true` is returned.) */
329
+ export async function recordResolution(
330
+ data: DataLayer,
331
+ userTaskKey: string,
332
+ patch: {
333
+ outcome: "resolved" | "abstained";
334
+ submittedVariables?: Record<string, unknown> | null;
335
+ selectedOption?: string | null;
336
+ rationale?: string | null;
337
+ evidence?: string | null;
338
+ options?: unknown[] | null;
339
+ agentJobKey?: string | null;
340
+ worker?: string | null;
341
+ model?: string | null;
342
+ },
343
+ owner?: string,
344
+ ): Promise<boolean> {
345
+ const params = [
346
+ patch.outcome,
347
+ patch.submittedVariables != null ? JSON.stringify(patch.submittedVariables) : null,
348
+ patch.selectedOption ?? null,
349
+ patch.rationale ?? null,
350
+ patch.evidence ?? null,
351
+ patch.options != null ? JSON.stringify(patch.options) : null,
352
+ patch.agentJobKey ?? null,
353
+ patch.worker ?? null,
354
+ patch.model ?? null,
355
+ userTaskKey,
356
+ ];
357
+ const res =
358
+ owner == null
359
+ ? await data.open().exec(
360
+ `UPDATE "escalation_auto_resolutions"
361
+ SET "outcome" = ?, "submitted_variables" = ?, "selected_option" = ?, "rationale" = ?, "evidence" = ?, "options" = ?,
362
+ "agent_job_key" = ?, "worker" = ?, "model" = ?, "claimed_by" = NULL, "claimed_until" = NULL
363
+ WHERE "user_task_key" = ? AND "outcome" = 'claimed'`,
364
+ params,
365
+ )
366
+ : await data.open().exec(
367
+ `UPDATE "escalation_auto_resolutions"
368
+ SET "outcome" = ?, "submitted_variables" = ?, "selected_option" = ?, "rationale" = ?, "evidence" = ?, "options" = ?,
369
+ "agent_job_key" = ?, "worker" = ?, "model" = ?, "claimed_by" = NULL, "claimed_until" = NULL
370
+ WHERE "user_task_key" = ? AND "outcome" = 'claimed' AND "claimed_by" = ? AND "claimed_until" IS NOT NULL AND "claimed_until" >= ?`,
371
+ [...params, owner, now()],
372
+ );
373
+ return res.changed > 0;
374
+ }
375
+
376
+ /** Re-acquire (renew) THIS run's own claim lease before finalising it (issue #889 review).
377
+ *
378
+ * Why it exists: the claim lease ({@link CLAIM_TTL_MS}) is a short CRASH-RECOVERY TTL, but the
379
+ * `senior:escalation-steward` agent task it must survive has NO bounded runtime (no interrupting
380
+ * boundary timer). On a long agent run the lease lapses before the record step runs, and
381
+ * {@link recordResolution}'s owner+lease CAS would then REJECT the expired claim and discard the
382
+ * agent's valid decision — the next tick would simply repeat the work. Renewing re-asserts the lease
383
+ * so the finalise CAS can win.
384
+ *
385
+ * Why owner-fenced but NOT expiry-gated: the whole point is to recover a LAPSED lease, so gating on
386
+ * an unexpired lease would defeat it. Scoping to `claimed_by = owner` is a sufficient exclusivity
387
+ * fence here because the steward's overlap guard (collect worker) blocks any OTHER tick from touching
388
+ * claims while this run is ACTIVE — so no overlapping tick can have released+reclaimed the row out
389
+ * from under this still-active run. A foreign reclaim (`claimed_by != owner`) or a released/finalised
390
+ * row (no open `claimed` row) therefore changes ZERO rows and reports `false`, so a stale/foreign run
391
+ * can never steal or resurrect a claim. Returns whether this run's claim was renewed. */
392
+ export async function renewClaim(
393
+ data: DataLayer,
394
+ userTaskKey: string,
395
+ owner: string,
396
+ ttlMs: number = CLAIM_TTL_MS,
397
+ ): Promise<boolean> {
398
+ const until = new Date(Date.now() + ttlMs).toISOString();
399
+ const res = await data.open().exec(
400
+ `UPDATE "escalation_auto_resolutions" SET "claimed_until" = ?
401
+ WHERE "user_task_key" = ? AND "outcome" = 'claimed' AND "claimed_by" = ?`,
402
+ [until, userTaskKey, owner],
403
+ );
404
+ return res.changed > 0;
405
+ }
406
+
407
+ /** Count a subject's prior steward RESOLUTIONS (for the loop guard). Only `resolved` rows count — an
408
+ * abstention did not apply an answer, so it must not trip the guard. Scoped by BOTH `subject_type`
409
+ * AND `subject_key` (issue #889 review): a bare `subject_key` collides across subject types (a plan
410
+ * `o/r#1` and a PR `o/r#1` share the key), so an unscoped count would let one subject's resolutions
411
+ * trip an UNRELATED subject's guard. */
412
+ export async function countSubjectResolutions(data: DataLayer, subjectType: string | null, subjectKey: string): Promise<number> {
413
+ const rows = await escalationAutoResolutions(data).find({ subject_key: subjectKey, outcome: "resolved" });
414
+ if (subjectType == null) return rows.length;
415
+ return rows.filter((r) => r.subject_type === subjectType).length;
416
+ }
417
+
418
+ /** The set of `user_task_key`s currently claimed by a live steward run (unexpired, unfinalised). */
419
+ export async function liveClaimedTaskKeys(data: DataLayer): Promise<Set<string>> {
420
+ const rows = await escalationAutoResolutions(data).find({ outcome: "claimed" });
421
+ const out = new Set<string>();
422
+ const at = now();
423
+ for (const r of rows) {
424
+ if (r.claimed_until != null && r.claimed_until >= at) out.add(r.user_task_key);
425
+ }
426
+ return out;
427
+ }
428
+
429
+ /** The set of `user_task_key`s the steward has ALREADY finally recorded (`resolved`/`abstained`).
430
+ * The claim fence's partial UNIQUE index only fences an OPEN (`claimed`) claim, so once a row is
431
+ * finalised it leaves that index — and because an abstention does NOT complete the user task, the
432
+ * task stays open and every later tick would otherwise re-claim the SAME `user_task_key` and record
433
+ * another abstention indefinitely (issue #889 review). Exclude already-final keys from eligibility so
434
+ * a task the steward already answered/abstained is never re-claimed. (A genuinely NEW escalation of
435
+ * the same work gets a NEW engine `user_task_key`, so this never blocks a fresh escalation.) */
436
+ export async function finalRecordedTaskKeys(data: DataLayer): Promise<Set<string>> {
437
+ const rows = await escalationAutoResolutions(data).all();
438
+ const out = new Set<string>();
439
+ for (const r of rows) {
440
+ if (r.outcome === "resolved" || r.outcome === "abstained") out.add(r.user_task_key);
441
+ }
442
+ return out;
443
+ }
444
+
445
+ /** Mark a record dismissed (reviewed, no action) or intervened (operator stepped in), with an optional
446
+ * note. Only a FINAL (`resolved`/`abstained`), still-`pending` record transitions — an in-flight
447
+ * `claimed` placeholder starts life `review_state='pending'` too, so without the outcome guard a
448
+ * guessed/old claim id could be "reviewed" before it finalises, letting its eventual resolution slip
449
+ * past the pending-review queue (matching the list/read-model, which only ever surfaces final rows).
450
+ * An already-reviewed record is left alone (idempotent). */
451
+ export async function reviewResolution(
452
+ data: DataLayer,
453
+ id: number,
454
+ action: "dismissed" | "intervened",
455
+ reviewedBy: string,
456
+ note: string | null,
457
+ ): Promise<boolean> {
458
+ const res = await data.open().exec(
459
+ `UPDATE "escalation_auto_resolutions"
460
+ SET "review_state" = ?, "reviewed_by" = ?, "reviewed_at" = ?, "review_note" = ?
461
+ WHERE "id" = ? AND "review_state" = 'pending' AND "outcome" IN ('resolved', 'abstained')`,
462
+ [action, reviewedBy, now(), note, id],
463
+ );
464
+ return res.changed === 1;
465
+ }
466
+
467
+ /** List auto-resolution records, newest first, optionally filtered by review state / subject / kind. */
468
+ export async function listAutoResolutions(
469
+ data: DataLayer,
470
+ filter: { reviewState?: string; subjectKey?: string; kind?: string } = {},
471
+ ): Promise<EscalationAutoResolution[]> {
472
+ let rows = await escalationAutoResolutions(data).all();
473
+ // Only FINAL records are reviewable — an unfinalised `claimed` placeholder is an in-flight steward
474
+ // action, not a reviewable decision, so it never reaches the view.
475
+ rows = rows.filter((r) => r.outcome === "resolved" || r.outcome === "abstained");
476
+ if (filter.reviewState) rows = rows.filter((r) => r.review_state === filter.reviewState);
477
+ if (filter.subjectKey) rows = rows.filter((r) => r.subject_key === filter.subjectKey);
478
+ if (filter.kind) rows = rows.filter((r) => r.escalation_kind === filter.kind || r.element_id === filter.kind);
479
+ return rows.sort((a, b) => (a.created_at < b.created_at ? 1 : a.created_at > b.created_at ? -1 : 0));
480
+ }
481
+
482
+ /** Count of PENDING (not-yet-reviewed) final records — the nav badge / list header. */
483
+ export async function pendingAutoResolutionCount(data: DataLayer): Promise<number> {
484
+ return (await listAutoResolutions(data, { reviewState: "pending" })).length;
485
+ }
486
+
487
+ /** Build the `processInstanceKey` → opted-in map across EVERY escalation-raising subject table
488
+ * (`feature_runs`, `plans`, `pull_requests`, `delivery_graph_runs`). The steward's eligibility reads
489
+ * the flag off the SUBJECT row keyed by the escalation's owning instance — the durable source of
490
+ * truth, immune to an agent-completion clobber of the process variable (#887/#888). A subject whose
491
+ * row is absent, has no `process_key`, or carries `auto_resolve_escalations != 1` maps to NOT opted
492
+ * in (fail-closed: work that didn't opt in is never touched). */
493
+ export async function optInByProcessKey(data: DataLayer): Promise<Map<string, boolean>> {
494
+ const out = new Map<string, boolean>();
495
+ const collect = async (table: string, key: string) => {
496
+ const rows: OptInSubjectRow[] = await data.table<OptInSubjectRow>(table, key).all();
497
+ for (const r of rows) {
498
+ if (r.process_key) out.set(String(r.process_key), r.auto_resolve_escalations === 1);
499
+ }
500
+ };
501
+ await collect("feature_runs", "feature_key");
502
+ await collect("plans", "plan_key");
503
+ await collect("pull_requests", "pr_key");
504
+ await collect("delivery_graph_runs", "run_key");
505
+ return out;
506
+ }
507
+
508
+ /** The same opt-in derivation keyed by the SUBJECT's primary key (`feature_key`/`plan_key`/`pr_key`/
509
+ * `run_key`) rather than its engine `process_key`. The collect step needs BOTH: an escalation that
510
+ * parks INSIDE a callActivity child cell (the shared `human-escalation`/`implement-cell`, ADR 0006 S4)
511
+ * is denormalised onto `user_tasks` with the CHILD's `process_key` (which no subject row tracks) but
512
+ * the PARENT run's `subject_key` (which `pollUserTasks` correlates via the root instance, #603/#633).
513
+ * Resolving eligibility off the subject key lets those child-cell escalations opt in exactly as their
514
+ * owning run did, where a `process_key`-only lookup would always read them as non-opted-in. */
515
+ export async function optInBySubjectKey(data: DataLayer): Promise<Map<string, boolean>> {
516
+ const out = new Map<string, boolean>();
517
+ // Each subject table's PRIMARY-KEY column differs, so read it generically: the row's key field is
518
+ // the one named `key`. The map is keyed by a `<subjectType>:<subjectKey>` COMPOUND key, never the
519
+ // bare subject key — a `feature_key`, `plan_key`, `pr_key` and `run_key` share one string space, so
520
+ // an un-scoped key would let one subject's opt-in (or opt-OUT) silently clobber a DIFFERENT subject
521
+ // type's row with a colliding key, making a non-opted-in task eligible (issue #889 review). A
522
+ // missing/blank key is dropped so it never matches a real escalation subject.
523
+ const collect = async (table: string, key: string, subjectType: string) => {
524
+ const rows: OptInSubjectRow[] = await data.table<OptInSubjectRow>(table, key).all();
525
+ for (const r of rows) {
526
+ // Read the subject's primary-key column generically (it differs per table). `keyOf` is a type
527
+ // guard, not an `as` cast, so the dynamic read stays type-safe.
528
+ const keyValue = keyOf(r, key);
529
+ if (keyValue) out.set(`${subjectType}:${keyValue}`, r.auto_resolve_escalations === 1);
530
+ }
531
+ };
532
+ await collect("feature_runs", "feature_key", "feature");
533
+ await collect("plans", "plan_key", "plan");
534
+ await collect("pull_requests", "pr_key", "pr");
535
+ await collect("delivery_graph_runs", "run_key", "delivery");
536
+ return out;
537
+ }
@@ -0,0 +1,123 @@
1
+ // Deterministic unit coverage for `pollEscalationSteward` (issue #889 review) — the steward tick's
2
+ // pacing + master-switch path, previously untested. Drives the exported poller against a stub
3
+ // EngineClient and asserts the four branches that decide whether the steward runs at all:
4
+ // • the master switch (`NANO_ESCALATION_STEWARD` off) — never starts an instance;
5
+ // • interval pacing (`NANO_ESCALATION_STEWARD_INTERVAL`) — a tick inside the interval is suppressed;
6
+ // • active-instance suppression — a still-ACTIVE prior run makes the tick a no-op;
7
+ // • failed-start retry — a `createInstance` throw leaves the tick un-advanced so the next pass retries.
8
+ //
9
+ // `pollEscalationSteward` paces itself via a MODULE-LEVEL `lastStewardTick`, so each test imports a
10
+ // FRESH (cache-busted) copy of `service.ts` to isolate that state — otherwise one test's tick would
11
+ // suppress the next test's first tick through the shared interval clock.
12
+ import { test } from "node:test";
13
+ import { assertEquals, assertRejects } from "#test-assert";
14
+ import type { EngineClient } from "@nanobpm/urban";
15
+
16
+ /** A stub EngineClient capturing createInstance calls and reporting a configurable ACTIVE set. */
17
+ function stubEngine(opts: { active?: string[]; failStart?: boolean } = {}) {
18
+ const created: { processDefinitionId: string; variables: Record<string, unknown> }[] = [];
19
+ const engine = {
20
+ searchProcessInstances: (filter?: { processDefinitionId?: string; state?: string }) =>
21
+ Promise.resolve(
22
+ filter?.processDefinitionId === "escalation-steward" && filter?.state === "ACTIVE"
23
+ ? (opts.active ?? []).map((k) => ({ processInstanceKey: k, state: "ACTIVE" }))
24
+ : [],
25
+ ),
26
+ createInstance: (input: { processDefinitionId: string; variables: Record<string, unknown> }) => {
27
+ if (opts.failStart) return Promise.reject(new Error("engine unavailable"));
28
+ created.push(input);
29
+ return Promise.resolve({ processInstanceKey: `PI-${created.length}` });
30
+ },
31
+ };
32
+ return { engine: engine as unknown as EngineClient, created };
33
+ }
34
+
35
+ /** Import a FRESH `service.ts` (isolated `lastStewardTick`) and return its poller. */
36
+ async function freshPoller(): Promise<(engine: EngineClient) => Promise<void>> {
37
+ const mod = await import(`./service.ts?iso=${Date.now()}-${Math.random()}`);
38
+ return mod.pollEscalationSteward;
39
+ }
40
+
41
+ /** Run `fn` with the given env vars set, restoring process.env afterwards. */
42
+ async function withEnv<T>(env: Record<string, string>, fn: () => Promise<T>): Promise<T> {
43
+ const prev: Record<string, string | undefined> = {};
44
+ for (const k of Object.keys(env)) prev[k] = process.env[k];
45
+ for (const [k, v] of Object.entries(env)) process.env[k] = v;
46
+ try {
47
+ return await fn();
48
+ } finally {
49
+ for (const k of Object.keys(env)) {
50
+ if (prev[k] === undefined) delete process.env[k];
51
+ else process.env[k] = prev[k];
52
+ }
53
+ }
54
+ }
55
+
56
+ const STEWARD_ENV = { NANO_ESCALATION_STEWARD: "1", NANO_ESCALATION_STEWARD_INTERVAL: "PT30M" };
57
+
58
+ test("pollEscalationSteward: the master switch OFF never starts an instance", async () => {
59
+ const poll = await freshPoller();
60
+ const { engine, created } = stubEngine();
61
+ await withEnv({ NANO_ESCALATION_STEWARD: "off" }, () => poll(engine));
62
+ assertEquals(created.length, 0, "disabled → no instance started");
63
+ });
64
+
65
+ test("pollEscalationSteward: a still-ACTIVE prior run suppresses the tick (overlap guard)", async () => {
66
+ const poll = await freshPoller();
67
+ const { engine, created } = stubEngine({ active: ["PI-PRIOR"] });
68
+ await withEnv(STEWARD_ENV, () => poll(engine));
69
+ assertEquals(created.length, 0, "an active prior run suppresses the start");
70
+ });
71
+
72
+ test("pollEscalationSteward: a failed start leaves the tick un-advanced so the next pass retries immediately", async () => {
73
+ const poll = await freshPoller();
74
+ // First pass: the engine start THROWS. The tick must NOT advance (lastStewardTick stays 0), so a
75
+ // second pass within the interval still attempts a start rather than suppressing retries.
76
+ const failing = stubEngine({ failStart: true });
77
+ await assertRejects(() => withEnv(STEWARD_ENV, () => poll(failing.engine)), Error, "engine unavailable");
78
+ assertEquals(failing.created.length, 0, "the failed start recorded nothing");
79
+
80
+ // Immediately retry (same module clock, well inside the interval) with a now-healthy engine: the
81
+ // un-advanced tick lets the start straight through.
82
+ const healthy = stubEngine();
83
+ await withEnv(STEWARD_ENV, () => poll(healthy.engine));
84
+ assertEquals(healthy.created.length, 1, "the retry is not suppressed by the failed attempt");
85
+ assertEquals(healthy.created[0].processDefinitionId, "escalation-steward");
86
+ });
87
+
88
+ test("pollEscalationSteward: interval pacing suppresses a tick that lands inside the interval", async () => {
89
+ const poll = await freshPoller();
90
+ const { engine, created } = stubEngine();
91
+ // First tick starts an instance and advances the module clock.
92
+ await withEnv(STEWARD_ENV, () => poll(engine));
93
+ assertEquals(created.length, 1, "the first tick starts an instance");
94
+ // A second tick immediately after (well inside the 30-min interval) is suppressed by pacing.
95
+ await withEnv(STEWARD_ENV, () => poll(engine));
96
+ assertEquals(created.length, 1, "a tick inside the interval is suppressed");
97
+ });
98
+
99
+ test("pollEscalationSteward: a zero/negative interval is clamped so the steward never free-runs every poll", async () => {
100
+ const poll = await freshPoller();
101
+ const { engine, created } = stubEngine();
102
+ // PT0S parses to 0 → clamped to the 30-min default. The first tick starts; the immediate second is
103
+ // still suppressed by the clamped interval (a 0-interval must never start every poll).
104
+ await withEnv({ NANO_ESCALATION_STEWARD: "1", NANO_ESCALATION_STEWARD_INTERVAL: "PT0S" }, () => poll(engine));
105
+ assertEquals(created.length, 1, "the first tick starts even with a zero interval");
106
+ await withEnv({ NANO_ESCALATION_STEWARD: "1", NANO_ESCALATION_STEWARD_INTERVAL: "PT0S" }, () => poll(engine));
107
+ assertEquals(created.length, 1, "a zero interval is clamped — the immediate re-tick is suppressed");
108
+ });
109
+
110
+ test("pollEscalationSteward: the steward start seeds the canonical agentSlaTimeout so its agent boundary can fire", async () => {
111
+ const poll = await freshPoller();
112
+ const { engine, created } = stubEngine();
113
+ await withEnv(STEWARD_ENV, () => poll(engine));
114
+ assertEquals(created.length, 1, "the tick starts an instance");
115
+ // The `be_steward_sla` interrupting timer boundary (escalation-steward.bpmn) evaluates
116
+ // `<bpmn:timeDuration>=agentSlaTimeout</…>` at timer creation (FEEL). If the start does not seed
117
+ // `agentSlaTimeout` the boundary can never arm and a hung steward agent parks the instance ACTIVE
118
+ // forever — the overlap guard then suppresses every later tick (issue #889 review). Assert the
119
+ // seeded variable is a non-blank ISO-8601 duration (the canonical AGENT_SLA_TIMEOUT default).
120
+ const seeded = created[0].variables.agentSlaTimeout;
121
+ assertEquals(typeof seeded, "string", "agentSlaTimeout is seeded as a string");
122
+ assertEquals(/^P/.test(seeded), true, `agentSlaTimeout is an ISO-8601 duration (got ${JSON.stringify(seeded)})`);
123
+ });