@nanobpm/nano-workforce 0.201.1 → 0.201.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md CHANGED
@@ -1,3 +1,9 @@
1
+ ## [0.201.2](https://github.com/nanobpm/nano-workforce/compare/v0.201.1...v0.201.2) (2026-10-08)
2
+
3
+ ### Bug Fixes
4
+
5
+ * distinguish never-started agent jobs from hung ones in SLA escalations ([#881](https://github.com/nanobpm/nano-workforce/issues/881)) ([da2bc79](https://github.com/nanobpm/nano-workforce/commit/da2bc7915c6cd8ef72bae7bdec2069d09ba091eb)), closes [#879](https://github.com/nanobpm/nano-workforce/issues/879) [#879](https://github.com/nanobpm/nano-workforce/issues/879) [#879](https://github.com/nanobpm/nano-workforce/issues/879)
6
+
1
7
  ## [0.201.1](https://github.com/nanobpm/nano-workforce/compare/v0.201.0...v0.201.1) (2026-10-07)
2
8
 
3
9
  ### Bug Fixes
@@ -0,0 +1,109 @@
1
+ // Issue #879: the activation ledger upsert must capture the FIRST activation only, be idempotent on a
2
+ // steady state, and never move/clear an earlier activation timestamp — so the SLA escalation reads a
3
+ // stable never-started-vs-activated signal regardless of how many poll passes observe the job.
4
+ import { test } from "node:test";
5
+ import { assertEquals } from "#test-assert";
6
+ import { getAgentJobActivation, recordAgentJobObservation } from "./agentActivationLedger.ts";
7
+ import { memDataFor } from "../test/worldDb.ts";
8
+
9
+ function ledger() {
10
+ return memDataFor(["119_agent_job_activations.sql"]).data;
11
+ }
12
+
13
+ test("first observation of a queued job records first_seen_at, no activation", async () => {
14
+ const data = ledger();
15
+ await recordAgentJobObservation(data, {
16
+ processInstanceKey: "PI-1",
17
+ jobType: "senior:feature",
18
+ worker: null,
19
+ now: "t0",
20
+ });
21
+ const row = await getAgentJobActivation(data, "PI-1", "senior:feature");
22
+ assertEquals(row?.first_seen_at, "t0");
23
+ assertEquals(row?.activated_at, null);
24
+ assertEquals(row?.worker, null);
25
+ });
26
+
27
+ test("a later observation with a worker records the activation", async () => {
28
+ const data = ledger();
29
+ await recordAgentJobObservation(data, { processInstanceKey: "PI-2", jobType: "senior:feature", worker: null, now: "t0" });
30
+ await recordAgentJobObservation(data, {
31
+ processInstanceKey: "PI-2",
32
+ jobType: "senior:feature",
33
+ worker: "agent-a",
34
+ now: "t1",
35
+ });
36
+ const row = await getAgentJobActivation(data, "PI-2", "senior:feature");
37
+ assertEquals(row?.first_seen_at, "t0");
38
+ assertEquals(row?.activated_at, "t1");
39
+ assertEquals(row?.worker, "agent-a");
40
+ });
41
+
42
+ test("the FIRST activation is sticky — a later re-observation never moves it", async () => {
43
+ const data = ledger();
44
+ await recordAgentJobObservation(data, { processInstanceKey: "PI-3", jobType: "senior:feature", worker: "agent-a", now: "t1" });
45
+ await recordAgentJobObservation(data, { processInstanceKey: "PI-3", jobType: "senior:feature", worker: "agent-b", now: "t2" });
46
+ const row = await getAgentJobActivation(data, "PI-3", "senior:feature");
47
+ assertEquals(row?.activated_at, "t1");
48
+ assertEquals(row?.worker, "agent-a");
49
+ });
50
+
51
+ test("observations are keyed by (instance, jobType) — distinct jobs do not collide", async () => {
52
+ const data = ledger();
53
+ await recordAgentJobObservation(data, { processInstanceKey: "PI-4", jobType: "senior:conformance", worker: null, now: "t0" });
54
+ await recordAgentJobObservation(data, { processInstanceKey: "PI-4", jobType: "senior:retro", worker: "agent-c", now: "t0" });
55
+ const conf = await getAgentJobActivation(data, "PI-4", "senior:conformance");
56
+ const retro = await getAgentJobActivation(data, "PI-4", "senior:retro");
57
+ assertEquals(conf?.activated_at, null);
58
+ assertEquals(retro?.worker, "agent-c");
59
+ });
60
+
61
+ test("an empty-string worker is treated as not-yet-activated", async () => {
62
+ const data = ledger();
63
+ await recordAgentJobObservation(data, { processInstanceKey: "PI-5", jobType: "senior:feature", worker: "", now: "t0" });
64
+ const row = await getAgentJobActivation(data, "PI-5", "senior:feature");
65
+ assertEquals(row?.activated_at, null);
66
+ assertEquals(row?.worker, null);
67
+ });
68
+
69
+ test("a new job key (a retry) RESETS the row — an activated attempt followed by an unactivated retry reads never-activated", async () => {
70
+ // Issue #879 review: implement-cell/merge-cell/merge-loop retry the SAME (instance, jobType), so a
71
+ // sticky first-activation would make a queued retry read the EARLIER attempt's worker and misdiagnose
72
+ // it as started/hung. The attempt's engine jobKey discriminates them.
73
+ const data = ledger();
74
+ // Attempt 1: activated by agent-a.
75
+ await recordAgentJobObservation(data, { processInstanceKey: "PI-6", jobType: "senior:feature", worker: "agent-a", jobKey: "J1", now: "t1" });
76
+ // A steady re-observation of the SAME attempt never moves the first activation.
77
+ await recordAgentJobObservation(data, { processInstanceKey: "PI-6", jobType: "senior:feature", worker: "agent-a", jobKey: "J1", now: "t2" });
78
+ let row = await getAgentJobActivation(data, "PI-6", "senior:feature");
79
+ assertEquals(row?.activated_at, "t1");
80
+ assertEquals(row?.worker, "agent-a");
81
+ // Attempt 2 (retry): a DIFFERENT jobKey, still queued → the row resets to the current attempt.
82
+ await recordAgentJobObservation(data, { processInstanceKey: "PI-6", jobType: "senior:feature", worker: null, jobKey: "J2", now: "t3" });
83
+ row = await getAgentJobActivation(data, "PI-6", "senior:feature");
84
+ assertEquals(row?.first_seen_at, "t3");
85
+ assertEquals(row?.activated_at, null);
86
+ assertEquals(row?.worker, null);
87
+ assertEquals(row?.job_key, "J2");
88
+ });
89
+
90
+ test("a new job key that is itself activated reflects the CURRENT attempt's worker, not the prior one", async () => {
91
+ const data = ledger();
92
+ await recordAgentJobObservation(data, { processInstanceKey: "PI-7", jobType: "senior:trial-merge", worker: "agent-a", jobKey: "J1", now: "t1" });
93
+ await recordAgentJobObservation(data, { processInstanceKey: "PI-7", jobType: "senior:trial-merge", worker: "agent-b", jobKey: "J2", now: "t2" });
94
+ const row = await getAgentJobActivation(data, "PI-7", "senior:trial-merge");
95
+ assertEquals(row?.activated_at, "t2");
96
+ assertEquals(row?.worker, "agent-b");
97
+ assertEquals(row?.job_key, "J2");
98
+ });
99
+
100
+ test("the SAME job key keeps the first-activation sticky across re-observations", async () => {
101
+ const data = ledger();
102
+ await recordAgentJobObservation(data, { processInstanceKey: "PI-8", jobType: "senior:feature", worker: null, jobKey: "J1", now: "t0" });
103
+ await recordAgentJobObservation(data, { processInstanceKey: "PI-8", jobType: "senior:feature", worker: "agent-a", jobKey: "J1", now: "t1" });
104
+ await recordAgentJobObservation(data, { processInstanceKey: "PI-8", jobType: "senior:feature", worker: "agent-b", jobKey: "J1", now: "t2" });
105
+ const row = await getAgentJobActivation(data, "PI-8", "senior:feature");
106
+ assertEquals(row?.first_seen_at, "t0");
107
+ assertEquals(row?.activated_at, "t1");
108
+ assertEquals(row?.worker, "agent-a");
109
+ });
@@ -0,0 +1,144 @@
1
+ // Agent-job activation ledger (issue #879) — the durable "did a worker ever pick this agent job up?"
2
+ // record every agent-task SLA-escalation reads to tell a NEVER-ACTIVATED (queue-starved) job apart
3
+ // from an ACTIVATED-but-hung one.
4
+ //
5
+ // Why it exists: an agent-task SLA boundary timer (`<bpmn:timeDuration>=agentSlaTimeout`) starts when
6
+ // the service task is ENTERED, so it bounds QUEUE time + RUN time together. Under fleet saturation a
7
+ // job can sit CREATED (no worker can take its type) for the whole budget and the boundary fires — but
8
+ // the escalation reported it as "hung or looping" and told the human to "check its progress" when
9
+ // there is none (issue #879: a `senior:feature` job queued 2h, never activated, then SLA-cancelled).
10
+ // The engine knows JobState::Created vs ::Activated, but once the boundary CANCELS the job that state
11
+ // is gone — so the observation must be captured WHILE the job is live and persisted. `pollAgentActivations`
12
+ // (app/service.ts) does exactly that, writing this ledger; the SLA-escalation recorders read it.
13
+ //
14
+ // The activation signal is the same one the review-round visibility uses (migration 005): the
15
+ // Camunda-8 `/v2/jobs/search` wire API collapses Activated -> CREATED (no ACTIVATED enum), so a leased
16
+ // job is recognised by its `worker` + `deadline` fields — a merely-queued job carries neither. This
17
+ // ledger generalises that to EVERY bounded agent job type and persists first-seen + first-activated.
18
+ //
19
+ // Advisory & best-effort (ADR 0056): this is an app-tier read model over the engine's job read model.
20
+ // It never activates/completes a job, publishes a message, or gates a BPMN sequence flow — it only
21
+ // enriches the human-facing escalation wording. A missing row (poller never observed the job) degrades
22
+ // to "activation unknown", never an error.
23
+ import type { DataLayer } from "@nanobpm/urban";
24
+
25
+ /** The agent job types that today carry an interrupting SLA timer boundary whose elapse escalates to a
26
+ * human with a "hung or looping" reason — the class issue #879 guards. Kept as one list so the
27
+ * activation poll's `/v2/jobs/search` type filter and the escalation recorders agree on exactly which
28
+ * jobs are tracked. (The convergence-loop / plan-fanout agent tasks are NOT yet bounded — pre-seeded
29
+ * for #868 — so they are deliberately absent until their boundaries land.) */
30
+ export const AGENT_SLA_JOB_TYPES: readonly string[] = [
31
+ "senior:feature", // implement-cell
32
+ "senior:trial-merge", // merge-cell
33
+ "senior:conformance", // retro conformance
34
+ "senior:retro", // retro synthesize
35
+ "senior:fix-ci", // merge-loop CI-fix cell (be_fixci_sla)
36
+ "senior:rebase", // merge-loop rebase cell (be_rebase_sla)
37
+ ];
38
+
39
+ /** One ledger row: whether (and when) a worker picked up an agent job for a given process instance. */
40
+ export interface AgentJobActivation {
41
+ /** Surrogate key `<process_instance_key>:<job_type>`. */
42
+ id: string;
43
+ process_instance_key: string;
44
+ job_type: string;
45
+ /** ISO ts the poller first observed the job (queued/CREATED). */
46
+ first_seen_at: string;
47
+ /** ISO ts first observed ACTIVATED (a leasing worker appeared); NULL ⇒ never started. */
48
+ activated_at: string | null;
49
+ /** The leasing worker's name at activation; NULL until activated. */
50
+ worker: string | null;
51
+ /** The engine job key of the attempt this row currently tracks; NULL when unknown (an old row or a
52
+ * caller that did not supply one). A poll observation carrying a DIFFERENT job key is a new attempt
53
+ * and resets the row — see {@link recordAgentJobObservation}. */
54
+ job_key: string | null;
55
+ updated_at: string;
56
+ }
57
+
58
+ /** The surrogate key for a (process instance, job type) pair — the agent job and its SLA-escalation
59
+ * recorder share one process instance, so both sides key the ledger identically. */
60
+ export function agentActivationId(processInstanceKey: string, jobType: string): string {
61
+ return `${processInstanceKey}:${jobType}`;
62
+ }
63
+
64
+ /** Record gateway over the activation ledger. */
65
+ export const agentActivations = (data: DataLayer) =>
66
+ data.table<AgentJobActivation>("agent_job_activations", "id");
67
+
68
+ /** Upsert one poll observation of an agent job. On first sight records `first_seen_at`; the FIRST time
69
+ * a leasing `worker` is observed it records `activated_at` + `worker` (a later re-observation of the
70
+ * SAME attempt never moves the first-activation timestamp). Idempotent on a steady state.
71
+ *
72
+ * Retry isolation (issue #879 review): the same (process instance, job type) runs MULTIPLE times within
73
+ * one instance when its cell retries (`ic_reImplement`, `mc_rerun`, merge-loop fix-ci/rebase reloops).
74
+ * Each attempt is a distinct engine job with its own `jobKey`, so when an observation carries a job key
75
+ * DIFFERENT from the one the row tracks, it is a NEW attempt and the row is RESET to it (first_seen_at /
76
+ * activated_at / worker), discarding the earlier attempt's activation. Without this a sticky
77
+ * first-activation row would make a later queued retry read the earlier attempt's worker and misdiagnose
78
+ * a queue-starved retry as started/hung, and its elapsed times would fold in the prior attempt + human
79
+ * wait. When the job key is unknown on either side the row keeps its prior first-sticky-activation
80
+ * behaviour (the caller cannot prove a new attempt). */
81
+ export async function recordAgentJobObservation(
82
+ data: DataLayer,
83
+ input: {
84
+ processInstanceKey: string;
85
+ jobType: string;
86
+ worker: string | null;
87
+ jobKey?: string | null;
88
+ now: string;
89
+ },
90
+ ): Promise<void> {
91
+ const table = agentActivations(data);
92
+ const id = agentActivationId(input.processInstanceKey, input.jobType);
93
+ const worker = input.worker && input.worker.length > 0 ? input.worker : null;
94
+ const jobKey = input.jobKey && input.jobKey.length > 0 ? input.jobKey : null;
95
+ const existing = await table.get(id);
96
+ if (!existing) {
97
+ await table.insert({
98
+ id,
99
+ process_instance_key: input.processInstanceKey,
100
+ job_type: input.jobType,
101
+ first_seen_at: input.now,
102
+ activated_at: worker ? input.now : null,
103
+ worker,
104
+ job_key: jobKey,
105
+ updated_at: input.now,
106
+ });
107
+ return;
108
+ }
109
+ const existingKey = existing.job_key && existing.job_key.length > 0 ? existing.job_key : null;
110
+ // A different engine job key for the same (instance, jobType) is a NEW attempt — reset the row so it
111
+ // reflects only the current attempt, never a stale earlier activation/wait.
112
+ if (jobKey && existingKey && jobKey !== existingKey) {
113
+ await table.update(id, {
114
+ first_seen_at: input.now,
115
+ activated_at: worker ? input.now : null,
116
+ worker,
117
+ job_key: jobKey,
118
+ updated_at: input.now,
119
+ });
120
+ return;
121
+ }
122
+ // Same attempt (or job key unknown): capture the FIRST activation only, and adopt a newly-known job
123
+ // key so a subsequent different one is recognised as a new attempt.
124
+ const patch: Partial<AgentJobActivation> = {};
125
+ if (worker && !existing.activated_at) {
126
+ patch.activated_at = input.now;
127
+ patch.worker = worker;
128
+ }
129
+ if (jobKey && !existingKey) patch.job_key = jobKey;
130
+ if (Object.keys(patch).length > 0) {
131
+ patch.updated_at = input.now;
132
+ await table.update(id, patch);
133
+ }
134
+ }
135
+
136
+ /** Read the activation record for a (process instance, job type), or `undefined` when the poller never
137
+ * observed the job. */
138
+ export async function getAgentJobActivation(
139
+ data: DataLayer,
140
+ processInstanceKey: string,
141
+ jobType: string,
142
+ ): Promise<AgentJobActivation | undefined> {
143
+ return agentActivations(data).get(agentActivationId(processInstanceKey, jobType));
144
+ }
@@ -0,0 +1,180 @@
1
+ // Issue #879: `pollAgentActivationsImpl` must persist, for every bounded agent job type, whether a
2
+ // worker ever leased the job — keyed by (processInstanceKey, jobType) — so a later SLA escalation can
3
+ // tell a never-activated (queue-starved) job from an activated one even after the boundary cancels it.
4
+ import { test } from "node:test";
5
+ import { assertEquals } from "#test-assert";
6
+ import { getAgentJobActivation } from "./agentActivationLedger.ts";
7
+ import { pollAgentActivationsImpl } from "./service.ts";
8
+ import { memDataFor } from "../test/worldDb.ts";
9
+
10
+ // biome-ignore lint/suspicious/noExplicitAny: test stub over the engine job-search wire shape.
11
+ type JobItem = any;
12
+
13
+ function jobsFetch(items: JobItem[], opts: { ignoreTypeFilter?: boolean } = {}) {
14
+ return (_url: string, init?: { body?: string }) => {
15
+ const body = init?.body ? JSON.parse(init.body) : {};
16
+ const t = body.filter?.type;
17
+ const wanted: string[] | null = Array.isArray(t?.$in) ? t.$in : typeof t === "string" ? [t] : null;
18
+ const matched = opts.ignoreTypeFilter || wanted == null ? items : items.filter((j) => wanted.includes(j.type ?? ""));
19
+ // Honour the `page: { from, limit }` paging contract so a pagination regression can observe
20
+ // whether the poller actually advances past the first page.
21
+ const from = typeof body.page?.from === "number" ? body.page.from : 0;
22
+ const limit = typeof body.page?.limit === "number" ? body.page.limit : matched.length;
23
+ const out = matched.slice(from, from + limit);
24
+ return Promise.resolve(
25
+ new Response(JSON.stringify({ items: out }), { status: 200, headers: { "content-type": "application/json" } }),
26
+ );
27
+ };
28
+ }
29
+
30
+ const headers = { "content-type": "application/json" };
31
+
32
+ test("poll records a queued (unleased) agent job as not-yet-activated", async () => {
33
+ const { data } = memDataFor(["119_agent_job_activations.sql"]);
34
+ const prev = globalThis.fetch;
35
+ globalThis.fetch = jobsFetch([
36
+ { type: "senior:feature", processInstanceKey: "PI-10", state: "CREATED" },
37
+ ]) as typeof fetch;
38
+ try {
39
+ await pollAgentActivationsImpl(data, "http://engine/v2", headers);
40
+ } finally {
41
+ globalThis.fetch = prev;
42
+ }
43
+ const row = await getAgentJobActivation(data, "PI-10", "senior:feature");
44
+ assertEquals(row?.activated_at, null);
45
+ assertEquals(row?.worker, null);
46
+ });
47
+
48
+ test("poll records an activation when a leasing worker appears, across passes", async () => {
49
+ const { data } = memDataFor(["119_agent_job_activations.sql"]);
50
+ const prev = globalThis.fetch;
51
+ try {
52
+ globalThis.fetch = jobsFetch([
53
+ { type: "senior:trial-merge", processInstanceKey: "PI-11", state: "CREATED" },
54
+ ]) as typeof fetch;
55
+ await pollAgentActivationsImpl(data, "http://engine/v2", headers);
56
+ assertEquals((await getAgentJobActivation(data, "PI-11", "senior:trial-merge"))?.activated_at, null);
57
+
58
+ globalThis.fetch = jobsFetch([
59
+ { type: "senior:trial-merge", processInstanceKey: "PI-11", worker: "agent-m", deadline: "2024-01-01T00:15:00Z", state: "CREATED" },
60
+ ]) as typeof fetch;
61
+ await pollAgentActivationsImpl(data, "http://engine/v2", headers);
62
+ const row = await getAgentJobActivation(data, "PI-11", "senior:trial-merge");
63
+ assertEquals(row?.worker, "agent-m");
64
+ assertEquals(typeof row?.activated_at === "string", true);
65
+ } finally {
66
+ globalThis.fetch = prev;
67
+ }
68
+ });
69
+
70
+ test("poll defensively excludes a non-agent job even if the wire $in filter is ignored", async () => {
71
+ const { data } = memDataFor(["119_agent_job_activations.sql"]);
72
+ const prev = globalThis.fetch;
73
+ globalThis.fetch = jobsFetch(
74
+ [{ type: "pr.capture-head", processInstanceKey: "PI-12", worker: "host-internal", state: "CREATED" }],
75
+ { ignoreTypeFilter: true },
76
+ ) as typeof fetch;
77
+ try {
78
+ await pollAgentActivationsImpl(data, "http://engine/v2", headers);
79
+ } finally {
80
+ globalThis.fetch = prev;
81
+ }
82
+ assertEquals(await getAgentJobActivation(data, "PI-12", "pr.capture-head"), undefined);
83
+ });
84
+
85
+ test("poll skips an item with no process instance to key on", async () => {
86
+ const { data } = memDataFor(["119_agent_job_activations.sql"]);
87
+ const prev = globalThis.fetch;
88
+ globalThis.fetch = jobsFetch([{ type: "senior:feature", state: "CREATED" }]) as typeof fetch;
89
+ try {
90
+ await pollAgentActivationsImpl(data, "http://engine/v2", headers);
91
+ } finally {
92
+ globalThis.fetch = prev;
93
+ }
94
+ assertEquals(await getAgentJobActivation(data, "", "senior:feature"), undefined);
95
+ });
96
+
97
+ test("poll leaves the ledger untouched when the engine search fails", async () => {
98
+ const { data } = memDataFor(["119_agent_job_activations.sql"]);
99
+ const prev = globalThis.fetch;
100
+ globalThis.fetch = (() =>
101
+ Promise.resolve(new Response("boom", { status: 500 }))) as typeof fetch;
102
+ try {
103
+ await pollAgentActivationsImpl(data, "http://engine/v2", headers);
104
+ } finally {
105
+ globalThis.fetch = prev;
106
+ }
107
+ assertEquals(await getAgentJobActivation(data, "PI-13", "senior:feature"), undefined);
108
+ });
109
+
110
+ test("poll records the merge-loop bounded agent jobs (senior:fix-ci, senior:rebase)", async () => {
111
+ // Regression for the #881 adversarial finding: merge-loop.bpmn arms an interrupting SLA boundary
112
+ // on its fix-ci and rebase agent tasks and routes the escalation through agentSlaEscalationQuestion
113
+ // — but if AGENT_SLA_JOB_TYPES omits those job types the poller never observes them and every such
114
+ // escalation degrades to "activation unknown" instead of the never-started/hung distinction.
115
+ const { data } = memDataFor(["119_agent_job_activations.sql"]);
116
+ const prev = globalThis.fetch;
117
+ globalThis.fetch = jobsFetch([
118
+ { type: "senior:fix-ci", processInstanceKey: "PI-20", state: "CREATED" },
119
+ { type: "senior:rebase", processInstanceKey: "PI-20", worker: "agent-r", deadline: "2024-01-01T00:15:00Z", state: "CREATED" },
120
+ ]) as typeof fetch;
121
+ try {
122
+ await pollAgentActivationsImpl(data, "http://engine/v2", headers);
123
+ } finally {
124
+ globalThis.fetch = prev;
125
+ }
126
+ const fixCi = await getAgentJobActivation(data, "PI-20", "senior:fix-ci");
127
+ assertEquals(fixCi?.activated_at, null);
128
+ const rebase = await getAgentJobActivation(data, "PI-20", "senior:rebase");
129
+ assertEquals(rebase?.worker, "agent-r");
130
+ assertEquals(typeof rebase?.activated_at === "string", true);
131
+ });
132
+
133
+ test("poll records jobs BEYOND the first page (pagination)", async () => {
134
+ // Issue #881 review: a single first-page read lets long-lived jobs occupy the page every pass while
135
+ // later jobs time out with no activation recorded. 450 queued jobs > one 200-item page must all land.
136
+ const { data } = memDataFor(["119_agent_job_activations.sql"]);
137
+ const many: JobItem[] = Array.from({ length: 450 }, (_v, i) => ({
138
+ type: "senior:feature",
139
+ processInstanceKey: `PI-${i}`,
140
+ jobKey: `J-${i}`,
141
+ state: "CREATED",
142
+ }));
143
+ const prev = globalThis.fetch;
144
+ globalThis.fetch = jobsFetch(many) as typeof fetch;
145
+ try {
146
+ await pollAgentActivationsImpl(data, "http://engine/v2", headers);
147
+ } finally {
148
+ globalThis.fetch = prev;
149
+ }
150
+ // A job on the first page, one straddling the page boundary, and the very last job are all recorded.
151
+ assertEquals((await getAgentJobActivation(data, "PI-0", "senior:feature"))?.first_seen_at != null, true);
152
+ assertEquals((await getAgentJobActivation(data, "PI-200", "senior:feature"))?.first_seen_at != null, true);
153
+ assertEquals((await getAgentJobActivation(data, "PI-449", "senior:feature"))?.first_seen_at != null, true);
154
+ });
155
+
156
+ test("poll resets the ledger to a retry (new jobKey) so an earlier attempt's activation is not replayed", async () => {
157
+ // Issue #881 review: a retrying agent task reuses (instance, jobType); a new engine jobKey is a new
158
+ // attempt and must reset the activation the recorder reads.
159
+ const { data } = memDataFor(["119_agent_job_activations.sql"]);
160
+ const prev = globalThis.fetch;
161
+ try {
162
+ globalThis.fetch = jobsFetch([
163
+ { type: "senior:feature", processInstanceKey: "PI-30", jobKey: "A1", worker: "agent-a", deadline: "2024-01-01T00:15:00Z", state: "CREATED" },
164
+ ]) as typeof fetch;
165
+ await pollAgentActivationsImpl(data, "http://engine/v2", headers);
166
+ assertEquals((await getAgentJobActivation(data, "PI-30", "senior:feature"))?.worker, "agent-a");
167
+
168
+ // The retry is a different jobKey, still queued → the recorder must see never-activated, not agent-a.
169
+ globalThis.fetch = jobsFetch([
170
+ { type: "senior:feature", processInstanceKey: "PI-30", jobKey: "A2", state: "CREATED" },
171
+ ]) as typeof fetch;
172
+ await pollAgentActivationsImpl(data, "http://engine/v2", headers);
173
+ const row = await getAgentJobActivation(data, "PI-30", "senior:feature");
174
+ assertEquals(row?.activated_at, null);
175
+ assertEquals(row?.worker, null);
176
+ assertEquals(row?.job_key, "A2");
177
+ } finally {
178
+ globalThis.fetch = prev;
179
+ }
180
+ });
@@ -0,0 +1,137 @@
1
+ // Red/green for issue #879: the agent-task SLA escalation must distinguish a NEVER-ACTIVATED
2
+ // (queue-starved) agent job from an ACTIVATED-but-hung one, name the job type, and say how long it
3
+ // waited — never the old blanket "it is hung or looping" for a job no worker ever picked up.
4
+ import { test } from "node:test";
5
+ import { assertEquals, assertStringIncludes } from "#test-assert";
6
+ import { recordAgentJobObservation } from "./agentActivationLedger.ts";
7
+ import {
8
+ agentSlaEscalationQuestion,
9
+ buildAgentSlaEscalationReason,
10
+ humanizeIsoDuration,
11
+ humanizeMs,
12
+ } from "./agentSlaEscalation.ts";
13
+ import { memDataFor } from "../test/worldDb.ts";
14
+
15
+ const RECOVERY = "Answer to resume, or abandon it.";
16
+
17
+ test("humanizeMs renders a compact span and floors sub-minute to seconds", () => {
18
+ assertEquals(humanizeMs(2 * 3600_000 + 5 * 60_000), "2h 5m");
19
+ assertEquals(humanizeMs(45 * 60_000), "45m");
20
+ assertEquals(humanizeMs(30_000), "30s");
21
+ assertEquals(humanizeMs(0), "less than a minute");
22
+ assertEquals(humanizeMs(-5), "less than a minute");
23
+ assertEquals(humanizeMs(Number.NaN), "less than a minute");
24
+ });
25
+
26
+ test("humanizeIsoDuration humanizes the SLA budget", () => {
27
+ assertEquals(humanizeIsoDuration("PT2H"), "2h");
28
+ assertEquals(humanizeIsoDuration("PT15M"), "15m");
29
+ });
30
+
31
+ test("never-observed-activated: queue starvation is reported hedged, NOT as hung or looping", () => {
32
+ const reason = buildAgentSlaEscalationReason({
33
+ jobType: "senior:feature",
34
+ sla: "PT2H",
35
+ // Observed queued, never leased (activated_at null) → the defect scenario.
36
+ activation: { firstSeenAt: "2024-01-01T00:00:00Z", activatedAt: null, worker: null },
37
+ now: "2024-01-01T02:00:00Z",
38
+ recovery: RECOVERY,
39
+ });
40
+ assertStringIncludes(reason, "senior:feature");
41
+ // Observational, hedged wording — never the over-confident "the agent never started".
42
+ assertStringIncludes(reason, "No worker was ever observed to pick up");
43
+ assertStringIncludes(reason, "queue starvation");
44
+ // Names how long it waited and the SLA budget.
45
+ assertStringIncludes(reason, "2h");
46
+ // Acknowledges the poll-window race rather than claiming certainty (issue #881 review).
47
+ assertStringIncludes(reason, "that would not have been recorded");
48
+ // Must NOT misdiagnose it as hung/looping.
49
+ assertEquals(reason.includes("may be hung or looping"), false);
50
+ assertEquals(reason.includes("agent never started"), false);
51
+ assertStringIncludes(reason, RECOVERY);
52
+ });
53
+
54
+ test("started-but-exceeded: a leased job may be hung or looping", () => {
55
+ const reason = buildAgentSlaEscalationReason({
56
+ jobType: "senior:trial-merge",
57
+ sla: "PT2H",
58
+ activation: {
59
+ firstSeenAt: "2024-01-01T00:00:00Z",
60
+ activatedAt: "2024-01-01T00:10:00Z",
61
+ worker: "agent-7",
62
+ },
63
+ now: "2024-01-01T02:10:00Z",
64
+ recovery: RECOVERY,
65
+ });
66
+ assertStringIncludes(reason, "senior:trial-merge");
67
+ assertStringIncludes(reason, "started");
68
+ assertStringIncludes(reason, "hung or looping");
69
+ // Reports both the queue wait and the run time, plus the worker.
70
+ assertStringIncludes(reason, "10m");
71
+ assertStringIncludes(reason, "agent-7");
72
+ assertEquals(reason.includes("never started"), false);
73
+ });
74
+
75
+ test("activation unknown: the poller never observed the job → honest ambiguity", () => {
76
+ const reason = buildAgentSlaEscalationReason({
77
+ jobType: "senior:retro",
78
+ sla: "PT2H",
79
+ activation: null,
80
+ now: "2024-01-01T02:00:00Z",
81
+ recovery: RECOVERY,
82
+ });
83
+ assertStringIncludes(reason, "senior:retro");
84
+ assertStringIncludes(reason, "no worker-activation was ever recorded");
85
+ assertEquals(reason.includes("hung or looping"), false);
86
+ });
87
+
88
+ test("agentSlaEscalationQuestion reads the ledger and distinguishes never-started", async () => {
89
+ const { data } = memDataFor(["119_agent_job_activations.sql"]);
90
+ // A queued-but-never-activated observation (worker null).
91
+ await recordAgentJobObservation(data, {
92
+ processInstanceKey: "PI-1",
93
+ jobType: "senior:feature",
94
+ worker: null,
95
+ now: "2024-01-01T00:00:00Z",
96
+ });
97
+ const q = await agentSlaEscalationQuestion(data, {
98
+ processInstanceKey: "PI-1",
99
+ jobType: "senior:feature",
100
+ sla: "PT2H",
101
+ recovery: RECOVERY,
102
+ now: "2024-01-01T02:00:00Z",
103
+ });
104
+ assertStringIncludes(q, "No worker was ever observed to pick up");
105
+ assertEquals(q.includes("may be hung or looping"), false);
106
+ });
107
+
108
+ test("agentSlaEscalationQuestion reads an activated observation as started-but-hung", async () => {
109
+ const { data } = memDataFor(["119_agent_job_activations.sql"]);
110
+ await recordAgentJobObservation(data, {
111
+ processInstanceKey: "PI-2",
112
+ jobType: "senior:feature",
113
+ worker: "agent-x",
114
+ now: "2024-01-01T00:05:00Z",
115
+ });
116
+ const q = await agentSlaEscalationQuestion(data, {
117
+ processInstanceKey: "PI-2",
118
+ jobType: "senior:feature",
119
+ sla: "PT2H",
120
+ recovery: RECOVERY,
121
+ now: "2024-01-01T02:05:00Z",
122
+ });
123
+ assertStringIncludes(q, "hung or looping");
124
+ assertStringIncludes(q, "agent-x");
125
+ });
126
+
127
+ test("agentSlaEscalationQuestion with no ledger row reports activation unknown", async () => {
128
+ const { data } = memDataFor(["119_agent_job_activations.sql"]);
129
+ const q = await agentSlaEscalationQuestion(data, {
130
+ processInstanceKey: "PI-404",
131
+ jobType: "senior:feature",
132
+ sla: "PT2H",
133
+ recovery: RECOVERY,
134
+ now: "2024-01-01T02:00:00Z",
135
+ });
136
+ assertStringIncludes(q, "no worker-activation was ever recorded");
137
+ });