@nanobpm/nano-workforce 0.201.0 → 0.201.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/SPEC.md +153 -0
- package/app/agentActivationLedger.test.ts +109 -0
- package/app/agentActivationLedger.ts +144 -0
- package/app/agentActivationsPoll.test.ts +180 -0
- package/app/agentCompletion.test.ts +52 -0
- package/app/agentCompletion.ts +20 -5
- package/app/agentSlaEscalation.test.ts +137 -0
- package/app/agentSlaEscalation.ts +169 -0
- package/app/agentSlaMarkerGuard.test.ts +172 -0
- package/app/contracts.ts +9 -0
- package/app/deliveryGraph.test.ts +251 -0
- package/app/deliveryGraph.ts +334 -1
- package/app/deliveryGraphCompiler.test.ts +1345 -30
- package/app/deliveryGraphCompiler.ts +882 -240
- package/app/deliveryGraphDeploy.test.ts +472 -3
- package/app/deliveryGraphRun.ts +2 -2
- package/app/deliveryHuman.test.ts +59 -5
- package/app/deliveryHuman.ts +115 -3
- package/app/deliveryRunner.test.ts +3 -3
- package/app/mergeLoopBehaviour.test.ts +46 -1
- package/app/redactText.ts +88 -0
- package/app/service.ts +110 -2
- package/db/migrations/119_agent_job_activations.sql +50 -0
- package/e2e/agent-sla-boundary.e2e.ts +51 -8
- package/e2e/delivery-graph-dispatch.e2e.ts +2 -1
- package/e2e/delivery-graph-node-scope.e2e.ts +358 -0
- package/e2e/delivery-graph.e2e.ts +11 -7
- package/package.json +1 -1
- package/resources/forms/delivery-escalation.form +41 -0
- package/resources/forms/delivery-human-generic.form +4 -3
- package/resources/processes/convergence-loop.bpmn +10 -0
- package/resources/processes/implement-cell.bpmn +4 -1
- package/resources/processes/merge-cell.bpmn +4 -1
- package/resources/processes/merge-loop.bpmn +7 -0
- package/resources/processes/plan-fanout.bpmn +5 -0
- package/resources/processes/retro.bpmn +5 -2
- package/test/derivation-parity/flows.ts +5 -3
- package/workers/conformance-record/worker.ts +15 -1
- package/workers/persist-escalation/worker.ts +22 -3
- package/workers/record-feature-escalation/worker.ts +28 -12
- package/workers/record-trial-merge/worker.ts +16 -1
package/app/deliveryHuman.ts
CHANGED
|
@@ -32,6 +32,7 @@
|
|
|
32
32
|
// binds and pins exactly the value the human handed forward.
|
|
33
33
|
import type { DeliveryFact, DeliveryNodeHuman } from "../nano-generated/api-io.d.ts";
|
|
34
34
|
import { DELIVERY_FACT_TYPES, type DeliveryFactType, FACT_NAME_MAX_LENGTH, FACT_NAME_PATTERN } from "./deliveryGraph.ts";
|
|
35
|
+
import { redactFreeText } from "./redactText.ts";
|
|
35
36
|
|
|
36
37
|
/** The single BPMN `bpmn:userTask` element id every `human` delivery-graph node schedules its work
|
|
37
38
|
* as (`resources/processes/delivery-human.bpmn`). One reusable engine-native body, instantiated once
|
|
@@ -54,6 +55,77 @@ export function isDeliveryHumanElement(elementId: string): boolean {
|
|
|
54
55
|
return elementId === DELIVERY_HUMAN_ELEMENT || elementId.startsWith(`${DELIVERY_HUMAN_ELEMENT}__`);
|
|
55
56
|
}
|
|
56
57
|
|
|
58
|
+
/** The id suffix the S4 compiler appends for a service node's bounded-timeout escalation TWIN user task
|
|
59
|
+
* (`delivery-human-task__<el>__esc`). */
|
|
60
|
+
export const DELIVERY_ESCALATION_TWIN_SUFFIX = "__esc";
|
|
61
|
+
|
|
62
|
+
/** The id suffix for an `agent` node's producer-contract escalation TWIN (`…__contract`). */
|
|
63
|
+
export const DELIVERY_CONTRACT_TWIN_SUFFIX = "__contract";
|
|
64
|
+
|
|
65
|
+
/** The KIND marker the S4 compiler appends to a `wait` gate's bounded-timeout escalation twin
|
|
66
|
+
* (`delivery-human-task__<el>__esc__wait`). A wait-gate escalation has NO retry semantics — its
|
|
67
|
+
* resolution is "supply the awaited value and continue", never "re-run the probe loop" — so the
|
|
68
|
+
* compiler renders the select-less {@link GENERIC_HUMAN_FORM} for it, NOT the retry-capable
|
|
69
|
+
* {@link ESCALATION_FORM} a service-node `__esc`/`__contract` twin renders. The bare `__esc` suffix is
|
|
70
|
+
* shared by both kinds, so a form inferred from the suffix ALONE misclassifies the wait gate as
|
|
71
|
+
* retryable (the #461 Tasks-inbox `form_key` fallback then renders a bogus "Retry this step" that the
|
|
72
|
+
* wait process silently ignores). Stamping the kind into the id lets {@link escalationFormId} DERIVE
|
|
73
|
+
* the compiled form from the id — no separate node-kind registry to drift (PR #863 review,
|
|
74
|
+
* app/agentCompletion.ts:186). */
|
|
75
|
+
export const DELIVERY_WAIT_ESCALATION_KIND_MARKER = "__wait";
|
|
76
|
+
|
|
77
|
+
/** The full id suffix of a `wait` gate's bounded-timeout escalation twin (`__esc` + the `__wait` kind
|
|
78
|
+
* marker). */
|
|
79
|
+
export const DELIVERY_WAIT_ESCALATION_TWIN_SUFFIX = `${DELIVERY_ESCALATION_TWIN_SUFFIX}${DELIVERY_WAIT_ESCALATION_KIND_MARKER}`;
|
|
80
|
+
|
|
81
|
+
/** True for a delivery-graph SERVICE-node escalation TWIN user task — the bounded-timeout `__esc` twin
|
|
82
|
+
* or an agent's producer-contract `__contract` twin — as opposed to a plain per-node human task
|
|
83
|
+
* (`delivery-human-task__<el>`). Unlike that per-node human task, which renders a DIFFERENT form per
|
|
84
|
+
* node (and so has no single static completion contract — left unvalidated, `ESCALATION_FORM_BY_ELEMENT`
|
|
85
|
+
* deliberately omits it), a twin ALWAYS renders a FIXED escalation form — the retry-capable
|
|
86
|
+
* {@link ESCALATION_FORM} (a strict superset of {@link GENERIC_HUMAN_FORM}: same `value`, an optional
|
|
87
|
+
* note field (`escalationNote` on the escalation form, `note` on the generic human form), PLUS
|
|
88
|
+
* the `decision` select) or the select-less generic form — so its completion variables CAN be validated
|
|
89
|
+
* against `ESCALATION_FORM`'s contract (no required fields, so the generic-form twins never false-reject;
|
|
90
|
+
* only a present-but-invalid `decision` is rejected). Matched on the convention suffix — single source of
|
|
91
|
+
* truth with the compiler's `escalationTaskElement`/`contractEscalationTaskElement` builders — requiring
|
|
92
|
+
* a NON-EMPTY `<el>` between prefix and suffix so the bare per-node base of a node literally named
|
|
93
|
+
* `esc`/`contract` (`delivery-human-task__esc`) is not misread as a twin. (A node id MAY itself end in
|
|
94
|
+
* `__esc` — `NODE_ID_PATTERN` allows it — so the twin of node `foo` and the base of a node named
|
|
95
|
+
* `foo__esc` collide on one id; validating that rare base against the superset escalation form is
|
|
96
|
+
* harmless, since the form has no required fields and only constrains a `decision` such a base never
|
|
97
|
+
* sets to an out-of-range value through its own select-less form.) */
|
|
98
|
+
export function isDeliveryEscalationTwin(elementId: string): boolean {
|
|
99
|
+
const prefix = `${DELIVERY_HUMAN_ELEMENT}__`;
|
|
100
|
+
if (!elementId.startsWith(prefix)) return false;
|
|
101
|
+
for (const suffix of [DELIVERY_WAIT_ESCALATION_TWIN_SUFFIX, DELIVERY_ESCALATION_TWIN_SUFFIX, DELIVERY_CONTRACT_TWIN_SUFFIX]) {
|
|
102
|
+
if (elementId.endsWith(suffix) && elementId.length > prefix.length + suffix.length) return true;
|
|
103
|
+
}
|
|
104
|
+
return false;
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
/** True for a `wait` gate's bounded-timeout escalation twin specifically (`…__esc__wait`) — the twin
|
|
108
|
+
* kind that renders the select-less {@link GENERIC_HUMAN_FORM} (no retry path), as opposed to a
|
|
109
|
+
* service-node `__esc`/`__contract` twin, which renders the retry-capable {@link ESCALATION_FORM}.
|
|
110
|
+
* This is the discriminator {@link escalationFormId} uses to DERIVE the compiled form from the id. */
|
|
111
|
+
export function isDeliveryWaitEscalationTwin(elementId: string): boolean {
|
|
112
|
+
const prefix = `${DELIVERY_HUMAN_ELEMENT}__`;
|
|
113
|
+
return (
|
|
114
|
+
elementId.startsWith(prefix) &&
|
|
115
|
+
elementId.endsWith(DELIVERY_WAIT_ESCALATION_TWIN_SUFFIX) &&
|
|
116
|
+
elementId.length > prefix.length + DELIVERY_WAIT_ESCALATION_TWIN_SUFFIX.length
|
|
117
|
+
);
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
/** Strip the trailing escalation-twin suffix (`__esc`, the wait-gate `__esc__wait`, or `__contract`)
|
|
121
|
+
* to recover the base per-node element id (`delivery-human-task__<el>`). The `__esc__wait` kind marker
|
|
122
|
+
* MUST be stripped before the bare `__esc` (it ends in `__esc`, so stripping `__esc` first would leave
|
|
123
|
+
* a dangling `__wait`). A non-twin id is returned unchanged. Single source of truth for the strip so
|
|
124
|
+
* every "resolve the twin to its base node" lookup agrees on the suffix set. */
|
|
125
|
+
export function stripDeliveryEscalationTwinSuffix(elementId: string): string {
|
|
126
|
+
return elementId.replace(/__(esc__wait|esc|contract)$/, "");
|
|
127
|
+
}
|
|
128
|
+
|
|
57
129
|
/** The read-model "Decision context" for a parked delivery-graph `human` node (issue #772). The Tasks
|
|
58
130
|
* surface renders the deployed `.form` against EMPTY data (it seeds no task-local variables), so the
|
|
59
131
|
* node's instruction cannot reach the operator through the form's readonly `prompt` field — it must
|
|
@@ -76,11 +148,45 @@ export function isDeliveryHumanElement(elementId: string): boolean {
|
|
|
76
148
|
export function deliveryHumanContextQuestion(
|
|
77
149
|
humanLabels: Record<string, string> | undefined,
|
|
78
150
|
elementId: string,
|
|
151
|
+
livePrompt?: string | null,
|
|
79
152
|
): string {
|
|
80
153
|
const labels = humanLabels ?? {};
|
|
81
|
-
const base = elementId
|
|
154
|
+
const base = stripDeliveryEscalationTwinSuffix(elementId);
|
|
82
155
|
const label = (labels[elementId] ?? labels[base] ?? "").trim();
|
|
83
|
-
return label || "A scheduled delivery-graph step is waiting to be completed.";
|
|
156
|
+
return label || (livePrompt ?? "").trim() || "A scheduled delivery-graph step is waiting to be completed.";
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
/** Whether a parked delivery task needs its LIVE engine prompt for the "Decision context": an
|
|
160
|
+
* `__esc`/`__contract` escalation twin of a bounded agent/wait/connector node has no stamped
|
|
161
|
+
* `human_labels` entry, so without the compiler's per-task `prompt` (node, reported status, missing
|
|
162
|
+
* emits, the agent's own report/question/transcript, and how to resolve) the operator saw only the
|
|
163
|
+
* static fallback — the "no actionable information" escalations of instance 171774. A real human
|
|
164
|
+
* node (or its `__esc` twin) keeps its authored instruction label. */
|
|
165
|
+
export function needsLiveDeliveryPrompt(humanLabels: Record<string, string> | undefined, elementId: string): boolean {
|
|
166
|
+
if (!/__(esc__wait|esc|contract)$/.test(elementId)) return false;
|
|
167
|
+
const labels = humanLabels ?? {};
|
|
168
|
+
const base = stripDeliveryEscalationTwinSuffix(elementId);
|
|
169
|
+
return !(labels[elementId] ?? labels[base] ?? "").trim();
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
/** Decode a `prompt` variable as the engine's variable search reports it (a JSON-encoded value) into
|
|
173
|
+
* plain text; `null` for an absent, non-string, or truncated preview (never show a clipped prompt as
|
|
174
|
+
* if it were whole — the caller falls back).
|
|
175
|
+
*
|
|
176
|
+
* The decoded text is REDACTED ({@link redactFreeText}) before it is returned: the escalation prompt
|
|
177
|
+
* embeds agent-controlled `summary`/`question`/`error`/`transcriptUrl` (the compiler's report
|
|
178
|
+
* sentence), and the caller persists the result as the operator-facing `user_tasks.question`. An
|
|
179
|
+
* agent report can carry a token-bearing URL or a credential copied into its report, so this path
|
|
180
|
+
* redacts exactly like the static `human_labels` instruction (which `buildHumanLabels` runs through
|
|
181
|
+
* `redactFreeText`, issue #778) — never store a live prompt raw (issue #863 review). */
|
|
182
|
+
export function decodeLivePrompt(row: { value: string; isTruncated?: boolean } | undefined): string | null {
|
|
183
|
+
if (!row || row.isTruncated) return null;
|
|
184
|
+
try {
|
|
185
|
+
const v: unknown = JSON.parse(row.value);
|
|
186
|
+
return typeof v === "string" && v.trim() ? redactFreeText(v.trim()) : null;
|
|
187
|
+
} catch {
|
|
188
|
+
return null;
|
|
189
|
+
}
|
|
84
190
|
}
|
|
85
191
|
|
|
86
192
|
/** The clickable link for a parked delivery-graph `human` node's Tasks-inbox row (issue #813): the
|
|
@@ -96,7 +202,7 @@ export function deliveryHumanContextUrl(
|
|
|
96
202
|
elementId: string,
|
|
97
203
|
): string | null {
|
|
98
204
|
const labels = humanLabels ?? {};
|
|
99
|
-
const base = elementId
|
|
205
|
+
const base = stripDeliveryEscalationTwinSuffix(elementId);
|
|
100
206
|
const instruction = labels[elementId] ?? labels[base] ?? "";
|
|
101
207
|
const url = firstHttpUrl(instruction);
|
|
102
208
|
// The stored `human_labels` are display-REDACTED (`redactFreeText` in `buildHumanLabels`): a URL that
|
|
@@ -179,6 +285,12 @@ export function firstHttpUrl(text: string | undefined | null): string | null {
|
|
|
179
285
|
* declared emitted fact, so a human node with no explicit/category form can STILL emit downstream. */
|
|
180
286
|
export const GENERIC_HUMAN_FORM = "delivery-human-generic";
|
|
181
287
|
|
|
288
|
+
/** The bounded service-node ESCALATION form (retry-node resolution): like the generic form plus the
|
|
289
|
+
* `decision` Resolution select (Continue / Retry this step). Kept SEPARATE from {@link GENERIC_HUMAN_FORM}
|
|
290
|
+
* — that form is shared with plain scheduled human steps and wait-gate escalations, which have no
|
|
291
|
+
* retry semantics and never read `decision`, so the select must not render there. */
|
|
292
|
+
export const ESCALATION_FORM = "delivery-escalation";
|
|
293
|
+
|
|
182
294
|
/** The "click done" category form: a degenerate no-emit acknowledgement ("now do X" → done). */
|
|
183
295
|
export const HUMAN_ACK_FORM = "delivery-human-ack";
|
|
184
296
|
|
|
@@ -238,12 +238,12 @@ test("producer contract required-emit subset (#761): a routing-only emit is NOT
|
|
|
238
238
|
const graph: DeliveryGraph = {
|
|
239
239
|
name: "routing-only producer",
|
|
240
240
|
nodes: [
|
|
241
|
-
{ id: "classify", kind: "agent", agent: { jobType: "senior:feature", prompt: "classify it" }, emits: [{ name: "
|
|
241
|
+
{ id: "classify", kind: "agent", agent: { jobType: "senior:feature", prompt: "classify it" }, emits: [{ name: "verdict", type: "string" }] },
|
|
242
242
|
{ id: "migrate", kind: "connector", connector: { target: "npm:install", dedupeKey: "m-1" } },
|
|
243
243
|
{ id: "release", kind: "connector", connector: { target: "npm:publish", dedupeKey: "r-1" } },
|
|
244
244
|
],
|
|
245
245
|
edges: [
|
|
246
|
-
{ from: "classify", to: "migrate", when: "classify.
|
|
246
|
+
{ from: "classify", to: "migrate", when: "classify.verdict", equals: "breaking" },
|
|
247
247
|
{ from: "classify", to: "release", default: true },
|
|
248
248
|
],
|
|
249
249
|
};
|
|
@@ -257,7 +257,7 @@ test("producer contract required-emit subset (#761): a routing-only emit is NOT
|
|
|
257
257
|
assert(!prompt.includes("populate each of these top-level fields"), "no required-emit sentence is rendered when every emit is routing-only");
|
|
258
258
|
// The classifier-emit contract STILL tells the agent to return the routing fact (and omit if undecidable).
|
|
259
259
|
assert(prompt.includes("Classifier emit contract"), "the classifier-emit contract is present");
|
|
260
|
-
assert(prompt.includes("`
|
|
260
|
+
assert(prompt.includes("`verdict`"), "the routing fact is still named by the classifier-emit contract");
|
|
261
261
|
});
|
|
262
262
|
|
|
263
263
|
|
|
@@ -28,7 +28,7 @@
|
|
|
28
28
|
// Plus the terminate semantics the refactor had to preserve: `MergeAbandoned`
|
|
29
29
|
// terminates the whole instance (it stays at the root, not inside a sub-process).
|
|
30
30
|
import { after, test } from "node:test";
|
|
31
|
-
import { assert, assertStringIncludes } from "#test-assert";
|
|
31
|
+
import { assert, assertEquals, assertStringIncludes } from "#test-assert";
|
|
32
32
|
import { readFileSync } from "node:fs";
|
|
33
33
|
import {
|
|
34
34
|
assertThatInstance,
|
|
@@ -562,6 +562,10 @@ test("the fix-ci agent SLA interrupts the sub-process and escalates", async () =
|
|
|
562
562
|
await assertThatUserTask(engine, { instance: byProcessId("merge-loop"), elementId: "wait-merge-answer" }).isCreated();
|
|
563
563
|
assertThatInstance(engine, byProcessId("merge-loop")).hasCompletedElements("merge-esc-attempt");
|
|
564
564
|
assertStringIncludes(String(escalation(engine).question ?? ""), "time budget (SLA)", "the SLA escalation must name the SLA trigger");
|
|
565
|
+
// #879 regression: the trusted fix-ci SLA arm must deliver its job-type marker to the shared
|
|
566
|
+
// persist-escalation sink so the worker builds the never-started-vs-hung diagnosis — a =null
|
|
567
|
+
// input pin on the shared recorder silently clobbered this (PR #881 adversarial review).
|
|
568
|
+
assertEquals(escalation(engine).agentSlaJobType, "senior:fix-ci", "the fix-ci SLA arm must reach the recorder with its job-type marker");
|
|
565
569
|
});
|
|
566
570
|
|
|
567
571
|
// ---------------------------------------------------------------------------
|
|
@@ -613,8 +617,49 @@ test("the rebase agent SLA interrupts the sub-process and escalates as not-merge
|
|
|
613
617
|
await engine.advanceTime(AGENT_SLA_MS + 1);
|
|
614
618
|
await assertThatUserTask(engine, { instance: byProcessId("merge-loop"), elementId: "wait-merge-answer" }).isCreated();
|
|
615
619
|
assertThatInstance(engine, byProcessId("merge-loop")).hasCompletedElements("merge-esc-conflict");
|
|
620
|
+
// #879 regression: the trusted rebase SLA arm must deliver its job-type marker to the shared sink.
|
|
621
|
+
assertEquals(escalation(engine).agentSlaJobType, "senior:rebase", "the rebase SLA arm must reach the recorder with its job-type marker");
|
|
616
622
|
});
|
|
617
623
|
|
|
624
|
+
test("a non-SLA escalation carrying a spoofed agentSlaJobType is scrubbed before the recorder (#879 fail-closed)", async () => {
|
|
625
|
+
// The c8ctl harness hoists an agent's result-JSON keys into process scope, so a NON-SLA rebase
|
|
626
|
+
// agent that returns a bogus `agentSlaJobType` must NOT be able to fabricate an SLA diagnosis on
|
|
627
|
+
// the shared persist-escalation sink. The loop head / non-SLA escalate end events clear it.
|
|
628
|
+
const engine = await boot({ responses: { "senior:rebase": { status: "blocked", agentSlaJobType: "senior:evil" } } });
|
|
629
|
+
await driveToRebase(engine);
|
|
630
|
+
await assertThatUserTask(engine, { instance: byProcessId("merge-loop"), elementId: "wait-merge-answer" }).isCreated();
|
|
631
|
+
assertThatInstance(engine, byProcessId("merge-loop")).hasCompletedElements("merge-esc-attempt");
|
|
632
|
+
const spoofed = escalation(engine).agentSlaJobType;
|
|
633
|
+
assert(spoofed === null || spoofed === undefined, `a spoofed agentSlaJobType must be scrubbed, got ${JSON.stringify(spoofed)}`);
|
|
634
|
+
});
|
|
635
|
+
|
|
636
|
+
test("a spoofed agentSlaJobType surviving the CI-reconcile re-check route is scrubbed before a later merge-esc-conflict (#881 Previously-missed)", async () => {
|
|
637
|
+
// The CI reconcile route (fix-ci blocked/pushed:false → ci-reconcile → end_ci_recheck →
|
|
638
|
+
// gw-merge-entry) re-enters the merge wait WITHOUT passing through the loop head `arm-merge` or
|
|
639
|
+
// the non-SLA escalate clears. A hoisted (spoofable) agentSlaJobType from that fix-ci result
|
|
640
|
+
// would otherwise survive into a later draft / budget-exhausted merge and fabricate an SLA
|
|
641
|
+
// never-started/hung diagnosis on the shared merge-esc-conflict sink with NO SLA timer having
|
|
642
|
+
// fired. `end_ci_recheck` must clear it (PR #881 review, "Previously missed" r5451284362).
|
|
643
|
+
const engine = await boot({
|
|
644
|
+
responses: { "senior:fix-ci": { status: "blocked", pushed: false, agentSlaJobType: "senior:evil" } },
|
|
645
|
+
});
|
|
646
|
+
await driveToCiFix(engine); // blocked/pushed:false → ci-reconcile → end_ci_recheck → re-check mergeable
|
|
647
|
+
assertThatInstance(engine, byProcessId("merge-loop"))
|
|
648
|
+
.isActive()
|
|
649
|
+
.hasActiveElement("wait-mergeable")
|
|
650
|
+
.hasCompletedElements("ci-reconcile")
|
|
651
|
+
.hasVariable("ciBlockedReconciled", true);
|
|
652
|
+
assert(!completedElementIds(engine).has("merge-esc-conflict"), "the reconcile pass must not escalate yet");
|
|
653
|
+
// A later merge-ready reports the PR as draft → f_m_mDraft → merge-esc-conflict reads the marker
|
|
654
|
+
// directly (the fix-ci agent does NOT run again on this route).
|
|
655
|
+
await engine.publishMessage({ name: "merge-ready", correlationKey: "pr-1", variables: { mergeState: "draft" } });
|
|
656
|
+
await assertThatUserTask(engine, { instance: byProcessId("merge-loop"), elementId: "wait-merge-answer" }).isCreated();
|
|
657
|
+
assertThatInstance(engine, byProcessId("merge-loop")).hasCompletedElements("merge-esc-conflict");
|
|
658
|
+
const spoofed = escalation(engine).agentSlaJobType;
|
|
659
|
+
assert(spoofed === null || spoofed === undefined, `the reconcile-route spoof must be scrubbed, got ${JSON.stringify(spoofed)}`);
|
|
660
|
+
// The human gets the genuine draft question, not a fabricated SLA diagnosis.
|
|
661
|
+
assertStringIncludes(String(escalation(engine).question ?? ""), "draft", "the escalation must be the genuine draft question, not an SLA diagnosis");
|
|
662
|
+
});
|
|
618
663
|
// ---------------------------------------------------------------------------
|
|
619
664
|
// Escalation user task (mergeEscalationUserTask #256, mergeEscalationQuestion #329/#454)
|
|
620
665
|
// ---------------------------------------------------------------------------
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
// Free-form-prose URL-credential redaction (`redactFreeText`) lives in this LOW-LEVEL helper module so
|
|
2
|
+
// both the delivery-graph compiler (`deliveryGraphCompiler.ts`, which embeds redacted prompts into the
|
|
3
|
+
// generated BPMN display name/documentation) AND `deliveryHuman.ts` (which redacts a decoded live
|
|
4
|
+
// prompt / human labels) can import it WITHOUT a cycle. Previously `redactFreeText` was defined in the
|
|
5
|
+
// compiler and `deliveryHuman.ts` imported the compiler solely for it — a direct `compiler ⇄ human`
|
|
6
|
+
// import cycle that reversed `deliveryHuman.ts`'s documented pure-helper layering and made module init
|
|
7
|
+
// depend on ESM cycle ordering (PR #863 review — thread deliveryGraphCompiler.ts / deliveryHuman.ts).
|
|
8
|
+
// This module depends only on the character-class strip (`deliveryGraph.ts`) and the canonical URL
|
|
9
|
+
// redactors (`readiness.ts`), both strictly lower layers, so nothing here can import back.
|
|
10
|
+
|
|
11
|
+
import { stripXmlInvalidChars } from "./deliveryGraph.ts";
|
|
12
|
+
import { redactEmbeddedCredential, redactString } from "./readiness.ts";
|
|
13
|
+
|
|
14
|
+
/** Redact credential-bearing pieces of any URL embedded in FREE-FORM prose (a node's authored
|
|
15
|
+
* `prompt`), IN PLACE. Unlike {@link redactString} — tuned for a single opaque target string, where it
|
|
16
|
+
* truncates from the first `?`/`#` to end-of-string — this finds each URL-shaped token *within*
|
|
17
|
+
* surrounding prose (`scheme://…` or a scheme-relative `//…`) and strips only that token's
|
|
18
|
+
* `user:pass@` userinfo and `?query`/`#fragment` via `redactString`, leaving the prose (and ordinary
|
|
19
|
+
* punctuation such as a `?` ending a sentence) intact. Applied to the DISPLAY name/documentation only —
|
|
20
|
+
* a prompt is up to 20 000 chars of arbitrary text that can embed a bearer token or private URL, and
|
|
21
|
+
* the deployed `<bpmn:documentation>`/name is visible to modeler/explorer readers; the RAW prompt still
|
|
22
|
+
* reaches the runtime job input (`appendPrompt`/`prompt`) unmodified (issue #778 review). Deterministic
|
|
23
|
+
* and total. */
|
|
24
|
+
export function redactFreeText(value: string): string {
|
|
25
|
+
// Strip XML-invalid display characters BEFORE tokenizing/redacting so every scan runs on the exact
|
|
26
|
+
// string the renderer emits. Otherwise a control char embedded in a URL (`//us\x0Ber:pass@…`) breaks
|
|
27
|
+
// the `//[^\s]+` token match, escapes redaction, then reconstructs the credential once `escapeXml`/
|
|
28
|
+
// `stripXmlInvalidChars` drops the control at render time (issue #778 review — same class as the
|
|
29
|
+
// connector/probe strip-before-classify fix).
|
|
30
|
+
const cleaned = stripXmlInvalidChars(value);
|
|
31
|
+
// Embedded credential userinfo: DERIVE from the ONE canonical redactor {@link redactEmbeddedCredential}
|
|
32
|
+
// (`EMBEDDED_CREDENTIAL_SRC` = `//[^/?#]*@`) rather than a bespoke belt heuristic. Its `[^/?#]` class
|
|
33
|
+
// spans spaces/TABs/newlines up to the LAST `@` before a `/`/`?`/`#`, so EVERY `//<userinfo>@` shape
|
|
34
|
+
// collapses uniformly to `//***@` — a colon-prefix `//user:pass@`, a colon-LESS bearer separated from
|
|
35
|
+
// its `@host` by a word or space (`//token part@host`, `//token @host`), a userinfo split by a raw
|
|
36
|
+
// CR/LF/TAB, and a malformed multi-`@` authority (`//user:pass@ss@host`) — with NO second "what is a
|
|
37
|
+
// credential" implementation that can drift from the canonical redactor/validator (`hasEmbeddedCredential`)
|
|
38
|
+
// (issue #783 review — thread deliveryGraphCompiler.ts:1140; the earlier bespoke colon-less bridge
|
|
39
|
+
// stopped on the first non-whitespace char after the space and leaked `//token part@host`). A `//…@`
|
|
40
|
+
// span is UNAMBIGUOUSLY a credential wherever it sits, so redacting a prose `//word …@host` too is the
|
|
41
|
+
// SAFE direction: the RAW prompt still reaches the runtime job input unmodified — only the operator-
|
|
42
|
+
// visible display doc loses the span. The `[^/?#]` bound also keeps the userinfo from swallowing a `?`
|
|
43
|
+
// marker (`//host?token=secret@tail` has no userinfo — its `@` rides the query), leaving that tail to
|
|
44
|
+
// the query/fragment belt below.
|
|
45
|
+
const credStripped = redactEmbeddedCredential(cleaned);
|
|
46
|
+
// Query/fragment across a whitespace BREAK: the primary `//[^\s]+` token below stops at the break, so a
|
|
47
|
+
// `//host/?\nTOKEN=secret` would leave the value visible in the XML-preserved doc. The linear belt walks
|
|
48
|
+
// each SPACE-bounded `//`-run (crossing an embedded CR/LF/TAB the primary token stopped at) and
|
|
49
|
+
// CONSERVATIVELY redacts its `?query`/`#fragment` tail through the span's space boundary — a continuation
|
|
50
|
+
// past the break is indistinguishable from a split value, so we never keep the far side (issue #778
|
|
51
|
+
// review — thread deliveryGraphCompiler.ts:1115).
|
|
52
|
+
const belted = redactQueryFragmentSpans(credStripped);
|
|
53
|
+
return belted.replace(/(?:[a-z][a-z0-9+.-]*:)?\/\/[^\s]+/gi, (m) => redactString(m));
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/** Linear (backtracking-free) newline-aware belt companion to {@link redactFreeText}'s primary
|
|
57
|
+
* whitespace-bounded pass. Each `//`-run is bounded by a literal SPACE (0x20) — so a single span may
|
|
58
|
+
* cross a CR/LF/TAB *inside* the URL that the primary `//[^\s]+` token stopped at. Credential userinfo is
|
|
59
|
+
* already collapsed to `//***@` by the canonical {@link redactEmbeddedCredential} before this runs, so the
|
|
60
|
+
* belt's SOLE remaining job is a `?query`/`#fragment` tail: it redacts CONSERVATIVELY from the first
|
|
61
|
+
* `?`/`#` marker through the span's space boundary — a continuation past an embedded break is
|
|
62
|
+
* indistinguishable from a split value, so the far side is never kept. A span with no `?`/`#` (ordinary
|
|
63
|
+
* prose — a `//comment` reference, an already-collapsed `//***@host`) is returned untouched. The scan is a
|
|
64
|
+
* single left-to-right walk using `indexOf`/`charCodeAt` over spans bounded by the next SPACE, with no
|
|
65
|
+
* regex backtracking, so a 20 000-char adversarial prompt cannot trigger catastrophic backtracking (issue
|
|
66
|
+
* #778 review). Deterministic and total. */
|
|
67
|
+
function redactQueryFragmentSpans(text: string): string {
|
|
68
|
+
let out = "";
|
|
69
|
+
let i = 0;
|
|
70
|
+
for (;;) {
|
|
71
|
+
const start = text.indexOf("//", i);
|
|
72
|
+
if (start < 0) return out + text.slice(i);
|
|
73
|
+
out += text.slice(i, start);
|
|
74
|
+
let end = start;
|
|
75
|
+
while (end < text.length && text.charCodeAt(end) !== 0x20 /* SPACE */) end++;
|
|
76
|
+
const span = text.slice(start, end);
|
|
77
|
+
const qMark = span.indexOf("?");
|
|
78
|
+
const hMark = span.indexOf("#");
|
|
79
|
+
const qi = qMark < 0 ? hMark : hMark < 0 ? qMark : Math.min(qMark, hMark);
|
|
80
|
+
// CONSERVATIVE: keep everything up to and including the `?`/`#` marker, then collapse the whole tail
|
|
81
|
+
// (any continuation past an embedded CR/LF/TAB) to `***`. A value/prose resuming after the break is
|
|
82
|
+
// INDISTINGUISHABLE from a split-credential continuation, so we never keep the post-break side; the RAW
|
|
83
|
+
// prompt still reaches the runtime job input unmodified (issue #778 review — thread
|
|
84
|
+
// deliveryGraphCompiler.ts:1115).
|
|
85
|
+
out += qi < 0 ? span : `${span.slice(0, qi + 1)}***`;
|
|
86
|
+
i = end;
|
|
87
|
+
}
|
|
88
|
+
}
|
package/app/service.ts
CHANGED
|
@@ -10,6 +10,7 @@
|
|
|
10
10
|
import type { DataLayer, EngineClient } from "@nanobpm/urban";
|
|
11
11
|
import { ABANDONED_STATUS, abandonUrl, mintAbandonToken, renderAbandonBrief } from "./abandon.ts";
|
|
12
12
|
import { matchAdjudication, prAdjudications, resetAdjudications } from "./adjudications.ts";
|
|
13
|
+
import { AGENT_SLA_JOB_TYPES, recordAgentJobObservation } from "./agentActivationLedger.ts";
|
|
13
14
|
import { completeEscalationAutoApplied, escalationFormId } from "./agentCompletion.ts";
|
|
14
15
|
import { AGENT_SLA_TIMEOUT } from "./agentSla.ts";
|
|
15
16
|
import {
|
|
@@ -34,7 +35,7 @@ import { deriveDelivery, EPIC_LIVE_STATUSES, TERMINAL_STATUSES } from "./deliver
|
|
|
34
35
|
import { DELIVERY_GRAPH_PROCESS_ID } from "./deliveryGraphCompiler.ts";
|
|
35
36
|
import { sweepExpiredProposals } from "./deliveryGraphProposals.ts";
|
|
36
37
|
import { deliveryGraphRuns, deriveDeliveryPhase, parseHumanLabels, reconcileOriginalInstanceBeforeRelaunch, reconcileStaleLaunchClaim } from "./deliveryGraphRun.ts";
|
|
37
|
-
import { deliveryHumanContextQuestion, deliveryHumanContextUrl, isDeliveryHumanElement } from "./deliveryHuman.ts";
|
|
38
|
+
import { decodeLivePrompt, deliveryHumanContextQuestion, deliveryHumanContextUrl, isDeliveryHumanElement, needsLiveDeliveryPrompt } from "./deliveryHuman.ts";
|
|
38
39
|
import { fleetSupportsDurableResume } from "./durableResume.ts";
|
|
39
40
|
import { deriveEpicPhaseLive, deriveTerminalEpicPhase } from "./epicPhase.ts";
|
|
40
41
|
import { deriveFeatureCompletion, deriveFeatureDelivery, FEATURE_BLOCKED_ELEMENT, FEATURE_ESCALATION_ELEMENT, FEATURE_RUN_STATUSES, type FeatureRunStatus, featureEscalations, featureRuns, featureRunsTracking, foldCompletedFeatureRun } from "./feature.ts";
|
|
@@ -1733,6 +1734,98 @@ interface JobSearchItem {
|
|
|
1733
1734
|
/** The job's `zeebe:taskDefinition` type — read to defensively re-filter the wire `$in` type
|
|
1734
1735
|
* filter so a non-agent job can never masquerade as a leasing convergence agent. */
|
|
1735
1736
|
type?: string;
|
|
1737
|
+
/** The owning process instance — keyed into the activation ledger (issue #879). */
|
|
1738
|
+
processInstanceKey?: string;
|
|
1739
|
+
/** The engine job key — the unique id of THIS job attempt. The activation ledger uses it to tell one
|
|
1740
|
+
* attempt of a retrying agent task from the next (issue #879 review). */
|
|
1741
|
+
jobKey?: string;
|
|
1742
|
+
}
|
|
1743
|
+
|
|
1744
|
+
/** One agent-activation ledger poll pass (issue #879). Unlike {@link pollJobActivationImpl} (which
|
|
1745
|
+
* tracks only the review-round for the pages grid), this records — for EVERY bounded agent job type —
|
|
1746
|
+
* whether a worker ever picked the job up, into the durable `agent_job_activations` ledger. When an
|
|
1747
|
+
* agent-task SLA boundary later cancels a job, the job's engine state is gone, so the escalation
|
|
1748
|
+
* recorder cannot ask the engine whether it was ever activated; this ledger, captured WHILE the job is
|
|
1749
|
+
* live, is what lets the escalation distinguish a NEVER-ACTIVATED (queue-starved) job from an
|
|
1750
|
+
* ACTIVATED-but-hung one.
|
|
1751
|
+
*
|
|
1752
|
+
* It reads the same activation signal as the review-round pass: the Camunda-8 `/v2/jobs/search` wire
|
|
1753
|
+
* API collapses Activated -> CREATED (no ACTIVATED enum), so a leased job is recognised by its
|
|
1754
|
+
* `worker`/`deadline` fields. One paged search across all bounded agent types per pass upserts the
|
|
1755
|
+
* ledger by (processInstanceKey, type), keyed to the current attempt by `jobKey`. Best-effort: a
|
|
1756
|
+
* transport failure leaves the ledger untouched and the next pass retries. */
|
|
1757
|
+
async function pollAgentActivations(
|
|
1758
|
+
data: DataLayer,
|
|
1759
|
+
restAddress: string,
|
|
1760
|
+
engineToken: string | undefined,
|
|
1761
|
+
) {
|
|
1762
|
+
const base = restAddress.replace(/\/+$/, "");
|
|
1763
|
+
const headers: Record<string, string> = { "content-type": "application/json" };
|
|
1764
|
+
if (engineToken) headers.authorization = `Bearer ${engineToken}`;
|
|
1765
|
+
await pollAgentActivationsImpl(data, base, headers);
|
|
1766
|
+
}
|
|
1767
|
+
|
|
1768
|
+
/** Testable core of {@link pollAgentActivations}: query the engine for every CREATED bounded-agent job
|
|
1769
|
+
* and upsert the activation ledger. Split out so tests can drive it with a stubbed `fetch`. */
|
|
1770
|
+
export async function pollAgentActivationsImpl(
|
|
1771
|
+
data: DataLayer,
|
|
1772
|
+
base: string,
|
|
1773
|
+
headers: Record<string, string>,
|
|
1774
|
+
) {
|
|
1775
|
+
// Page through EVERY matching open job: a single first-page read would, under fleet saturation, let
|
|
1776
|
+
// long-lived jobs occupy the page on every pass while later jobs time out with their activation never
|
|
1777
|
+
// recorded — exactly when the never-started/hung distinction matters most. Mirror the paging contract
|
|
1778
|
+
// the other engine sweeps use (`page: { from, limit }`, stop on a short page). (#881 review)
|
|
1779
|
+
const ts = now();
|
|
1780
|
+
const limit = 200;
|
|
1781
|
+
let from = 0;
|
|
1782
|
+
for (let guard = 0; guard < 1000; guard++) {
|
|
1783
|
+
let items: JobSearchItem[] = [];
|
|
1784
|
+
try {
|
|
1785
|
+
const res = await fetch(`${base}/jobs/search`, {
|
|
1786
|
+
method: "POST",
|
|
1787
|
+
headers,
|
|
1788
|
+
body: JSON.stringify({
|
|
1789
|
+
filter: {
|
|
1790
|
+
type: { $in: [...AGENT_SLA_JOB_TYPES] },
|
|
1791
|
+
// A merely-created (queued) AND an activated-but-not-yet-completed job both project to
|
|
1792
|
+
// CREATED on the wire (Camunda has no ACTIVATED enum) — so this one filter captures both,
|
|
1793
|
+
// and the leasing `worker` field discriminates them at upsert time.
|
|
1794
|
+
state: "CREATED",
|
|
1795
|
+
},
|
|
1796
|
+
page: { from, limit },
|
|
1797
|
+
}),
|
|
1798
|
+
});
|
|
1799
|
+
if (!res.ok) return; // engine unhappy → keep the ledger as-is, retry next pass
|
|
1800
|
+
// biome-ignore lint/plugin: runtime/framework contract boundary for external data shape
|
|
1801
|
+
const body = (await res.json()) as { items?: JobSearchItem[] };
|
|
1802
|
+
items = body.items ?? [];
|
|
1803
|
+
} catch (err) {
|
|
1804
|
+
console.error(`[poller] agent-activation search: ${err}`);
|
|
1805
|
+
return;
|
|
1806
|
+
}
|
|
1807
|
+
for (const j of items) {
|
|
1808
|
+
const jobType = j.type ?? "";
|
|
1809
|
+
const pik = j.processInstanceKey ?? "";
|
|
1810
|
+
// Defensively re-filter the wire `$in` type filter and require a process instance to key on.
|
|
1811
|
+
if (pik.length === 0 || !AGENT_SLA_JOB_TYPES.includes(jobType)) continue;
|
|
1812
|
+
const worker = typeof j.worker === "string" && j.worker.length > 0 ? j.worker : null;
|
|
1813
|
+
const jobKey = typeof j.jobKey === "string" && j.jobKey.length > 0 ? j.jobKey : null;
|
|
1814
|
+
try {
|
|
1815
|
+
await recordAgentJobObservation(data, {
|
|
1816
|
+
processInstanceKey: pik,
|
|
1817
|
+
jobType,
|
|
1818
|
+
worker,
|
|
1819
|
+
jobKey,
|
|
1820
|
+
now: ts,
|
|
1821
|
+
});
|
|
1822
|
+
} catch (err) {
|
|
1823
|
+
console.error(`[poller] agent-activation upsert ${pik}:${jobType}: ${err}`);
|
|
1824
|
+
}
|
|
1825
|
+
}
|
|
1826
|
+
if (items.length < limit) break; // last page
|
|
1827
|
+
from += items.length;
|
|
1828
|
+
}
|
|
1736
1829
|
}
|
|
1737
1830
|
|
|
1738
1831
|
/** One job-activation poll pass. The `converging` status means the process is parked at a
|
|
@@ -3316,7 +3409,21 @@ export async function pollUserTasks(
|
|
|
3316
3409
|
// null), so this is the only link a delivery human task can offer.
|
|
3317
3410
|
let deliverySubjectUrl: string | null = null;
|
|
3318
3411
|
if (question === null && isDeliveryHumanElement(elementId)) {
|
|
3319
|
-
|
|
3412
|
+
let livePrompt: string | null = null;
|
|
3413
|
+
if (needsLiveDeliveryPrompt(subj?.humanLabels, elementId)) {
|
|
3414
|
+
// An agent/wait/connector escalation twin: read the task's OWN compiler-rendered `prompt`, local
|
|
3415
|
+
// to the user task's element instance (fail-open to the static fallback on any read error).
|
|
3416
|
+
try {
|
|
3417
|
+
const [ei] = await engine.searchElementInstances({ processInstanceKey, elementId, state: "ACTIVE" });
|
|
3418
|
+
if (ei) {
|
|
3419
|
+
const [row] = await engine.searchVariables({ scopeKey: String(ei.elementInstanceKey), name: "prompt" });
|
|
3420
|
+
livePrompt = decodeLivePrompt(row);
|
|
3421
|
+
}
|
|
3422
|
+
} catch (err) {
|
|
3423
|
+
console.error(`[poller] delivery escalation prompt (${processInstanceKey}/${elementId}): ${err}`);
|
|
3424
|
+
}
|
|
3425
|
+
}
|
|
3426
|
+
question = deliveryHumanContextQuestion(subj?.humanLabels, elementId, livePrompt);
|
|
3320
3427
|
deliverySubjectUrl = deliveryHumanContextUrl(subj?.humanLabels, elementId);
|
|
3321
3428
|
}
|
|
3322
3429
|
return { userTaskKey, elementId, subjectType, subjectKey, subjectTitle: subj?.title ?? null, subjectUrl: subj?.url ?? deliverySubjectUrl, question, processKey: processInstanceKey, formKey: resolvedFormKey };
|
|
@@ -3703,6 +3810,7 @@ export async function pollOnce(
|
|
|
3703
3810
|
await pollWaveGatesImpl(data, engine, token, base, headers);
|
|
3704
3811
|
await pollCapabilityGatesImpl(data, engine, base, headers);
|
|
3705
3812
|
await pollJobActivation(data, engineRest.restAddress, engineRest.token);
|
|
3813
|
+
await pollAgentActivations(data, engineRest.restAddress, engineRest.token);
|
|
3706
3814
|
await pollIncidents(data, engineRest.restAddress, engineRest.token);
|
|
3707
3815
|
}
|
|
3708
3816
|
}
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
-- 119_agent_job_activations.sql — issue #879: distinguish a NEVER-ACTIVATED (queue-starved) agent
|
|
2
|
+
-- job from an ACTIVATED-but-hung one when an agent-task SLA boundary fires.
|
|
3
|
+
--
|
|
4
|
+
-- The agent-task SLA timer (`<bpmn:timeDuration>=agentSlaTimeout`) starts when the service task is
|
|
5
|
+
-- ENTERED, so it measures QUEUE time + RUN time. When no worker can take the job type (fleet
|
|
6
|
+
-- saturation / no capacity), the job sits CREATED (never leased) for the whole budget and the
|
|
7
|
+
-- boundary fires — yet the escalation reported it as "hung or looping", telling the human to "check
|
|
8
|
+
-- its progress" when there is none. (The reported case: a `senior:feature` job queued 2h, never
|
|
9
|
+
-- activated, then cancelled by its SLA boundary.)
|
|
10
|
+
--
|
|
11
|
+
-- The engine tracks JobState::Created vs ::Activated, but the Camunda-8 `/v2/jobs/search` wire API
|
|
12
|
+
-- collapses Activated -> CREATED (no ACTIVATED enum value); activation is read off the `worker` +
|
|
13
|
+
-- `deadline` fields — an activated job carries a leasing worker's name and a lock deadline, a
|
|
14
|
+
-- merely-queued one carries neither (same signal `pull_requests.active_worker`/`lease_until` use for
|
|
15
|
+
-- the review-round, migration 005). This ledger generalises that observation to EVERY bounded agent
|
|
16
|
+
-- job type (not just the review-round) and PERSISTS it, so when the SLA boundary cancels the job the
|
|
17
|
+
-- escalation recorder can still tell whether a worker ever picked it up.
|
|
18
|
+
--
|
|
19
|
+
-- The poller (`pollAgentActivations`, app/service.ts) upserts one row per (process_instance_key,
|
|
20
|
+
-- job_type) each pass: `first_seen_at` on first observation (job CREATED), and `activated_at` +
|
|
21
|
+
-- `worker` the first time it observes a leasing worker. The SLA-escalation recorders read it by the
|
|
22
|
+
-- same key (the agent job and its escalation recorder share one process instance) to word the
|
|
23
|
+
-- escalation accurately — "never started (queue starvation)" vs "started but exceeded its budget" —
|
|
24
|
+
-- including the job type and how long it waited.
|
|
25
|
+
--
|
|
26
|
+
-- `job_key` holds the engine job key of the CURRENT attempt so the row tracks that attempt, not a
|
|
27
|
+
-- stale earlier one: the same (process_instance_key, job_type) runs MULTIPLE times within one instance
|
|
28
|
+
-- when its cell retries (implement-cell `ic_reImplement`, merge-cell `mc_rerun`, merge-loop fix-ci /
|
|
29
|
+
-- rebase reloops). Keyed only by (instance, job_type) a sticky first-activation would make a later
|
|
30
|
+
-- queued retry read the EARLIER attempt's worker and misdiagnose it as started/hung. When the poller
|
|
31
|
+
-- observes a different `job_key` for the pair it RESETS the row to the new attempt (first_seen_at /
|
|
32
|
+
-- activated_at / worker), so the escalation always reflects the attempt the SLA boundary cancelled.
|
|
33
|
+
--
|
|
34
|
+
-- Keyed by a surrogate TEXT id (`<process_instance_key>:<job_type>`) so the single-pk `data.table`
|
|
35
|
+
-- accessor can upsert it; the (process_instance_key, job_type) pair is unique per id. FK-free by
|
|
36
|
+
-- design — the owning instance lives in the durable ENGINE store, not app.db, so there is no app-tier
|
|
37
|
+
-- parent row to reference (mirrors `worker_durable_resume`/`worker_harness_protocol`). EXPAND
|
|
38
|
+
-- (additive) phase: one new table; nothing is dropped or renamed. The runner wraps each file in its
|
|
39
|
+
-- own transaction, so this file must NOT contain BEGIN/COMMIT.
|
|
40
|
+
CREATE TABLE IF NOT EXISTS agent_job_activations (
|
|
41
|
+
id TEXT PRIMARY KEY, -- surrogate: `<process_instance_key>:<job_type>`
|
|
42
|
+
process_instance_key TEXT NOT NULL, -- the engine instance the agent job (and its SLA recorder) runs in
|
|
43
|
+
job_type TEXT NOT NULL, -- the agent job's `<zeebe:taskDefinition>` type (e.g. senior:feature)
|
|
44
|
+
first_seen_at TEXT NOT NULL, -- ISO ts the poller first observed the job (CREATED/queued)
|
|
45
|
+
activated_at TEXT, -- ISO ts first observed ACTIVATED (leasing worker seen); NULL = never started
|
|
46
|
+
worker TEXT, -- the leasing worker's name at activation; NULL until activated
|
|
47
|
+
job_key TEXT, -- the engine job key of the CURRENT attempt; a new key RESETS the row (retry isolation)
|
|
48
|
+
updated_at TEXT NOT NULL
|
|
49
|
+
);
|
|
50
|
+
CREATE INDEX IF NOT EXISTS ix_agent_job_activations_instance ON agent_job_activations (process_instance_key);
|
|
@@ -307,13 +307,25 @@ describe("agent-task SLA boundary — rerouted recorder→escalation arms (#849
|
|
|
307
307
|
);
|
|
308
308
|
const task = await openTaskById(app, processKey, "trial-merge-decision");
|
|
309
309
|
assert.ok(task?.userTaskKey, "the merge-cell parked a completable trial-merge decision task");
|
|
310
|
+
|
|
311
|
+
// UNKNOWN activation fixture (issue #881 review): no activation-ledger row exists for this attempt
|
|
312
|
+
// (the in-test poller records none), so the recorder must word the escalation honestly as
|
|
313
|
+
// "activation unknown" — never the removed blanket "hung or looping".
|
|
314
|
+
const slaAudit = (
|
|
315
|
+
await app.db
|
|
316
|
+
.table<{ id: number; plan_key: string; summary: string | null }>("plan_trial_merges", "id")
|
|
317
|
+
.find({ plan_key: "owner/repo#1" })
|
|
318
|
+
).sort((a, b) => b.id - a.id)[0];
|
|
319
|
+
assert.ok(
|
|
320
|
+
(slaAudit?.summary ?? "").includes("no worker-activation was ever recorded"),
|
|
321
|
+
`an unobserved job escalates with honest ambiguity (got: ${slaAudit?.summary})`,
|
|
322
|
+
);
|
|
310
323
|
} finally {
|
|
311
324
|
await app.stop();
|
|
312
325
|
rmSync(dbDir, { recursive: true, force: true });
|
|
313
326
|
}
|
|
314
327
|
});
|
|
315
328
|
|
|
316
|
-
// Regression for PR #864 review r3: on an SLA during a RERUN the recorder must not replay the
|
|
317
329
|
// prior COMPLETED attempt's process-scope `result`/`summary`/`conflicts`/`failing`. The
|
|
318
330
|
// `=null` ioMapping on `trial-merge` is task-local, so without a dedicated SLA recorder the
|
|
319
331
|
// interrupted second attempt would inherit attempt one's stale verdict. Drive ONE instance: a
|
|
@@ -381,6 +393,19 @@ describe("agent-task SLA boundary — rerouted recorder→escalation arms (#849
|
|
|
381
393
|
);
|
|
382
394
|
|
|
383
395
|
// The second agent hangs; the SLA fires. The DEDICATED SLA recorder must pin the TIMEOUT verdict.
|
|
396
|
+
// Seed an ACTIVATED activation-ledger fixture for THIS attempt (issue #881 review) so the recorder
|
|
397
|
+
// reads the ledger and produces the activation-aware "started but hung" reason — exercising the
|
|
398
|
+
// builder's activated branch end-to-end through record-trial-merge-sla, not just the pure builder.
|
|
399
|
+
await app.db.table("agent_job_activations", "id").insert({
|
|
400
|
+
id: `${processKey}:senior:trial-merge`,
|
|
401
|
+
process_instance_key: processKey,
|
|
402
|
+
job_type: "senior:trial-merge",
|
|
403
|
+
first_seen_at: "2024-01-01T00:00:00Z",
|
|
404
|
+
activated_at: "2024-01-01T00:05:00Z",
|
|
405
|
+
worker: "agent-stuck",
|
|
406
|
+
job_key: "rerun-attempt",
|
|
407
|
+
updated_at: "2024-01-01T00:05:00Z",
|
|
408
|
+
});
|
|
384
409
|
await advancePastTimer(app, PAST_AGENT_SLA_MS);
|
|
385
410
|
flows = takenFlows(app);
|
|
386
411
|
assert.ok(
|
|
@@ -402,11 +427,15 @@ describe("agent-task SLA boundary — rerouted recorder→escalation arms (#849
|
|
|
402
427
|
.find({ plan_key: "owner/repo#1" });
|
|
403
428
|
const slaRow = audits.sort((a, b) => b.id - a.id)[0];
|
|
404
429
|
assert.equal(slaRow.result, "suite-failed", "the SLA row records the timeout verdict");
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
430
|
+
// Activation-aware summary: names the job type, the leasing worker, and the hung diagnosis — NOT
|
|
431
|
+
// the stale first-attempt summary, and NOT the removed blanket wording.
|
|
432
|
+
assert.ok(
|
|
433
|
+
(slaRow.summary ?? "").includes("senior:trial-merge") &&
|
|
434
|
+
(slaRow.summary ?? "").includes("hung or looping") &&
|
|
435
|
+
(slaRow.summary ?? "").includes("agent-stuck"),
|
|
436
|
+
`the SLA row pins the activation-aware timeout summary (got: ${slaRow.summary})`,
|
|
409
437
|
);
|
|
438
|
+
assert.equal((slaRow.summary ?? "").includes("stale prior-attempt summary"), false, "not the stale summary");
|
|
410
439
|
assert.equal(slaRow.conflicts, null, "the SLA row clears the stale prior-attempt conflicts");
|
|
411
440
|
assert.equal(slaRow.failing, null, "the SLA row clears the stale prior-attempt failing list");
|
|
412
441
|
} finally {
|
|
@@ -552,7 +581,20 @@ describe("agent-task SLA boundary — rerouted recorder→escalation arms (#849
|
|
|
552
581
|
assert.equal(filed?.status, "filed", "the conformance verdict was filed before synthesize");
|
|
553
582
|
assert.equal(filed?.slices_met, 3);
|
|
554
583
|
|
|
555
|
-
// Fire the synthesize SLA.
|
|
584
|
+
// Fire the synthesize SLA. Seed a QUEUED (never-activated) activation-ledger fixture for the
|
|
585
|
+
// synthesize (senior:retro) attempt (issue #881 review) so the recorder reads it and APPENDS the
|
|
586
|
+
// activation-aware never-observed-activated reason — exercising the builder's queued branch
|
|
587
|
+
// end-to-end through conformance-record, not just the pure builder.
|
|
588
|
+
await app.db.table("agent_job_activations", "id").insert({
|
|
589
|
+
id: `${processKey}:senior:retro`,
|
|
590
|
+
process_instance_key: processKey,
|
|
591
|
+
job_type: "senior:retro",
|
|
592
|
+
first_seen_at: "2024-01-01T00:00:00Z",
|
|
593
|
+
activated_at: null,
|
|
594
|
+
worker: null,
|
|
595
|
+
job_key: "synthesize-attempt",
|
|
596
|
+
updated_at: "2024-01-01T00:00:00Z",
|
|
597
|
+
});
|
|
556
598
|
await advancePastTimer(app, PAST_AGENT_SLA_MS);
|
|
557
599
|
const flows = takenFlows(app);
|
|
558
600
|
assert.ok(
|
|
@@ -587,8 +629,9 @@ describe("agent-task SLA boundary — rerouted recorder→escalation arms (#849
|
|
|
587
629
|
`the audit summary is preserved and the SLA reason appended (got: ${row?.summary})`,
|
|
588
630
|
);
|
|
589
631
|
assert.ok(
|
|
590
|
-
(row?.summary ?? "").includes("
|
|
591
|
-
|
|
632
|
+
(row?.summary ?? "").includes("No worker was ever observed to pick up") &&
|
|
633
|
+
(row?.summary ?? "").includes("senior:retro"),
|
|
634
|
+
`the activation-aware synthesis-timeout context is appended to the summary (got: ${row?.summary})`,
|
|
592
635
|
);
|
|
593
636
|
} finally {
|
|
594
637
|
await app.stop();
|