@nanobpm/nano-workforce 0.201.5 → 0.203.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/SPEC.md +48 -0
- package/app/admittedStaging.test.ts +71 -0
- package/app/agentActivationLedger.ts +5 -0
- package/app/agentCompletion.test.ts +33 -0
- package/app/agentCompletion.ts +108 -2
- package/app/agentic/README.md +48 -0
- package/app/agentic/claim-registry.test.ts +61 -0
- package/app/agentic/claim-registry.ts +35 -1
- package/app/agentic/families/claim.family.test.ts +216 -0
- package/app/agentic/families/claim.family.ts +100 -4
- package/app/agentic/vocab/agent-sla-boundary.test.ts +6 -1
- package/app/contracts.ts +31 -0
- package/app/deliveryGraphDispatch.ts +6 -0
- package/app/deliveryGraphReadModel.test.ts +2 -2
- package/app/deliveryGraphRun.ts +18 -0
- package/app/deliveryRunner.ts +9 -1
- package/app/escalationSteward.test.ts +452 -0
- package/app/escalationSteward.ts +537 -0
- package/app/escalationStewardPoll.test.ts +123 -0
- package/app/feature.test.ts +16 -0
- package/app/feature.ts +11 -0
- package/app/featureDelivery.test.ts +23 -0
- package/app/mcpToolSurface.ts +17 -1
- package/app/plan.ts +49 -6
- package/app/planLowering.test.ts +26 -1
- package/app/planLowering.ts +5 -0
- package/app/promotionPoll.test.ts +19 -0
- package/app/pullRequestReadModel.test.ts +1 -1
- package/app/service.test.ts +36 -0
- package/app/service.ts +139 -5
- package/db/migrations/120_escalation_auto_resolutions.sql +229 -0
- package/db/migrations/121_delivery_graph_read_model_auto_resolve.sql +54 -0
- package/db/migrations/122_pull_requests_read_model_auto_resolve.sql +56 -0
- package/db/migrations/123_escalation_auto_resolutions_selected_option.sql +47 -0
- package/db/migrations/124_admitted_epics_auto_resolve.sql +21 -0
- package/docs/agent-guide.md +69 -30
- package/docs/agentic-cockpit.md +34 -0
- package/docs/mcp-runbook.md +2 -0
- package/e2e/escalation-steward.e2e.ts +211 -0
- package/nano.app.json +10 -1
- package/openapi.yaml +365 -0
- package/operations/dispatchDeliveryGraph.ts +6 -1
- package/operations/listActivePrs.test.ts +60 -1
- package/operations/listActivePrs.ts +3 -3
- package/operations/listAutoResolutions.ts +83 -0
- package/operations/reviewAutoResolution.test.ts +137 -0
- package/operations/reviewAutoResolution.ts +42 -0
- package/operations/startConvergenceLoop.ts +5 -1
- package/operations/startEpicSet.admission.integration.test.ts +2 -2
- package/operations/startEpicSet.ts +8 -2
- package/operations/startFeature.ts +5 -0
- package/operations/startPlanFanout.ts +4 -1
- package/package.json +1 -1
- package/pages/_nav.json +12 -0
- package/pages/auto-resolutions.page.json +191 -0
- package/pages/board.page.json +17 -0
- package/pages/cockpit.page.json +17 -0
- package/pages/delivery-graph-detail.page.json +17 -0
- package/pages/delivery-graphs/staged.mount.js +17 -5
- package/pages/delivery-graphs.page.json +17 -0
- package/pages/epic-detail.page.json +17 -0
- package/pages/epic.page.json +19 -1
- package/pages/feature.page.json +19 -1
- package/pages/home.page.json +22 -0
- package/pages/lineage.page.json +17 -0
- package/pages/mcp.page.json +17 -0
- package/pages/overview.page.json +17 -0
- package/pages/tasks.page.json +17 -0
- package/pages/velocity.page.json +17 -0
- package/resources/processes/escalation-steward.bpmn +162 -0
- package/resources/prompts/escalation-steward.md +87 -0
- package/test/delivery-graphs-staged-embed.test.ts +13 -0
- package/test/derivation-parity/derivation-parity.test.ts +1 -0
- package/test/derivation-parity/flows.ts +12 -0
- package/workers/converge-feature/worker.ts +6 -1
- package/workers/delivery-connector/worker.test.ts +72 -1
- package/workers/delivery-connector/worker.ts +53 -2
- package/workers/escalation-steward-collect/worker.test.ts +180 -0
- package/workers/escalation-steward-collect/worker.ts +99 -0
- package/workers/escalation-steward-record/worker.test.ts +740 -0
- package/workers/escalation-steward-record/worker.ts +428 -0
- package/workers/record-wave/worker.ts +5 -1
|
@@ -56,6 +56,10 @@ function releaseFrame(instance: string, jobKey: string): Frame {
|
|
|
56
56
|
return { lane: "control", family: RELEASE_FAMILY, seq: 1, payload: { instance, jobKey } };
|
|
57
57
|
}
|
|
58
58
|
|
|
59
|
+
function resyncFrame(instance: string, jobKeys: string[]): Frame {
|
|
60
|
+
return { lane: "control", family: CLAIM_FAMILY, seq: 1, payload: { instance, jobKeys } };
|
|
61
|
+
}
|
|
62
|
+
|
|
59
63
|
function mountClaim(): { hub: AgenticHub; transport: ReturnType<typeof memTransport> } {
|
|
60
64
|
const transport = memTransport();
|
|
61
65
|
const hub = new AgenticHub({ transport: transport.transport, authenticator, sweepIntervalMs: 0 });
|
|
@@ -189,6 +193,218 @@ test("a malformed claim payload is rejected without touching the registry (advis
|
|
|
189
193
|
}
|
|
190
194
|
});
|
|
191
195
|
|
|
196
|
+
test("a resync snapshot frame REPLACES an instance's claims — a RELEASE lost during an outage no longer strands the job", async () => {
|
|
197
|
+
const { transport } = mountClaim();
|
|
198
|
+
try {
|
|
199
|
+
const peer = fakeConn("c1", "host-supervisor");
|
|
200
|
+
transport.connect(peer.conn);
|
|
201
|
+
await flush();
|
|
202
|
+
peer.feed(claimFrame("wk-a", "8420"));
|
|
203
|
+
peer.feed(claimFrame("wk-a", "8421"));
|
|
204
|
+
await flush();
|
|
205
|
+
|
|
206
|
+
const reg = currentClaimRegistry();
|
|
207
|
+
assert(reg);
|
|
208
|
+
assertEquals(reg.jobKeysFor("wk-a"), ["8420", "8421"]);
|
|
209
|
+
|
|
210
|
+
// The hub missed the `release` for 8420 during an outage. On reconnect the supervisor restates its
|
|
211
|
+
// authoritative snapshot: wk-a now owns only 8421. The non-re-asserted 8420 is dropped.
|
|
212
|
+
peer.feed(resyncFrame("wk-a", ["8421"]));
|
|
213
|
+
await flush();
|
|
214
|
+
assertEquals(reg.jobKeysFor("wk-a"), ["8421"], "the stranded claim was dropped on resync");
|
|
215
|
+
assertEquals(reg.count(), 1);
|
|
216
|
+
} finally {
|
|
217
|
+
family.teardown?.();
|
|
218
|
+
}
|
|
219
|
+
});
|
|
220
|
+
|
|
221
|
+
test("a resync snapshot does NOT blank a jobKey that moved to another instance", async () => {
|
|
222
|
+
const { transport } = mountClaim();
|
|
223
|
+
try {
|
|
224
|
+
const peer = fakeConn("c1", "host-supervisor");
|
|
225
|
+
transport.connect(peer.conn);
|
|
226
|
+
await flush();
|
|
227
|
+
peer.feed(claimFrame("wk-a", "8420"));
|
|
228
|
+
peer.feed(claimFrame("wk-b", "8420")); // 8420 moved to wk-b
|
|
229
|
+
await flush();
|
|
230
|
+
|
|
231
|
+
// wk-a's authoritative resync no longer mentions 8420 — and must not blank the live owner wk-b.
|
|
232
|
+
peer.feed(resyncFrame("wk-a", []));
|
|
233
|
+
await flush();
|
|
234
|
+
|
|
235
|
+
const reg = currentClaimRegistry();
|
|
236
|
+
assert(reg);
|
|
237
|
+
assertEquals(reg.jobKeysFor("wk-b"), ["8420"], "the current owner's claim survives");
|
|
238
|
+
assertEquals(reg.ownerOf("8420"), "wk-b");
|
|
239
|
+
} finally {
|
|
240
|
+
family.teardown?.();
|
|
241
|
+
}
|
|
242
|
+
});
|
|
243
|
+
|
|
244
|
+
test("a supervisor that never sends a resync snapshot keeps today's additive claim behaviour", async () => {
|
|
245
|
+
const { transport } = mountClaim();
|
|
246
|
+
try {
|
|
247
|
+
const peer = fakeConn("c1", "host-supervisor");
|
|
248
|
+
transport.connect(peer.conn);
|
|
249
|
+
await flush();
|
|
250
|
+
// Only per-job claims, no snapshot frame — a claim is never dropped except by an explicit release.
|
|
251
|
+
peer.feed(claimFrame("wk-a", "8420"));
|
|
252
|
+
peer.feed(claimFrame("wk-a", "8421"));
|
|
253
|
+
await flush();
|
|
254
|
+
|
|
255
|
+
const reg = currentClaimRegistry();
|
|
256
|
+
assert(reg);
|
|
257
|
+
assertEquals(reg.jobKeysFor("wk-a"), ["8420", "8421"], "additive — neither claim is dropped");
|
|
258
|
+
assertEquals(reg.count(), 2);
|
|
259
|
+
} finally {
|
|
260
|
+
family.teardown?.();
|
|
261
|
+
}
|
|
262
|
+
});
|
|
263
|
+
|
|
264
|
+
test("a resync snapshot with a non-string instance is ignored (advisory, connection kept)", async () => {
|
|
265
|
+
const { transport } = mountClaim();
|
|
266
|
+
try {
|
|
267
|
+
const peer = fakeConn("c1", "host-supervisor");
|
|
268
|
+
transport.connect(peer.conn);
|
|
269
|
+
await flush();
|
|
270
|
+
peer.feed(claimFrame("wk-a", "8420"));
|
|
271
|
+
await flush();
|
|
272
|
+
// A malformed snapshot (no instance) is rejected without touching the registry.
|
|
273
|
+
peer.feed({ lane: "control", family: CLAIM_FAMILY, seq: 1, payload: { jobKeys: ["8421"] } });
|
|
274
|
+
await flush();
|
|
275
|
+
|
|
276
|
+
const reg = currentClaimRegistry();
|
|
277
|
+
assert(reg);
|
|
278
|
+
assertEquals(reg.jobKeysFor("wk-a"), ["8420"], "the registry was untouched by the malformed snapshot");
|
|
279
|
+
assertEquals(reg.count(), 1);
|
|
280
|
+
} finally {
|
|
281
|
+
family.teardown?.();
|
|
282
|
+
}
|
|
283
|
+
});
|
|
284
|
+
|
|
285
|
+
test("a snapshot frame carrying BOTH jobKeys and a malformed jobKey is rejected — the snapshot branch never bypasses validation", async () => {
|
|
286
|
+
const { transport } = mountClaim();
|
|
287
|
+
try {
|
|
288
|
+
const peer = fakeConn("c1", "host-supervisor");
|
|
289
|
+
transport.connect(peer.conn);
|
|
290
|
+
await flush();
|
|
291
|
+
peer.feed(claimFrame("wk-a", "8420"));
|
|
292
|
+
await flush();
|
|
293
|
+
// The bypass the reviewer flagged: `jobKeys` present routed the frame straight to replaceInstance,
|
|
294
|
+
// so a smuggled empty `jobKey` was never validated. The union validator rejects it.
|
|
295
|
+
peer.feed({ lane: "control", family: CLAIM_FAMILY, seq: 1, payload: { instance: "wk-a", jobKey: "", jobKeys: ["8421"] } });
|
|
296
|
+
await flush();
|
|
297
|
+
|
|
298
|
+
const reg = currentClaimRegistry();
|
|
299
|
+
assert(reg);
|
|
300
|
+
assertEquals(reg.jobKeysFor("wk-a"), ["8420"], "the malformed both-fields frame touched nothing");
|
|
301
|
+
assertEquals(reg.count(), 1);
|
|
302
|
+
} finally {
|
|
303
|
+
family.teardown?.();
|
|
304
|
+
}
|
|
305
|
+
});
|
|
306
|
+
|
|
307
|
+
test("a frame with a valid jobKey but a PRESENT malformed jobKeys is rejected — a broken snapshot can't launder as a per-job claim", async () => {
|
|
308
|
+
const { transport } = mountClaim();
|
|
309
|
+
try {
|
|
310
|
+
const peer = fakeConn("c1", "host-supervisor");
|
|
311
|
+
transport.connect(peer.conn);
|
|
312
|
+
await flush();
|
|
313
|
+
peer.feed(claimFrame("wk-a", "8420"));
|
|
314
|
+
await flush();
|
|
315
|
+
// The reviewer's case: `jobKeys` is PRESENT but malformed ("invalid"), while `jobKey` is valid.
|
|
316
|
+
// Presence — not validity — selects the snapshot variant, so a broken `jobKeys` must be rejected,
|
|
317
|
+
// never treated as absent and accepted as a bare per-job claim that mutates ownership.
|
|
318
|
+
peer.feed({ lane: "control", family: CLAIM_FAMILY, seq: 1, payload: { instance: "wk-a", jobKey: "8421", jobKeys: "invalid" } });
|
|
319
|
+
await flush();
|
|
320
|
+
|
|
321
|
+
const reg = currentClaimRegistry();
|
|
322
|
+
assert(reg);
|
|
323
|
+
assertEquals(reg.jobKeysFor("wk-a"), ["8420"], "the malformed-snapshot frame touched nothing");
|
|
324
|
+
assertEquals(reg.count(), 1);
|
|
325
|
+
} finally {
|
|
326
|
+
family.teardown?.();
|
|
327
|
+
}
|
|
328
|
+
});
|
|
329
|
+
|
|
330
|
+
test("a snapshot frame with a PRESENT malformed jobKeys array is rejected as a broken snapshot (presence is the discriminant)", async () => {
|
|
331
|
+
const { transport } = mountClaim();
|
|
332
|
+
try {
|
|
333
|
+
const peer = fakeConn("c1", "host-supervisor");
|
|
334
|
+
transport.connect(peer.conn);
|
|
335
|
+
await flush();
|
|
336
|
+
peer.feed(claimFrame("wk-a", "8420"));
|
|
337
|
+
await flush();
|
|
338
|
+
// `jobKeys` present but not a string array → a broken snapshot; it is rejected outright, never
|
|
339
|
+
// mistaken for a per-job claim. (No `jobKey` here either, but presence alone is enough to reject.)
|
|
340
|
+
peer.feed({ lane: "control", family: CLAIM_FAMILY, seq: 1, payload: { instance: "wk-a", jobKeys: "8421" } });
|
|
341
|
+
await flush();
|
|
342
|
+
|
|
343
|
+
const reg = currentClaimRegistry();
|
|
344
|
+
assert(reg);
|
|
345
|
+
assertEquals(reg.jobKeysFor("wk-a"), ["8420"], "the malformed frame touched nothing");
|
|
346
|
+
assertEquals(reg.count(), 1);
|
|
347
|
+
} finally {
|
|
348
|
+
family.teardown?.();
|
|
349
|
+
}
|
|
350
|
+
});
|
|
351
|
+
|
|
352
|
+
test("an EMPTY snapshot is a valid authoritative resync — the supervisor asserting it now owns nothing", async () => {
|
|
353
|
+
const { transport } = mountClaim();
|
|
354
|
+
try {
|
|
355
|
+
const peer = fakeConn("c1", "host-supervisor");
|
|
356
|
+
transport.connect(peer.conn);
|
|
357
|
+
await flush();
|
|
358
|
+
peer.feed(claimFrame("wk-a", "8420"));
|
|
359
|
+
await flush();
|
|
360
|
+
|
|
361
|
+
const reg = currentClaimRegistry();
|
|
362
|
+
assert(reg);
|
|
363
|
+
assertEquals(reg.jobKeysFor("wk-a"), ["8420"]);
|
|
364
|
+
|
|
365
|
+
// The producer emits one snapshot per instance INCLUDING empty sets — an empty `jobKeys` is the
|
|
366
|
+
// meaningful "owns nothing now" resync, not a malformed frame.
|
|
367
|
+
peer.feed(resyncFrame("wk-a", []));
|
|
368
|
+
await flush();
|
|
369
|
+
assertEquals(reg.jobKeysFor("wk-a"), [], "the empty snapshot cleared the instance's claims");
|
|
370
|
+
assertEquals(reg.count(), 0);
|
|
371
|
+
} finally {
|
|
372
|
+
family.teardown?.();
|
|
373
|
+
}
|
|
374
|
+
});
|
|
375
|
+
|
|
376
|
+
test("a snapshot whose jobKeys contains an EMPTY-STRING element is rejected — it can't masquerade as an 'owns nothing' resync and clear real claims", async () => {
|
|
377
|
+
const { transport } = mountClaim();
|
|
378
|
+
try {
|
|
379
|
+
const peer = fakeConn("c1", "host-supervisor");
|
|
380
|
+
transport.connect(peer.conn);
|
|
381
|
+
await flush();
|
|
382
|
+
peer.feed(claimFrame("wk-a", "8420"));
|
|
383
|
+
await flush();
|
|
384
|
+
|
|
385
|
+
const reg = currentClaimRegistry();
|
|
386
|
+
assert(reg);
|
|
387
|
+
assertEquals(reg.jobKeysFor("wk-a"), ["8420"]);
|
|
388
|
+
|
|
389
|
+
// The empty-ARRAY resync is valid ("owns nothing"), but a present `jobKeys` with an empty-STRING
|
|
390
|
+
// element is a BROKEN snapshot — `replaceInstance` silently skips empty keys, so `[""]` would
|
|
391
|
+
// otherwise masquerade as an authoritative empty snapshot and clear every real claim. It is held
|
|
392
|
+
// to the same non-empty bar as the per-job `jobKey` and rejected outright, touching nothing.
|
|
393
|
+
peer.feed({ lane: "control", family: CLAIM_FAMILY, seq: 1, payload: { instance: "wk-a", jobKeys: [""] } });
|
|
394
|
+
await flush();
|
|
395
|
+
assertEquals(reg.jobKeysFor("wk-a"), ["8420"], "the sole-empty-string snapshot did not clear the real claim");
|
|
396
|
+
assertEquals(reg.count(), 1);
|
|
397
|
+
|
|
398
|
+
// A MIXED array with one empty element is equally a broken snapshot — fail closed, not partial.
|
|
399
|
+
peer.feed({ lane: "control", family: CLAIM_FAMILY, seq: 1, payload: { instance: "wk-a", jobKeys: ["8421", ""] } });
|
|
400
|
+
await flush();
|
|
401
|
+
assertEquals(reg.jobKeysFor("wk-a"), ["8420"], "the mixed empty-element snapshot touched nothing — not even the valid element");
|
|
402
|
+
assertEquals(reg.count(), 1);
|
|
403
|
+
} finally {
|
|
404
|
+
family.teardown?.();
|
|
405
|
+
}
|
|
406
|
+
});
|
|
407
|
+
|
|
192
408
|
test("teardown clears the singleton", () => {
|
|
193
409
|
mountClaim();
|
|
194
410
|
assert(currentClaimRegistry() !== undefined);
|
|
@@ -13,7 +13,11 @@
|
|
|
13
13
|
// the frame, NOT the connection id (`conn.id`); that is what lets one per-host supervisor multiplex N
|
|
14
14
|
// distinct workers' ownership over a single connection. On a reconnect the supervisor re-`register`s
|
|
15
15
|
// every worker and re-`claim`s every active jobKey, and the (idempotent) claim handler rebuilds the
|
|
16
|
-
// registry from that resync.
|
|
16
|
+
// registry from that resync. A supervisor may additionally close its resync with an AUTHORITATIVE
|
|
17
|
+
// snapshot frame (`claim` carrying the full `jobKeys` set for an instance, issue #893 — the
|
|
18
|
+
// `claim.snapshot` wire contract in `app/contracts.ts`): the handler then `replaceInstance`s, dropping
|
|
19
|
+
// any claim the supervisor did NOT re-assert — so a `release` lost during a hub outage no longer
|
|
20
|
+
// strands the job. A supervisor that omits the snapshot keeps the pre-#893 additive behaviour.
|
|
17
21
|
//
|
|
18
22
|
// Liveness: a worker holding a claim reads "working" even with ZERO transcript — visibility no longer
|
|
19
23
|
// depends on terminal bytes landing/correlating. A bounded-memory maintenance tick reconciles the
|
|
@@ -46,6 +50,81 @@ function readString(value: unknown, key: string): string | undefined {
|
|
|
46
50
|
return typeof field === "string" ? field : undefined;
|
|
47
51
|
}
|
|
48
52
|
|
|
53
|
+
/**
|
|
54
|
+
* The classification of a `claim` frame's `jobKeys` field. Its PRESENCE — NOT its validity — is the
|
|
55
|
+
* discriminant that tells an authoritative resync snapshot (`{ instance, jobKeys }`, issue #893) apart
|
|
56
|
+
* from a classic per-job claim (`{ instance, jobKey }`). So a PRESENT-but-malformed `jobKeys` is a
|
|
57
|
+
* BROKEN snapshot and must be rejected; it must never be treated as "absent" and laundered through the
|
|
58
|
+
* per-job validator (which tolerates the extra field), because that would let a malformed snapshot
|
|
59
|
+
* field mutate ownership on a frame that merely carries a valid `jobKey`.
|
|
60
|
+
*/
|
|
61
|
+
type JobKeysField =
|
|
62
|
+
| { present: false }
|
|
63
|
+
| { present: true; malformed: true }
|
|
64
|
+
| { present: true; malformed: false; jobKeys: readonly string[] };
|
|
65
|
+
|
|
66
|
+
/**
|
|
67
|
+
* Classify a frame payload's `jobKeys` field. It rides the existing `claim` wire family as a
|
|
68
|
+
* forward-compatible enrichment — the protocol's structural validators tolerate the extra field, so an
|
|
69
|
+
* older hub ignores it and keeps additive behaviour. This is the SINGLE source that both the validator
|
|
70
|
+
* and the handler branch on, so the "present ⇒ snapshot" discriminant cannot drift between them.
|
|
71
|
+
*/
|
|
72
|
+
function readJobKeysField(value: unknown): JobKeysField {
|
|
73
|
+
if (!value || typeof value !== "object") return { present: false };
|
|
74
|
+
if (!Object.hasOwn(value, "jobKeys")) return { present: false };
|
|
75
|
+
const field = Object.getOwnPropertyDescriptor(value, "jobKeys")?.value;
|
|
76
|
+
// Each element must be a NON-EMPTY string — the canonical per-job validator requires a non-empty
|
|
77
|
+
// `jobKey`, so a snapshot entry is held to the same bar. An empty-string element is NOT a benign
|
|
78
|
+
// "owns nothing" signal (that is the empty ARRAY): `replaceInstance` silently skips empty keys, so
|
|
79
|
+
// `{ instance, jobKeys: [""] }` would otherwise masquerade as an authoritative empty snapshot and
|
|
80
|
+
// clear every real claim. Fail closed — a present `jobKeys` with any empty element is a broken
|
|
81
|
+
// snapshot and is rejected, never laundered through the per-job validator.
|
|
82
|
+
if (!Array.isArray(field) || !field.every((k) => typeof k === "string" && k !== "")) {
|
|
83
|
+
return { present: true, malformed: true };
|
|
84
|
+
}
|
|
85
|
+
return { present: true, malformed: false, jobKeys: field };
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
/**
|
|
89
|
+
* The `claim` family carries TWO payload variants (issue #893 — the `claim.snapshot` wire contract in
|
|
90
|
+
* `app/contracts.ts`): the classic per-job claim `{ instance, jobKey }` and the authoritative resync
|
|
91
|
+
* snapshot `{ instance, jobKeys }`. The pinned `@nanobpm/agentic` validator predates the snapshot
|
|
92
|
+
* variant and REQUIRES a non-empty `jobKey`, so it cannot validate the union — routing a snapshot
|
|
93
|
+
* through `validatePayload` would reject the very frame that closes a resync. Validate the union
|
|
94
|
+
* locally instead: a payload is well-formed iff it is EITHER a valid snapshot (non-empty `instance`
|
|
95
|
+
* plus a `jobKeys` array of NON-EMPTY strings — an EMPTY array is the meaningful "owns nothing now"
|
|
96
|
+
* resync) OR a valid per-job claim per the canonical validator. This is a UNION with a VALIDATED
|
|
97
|
+
* OVERLAP, not an exclusive-or: a frame carrying BOTH fields is accepted, but only when BOTH halves
|
|
98
|
+
* validate — it is validated as a snapshot AND must still pass the per-job validator, so a malformed
|
|
99
|
+
* `jobKey` can no longer smuggle past on a snapshot frame.
|
|
100
|
+
*/
|
|
101
|
+
function validateClaimPayload(payload: unknown): { ok: boolean; codes: string[] } {
|
|
102
|
+
const field = readJobKeysField(payload);
|
|
103
|
+
if (field.present) {
|
|
104
|
+
// PRESENCE selects the snapshot variant unconditionally. A present-but-malformed `jobKeys` is a
|
|
105
|
+
// broken snapshot — reject it here rather than letting it fall through to the per-job validator,
|
|
106
|
+
// which would accept it (extra fields tolerated) on any frame that also carries a valid `jobKey`.
|
|
107
|
+
if (field.malformed) {
|
|
108
|
+
return { ok: false, codes: ["bad-jobKeys"] };
|
|
109
|
+
}
|
|
110
|
+
const instance = readString(payload, "instance");
|
|
111
|
+
if (instance) {
|
|
112
|
+
// A frame carrying BOTH variants must satisfy both — the snapshot branch must never launder a
|
|
113
|
+
// malformed per-job field past validation. (readJobKeysField defined the value as an object.)
|
|
114
|
+
if (typeof payload === "object" && payload !== null && !Object.hasOwn(payload, "jobKey")) {
|
|
115
|
+
return { ok: true, codes: [] };
|
|
116
|
+
}
|
|
117
|
+
const both = validatePayload(CLAIM_FAMILY, payload);
|
|
118
|
+
return both.ok ? { ok: true, codes: [] } : { ok: false, codes: both.errors.map((e) => e.code) };
|
|
119
|
+
}
|
|
120
|
+
// A snapshot-shaped payload with a missing/empty instance is NOT a valid per-job claim either
|
|
121
|
+
// (jobKey is absent) — report the snapshot violation precisely.
|
|
122
|
+
return { ok: false, codes: ["bad-instance"] };
|
|
123
|
+
}
|
|
124
|
+
const result = validatePayload(CLAIM_FAMILY, payload);
|
|
125
|
+
return result.ok ? { ok: true, codes: [] } : { ok: false, codes: result.errors.map((e) => e.code) };
|
|
126
|
+
}
|
|
127
|
+
|
|
49
128
|
interface MountState {
|
|
50
129
|
readonly registry: ClaimRegistry;
|
|
51
130
|
timer: ReturnType<typeof setTimeout> | undefined;
|
|
@@ -72,9 +151,26 @@ export const family: AgenticFamily = {
|
|
|
72
151
|
// malformed payload is rejected before it touches the registry (advisory: logged, connection
|
|
73
152
|
// kept). Both mutations are idempotent, matching the wire contract.
|
|
74
153
|
ctx.hub.registerFamilyHandler(CLAIM_FAMILY, (frame: Frame, _conn: HubConnection) => {
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
154
|
+
// An authoritative resync SNAPSHOT (`{ instance, jobKeys }`, issue #893): the reconnecting
|
|
155
|
+
// supervisor restates the full current ownership for one instance and its view WINS —
|
|
156
|
+
// `replaceInstance` drops every claim it did NOT re-assert (so a `release` lost during a hub
|
|
157
|
+
// outage no longer strands the job). A supervisor that never sends it keeps additive behaviour.
|
|
158
|
+
// EVERY claim frame is validated first — as the snapshot/per-job union (a both-fields frame is
|
|
159
|
+
// accepted only when both halves validate) — so the snapshot
|
|
160
|
+
// branch never bypasses validation (a malformed frame is dropped before it touches the
|
|
161
|
+
// registry, exactly like a malformed per-job claim).
|
|
162
|
+
const valid = validateClaimPayload(frame.payload);
|
|
163
|
+
if (!valid.ok) {
|
|
164
|
+
ctx.log.warn("agentic claim: malformed payload", { errors: valid.codes });
|
|
165
|
+
return;
|
|
166
|
+
}
|
|
167
|
+
const field = readJobKeysField(frame.payload);
|
|
168
|
+
if (field.present && !field.malformed) {
|
|
169
|
+
// Validated above, so the instance is present and non-empty (and a malformed `jobKeys` was
|
|
170
|
+
// already rejected, so `field.present && field.malformed` can never reach the registry).
|
|
171
|
+
const instance = readString(frame.payload, "instance");
|
|
172
|
+
if (!instance) return;
|
|
173
|
+
registry.replaceInstance(instance, field.jobKeys);
|
|
78
174
|
return;
|
|
79
175
|
}
|
|
80
176
|
const instance = readString(frame.payload, "instance");
|
|
@@ -150,7 +150,12 @@ test("externalAgentTasksMissingSlaBoundary rejects a timer with no timeDuration,
|
|
|
150
150
|
// exactly these processes so a regression on a *bounded* process is caught while the deferred
|
|
151
151
|
// two stay out. merge-loop.bpmn's agent tasks (`fix-ci` / `rebase`) were bounded in an earlier
|
|
152
152
|
// #849 commit, so it belongs here too — every already-bounded process stays regression-guarded.
|
|
153
|
-
|
|
153
|
+
// escalation-steward.bpmn's `steward-agent` was bounded for the #889 review (a hung steward
|
|
154
|
+
// instance otherwise parks ACTIVE forever, suppressing every later tick via the poller's overlap
|
|
155
|
+
// guard and wedging claim recovery); its boundary routes to `done` (the steward has no escalation
|
|
156
|
+
// path of its own — on SLA the tick ends and the 15-min claim TTL frees its claims for the next
|
|
157
|
+
// tick), which satisfies the guard's "routes somewhere" requirement.
|
|
158
|
+
const BOUNDED_PROCESSES = ["implement-cell.bpmn", "merge-cell.bpmn", "merge-loop.bpmn", "retro.bpmn", "escalation-steward.bpmn"] as const;
|
|
154
159
|
|
|
155
160
|
test("DEFECT-CLASS GUARD: every external agent task in a bounded process carries an interrupting timer SLA boundary", () => {
|
|
156
161
|
let anyAgentTasks = false;
|
package/app/contracts.ts
CHANGED
|
@@ -179,6 +179,29 @@ export const ENV_CONTRACTS = {
|
|
|
179
179
|
semantics: "How long to wait for a review before nudging/escalating (FEEL/ISO-8601 duration).",
|
|
180
180
|
default: "PT30M",
|
|
181
181
|
},
|
|
182
|
+
NANO_ESCALATION_STEWARD: {
|
|
183
|
+
category: "env",
|
|
184
|
+
name: "NANO_ESCALATION_STEWARD",
|
|
185
|
+
owner: "app/escalationSteward.ts",
|
|
186
|
+
semantics:
|
|
187
|
+
"Escalation-steward master switch (issue #889): `off`/`0`/`false`/`no` disables the timer loop entirely; anything else (or unset) leaves it enabled.",
|
|
188
|
+
},
|
|
189
|
+
NANO_ESCALATION_STEWARD_INTERVAL: {
|
|
190
|
+
category: "env",
|
|
191
|
+
name: "NANO_ESCALATION_STEWARD_INTERVAL",
|
|
192
|
+
owner: "app/escalationSteward.ts",
|
|
193
|
+
semantics:
|
|
194
|
+
"How often the escalation-steward timer fires (FEEL/ISO-8601 duration or `R/PT…` cycle), pacing the steward tick (issue #889).",
|
|
195
|
+
default: "R/PT30M",
|
|
196
|
+
},
|
|
197
|
+
NANO_ESCALATION_STEWARD_MIN_AGE_MS: {
|
|
198
|
+
category: "env",
|
|
199
|
+
name: "NANO_ESCALATION_STEWARD_MIN_AGE_MS",
|
|
200
|
+
owner: "app/escalationSteward.ts",
|
|
201
|
+
semantics:
|
|
202
|
+
"Minimum age (ms) an open escalation must reach before the steward considers it, so a human already answering wins (issue #889).",
|
|
203
|
+
default: "300000",
|
|
204
|
+
},
|
|
182
205
|
NANO_PR_REVIEW_NUDGE_MINUTES: {
|
|
183
206
|
category: "env",
|
|
184
207
|
name: "NANO_PR_REVIEW_NUDGE_MINUTES",
|
|
@@ -443,6 +466,14 @@ export const WIRE_CONTRACTS = {
|
|
|
443
466
|
"Op-tagged relay control frame a worker terminal chunk producer emits and the hub consumes. The op-tagged shape superseded the legacy positional `{stream, offset, chunk}` frame (nano-ide #234/#236); a producer must emit the op-tagged shape or the hub rejects it as `malformed relay message payload`.",
|
|
444
467
|
shape: '{ op: "produce", incarnation: number, stream: string, offset: number, chunk: string }',
|
|
445
468
|
},
|
|
469
|
+
"claim.snapshot": {
|
|
470
|
+
category: "wire",
|
|
471
|
+
name: "claim.snapshot",
|
|
472
|
+
owner: "app/agentic/families/claim.family.ts (hub consumer); nano-supervisor#63 (producer)",
|
|
473
|
+
semantics:
|
|
474
|
+
"Authoritative reconnect-resync snapshot riding the EXISTING `claim` wire family (wire code 8, nano-ide#542) as a forward-compatible enrichment — the issue #893 resync boundary. A reconnecting supervisor closes its per-instance resync with ONE snapshot frame carrying the full current jobKey set for that instance; the hub validates it (as a snapshot/per-job UNION with a VALIDATED OVERLAP, not an exclusive-or — a both-fields frame is accepted only when BOTH halves validate; the pinned `@nanobpm/agentic` validator predates this variant and requires a non-empty `jobKey`, so the union is validated hub-side until nano-ide versions the shape into the canonical protocol payloads) and calls `ClaimRegistry.replaceInstance(instance, jobKeys)`, dropping every claim the supervisor did NOT re-assert so a `release` lost during a hub outage no longer strands the job. `jobKeys` MAY be the empty ARRAY (the meaningful 'owns nothing now' resync — this is why the producer emits one snapshot per instance INCLUDING empty sets), but every ELEMENT must be a NON-EMPTY string (held to the same bar as the per-job `jobKey`); a present `jobKeys` that is a non-array, has a non-string element, or has an empty-string element is a BROKEN snapshot and is rejected (fail closed), never laundered through the per-job validator. The frame's explicit `instance` is the owning worker, never the connection id. Compatibility: the protocol's structural validators tolerate the extra field, so an older hub ignores it and keeps additive behaviour; a supervisor that never sends a snapshot keeps the pre-#893 additive behaviour. Distinct from the classic per-job claim `{ instance, jobKey }` (the `@nanobpm/agentic` `ClaimPayload`) — consume this ONE shape for the resync boundary, do not re-declare a synonym.",
|
|
475
|
+
shape: '{ instance: string, jobKeys: string[] } — a `claim` (wire code 8) frame whose `jobKeys` array PRESENCE is the discriminant; a UNION with the per-job `{ instance: string, jobKey: string }` (a both-fields frame is accepted only when both halves validate, not an exclusive-or)',
|
|
476
|
+
},
|
|
446
477
|
"io.nanobpm.agentTask.repository": {
|
|
447
478
|
category: "wire",
|
|
448
479
|
name: "io.nanobpm.agentTask.repository",
|
|
@@ -227,6 +227,9 @@ export async function dispatchDeliveryGraphRun(
|
|
|
227
227
|
title,
|
|
228
228
|
humanLabels: buildHumanLabels(compiled),
|
|
229
229
|
createdAt: existing?.created_at,
|
|
230
|
+
// Opt-in escalation steward (issue #889): persist the run's opt-in so the steward's eligibility
|
|
231
|
+
// derivation reads it off the durable row.
|
|
232
|
+
autoResolveEscalations: options.autoResolveEscalations === true,
|
|
230
233
|
};
|
|
231
234
|
|
|
232
235
|
// Claim the run durably BEFORE the side effect (the at-most-once dispatch fence). A losing racer
|
|
@@ -344,6 +347,9 @@ export async function dispatchDeliveryGraphRun(
|
|
|
344
347
|
repository: options.repository,
|
|
345
348
|
baseBranch: options.baseBranch,
|
|
346
349
|
repoless: options.repoless,
|
|
350
|
+
// Opt-in escalation steward (issue #889): seeded as a run-root variable so every node's
|
|
351
|
+
// escalation inherits it (and the delivery-connector propagates it to any enrolled PR).
|
|
352
|
+
autoResolveEscalations: options.autoResolveEscalations,
|
|
347
353
|
});
|
|
348
354
|
} catch (err) {
|
|
349
355
|
await markClaimFailed();
|
|
@@ -39,7 +39,7 @@ import { applyMigrationSet, readMigrationSetFromDisk } from "../test/migrations.
|
|
|
39
39
|
const MIG = (name: string) => readFileSync(fileURLToPath(new URL(`../db/migrations/${name}`, import.meta.url)), "utf8");
|
|
40
40
|
const PAGE = (name: string) => JSON.parse(readFileSync(fileURLToPath(new URL(`../pages/${name}`, import.meta.url)), "utf8"));
|
|
41
41
|
|
|
42
|
-
const READ_MODEL_MIGRATION = "
|
|
42
|
+
const READ_MODEL_MIGRATION = "121_delivery_graph_read_model_auto_resolve.sql";
|
|
43
43
|
|
|
44
44
|
// A minimal in-memory DB carrying the base `delivery_graph_runs` / `pull_requests` shapes the VIEW
|
|
45
45
|
// reads, plus stand-ins for the managed `<table>__tracking` derived VIEWs urban provisions at mount
|
|
@@ -53,7 +53,7 @@ function viewDb(): DatabaseSync {
|
|
|
53
53
|
run_key TEXT PRIMARY KEY, process_key TEXT, process_definition_id TEXT, digest TEXT,
|
|
54
54
|
status TEXT, side_effecting INTEGER, node_count INTEGER, human_node_count INTEGER,
|
|
55
55
|
side_effect_count INTEGER, title TEXT, phase TEXT, phase_node_id TEXT, human_labels TEXT,
|
|
56
|
-
created_at TEXT, updated_at TEXT, acknowledged_at TEXT, derived_status_override TEXT);
|
|
56
|
+
created_at TEXT, updated_at TEXT, acknowledged_at TEXT, auto_resolve_escalations INTEGER, derived_status_override TEXT);
|
|
57
57
|
CREATE TABLE pull_requests (pr_key TEXT PRIMARY KEY, root_request_key TEXT, status TEXT,
|
|
58
58
|
derived_status_override TEXT);`,
|
|
59
59
|
);
|
package/app/deliveryGraphRun.ts
CHANGED
|
@@ -28,6 +28,7 @@ import type { CompileDeliveryGraphResult } from "../nano-generated/api-io.d.ts";
|
|
|
28
28
|
import { isUniqueConstraintFence } from "./dbFence.ts";
|
|
29
29
|
import { redactFreeText } from "./deliveryGraphCompiler.ts";
|
|
30
30
|
import { DELIVERY_HUMAN_ELEMENT, isDeliveryHumanElement, stripDeliveryEscalationTwinSuffix } from "./deliveryHuman.ts";
|
|
31
|
+
import { derivedTrackingTable } from "./instanceTracking.ts";
|
|
31
32
|
|
|
32
33
|
const now = () => new Date().toISOString();
|
|
33
34
|
|
|
@@ -52,6 +53,9 @@ export interface DeliveryGraphRun {
|
|
|
52
53
|
* the `delivery_graph_read_model` VIEW folds its `list_bucket` to 'history'; NULL while the run is
|
|
53
54
|
* live or terminal-but-undismissed (it stays in Active until an operator ticks it off). */
|
|
54
55
|
acknowledged_at: string | null;
|
|
56
|
+
/** Opt-in escalation steward (120_escalation_auto_resolutions.sql, issue #889): 1 ⇒ this run's
|
|
57
|
+
* escalations are eligible for steward auto-resolution. Default 0. */
|
|
58
|
+
auto_resolve_escalations: number;
|
|
55
59
|
}
|
|
56
60
|
|
|
57
61
|
/** The run lifecycle. `awaiting-approval` is RESERVED but no longer produced (issue #460 moved dispatch
|
|
@@ -98,6 +102,18 @@ export const DELIVERY_PHASE = {
|
|
|
98
102
|
export const deliveryGraphRuns = (data: DataLayer) =>
|
|
99
103
|
data.table<DeliveryGraphRun>("delivery_graph_runs", "run_key");
|
|
100
104
|
|
|
105
|
+
/** A `delivery_graph_runs` row as seen through its derived tracking VIEW
|
|
106
|
+
* (`delivery_graph_runs__tracking`): the base columns plus urban's ADR-0065 `derived_status`, which
|
|
107
|
+
* folds the reconciler's terminal edge (out-of-band terminate → `failed`) over the worker-owned
|
|
108
|
+
* transient. */
|
|
109
|
+
export type TrackedDeliveryGraphRun = DeliveryGraphRun & { derived_status: string };
|
|
110
|
+
/** Read-only accessor over the delivery-run derived tracking VIEW. Use this — and read
|
|
111
|
+
* `derived_status`, not `status` — for any terminal-edge classification (e.g. the `/status` active
|
|
112
|
+
* list), so an out-of-band-terminated run (whose base row is still `running`) is correctly seen as
|
|
113
|
+
* `failed`. Mirrors `prsTracking` / `featureRunsTracking`. Writes stay on `deliveryGraphRuns`. */
|
|
114
|
+
export const deliveryGraphRunsTracking = (data: DataLayer) =>
|
|
115
|
+
derivedTrackingTable<TrackedDeliveryGraphRun>(data, "delivery_graph_runs", "run_key");
|
|
116
|
+
|
|
101
117
|
/** One row of the `delivery_graph_run_identity` SIDE table (migration 113) — the run's LOSSLESS
|
|
102
118
|
* content-identity fingerprint, kept OFF the heavily-projected `delivery_graph_runs` base row so no
|
|
103
119
|
* drift-guarded read-model/compat view has to carry it. Keyed by `run_key`. */
|
|
@@ -493,6 +509,7 @@ export function buildDeliveryGraphRunRow(input: {
|
|
|
493
509
|
processDefinitionId?: string | null;
|
|
494
510
|
humanLabels?: Record<string, string>;
|
|
495
511
|
createdAt?: string;
|
|
512
|
+
autoResolveEscalations?: boolean;
|
|
496
513
|
}): DeliveryGraphRun {
|
|
497
514
|
const at = now();
|
|
498
515
|
return {
|
|
@@ -509,6 +526,7 @@ export function buildDeliveryGraphRunRow(input: {
|
|
|
509
526
|
phase: input.phase,
|
|
510
527
|
phase_node_id: null,
|
|
511
528
|
human_labels: input.humanLabels ? JSON.stringify(input.humanLabels) : null,
|
|
529
|
+
auto_resolve_escalations: input.autoResolveEscalations === true ? 1 : 0,
|
|
512
530
|
created_at: input.createdAt ?? at,
|
|
513
531
|
updated_at: at,
|
|
514
532
|
acknowledged_at: null,
|
package/app/deliveryRunner.ts
CHANGED
|
@@ -55,6 +55,10 @@ export interface DeliveryRunTimeouts {
|
|
|
55
55
|
probePollEvery?: string;
|
|
56
56
|
/** Optional explicit assignee for `human` nodes + escalation tasks (else candidate-group routed). */
|
|
57
57
|
escalationAssignee?: string | null;
|
|
58
|
+
/** Opt-in escalation steward (issue #889): when true, this run's escalations (human nodes, service
|
|
59
|
+
* node escalation twins, and any `converge*` connector's enrolled PR) become eligible for steward
|
|
60
|
+
* auto-resolution. Seeded as a run-root variable and persisted on the `delivery_graph_runs` row. */
|
|
61
|
+
autoResolveEscalations?: boolean;
|
|
58
62
|
}
|
|
59
63
|
|
|
60
64
|
export interface DeliveryRunOptions extends DeliveryRunTimeouts {
|
|
@@ -96,7 +100,7 @@ export interface DeliveryRunOptions extends DeliveryRunTimeouts {
|
|
|
96
100
|
repoless?: boolean;
|
|
97
101
|
}
|
|
98
102
|
|
|
99
|
-
const DEFAULTS: Required<Omit<DeliveryRunTimeouts, "escalationAssignee">> = {
|
|
103
|
+
const DEFAULTS: Required<Omit<DeliveryRunTimeouts, "escalationAssignee" | "autoResolveEscalations">> = {
|
|
100
104
|
nodeTimeout: DELIVERY_NODE_DEFAULT_TIMEOUT,
|
|
101
105
|
probeTimeout: "PT30M",
|
|
102
106
|
probePollEvery: msToIsoDuration(DEFAULT_EVERY_MS),
|
|
@@ -253,6 +257,10 @@ export async function runDeliveryGraph(
|
|
|
253
257
|
// bounds a node's whole retry budget; this seeds the shared `agentSlaTimeout` knob so any
|
|
254
258
|
// agent cell modelled with the fleet-wide SLA boundary resolves it too.
|
|
255
259
|
agentSlaTimeout: AGENT_SLA_TIMEOUT,
|
|
260
|
+
// Opt-in escalation steward (issue #889): seeded as a run-root variable so every node's
|
|
261
|
+
// escalation (and the delivery-connector's `converge*` enrollment) inherits the opt-in. The
|
|
262
|
+
// durable record the steward's eligibility reads is the `delivery_graph_runs` row (dispatch).
|
|
263
|
+
autoResolveEscalations: options.autoResolveEscalations === true,
|
|
256
264
|
},
|
|
257
265
|
});
|
|
258
266
|
// The engine can yield a numeric key; `DeliveryRunHandle.processInstanceKey` is typed `string` and
|