@teamscala/service-boot 0.23.428 → 0.23.430
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/handlers/platform-fault-escalation/config.ts +6 -0
- package/src/handlers/platform-fault-escalation/delivery.ts +53 -17
- package/src/handlers/platform-fault-escalation-logic.ts +26 -0
- package/src/handlers/platform-fault-escalation-refresh.test.ts +85 -0
- package/src/handlers/platform-fault-escalation.ts +21 -1
package/package.json
CHANGED
|
@@ -33,6 +33,12 @@ export const FaultEscalationConfigSchema = v.object({
|
|
|
33
33
|
),
|
|
34
34
|
),
|
|
35
35
|
resolve_after_minutes: v.number(),
|
|
36
|
+
/** Kinds a standing agent lane answers, which therefore escalate to the
|
|
37
|
+
* work item WITHOUT paging the operator (operator ruling 2026-08-28: "why
|
|
38
|
+
* is the WhatsApp PA messaging me about stuff that you should be doing").
|
|
39
|
+
* The episode, the comment and the durable record are unchanged — only the
|
|
40
|
+
* phone is spared, so silencing a page never silences the signal. */
|
|
41
|
+
agent_handled_kinds: v.optional(v.array(v.string())),
|
|
36
42
|
/** Per-kind volume sources closing the quiet-vs-dead gap: a fault kind
|
|
37
43
|
* whose producer can DIE maps to the kind whose row VOLUME proves the
|
|
38
44
|
* pipeline alive. Without a source, resolution on quiet is unchanged;
|
|
@@ -18,6 +18,7 @@ import {
|
|
|
18
18
|
composeEscalationMessage,
|
|
19
19
|
composeZeroVolumeMessage,
|
|
20
20
|
extractWhatsAppMessageId,
|
|
21
|
+
pagesOperator,
|
|
21
22
|
type EscalationEpisode,
|
|
22
23
|
type FaultAggregate,
|
|
23
24
|
type ZeroVolumeSource,
|
|
@@ -53,7 +54,7 @@ export async function deliverEscalation(
|
|
|
53
54
|
kind: string,
|
|
54
55
|
aggregate: FaultAggregate,
|
|
55
56
|
config: FaultEscalationConfig,
|
|
56
|
-
): Promise<{ messageId: string; workItemId: string }> {
|
|
57
|
+
): Promise<{ messageId: string | null; workItemId: string }> {
|
|
57
58
|
const now = new Date();
|
|
58
59
|
|
|
59
60
|
// 1. Claim the episode BEFORE sending, so a send failure leaves a durable
|
|
@@ -80,8 +81,11 @@ export async function deliverEscalation(
|
|
|
80
81
|
// that picks the device and records the send in messaging_logs. The retired
|
|
81
82
|
// api_gowa_message tool (deactivated 2026-08-26) is no longer callable;
|
|
82
83
|
// naming it here left every escalation send failing with "Unknown tool".
|
|
83
|
-
|
|
84
|
-
|
|
84
|
+
|
|
85
|
+
// PAGING IS PER-KIND (operator ruling 2026-08-28). Only the phone is
|
|
86
|
+
// skipped; episode, comment and record are identical, so a silenced page
|
|
87
|
+
// never becomes a silenced signal.
|
|
88
|
+
const paging = pagesOperator(kind, config.agent_handled_kinds);
|
|
85
89
|
const copy = config.copy[kind] ?? config.default_copy;
|
|
86
90
|
const { sample, lastEvent } = await fetchLatestEvidence(kind, config.sample_fields?.[kind] ?? []);
|
|
87
91
|
const message = composeEscalationMessage({
|
|
@@ -93,23 +97,31 @@ export async function deliverEscalation(
|
|
|
93
97
|
recommendation: copy.recommendation,
|
|
94
98
|
});
|
|
95
99
|
|
|
96
|
-
const { userId, organisationId } = await getSystemIdentity();
|
|
97
|
-
const systemCtx: ExecutionContext = { userId, organisationId } as ExecutionContext;
|
|
98
|
-
const { forwardToolToCentralMcp } = await import("@teamscala/chat-completion/central-tool-forwarder");
|
|
99
|
-
const sendResult = await forwardToolToCentralMcp(
|
|
100
|
-
"messaging_whatsapp_send",
|
|
101
|
-
{ phone: operatorPhone, message },
|
|
102
|
-
systemCtx,
|
|
103
|
-
);
|
|
104
|
-
|
|
105
100
|
// 3. VERIFY delivery before recording it: the send must return a WhatsApp
|
|
106
101
|
// message id. No id => throw, the claim stays undelivered, next sweep retries.
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
102
|
+
// An agent-handled kind sends nothing, so there is no id to verify and none
|
|
103
|
+
// is recorded — absence here means "never paged", never "paged unverified".
|
|
104
|
+
let messageId: string | null = null;
|
|
105
|
+
if (paging) {
|
|
106
|
+
const bot = await resolveOperatorRelay();
|
|
107
|
+
const operatorPhone = (bot.flow_config as { operator_phone?: string }).operator_phone!;
|
|
108
|
+
const { userId, organisationId } = await getSystemIdentity();
|
|
109
|
+
const systemCtx: ExecutionContext = { userId, organisationId } as ExecutionContext;
|
|
110
|
+
const { forwardToolToCentralMcp } = await import(
|
|
111
|
+
"@teamscala/chat-completion/central-tool-forwarder"
|
|
112
112
|
);
|
|
113
|
+
const sendResult = await forwardToolToCentralMcp(
|
|
114
|
+
"messaging_whatsapp_send",
|
|
115
|
+
{ phone: operatorPhone, message },
|
|
116
|
+
systemCtx,
|
|
117
|
+
);
|
|
118
|
+
messageId = extractWhatsAppMessageId(sendResult);
|
|
119
|
+
if (!messageId) {
|
|
120
|
+
throw new Error(
|
|
121
|
+
`platform-fault-escalation: WhatsApp send for '${kind}' returned no message id — not recording delivery. ` +
|
|
122
|
+
`Response: ${JSON.stringify(sendResult).slice(0, 400)}`,
|
|
123
|
+
);
|
|
124
|
+
}
|
|
113
125
|
}
|
|
114
126
|
|
|
115
127
|
// 4. Durable record: a COMMENT on the orchestrator's work item — an
|
|
@@ -178,6 +190,30 @@ export async function resolveEpisode(
|
|
|
178
190
|
});
|
|
179
191
|
}
|
|
180
192
|
|
|
193
|
+
/**
|
|
194
|
+
* Refresh an OPEN episode's window facts from the live aggregate.
|
|
195
|
+
*
|
|
196
|
+
* Escalation wrote the row once and never again, so its facts froze while the
|
|
197
|
+
* fault kept firing (2026-08-28: fifteen hours stale, 1,150 later events).
|
|
198
|
+
* That also corrupts RESOLUTION, since `decideSweep` falls back to
|
|
199
|
+
* `lastSeenAt` once a kind leaves the window. Fields mirror the escalation
|
|
200
|
+
* upsert. `last_updated_at` is untouched: it gates the resolve-hold dedupe.
|
|
201
|
+
*/
|
|
202
|
+
export async function refreshOpenEpisode(
|
|
203
|
+
kind: string,
|
|
204
|
+
aggregate: { firstSeen: Date; lastSeen: Date; count: number },
|
|
205
|
+
): Promise<void> {
|
|
206
|
+
await (await orpc()).platform_fault_escalations.update({
|
|
207
|
+
where: { kind },
|
|
208
|
+
data: {
|
|
209
|
+
first_seen_at: aggregate.firstSeen.toISOString(),
|
|
210
|
+
last_seen_at: aggregate.lastSeen.toISOString(),
|
|
211
|
+
event_count: aggregate.count,
|
|
212
|
+
updated_at: new Date().toISOString(),
|
|
213
|
+
},
|
|
214
|
+
});
|
|
215
|
+
}
|
|
216
|
+
|
|
181
217
|
/** Send one message to the operator through the verified WhatsApp relay and
|
|
182
218
|
* record it as a comment on the orchestrator's work item. Returns the
|
|
183
219
|
* verified ids; throws when the send carries no message id so the caller's
|
|
@@ -229,6 +229,32 @@ export function decideSweep(
|
|
|
229
229
|
}
|
|
230
230
|
|
|
231
231
|
/** Human "N min" span, rounded — for the phone-sized escalation message. */
|
|
232
|
+
/**
|
|
233
|
+
* Do this OPEN episode's stored window facts lag the live aggregate?
|
|
234
|
+
*
|
|
235
|
+
* Both NEGATIVE answers carry weight: a quiet kind keeps its last true
|
|
236
|
+
* sighting (the value resolve falls back to), and an aggregate no newer than
|
|
237
|
+
* the row is never written, so a steady-state sweep writes nothing.
|
|
238
|
+
*/
|
|
239
|
+
export function episodeNeedsRefresh(
|
|
240
|
+
episode: Pick<EscalationEpisode, "lastSeenAt">,
|
|
241
|
+
aggregate: { lastSeen: Date } | undefined,
|
|
242
|
+
): boolean {
|
|
243
|
+
if (!aggregate) return false;
|
|
244
|
+
return aggregate.lastSeen.getTime() > episode.lastSeenAt.getTime();
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
/**
|
|
248
|
+
* Does this fault kind page the operator, or land on the work item alone?
|
|
249
|
+
*
|
|
250
|
+
* DEFAULT IS TO PAGE. An unlisted kind reaches the phone, so forgetting to
|
|
251
|
+
* classify a new fault fails loudly rather than silently — the opposite
|
|
252
|
+
* default would let a kind be silenced by omission.
|
|
253
|
+
*/
|
|
254
|
+
export function pagesOperator(kind: string, agentHandledKinds?: string[]): boolean {
|
|
255
|
+
return !(agentHandledKinds ?? []).includes(kind);
|
|
256
|
+
}
|
|
257
|
+
|
|
232
258
|
export function formatSpanMinutes(minutes: number): string {
|
|
233
259
|
if (minutes < 60) return `${Math.round(minutes)} min`;
|
|
234
260
|
const hours = minutes / 60;
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @system server-boot
|
|
3
|
+
* @status handwritten
|
|
4
|
+
* @edit edit directly
|
|
5
|
+
*
|
|
6
|
+
* Unit tests for episode FACT CURRENCY — a separate concern from the
|
|
7
|
+
* threshold/dedup semantics locked by `platform-fault-escalation.test.ts`,
|
|
8
|
+
* and kept in its own file because that one already sits over the size
|
|
9
|
+
* budget.
|
|
10
|
+
*
|
|
11
|
+
* WHAT THIS LOCKS. The sweep escalated once and then never wrote the row
|
|
12
|
+
* again, so an open episode's window facts froze at their escalation-time
|
|
13
|
+
* values while the fault kept firing. Measured 2026-08-28: the
|
|
14
|
+
* `host-command-bus.command.failed` row read `last_seen_at` fifteen hours old
|
|
15
|
+
* with 1,150 further events recorded after it, and a reader — this lane
|
|
16
|
+
* included — concluded from the row that the fault had stopped.
|
|
17
|
+
*
|
|
18
|
+
* The stale value is not only cosmetic: `decideSweep` falls back to
|
|
19
|
+
* `episode.lastSeenAt` once a kind drops out of the window, so a frozen
|
|
20
|
+
* timestamp makes a just-gone-quiet fault look quiet for hours and closes the
|
|
21
|
+
* episode immediately instead of after its configured resolve window.
|
|
22
|
+
*/
|
|
23
|
+
import { describe, expect, test } from "bun:test";
|
|
24
|
+
import { episodeNeedsRefresh, pagesOperator } from "./platform-fault-escalation-logic.ts";
|
|
25
|
+
|
|
26
|
+
const at = (iso: string) => ({ lastSeenAt: new Date(iso) });
|
|
27
|
+
const seen = (iso: string) => ({ lastSeen: new Date(iso) });
|
|
28
|
+
|
|
29
|
+
describe("episodeNeedsRefresh", () => {
|
|
30
|
+
test("a still-firing fault whose row has fallen behind is refreshed", () => {
|
|
31
|
+
// The measured incident, in miniature: row frozen at escalation time,
|
|
32
|
+
// events still arriving fifteen hours later.
|
|
33
|
+
expect(episodeNeedsRefresh(at("2026-08-27T12:25:03Z"), seen("2026-08-28T03:00:41Z"))).toBe(
|
|
34
|
+
true,
|
|
35
|
+
);
|
|
36
|
+
});
|
|
37
|
+
|
|
38
|
+
test("a kind that has gone quiet keeps its last true sighting", () => {
|
|
39
|
+
// No aggregate means nothing landed in the window. The stored value must
|
|
40
|
+
// STAND rather than advance, because it is the reference the resolve
|
|
41
|
+
// decision falls back to — advancing it here would hold every quiet
|
|
42
|
+
// episode open forever.
|
|
43
|
+
expect(episodeNeedsRefresh(at("2026-08-27T12:25:03Z"), undefined)).toBe(false);
|
|
44
|
+
});
|
|
45
|
+
|
|
46
|
+
test("an unchanged fault is not written back", () => {
|
|
47
|
+
// Equal timestamps must not write: a sweep over steady state performs
|
|
48
|
+
// zero updates, so the refresh cannot become a per-sweep write storm.
|
|
49
|
+
const same = "2026-08-28T03:00:41Z";
|
|
50
|
+
expect(episodeNeedsRefresh(at(same), seen(same))).toBe(false);
|
|
51
|
+
});
|
|
52
|
+
|
|
53
|
+
test("an aggregate OLDER than the row never rewinds it", () => {
|
|
54
|
+
// A shrinking window can report an older newest-event than the row
|
|
55
|
+
// already holds. Rewinding would fabricate quiet time and resolve a
|
|
56
|
+
// live fault early.
|
|
57
|
+
expect(episodeNeedsRefresh(at("2026-08-28T03:00:41Z"), seen("2026-08-28T01:00:00Z"))).toBe(
|
|
58
|
+
false,
|
|
59
|
+
);
|
|
60
|
+
});
|
|
61
|
+
});
|
|
62
|
+
|
|
63
|
+
describe("pagesOperator", () => {
|
|
64
|
+
// Operator ruling 2026-08-28: "why is the WhatsApp PA messaging me about
|
|
65
|
+
// stuff that you should be doing". A kind a standing lane answers escalates
|
|
66
|
+
// to the work item; the phone is spared.
|
|
67
|
+
const AGENT_HANDLED = ["guard.fail.open", "host-command-bus.command.failed"];
|
|
68
|
+
|
|
69
|
+
test("a kind the maintenance lane answers does not page", () => {
|
|
70
|
+
expect(pagesOperator("guard.fail.open", AGENT_HANDLED)).toBe(false);
|
|
71
|
+
expect(pagesOperator("host-command-bus.command.failed", AGENT_HANDLED)).toBe(false);
|
|
72
|
+
});
|
|
73
|
+
|
|
74
|
+
test("an unlisted kind still pages", () => {
|
|
75
|
+
expect(pagesOperator("supervisor.halt.entered", AGENT_HANDLED)).toBe(true);
|
|
76
|
+
});
|
|
77
|
+
|
|
78
|
+
test("DEFAULT IS TO PAGE when nothing is configured", () => {
|
|
79
|
+
// The load-bearing direction: a kind is silenced only by being named.
|
|
80
|
+
// If the list is missing or empty, every fault reaches the operator, so
|
|
81
|
+
// a new kind can never be silenced by omission.
|
|
82
|
+
expect(pagesOperator("guard.fail.open", undefined)).toBe(true);
|
|
83
|
+
expect(pagesOperator("guard.fail.open", [])).toBe(true);
|
|
84
|
+
});
|
|
85
|
+
});
|
|
@@ -20,6 +20,7 @@ import { getAppLogger } from "@teamscala/logger/app-loggers";
|
|
|
20
20
|
import { pgBossHandlerRegistry } from "../pg-boss-handler-registry.ts";
|
|
21
21
|
import {
|
|
22
22
|
decideSweep,
|
|
23
|
+
episodeNeedsRefresh,
|
|
23
24
|
type EscalationEpisode,
|
|
24
25
|
type KindThresholdOverride,
|
|
25
26
|
type ZeroVolumeInputs,
|
|
@@ -37,6 +38,7 @@ import {
|
|
|
37
38
|
clearZeroVolumeHold,
|
|
38
39
|
deliverEscalation,
|
|
39
40
|
deliverZeroVolumeHold,
|
|
41
|
+
refreshOpenEpisode,
|
|
40
42
|
resolveEpisode,
|
|
41
43
|
} from "./platform-fault-escalation/delivery.ts";
|
|
42
44
|
|
|
@@ -117,7 +119,7 @@ async function platformFaultEscalationSweep(ctx: { config: ScheduledJobConfig })
|
|
|
117
119
|
try {
|
|
118
120
|
const delivered = await deliverEscalation(aggregate.kind, aggregate, config);
|
|
119
121
|
log.warn(
|
|
120
|
-
`[platform-fault-escalation] ESCALATED '${aggregate.kind}' — WhatsApp message ${delivered.messageId}, work item ${delivered.workItemId}`,
|
|
122
|
+
`[platform-fault-escalation] ESCALATED '${aggregate.kind}' — ${delivered.messageId ? `WhatsApp message ${delivered.messageId}` : "work item only (agent-handled kind, operator not paged)"}, work item ${delivered.workItemId}`,
|
|
121
123
|
);
|
|
122
124
|
} catch (err) {
|
|
123
125
|
const reason = err instanceof Error ? err.message : String(err);
|
|
@@ -125,6 +127,24 @@ async function platformFaultEscalationSweep(ctx: { config: ScheduledJobConfig })
|
|
|
125
127
|
recordEvent({ kind: "platform.fault.escalation_failed", payload: { event_kind: aggregate.kind, reason } });
|
|
126
128
|
}
|
|
127
129
|
}
|
|
130
|
+
// AN OPEN EPISODE KEEPS ITS FACTS CURRENT. Escalation is a TRANSITION and
|
|
131
|
+
// fires once (`emit-transitions-not-conditions`) — but the ROW is the
|
|
132
|
+
// durable record of the condition, and a record that stops updating starts
|
|
133
|
+
// lying. Refreshing is not emitting: nothing is delivered here, no comment
|
|
134
|
+
// is written, no message is sent. Only the stored window facts move.
|
|
135
|
+
const aggregateByKind = new Map(aggregates.map((aggregate) => [aggregate.kind, aggregate]));
|
|
136
|
+
for (const episode of episodes) {
|
|
137
|
+
const aggregate = aggregateByKind.get(episode.kind);
|
|
138
|
+
if (!episodeNeedsRefresh(episode, aggregate) || !aggregate) continue;
|
|
139
|
+
try {
|
|
140
|
+
await refreshOpenEpisode(episode.kind, aggregate);
|
|
141
|
+
} catch (err) {
|
|
142
|
+
log.error(
|
|
143
|
+
`[platform-fault-escalation] refresh for '${episode.kind}' failed (facts stay stale for this sweep, retried next): ${err instanceof Error ? err.message : String(err)}`,
|
|
144
|
+
);
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
|
|
128
148
|
for (const { episode, quietMinutes } of decision.zeroVolumeHold) {
|
|
129
149
|
const source = volumeSources[episode.kind];
|
|
130
150
|
if (!source) continue;
|