@teamscala/service-boot 0.23.406 → 0.23.408
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json
CHANGED
|
@@ -3,25 +3,9 @@
|
|
|
3
3
|
* @status handwritten
|
|
4
4
|
* @edit edit directly
|
|
5
5
|
*
|
|
6
|
-
* Pure decision logic for the platform-fault escalation sweep
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
* dependencies (same shape as whatsapp-liveness-logic).
|
|
10
|
-
*
|
|
11
|
-
* The semantics this locks (from the 2026-08-15 "760 errors escalated to
|
|
12
|
-
* nobody" incident brief):
|
|
13
|
-
* - Threshold on DURATION and RECURRENCE, never a single event. A kind
|
|
14
|
-
* qualifies as a sustained fault only when BOTH hold inside the lookback
|
|
15
|
-
* window: at least `min_count` occurrences (recurrence) spread over at
|
|
16
|
-
* least `min_span_minutes` (duration). One failed push is noise; the same
|
|
17
|
-
* kind recurring for half an hour is an outage.
|
|
18
|
-
* - One escalation per distinct fault kind per episode. An episode is open
|
|
19
|
-
* from first qualification until `resolve_after_minutes` of quiet; while
|
|
20
|
-
* open, the fault never re-fires — it only appends throttled updates.
|
|
21
|
-
* - Delivery is verified, not queued: an episode row with `escalated_at`
|
|
22
|
-
* NULL is a CLAIM, not an escalation. The sweep re-attempts delivery for
|
|
23
|
-
* claimed-but-undelivered episodes until the WhatsApp send returns a
|
|
24
|
-
* message id.
|
|
6
|
+
* Pure decision logic for the platform-fault escalation sweep — full
|
|
7
|
+
* contract in curated `apps/escalation.md`. Episodes emit on TRANSITIONS
|
|
8
|
+
* ONLY; a future-dated `lastSeen` is never recurrence evidence.
|
|
25
9
|
*/
|
|
26
10
|
|
|
27
11
|
export interface FaultAggregate {
|
|
@@ -29,6 +13,10 @@ export interface FaultAggregate {
|
|
|
29
13
|
count: number;
|
|
30
14
|
firstSeen: Date;
|
|
31
15
|
lastSeen: Date;
|
|
16
|
+
/** Id of the newest row backing `lastSeen`, when the caller resolved it —
|
|
17
|
+
* the resolvable handle the escalation cites so "last at T" is checkable
|
|
18
|
+
* against a real event_log row, not a computed or cached value. */
|
|
19
|
+
lastEventId?: string | null;
|
|
32
20
|
}
|
|
33
21
|
|
|
34
22
|
export interface EscalationEpisode {
|
|
@@ -50,8 +38,6 @@ export interface FaultEscalationPolicy {
|
|
|
50
38
|
minSpanMinutes: number;
|
|
51
39
|
/** Quiet period that closes an episode and re-arms escalation, minutes. */
|
|
52
40
|
resolveAfterMinutes: number;
|
|
53
|
-
/** Minimum spacing between throttled updates on an open episode, minutes. */
|
|
54
|
-
updateAfterMinutes: number;
|
|
55
41
|
}
|
|
56
42
|
|
|
57
43
|
/** Per-kind threshold overrides — a low-rate-but-persistent kind (guard.fail.open
|
|
@@ -65,8 +51,6 @@ export interface KindThresholdOverride {
|
|
|
65
51
|
export interface SweepDecision {
|
|
66
52
|
/** Sustained faults needing a FIRST delivery (new episode, or a claim whose send failed). */
|
|
67
53
|
escalate: Array<{ aggregate: FaultAggregate; episode: EscalationEpisode | null }>;
|
|
68
|
-
/** Open, delivered episodes still recurring — append evidence, never re-send. */
|
|
69
|
-
update: Array<{ aggregate: FaultAggregate; episode: EscalationEpisode }>;
|
|
70
54
|
/** Open episodes whose kind went quiet past the resolve window — close them. */
|
|
71
55
|
resolve: Array<{ episode: EscalationEpisode; quietMinutes: number }>;
|
|
72
56
|
}
|
|
@@ -88,9 +72,13 @@ function isSustained(
|
|
|
88
72
|
// happening. Without this the 720-minute lookback keeps a dead fault
|
|
89
73
|
// qualifying for hours, and since a resolved episode reads as no episode
|
|
90
74
|
// it re-escalates on every sweep until its events age out (2026-08-26).
|
|
75
|
+
// A future `lastSeen` (producer clock skew / mocked test clock, measured
|
|
76
|
+
// 2026-08-26: "still recurring" citing a timestamp 44 min in its own
|
|
77
|
+
// future) makes minutesBetween go NEGATIVE — refuse it as evidence.
|
|
91
78
|
return (
|
|
92
79
|
aggregate.count >= minCount &&
|
|
93
80
|
minutesBetween(aggregate.firstSeen, aggregate.lastSeen) >= minSpan &&
|
|
81
|
+
aggregate.lastSeen.getTime() <= now.getTime() &&
|
|
94
82
|
minutesBetween(aggregate.lastSeen, now) < policy.resolveAfterMinutes
|
|
95
83
|
);
|
|
96
84
|
}
|
|
@@ -110,7 +98,7 @@ export function decideSweep(
|
|
|
110
98
|
): SweepDecision {
|
|
111
99
|
const byKind = new Map(episodes.map((e) => [e.kind, e]));
|
|
112
100
|
const aggregateByKind = new Map(aggregates.map((a) => [a.kind, a]));
|
|
113
|
-
const decision: SweepDecision = { escalate: [],
|
|
101
|
+
const decision: SweepDecision = { escalate: [], resolve: [] };
|
|
114
102
|
|
|
115
103
|
for (const aggregate of aggregates) {
|
|
116
104
|
const episode = byKind.get(aggregate.kind);
|
|
@@ -126,19 +114,23 @@ export function decideSweep(
|
|
|
126
114
|
// verified).
|
|
127
115
|
if (!episode.escalatedAt) {
|
|
128
116
|
decision.escalate.push({ aggregate, episode });
|
|
129
|
-
continue;
|
|
130
|
-
}
|
|
131
|
-
// Open, delivered, still recurring — throttled evidence update only.
|
|
132
|
-
const lastUpdate = episode.lastUpdatedAt ?? episode.escalatedAt;
|
|
133
|
-
if (minutesBetween(lastUpdate, now) >= policy.updateAfterMinutes) {
|
|
134
|
-
decision.update.push({ aggregate, episode });
|
|
135
117
|
}
|
|
118
|
+
// An open, delivered episode emits NOTHING while it recurs: the open
|
|
119
|
+
// emission already happened and the close emission fires when the kind
|
|
120
|
+
// goes quiet. A cadence-driven "still recurring" repeat is exactly the
|
|
121
|
+
// silenced-alarm shape `emit-transitions-not-conditions` prohibits.
|
|
136
122
|
}
|
|
137
123
|
|
|
138
124
|
for (const episode of episodes) {
|
|
139
125
|
if (episode.resolvedAt) continue;
|
|
140
126
|
const aggregate = aggregateByKind.get(episode.kind);
|
|
141
|
-
|
|
127
|
+
// A future-dated aggregate lastSeen (producer clock skew) cannot serve
|
|
128
|
+
// as the quiet reference — it would freeze quietMinutes negative and
|
|
129
|
+
// hold the episode open forever while the kind is actually quiet. Fall
|
|
130
|
+
// back to the episode's own last_seen_at, which the bounded aggregate
|
|
131
|
+
// wrote and is therefore never in the future.
|
|
132
|
+
const quietReference =
|
|
133
|
+
aggregate && aggregate.lastSeen.getTime() <= now.getTime() ? aggregate.lastSeen : episode.lastSeenAt;
|
|
142
134
|
const quietMinutes = minutesBetween(quietReference, now);
|
|
143
135
|
if (quietMinutes >= policy.resolveAfterMinutes) {
|
|
144
136
|
decision.resolve.push({ episode, quietMinutes });
|
|
@@ -176,39 +168,56 @@ export function extractGowaMessageId(sendResult: unknown): string | null {
|
|
|
176
168
|
return null;
|
|
177
169
|
}
|
|
178
170
|
|
|
171
|
+
/**
|
|
172
|
+
* One sampled payload value rendered for the message's evidence line: a
|
|
173
|
+
* kind+count escalation is a pointer to evidence, not evidence (2026-08-26:
|
|
174
|
+
* the llm.provider.call.failed payloads held our own bounded-output contract
|
|
175
|
+
* violations while the escalation's impact line named healthy providers).
|
|
176
|
+
* JSON-encoded when non-string so structure survives, whitespace-collapsed
|
|
177
|
+
* to one line, credential shapes redacted before truncation (a hard cut can
|
|
178
|
+
* never expose the tail of a secret the length limit happened to spare),
|
|
179
|
+
* hard-truncated with an ellipsis so the reader knows the cut happened.
|
|
180
|
+
*/
|
|
181
|
+
export function formatEscalationSample(value: unknown): string {
|
|
182
|
+
const maxChars = 200;
|
|
183
|
+
const redactions: ReadonlyArray<[RegExp, string]> = [
|
|
184
|
+
[/\b(?:sk|gsk|ghp|gho|xox[baprs])[-_][A-Za-z0-9_-]{8,}/g, "[redacted]"],
|
|
185
|
+
[/\b(api[_-]?key|token|password|secret|authorization)\b["'=:\s]+(?:(?:bearer|basic)\s+)?[A-Za-z0-9._~+/=-]{4,}/gi, "$1=[redacted]"],
|
|
186
|
+
[/Bearer\s+[A-Za-z0-9._~+/=-]{8,}/gi, "Bearer [redacted]"],
|
|
187
|
+
];
|
|
188
|
+
const raw = typeof value === "string" ? value : (JSON.stringify(value) ?? "");
|
|
189
|
+
if (raw.length === 0) return "";
|
|
190
|
+
const redacted = redactions.reduce((acc, [pattern, replacement]) => acc.replace(pattern, replacement), raw);
|
|
191
|
+
const collapsed = redacted.replace(/\s+/g, " ").trim();
|
|
192
|
+
return collapsed.length > maxChars ? `${collapsed.slice(0, maxChars)}…` : collapsed;
|
|
193
|
+
}
|
|
194
|
+
|
|
179
195
|
/**
|
|
180
196
|
* The escalation message — escalation-routing's three-field contract adapted
|
|
181
|
-
* to an automated detector: what was OBSERVED (evidence), what it
|
|
182
|
-
* (impact), and the RECOMMENDATION. Short: the channel is a phone.
|
|
197
|
+
* to an automated detector: what was OBSERVED (evidence + sample), what it
|
|
198
|
+
* MEANS (impact), and the RECOMMENDATION. Short: the channel is a phone. The
|
|
199
|
+
* sample line is the triage evidence — `sample_fields` whitelisted per kind
|
|
200
|
+
* by the caller, formatted by formatEscalationSample; empty means the kind
|
|
201
|
+
* samples nothing or the latest payload carried none of its fields.
|
|
183
202
|
*/
|
|
184
203
|
export function composeEscalationMessage(input: {
|
|
185
204
|
kind: string;
|
|
186
205
|
aggregate: FaultAggregate;
|
|
187
206
|
impact: string;
|
|
188
207
|
recommendation: string;
|
|
208
|
+
sample: string;
|
|
209
|
+
/** ISO timestamp + event id of the newest backing row, when resolved —
|
|
210
|
+
* cited verbatim so the reader can resolve the claim against event_log. */
|
|
211
|
+
lastEvent?: { occurredAt: string; eventId: string } | null;
|
|
189
212
|
}): string {
|
|
190
213
|
const span = formatSpanMinutes(
|
|
191
214
|
(input.aggregate.lastSeen.getTime() - input.aggregate.firstSeen.getTime()) / 60_000,
|
|
192
215
|
);
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
`Observed: ${input.aggregate.count} errors over ${span},
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
/** The work-item description carries the same three fields plus the evidence. */
|
|
202
|
-
export function composeEscalationDescription(input: {
|
|
203
|
-
aggregate: FaultAggregate;
|
|
204
|
-
impact: string;
|
|
205
|
-
recommendation: string;
|
|
206
|
-
}): string {
|
|
207
|
-
return [
|
|
208
|
-
"Automated platform-fault escalation (platform-fault-escalation sweep).",
|
|
209
|
-
`Observed: ${input.aggregate.count} '${input.aggregate.kind}' events in event_log from ${input.aggregate.firstSeen.toISOString()} to ${input.aggregate.lastSeen.toISOString()}, still recurring.`,
|
|
210
|
-
`Impact: ${input.impact}`,
|
|
211
|
-
`Recommendation: ${input.recommendation}`,
|
|
212
|
-
"Delivery: WhatsApp via the scala PA. This task closes automatically when the fault stops for the resolve window; reply via the PA to rule otherwise.",
|
|
213
|
-
].join("\n");
|
|
216
|
+
const observed = input.lastEvent
|
|
217
|
+
? `Observed: ${input.aggregate.count} errors over ${span}, last ${input.lastEvent.occurredAt} (event ${input.lastEvent.eventId.slice(0, 8)}).`
|
|
218
|
+
: `Observed: ${input.aggregate.count} errors over ${span}, last ${input.aggregate.lastSeen.toISOString()}.`;
|
|
219
|
+
const lines = [`🔴 Platform fault: ${input.kind}`, observed];
|
|
220
|
+
if (input.sample) lines.push(`Sample: ${input.sample}`);
|
|
221
|
+
lines.push(`Impact: ${input.impact}`, `Recommendation: ${input.recommendation}`);
|
|
222
|
+
return lines.join("\n");
|
|
214
223
|
}
|
|
@@ -13,6 +13,7 @@ import {
|
|
|
13
13
|
composeEscalationMessage,
|
|
14
14
|
decideSweep,
|
|
15
15
|
extractGowaMessageId,
|
|
16
|
+
formatEscalationSample,
|
|
16
17
|
type EscalationEpisode,
|
|
17
18
|
type FaultAggregate,
|
|
18
19
|
type FaultEscalationPolicy,
|
|
@@ -23,7 +24,6 @@ const POLICY: FaultEscalationPolicy = {
|
|
|
23
24
|
minCount: 10,
|
|
24
25
|
minSpanMinutes: 25,
|
|
25
26
|
resolveAfterMinutes: 30,
|
|
26
|
-
updateAfterMinutes: 30,
|
|
27
27
|
};
|
|
28
28
|
|
|
29
29
|
function agg(kind: string, count: number, firstSeenMinutesAgo: number, lastSeenMinutesAgo: number): FaultAggregate {
|
|
@@ -101,29 +101,20 @@ describe("decideSweep — dedup (one escalation per fault, no re-fire)", () => {
|
|
|
101
101
|
expect(decision.escalate).toHaveLength(0);
|
|
102
102
|
});
|
|
103
103
|
|
|
104
|
-
test("open delivered episode
|
|
104
|
+
test("open delivered episode emits NOTHING while it recurs — transitions, not cadence", () => {
|
|
105
|
+
// 2026-08-26: the hourly "still recurring" comment made the alarm
|
|
106
|
+
// skippable noise and hid a real signal. An open, delivered episode now
|
|
107
|
+
// stays silent until the close transition fires (see the resolve suite).
|
|
105
108
|
const sustained = agg("llm.provider.call.failed", 300, 720, 0);
|
|
106
|
-
const
|
|
109
|
+
const longOpen = episode({
|
|
107
110
|
kind: "llm.provider.call.failed",
|
|
108
|
-
escalatedAt: new Date(NOW.getTime() - 90 * 60_000),
|
|
111
|
+
escalatedAt: new Date(NOW.getTime() - 90 * 60_000),
|
|
109
112
|
lastUpdatedAt: new Date(NOW.getTime() - 90 * 60_000),
|
|
110
113
|
});
|
|
111
|
-
const decision = decideSweep([sustained], [
|
|
112
|
-
expect(decision.escalate).toHaveLength(0);
|
|
113
|
-
expect(decision.update).toHaveLength(1);
|
|
114
|
-
expect(decision.update[0]!.episode.workItemId).toBe("wi-1");
|
|
115
|
-
});
|
|
116
|
-
|
|
117
|
-
test("open episode inside the update window gets NOTHING (dedup window)", () => {
|
|
118
|
-
const sustained = agg("llm.provider.call.failed", 300, 720, 0);
|
|
119
|
-
const open = episode({
|
|
120
|
-
kind: "llm.provider.call.failed",
|
|
121
|
-
escalatedAt: new Date(NOW.getTime() - 10 * 60_000),
|
|
122
|
-
lastUpdatedAt: new Date(NOW.getTime() - 10 * 60_000),
|
|
123
|
-
});
|
|
124
|
-
const decision = decideSweep([sustained], [open], POLICY, NOW);
|
|
114
|
+
const decision = decideSweep([sustained], [longOpen], POLICY, NOW);
|
|
125
115
|
expect(decision.escalate).toHaveLength(0);
|
|
126
|
-
expect(decision.
|
|
116
|
+
expect(decision.resolve).toHaveLength(0);
|
|
117
|
+
expect(Object.keys(decision).sort()).toEqual(["escalate", "resolve"]);
|
|
127
118
|
});
|
|
128
119
|
|
|
129
120
|
test("a claimed-but-undelivered episode is RETRIED, not counted as escalated", () => {
|
|
@@ -167,6 +158,34 @@ describe("decideSweep — resolution + re-arm", () => {
|
|
|
167
158
|
});
|
|
168
159
|
});
|
|
169
160
|
|
|
161
|
+
describe("decideSweep — a future-dated lastSeen is not recurrence evidence", () => {
|
|
162
|
+
// 2026-08-26: an escalation sent 08:20Z cited "last at 09:04:54Z" — 44
|
|
163
|
+
// minutes in its own future. The rows existed (producer envelopes whose
|
|
164
|
+
// clock ran 70 minutes ahead), and `minutesBetween(lastSeen, now)` went
|
|
165
|
+
// negative, passing every recency check while the kind was actually quiet.
|
|
166
|
+
// A timestamp that has not happened yet cannot evidence a fault happening
|
|
167
|
+
// now — no matter how the aggregate was built.
|
|
168
|
+
test("a future-dated aggregate does not escalate", () => {
|
|
169
|
+
const phantom = agg("guard.fail.open", 40, 720, -44);
|
|
170
|
+
const decision = decideSweep([phantom], [], POLICY, NOW);
|
|
171
|
+
expect(decision.escalate).toHaveLength(0);
|
|
172
|
+
});
|
|
173
|
+
|
|
174
|
+
test("a future-dated aggregate does not hold an open episode open past the quiet window", () => {
|
|
175
|
+
// lastSeen 44 min in the future means the kind's real events are quiet:
|
|
176
|
+
// the episode's own last_seen_at (honest) drives the resolve decision,
|
|
177
|
+
// and the phantom aggregate must not veto it.
|
|
178
|
+
const quietEpisode = episode({
|
|
179
|
+
kind: "guard.fail.open",
|
|
180
|
+
lastSeenAt: new Date(NOW.getTime() - 60 * 60_000),
|
|
181
|
+
});
|
|
182
|
+
const phantom = agg("guard.fail.open", 40, 720, -44);
|
|
183
|
+
const decision = decideSweep([phantom], [quietEpisode], POLICY, NOW);
|
|
184
|
+
expect(decision.escalate).toHaveLength(0);
|
|
185
|
+
expect(decision.resolve).toHaveLength(1);
|
|
186
|
+
});
|
|
187
|
+
});
|
|
188
|
+
|
|
170
189
|
describe("extractGowaMessageId — delivery verification", () => {
|
|
171
190
|
test("the canonical GOWA SendResponse shape yields results.message_id", () => {
|
|
172
191
|
expect(extractGowaMessageId({ code: "SUCCESS", results: { message_id: "3EB089B9D6ADD58153C561", status: "sent" } })).toBe(
|
|
@@ -194,6 +213,8 @@ describe("composeEscalationMessage", () => {
|
|
|
194
213
|
aggregate: agg("host-command-bus.command.failed", 490, 720, 0),
|
|
195
214
|
impact: "scala-tools verbs are failing host-wide",
|
|
196
215
|
recommendation: "check oxmgr logs for the bus consumer",
|
|
216
|
+
sample: "",
|
|
217
|
+
lastEvent: null,
|
|
197
218
|
});
|
|
198
219
|
expect(message).toContain("Platform fault: host-command-bus.command.failed");
|
|
199
220
|
expect(message).toContain("Observed: 490 errors over 12 h");
|
|
@@ -201,6 +222,74 @@ describe("composeEscalationMessage", () => {
|
|
|
201
222
|
expect(message).toContain("Recommendation: check oxmgr logs for the bus consumer");
|
|
202
223
|
expect(message.split("\n")).toHaveLength(4);
|
|
203
224
|
});
|
|
225
|
+
|
|
226
|
+
// 2026-08-26: an escalation sent 08:20Z cited "last at 09:04:54Z", a
|
|
227
|
+
// timestamp 44 minutes in its own future matching no event the operator
|
|
228
|
+
// could find. The Observed line now cites the newest backing row's own
|
|
229
|
+
// occurred_at and id, so "last at T" resolves to a real event_log row.
|
|
230
|
+
test("cites the newest backing row's timestamp and event id — resolvable evidence", () => {
|
|
231
|
+
const message = composeEscalationMessage({
|
|
232
|
+
kind: "guard.fail.open",
|
|
233
|
+
aggregate: agg("guard.fail.open", 490, 720, 0),
|
|
234
|
+
impact: "guard enforcement is failing open",
|
|
235
|
+
recommendation: "check the guard daemon",
|
|
236
|
+
sample: "",
|
|
237
|
+
lastEvent: { occurredAt: "2026-08-26T08:20:29Z", eventId: "01a03b79-c124-748f-a79f-a9a436729544" },
|
|
238
|
+
});
|
|
239
|
+
expect(message).toContain("last 2026-08-26T08:20:29Z (event 01a03b79)");
|
|
240
|
+
});
|
|
241
|
+
|
|
242
|
+
// 2026-08-26: three escalations reached the orchestrator seat carrying
|
|
243
|
+
// kind, count and duration only — triaging each required a hand event_log
|
|
244
|
+
// query, and the llm.provider.call.failed impact line pointed AWAY from
|
|
245
|
+
// the defect (payloads held our own 'Invalid length: Expected <=1200 but
|
|
246
|
+
// received 1255' contract violations recorded against healthy providers).
|
|
247
|
+
// The sample line carries the evidence in the first read.
|
|
248
|
+
test("carries the sampled payload evidence as its own line", () => {
|
|
249
|
+
const message = composeEscalationMessage({
|
|
250
|
+
kind: "llm.provider.call.failed",
|
|
251
|
+
aggregate: agg("llm.provider.call.failed", 26, 560, 0),
|
|
252
|
+
sample: 'status=429; message=Usage limit reached for 5 hour. Your limit will reset at 2026-08-26 01:19:03',
|
|
253
|
+
impact: "AI chat/completion calls are failing",
|
|
254
|
+
recommendation: "check provider quota",
|
|
255
|
+
lastEvent: null,
|
|
256
|
+
});
|
|
257
|
+
const lines = message.split("\n");
|
|
258
|
+
expect(lines).toHaveLength(5);
|
|
259
|
+
expect(lines[2]).toContain("Sample: status=429");
|
|
260
|
+
expect(lines[2]).toContain("reset at 2026-08-26 01:19:03");
|
|
261
|
+
});
|
|
262
|
+
});
|
|
263
|
+
|
|
264
|
+
describe("formatEscalationSample — evidence truncated hard, secrets never shipped", () => {
|
|
265
|
+
test("non-string values are JSON-encoded so structure survives the phone channel", () => {
|
|
266
|
+
expect(formatEscalationSample(429)).toBe("429");
|
|
267
|
+
expect(formatEscalationSample(true)).toBe("true");
|
|
268
|
+
expect(formatEscalationSample({ expected: 1200, received: 1255 })).toBe('{"expected":1200,"received":1255}');
|
|
269
|
+
});
|
|
270
|
+
|
|
271
|
+
test("truncates hard at the bound with an ellipsis so the reader knows the cut happened", () => {
|
|
272
|
+
const formatted = formatEscalationSample("x".repeat(500));
|
|
273
|
+
expect(formatted.length).toBe(201); // 200 chars + ellipsis
|
|
274
|
+
expect(formatted.endsWith("…")).toBe(true);
|
|
275
|
+
});
|
|
276
|
+
|
|
277
|
+
test("redacts credential shapes before truncation — a cut can never expose a secret tail", () => {
|
|
278
|
+
expect(formatEscalationSample("key sk-abcdefghijklmnopqrstuvwxyz failed")).toBe("key [redacted] failed");
|
|
279
|
+
expect(formatEscalationSample("Authorization: Bearer abcdefghijklmnop")).toBe(
|
|
280
|
+
"Authorization=[redacted]",
|
|
281
|
+
);
|
|
282
|
+
expect(formatEscalationSample("api_key=abcdefgh1234 rejected")).toBe("api_key=[redacted] rejected");
|
|
283
|
+
});
|
|
284
|
+
|
|
285
|
+
test("collapses whitespace to one line — the sample is one message line", () => {
|
|
286
|
+
expect(formatEscalationSample("line one\nline two\t tabbed")).toBe("line one line two tabbed");
|
|
287
|
+
});
|
|
288
|
+
|
|
289
|
+
test("empty input yields an empty sample (no Sample line)", () => {
|
|
290
|
+
expect(formatEscalationSample(undefined)).toBe("");
|
|
291
|
+
expect(formatEscalationSample("")).toBe("");
|
|
292
|
+
});
|
|
204
293
|
});
|
|
205
294
|
|
|
206
295
|
describe("decideSweep — qualification asks 'is this happening', not 'did this happen'", () => {
|
|
@@ -3,45 +3,10 @@
|
|
|
3
3
|
* @status handwritten @indivisible-unit
|
|
4
4
|
* @edit edit directly
|
|
5
5
|
*
|
|
6
|
-
* The `platform-fault-escalation` pg-boss handler
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
* llm.provider.error) 231, guard.fail.open 16, scale-to-zero.wake_failed 12,
|
|
11
|
-
* listen.connect_failed 11). Emission works and observability-is-passive is satisfied; nothing
|
|
12
|
-
* acted. This sweep makes RECEIVING have CONSEQUENCE: sustained platform
|
|
13
|
-
* faults escalate to the OPERATOR over the existing PA path (WhatsApp, the
|
|
14
|
-
* scala PA operator-relay bot — escalation-routing's channel; no second
|
|
15
|
-
* channel invented), with the durable record as a work_items task the PA can
|
|
16
|
-
* read back and the operator can answer on.
|
|
17
|
-
*
|
|
18
|
-
* THIS IS NOT A DRIFT-DETECTOR PATROLLING INSTANCES. It reacts only to
|
|
19
|
-
* components reporting THEIR OWN failure — event_log kinds that already
|
|
20
|
-
* assert a fault about their emitter. Thresholds (duration + recurrence,
|
|
21
|
-
* never a single event), the watched-kind patterns, the resolve window and
|
|
22
|
-
* the per-kind impact/recommendation copy are all in the
|
|
23
|
-
* `config/platform-fault-escalation` registry row — changing policy is a row
|
|
24
|
-
* edit, not a redeploy. Episode dedup lives in the
|
|
25
|
-
* `platform_fault_escalations` table (migration 501): one escalation per
|
|
26
|
-
* distinct fault kind per episode; an already-escalated-and-unresolved fault
|
|
27
|
-
* only appends throttled work-item comments, never a second message.
|
|
28
|
-
*
|
|
29
|
-
* DELIVERY IS VERIFIED, NOT QUEUED. The WhatsApp send goes through the
|
|
30
|
-
* central `/mcp` loopback (`api_gowa_message` sendMessage — the single
|
|
31
|
-
* sealed GOWA-credential surface, same as whatsapp-liveness) and an episode
|
|
32
|
-
* only carries `escalated_at` once the send returned a WhatsApp message id;
|
|
33
|
-
* before that the row is a CLAIM the next sweep retries. Success is recorded
|
|
34
|
-
* only after readback of the delivery facts (message id + work item id).
|
|
35
|
-
*
|
|
36
|
-
* WHY A TICK EXISTS AT ALL, given observability-is-passive and
|
|
37
|
-
* no-polling-for-pushable-state: the trigger is DURATION + RECURRENCE of
|
|
38
|
-
* events already durably logged — a fact about the PATTERN of past events,
|
|
39
|
-
* not a state transition any single event can announce. The sweep reads one
|
|
40
|
-
* server-side aggregate (GROUP BY kind) per run against indexed event_log;
|
|
41
|
-
* while the platform is healthy it returns zero rows and costs nothing
|
|
42
|
-
* beyond the tick (idle-costs-nothing).
|
|
43
|
-
*
|
|
44
|
-
* Registered under slug `@teamscala/service-boot:platform-fault-escalation`.
|
|
6
|
+
* The `platform-fault-escalation` pg-boss handler. Full contract in curated
|
|
7
|
+
* `apps/escalation.md` — episodes EMIT ON TRANSITIONS ONLY; evidence bounded
|
|
8
|
+
* `occurred_at <= now()` and cited by resolvable event id; delivery verified
|
|
9
|
+
* by WhatsApp message id before `escalated_at` is recorded.
|
|
45
10
|
*/
|
|
46
11
|
import type { ExecutionContext } from "@teamscala/os/contracts/mcp";
|
|
47
12
|
import type { ScheduledJobConfig } from "@teamscala/db-validation/registry-schemas/scheduled-job";
|
|
@@ -52,10 +17,10 @@ import { recordEvent } from "@teamscala/event-log/record-event";
|
|
|
52
17
|
import { getAppLogger } from "@teamscala/logger/app-loggers";
|
|
53
18
|
import { pgBossHandlerRegistry } from "../pg-boss-handler-registry.ts";
|
|
54
19
|
import {
|
|
55
|
-
composeEscalationDescription,
|
|
56
20
|
composeEscalationMessage,
|
|
57
21
|
decideSweep,
|
|
58
22
|
extractGowaMessageId,
|
|
23
|
+
formatEscalationSample,
|
|
59
24
|
type EscalationEpisode,
|
|
60
25
|
type FaultAggregate,
|
|
61
26
|
type KindThresholdOverride,
|
|
@@ -81,7 +46,6 @@ const FaultEscalationConfigSchema = v.object({
|
|
|
81
46
|
),
|
|
82
47
|
),
|
|
83
48
|
resolve_after_minutes: v.number(),
|
|
84
|
-
update_after_minutes: v.number(),
|
|
85
49
|
/** The orchestrator's work item — escalations are COMMENTS on it, never
|
|
86
50
|
* task rows (the fleet-task-placement CHECK constraint refuses a system
|
|
87
51
|
* task without session identity, and an escalation is a finding, not an
|
|
@@ -90,6 +54,10 @@ const FaultEscalationConfigSchema = v.object({
|
|
|
90
54
|
/** Per-kind copy for the message (impact + recommendation). Falls back to default_copy. */
|
|
91
55
|
copy: v.record(v.string(), v.object({ impact: v.string(), recommendation: v.string() })),
|
|
92
56
|
default_copy: v.object({ impact: v.string(), recommendation: v.string() }),
|
|
57
|
+
/** Per-kind payload-field whitelist sampled into the message's evidence
|
|
58
|
+
* line — named fields ONLY, never the whole payload: some kinds embed
|
|
59
|
+
* request context, and the escalation channel is WhatsApp. */
|
|
60
|
+
sample_fields: v.optional(v.record(v.string(), v.array(v.string()))),
|
|
93
61
|
});
|
|
94
62
|
|
|
95
63
|
type FaultEscalationConfig = v.InferOutput<typeof FaultEscalationConfigSchema>;
|
|
@@ -128,8 +96,9 @@ async function getSystemIdentity(): Promise<{ userId: string | undefined; organi
|
|
|
128
96
|
* BACKEND service, where the page-server client is never registered — the
|
|
129
97
|
* empty-router fallback returns a client with no model namespaces (the
|
|
130
98
|
* TypeError this replaced). Per `no-direct-prisma-outside-orpc`, every WRITE
|
|
131
|
-
* in this handler routes through this client; the
|
|
132
|
-
* event_log recurrence aggregate
|
|
99
|
+
* in this handler routes through this client; the two raw reads (the
|
|
100
|
+
* event_log recurrence aggregate and the latest-payload sample, shapes no
|
|
101
|
+
* generated CRUD expresses) use
|
|
133
102
|
* the assigned-client form with $queryRawUnsafe. Timestamps are ISO strings,
|
|
134
103
|
* never Date instances (the generated valibot schemas require it).
|
|
135
104
|
*/
|
|
@@ -184,9 +153,6 @@ interface OrpcSurface {
|
|
|
184
153
|
messaging_bots: {
|
|
185
154
|
findFirst(args: Record<string, unknown>): Promise<Record<string, unknown> | null>;
|
|
186
155
|
};
|
|
187
|
-
work_items: {
|
|
188
|
-
create(args: { data: Record<string, unknown>; select?: Record<string, unknown> }): Promise<{ id: string }>;
|
|
189
|
-
};
|
|
190
156
|
comments: {
|
|
191
157
|
create(args: { data: Record<string, unknown> }): Promise<{ id: string }>;
|
|
192
158
|
};
|
|
@@ -201,6 +167,11 @@ async function fetchAggregates(patterns: string[], windowMinutes: number): Promi
|
|
|
201
167
|
// Patterns come from the valibot-validated registry row and are regex-only
|
|
202
168
|
// by construction below; embedding them as one alternation keeps the query
|
|
203
169
|
// parameter-free on the pattern side while the window is a real parameter.
|
|
170
|
+
// The window is bounded on BOTH sides by the database's own statement
|
|
171
|
+
// clock: `occurred_at` is producer-claimed, and an unbounded top edge
|
|
172
|
+
// admits rows stamped in the future by a skewed or mocked producer clock —
|
|
173
|
+
// which then fabricate "still recurring" for a quiet fault (measured
|
|
174
|
+
// 2026-08-26). A future row is not evidence of anything happening now.
|
|
204
175
|
const regexClass = /^[A-Za-z0-9_\\^$.*+?()[\]{}|-]+$/;
|
|
205
176
|
for (const pattern of patterns) {
|
|
206
177
|
if (!regexClass.test(pattern)) {
|
|
@@ -210,10 +181,11 @@ async function fetchAggregates(patterns: string[], windowMinutes: number): Promi
|
|
|
210
181
|
const combined = `(${patterns.join("|")})`;
|
|
211
182
|
const rows = await prisma.$queryRawUnsafe<EventLogAggregateRow[]>(
|
|
212
183
|
`SELECT kind, count(*)::bigint AS n, min(occurred_at) AS first_seen, max(occurred_at) AS last_seen
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
184
|
+
FROM event_log
|
|
185
|
+
WHERE occurred_at > now() - ($1::int * interval '1 minute')
|
|
186
|
+
AND occurred_at <= now()
|
|
187
|
+
AND kind ~ $2::text
|
|
188
|
+
GROUP BY kind`,
|
|
217
189
|
windowMinutes,
|
|
218
190
|
combined,
|
|
219
191
|
);
|
|
@@ -225,6 +197,70 @@ async function fetchAggregates(patterns: string[], windowMinutes: number): Promi
|
|
|
225
197
|
}));
|
|
226
198
|
}
|
|
227
199
|
|
|
200
|
+
/**
|
|
201
|
+
* The message's evidence: the whitelisted sample fields AND the row identity
|
|
202
|
+
* (id + occurred_at) of the newest payload for the kind. Field names come
|
|
203
|
+
* from the valibot-validated config row and are constrained to bare
|
|
204
|
+
* identifiers below, so the JSON path cannot break out of the string. The
|
|
205
|
+
* `occurred_at <= now()` bound matches the aggregate's top edge: the cited
|
|
206
|
+
* "last" row must be a real, non-future event whose id rides in the message
|
|
207
|
+
* so the claim resolves against event_log.
|
|
208
|
+
*/
|
|
209
|
+
async function fetchLatestEvidence(
|
|
210
|
+
kind: string,
|
|
211
|
+
fields: string[],
|
|
212
|
+
): Promise<{ sample: string; lastEvent: { occurredAt: string; eventId: string } | null }> {
|
|
213
|
+
const prisma = getPrisma();
|
|
214
|
+
if (fields.length === 0) {
|
|
215
|
+
const rows = await prisma.$queryRawUnsafe<Array<{ id: unknown; occurred_at: Date }>>(
|
|
216
|
+
`SELECT id::text, occurred_at
|
|
217
|
+
FROM event_log
|
|
218
|
+
WHERE kind = $1::text AND occurred_at <= now()
|
|
219
|
+
ORDER BY occurred_at DESC
|
|
220
|
+
LIMIT 1`,
|
|
221
|
+
kind,
|
|
222
|
+
);
|
|
223
|
+
const top = rows[0];
|
|
224
|
+
return { sample: "", lastEvent: top ? { occurredAt: isoSeconds(top.occurred_at), eventId: String(top.id) } : null };
|
|
225
|
+
}
|
|
226
|
+
for (const field of fields) {
|
|
227
|
+
if (!/^[A-Za-z0-9_]+$/.test(field)) {
|
|
228
|
+
throw new Error(`config/${CONFIG_SLUG}: sample_fields entry is not a bare field name: ${field}`);
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
const selection = fields.map((field) => `payload_json->'${field}' AS "${field}"`).join(", ");
|
|
232
|
+
const rows = await prisma.$queryRawUnsafe<Array<Record<string, unknown>>>(
|
|
233
|
+
`SELECT id::text, occurred_at, ${selection}
|
|
234
|
+
FROM event_log
|
|
235
|
+
WHERE kind = $1::text AND occurred_at <= now()
|
|
236
|
+
ORDER BY occurred_at DESC
|
|
237
|
+
LIMIT 1`,
|
|
238
|
+
kind,
|
|
239
|
+
);
|
|
240
|
+
const row = rows[0];
|
|
241
|
+
if (!row) return { sample: "", lastEvent: null };
|
|
242
|
+
const parts: string[] = [];
|
|
243
|
+
for (const field of fields) {
|
|
244
|
+
const value = row[field];
|
|
245
|
+
if (value === null || value === undefined) continue;
|
|
246
|
+
const formatted = formatEscalationSample(value);
|
|
247
|
+
if (formatted) parts.push(`${field}=${formatted}`);
|
|
248
|
+
}
|
|
249
|
+
return {
|
|
250
|
+
sample: parts.join("; "),
|
|
251
|
+
lastEvent: {
|
|
252
|
+
occurredAt: isoSeconds(row.occurred_at as string | Date),
|
|
253
|
+
eventId: String(row.id),
|
|
254
|
+
},
|
|
255
|
+
};
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
/** ISO-8601 to whole seconds — the same precision event_log stores, so the
|
|
259
|
+
* cited string round-trips a `WHERE occurred_at = '<cited>'` query. */
|
|
260
|
+
function isoSeconds(value: Date | string): string {
|
|
261
|
+
return new Date(value).toISOString().replace(/\.\d{3}Z$/, "Z");
|
|
262
|
+
}
|
|
263
|
+
|
|
228
264
|
async function resolveOperatorRelay(): Promise<OperatorRelayBot> {
|
|
229
265
|
const bot = (await (await orpc()).messaging_bots.findFirst({
|
|
230
266
|
where: {
|
|
@@ -284,9 +320,12 @@ async function deliverEscalation(
|
|
|
284
320
|
const bot = await resolveOperatorRelay();
|
|
285
321
|
const operatorPhone = (bot.flow_config as { operator_phone?: string }).operator_phone!;
|
|
286
322
|
const copy = config.copy[kind] ?? config.default_copy;
|
|
323
|
+
const { sample, lastEvent } = await fetchLatestEvidence(kind, config.sample_fields?.[kind] ?? []);
|
|
287
324
|
const message = composeEscalationMessage({
|
|
288
325
|
kind,
|
|
289
326
|
aggregate,
|
|
327
|
+
sample,
|
|
328
|
+
lastEvent,
|
|
290
329
|
impact: copy.impact,
|
|
291
330
|
recommendation: copy.recommendation,
|
|
292
331
|
});
|
|
@@ -325,12 +364,7 @@ async function deliverEscalation(
|
|
|
325
364
|
const comment = await (await orpc()).comments.create({
|
|
326
365
|
data: {
|
|
327
366
|
work_item_id: config.orchestrator_work_item_id,
|
|
328
|
-
content:
|
|
329
|
-
kind,
|
|
330
|
-
aggregate,
|
|
331
|
-
impact: copy.impact,
|
|
332
|
-
recommendation: copy.recommendation,
|
|
333
|
-
}),
|
|
367
|
+
content: message,
|
|
334
368
|
source_cli_session_id: null,
|
|
335
369
|
},
|
|
336
370
|
});
|
|
@@ -353,6 +387,9 @@ async function deliverEscalation(
|
|
|
353
387
|
payload: {
|
|
354
388
|
event_kind: kind,
|
|
355
389
|
count: aggregate.count,
|
|
390
|
+
sample,
|
|
391
|
+
last_event_id: lastEvent?.eventId ?? null,
|
|
392
|
+
last_event_occurred_at: lastEvent?.occurredAt ?? null,
|
|
356
393
|
work_item_id: config.orchestrator_work_item_id,
|
|
357
394
|
delivered_comment_id: comment.id,
|
|
358
395
|
delivered_message_id: messageId,
|
|
@@ -361,33 +398,6 @@ async function deliverEscalation(
|
|
|
361
398
|
return { messageId, workItemId: config.orchestrator_work_item_id };
|
|
362
399
|
}
|
|
363
400
|
|
|
364
|
-
async function updateEpisode(aggregate: FaultAggregate, episode: EscalationEpisode): Promise<void> {
|
|
365
|
-
const now = new Date();
|
|
366
|
-
await (await orpc()).platform_fault_escalations.update({
|
|
367
|
-
where: { kind: episode.kind },
|
|
368
|
-
data: {
|
|
369
|
-
last_seen_at: aggregate.lastSeen.toISOString(),
|
|
370
|
-
event_count: aggregate.count,
|
|
371
|
-
last_updated_at: now.toISOString(),
|
|
372
|
-
updated_at: now.toISOString(),
|
|
373
|
-
},
|
|
374
|
-
});
|
|
375
|
-
if (episode.workItemId) {
|
|
376
|
-
await (await orpc()).comments.create({
|
|
377
|
-
data: {
|
|
378
|
-
work_item_id: episode.workItemId,
|
|
379
|
-
content: `Still recurring: ${aggregate.count} events, last at ${aggregate.lastSeen.toISOString()}. No new escalation (already escalated and unresolved).`,
|
|
380
|
-
sender_role: "system",
|
|
381
|
-
sender_name: "platform-fault-escalation",
|
|
382
|
-
},
|
|
383
|
-
});
|
|
384
|
-
}
|
|
385
|
-
recordEvent({
|
|
386
|
-
kind: "platform.fault.updated",
|
|
387
|
-
payload: { event_kind: episode.kind, count: aggregate.count, work_item_id: episode.workItemId },
|
|
388
|
-
});
|
|
389
|
-
}
|
|
390
|
-
|
|
391
401
|
async function resolveEpisode(episode: EscalationEpisode, quietMinutes: number): Promise<void> {
|
|
392
402
|
const now = new Date();
|
|
393
403
|
await (await orpc()).platform_fault_escalations.update({
|
|
@@ -456,7 +466,6 @@ async function platformFaultEscalationSweep(ctx: { config: ScheduledJobConfig })
|
|
|
456
466
|
minCount: config.min_count,
|
|
457
467
|
minSpanMinutes: config.min_span_minutes,
|
|
458
468
|
resolveAfterMinutes: config.resolve_after_minutes,
|
|
459
|
-
updateAfterMinutes: config.update_after_minutes,
|
|
460
469
|
},
|
|
461
470
|
now,
|
|
462
471
|
kindThresholds,
|
|
@@ -477,13 +486,6 @@ async function platformFaultEscalationSweep(ctx: { config: ScheduledJobConfig })
|
|
|
477
486
|
recordEvent({ kind: "platform.fault.escalation_failed", payload: { event_kind: aggregate.kind, reason } });
|
|
478
487
|
}
|
|
479
488
|
}
|
|
480
|
-
for (const { aggregate, episode } of decision.update) {
|
|
481
|
-
try {
|
|
482
|
-
await updateEpisode(aggregate, episode);
|
|
483
|
-
} catch (err) {
|
|
484
|
-
log.error(`[platform-fault-escalation] update for '${episode.kind}' failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
485
|
-
}
|
|
486
|
-
}
|
|
487
489
|
for (const { episode, quietMinutes } of decision.resolve) {
|
|
488
490
|
try {
|
|
489
491
|
await resolveEpisode(episode, quietMinutes);
|
|
@@ -499,18 +501,17 @@ async function platformFaultEscalationSweep(ctx: { config: ScheduledJobConfig })
|
|
|
499
501
|
payload: {
|
|
500
502
|
watched_kinds: aggregates.length,
|
|
501
503
|
escalated: decision.escalate.length,
|
|
502
|
-
updated: decision.update.length,
|
|
503
504
|
resolved: decision.resolve.length,
|
|
504
505
|
},
|
|
505
506
|
});
|
|
506
|
-
if (decision.escalate.length > 0 || decision.
|
|
507
|
+
if (decision.escalate.length > 0 || decision.resolve.length > 0) {
|
|
507
508
|
log.info(
|
|
508
|
-
`[platform-fault-escalation] sweep — ${aggregates.length} kind(s) watched, ${decision.escalate.length} escalated, ${decision.
|
|
509
|
+
`[platform-fault-escalation] sweep — ${aggregates.length} kind(s) watched, ${decision.escalate.length} escalated, ${decision.resolve.length} resolved`,
|
|
509
510
|
);
|
|
510
511
|
}
|
|
511
|
-
}
|
|
512
|
+
}
|
|
512
513
|
|
|
513
|
-
pgBossHandlerRegistry.register(
|
|
514
|
+
pgBossHandlerRegistry.register(
|
|
514
515
|
"@teamscala/service-boot:platform-fault-escalation",
|
|
515
516
|
platformFaultEscalationSweep as (ctx: { config: ScheduledJobConfig }) => Promise<void>,
|
|
516
517
|
);
|