@teamscala/service-boot 0.23.406 → 0.23.408

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -6,7 +6,7 @@
6
6
  },
7
7
  "name": "@teamscala/service-boot",
8
8
  "type": "module",
9
- "version": "0.23.406",
9
+ "version": "0.23.408",
10
10
  "sideEffects": false,
11
11
  "imports": {
12
12
  "#service-boot/*": "./src/*",
@@ -1,6 +1,6 @@
1
1
  /**
2
2
  * @system service-boot
3
- * @status handwritten
3
+ * @status handwritten @indivisible-unit
4
4
  * @edit edit directly
5
5
  *
6
6
  * Public ORPC procedures for DocuSign-style agreement signing, reached by the
@@ -3,25 +3,9 @@
3
3
  * @status handwritten
4
4
  * @edit edit directly
5
5
  *
6
- * Pure decision logic for the platform-fault escalation sweep
7
- * (`platform-fault-escalation` handler). Split from the I/O shell so the
8
- * threshold/dedup/episode semantics are unit-testable with zero adapter
9
- * dependencies (same shape as whatsapp-liveness-logic).
10
- *
11
- * The semantics this locks (from the 2026-08-15 "760 errors escalated to
12
- * nobody" incident brief):
13
- * - Threshold on DURATION and RECURRENCE, never a single event. A kind
14
- * qualifies as a sustained fault only when BOTH hold inside the lookback
15
- * window: at least `min_count` occurrences (recurrence) spread over at
16
- * least `min_span_minutes` (duration). One failed push is noise; the same
17
- * kind recurring for half an hour is an outage.
18
- * - One escalation per distinct fault kind per episode. An episode is open
19
- * from first qualification until `resolve_after_minutes` of quiet; while
20
- * open, the fault never re-fires — it only appends throttled updates.
21
- * - Delivery is verified, not queued: an episode row with `escalated_at`
22
- * NULL is a CLAIM, not an escalation. The sweep re-attempts delivery for
23
- * claimed-but-undelivered episodes until the WhatsApp send returns a
24
- * message id.
6
+ * Pure decision logic for the platform-fault escalation sweep — full
7
+ * contract in curated `apps/escalation.md`. Episodes emit on TRANSITIONS
8
+ * ONLY; a future-dated `lastSeen` is never recurrence evidence.
25
9
  */
26
10
 
27
11
  export interface FaultAggregate {
@@ -29,6 +13,10 @@ export interface FaultAggregate {
29
13
  count: number;
30
14
  firstSeen: Date;
31
15
  lastSeen: Date;
16
+ /** Id of the newest row backing `lastSeen`, when the caller resolved it —
17
+ * the resolvable handle the escalation cites so "last at T" is checkable
18
+ * against a real event_log row, not a computed or cached value. */
19
+ lastEventId?: string | null;
32
20
  }
33
21
 
34
22
  export interface EscalationEpisode {
@@ -50,8 +38,6 @@ export interface FaultEscalationPolicy {
50
38
  minSpanMinutes: number;
51
39
  /** Quiet period that closes an episode and re-arms escalation, minutes. */
52
40
  resolveAfterMinutes: number;
53
- /** Minimum spacing between throttled updates on an open episode, minutes. */
54
- updateAfterMinutes: number;
55
41
  }
56
42
 
57
43
  /** Per-kind threshold overrides — a low-rate-but-persistent kind (guard.fail.open
@@ -65,8 +51,6 @@ export interface KindThresholdOverride {
65
51
  export interface SweepDecision {
66
52
  /** Sustained faults needing a FIRST delivery (new episode, or a claim whose send failed). */
67
53
  escalate: Array<{ aggregate: FaultAggregate; episode: EscalationEpisode | null }>;
68
- /** Open, delivered episodes still recurring — append evidence, never re-send. */
69
- update: Array<{ aggregate: FaultAggregate; episode: EscalationEpisode }>;
70
54
  /** Open episodes whose kind went quiet past the resolve window — close them. */
71
55
  resolve: Array<{ episode: EscalationEpisode; quietMinutes: number }>;
72
56
  }
@@ -88,9 +72,13 @@ function isSustained(
88
72
  // happening. Without this the 720-minute lookback keeps a dead fault
89
73
  // qualifying for hours, and since a resolved episode reads as no episode
90
74
  // it re-escalates on every sweep until its events age out (2026-08-26).
75
+ // A future `lastSeen` (producer clock skew / mocked test clock, measured
76
+ // 2026-08-26: "still recurring" citing a timestamp 44 min in its own
77
+ // future) makes minutesBetween go NEGATIVE — refuse it as evidence.
91
78
  return (
92
79
  aggregate.count >= minCount &&
93
80
  minutesBetween(aggregate.firstSeen, aggregate.lastSeen) >= minSpan &&
81
+ aggregate.lastSeen.getTime() <= now.getTime() &&
94
82
  minutesBetween(aggregate.lastSeen, now) < policy.resolveAfterMinutes
95
83
  );
96
84
  }
@@ -110,7 +98,7 @@ export function decideSweep(
110
98
  ): SweepDecision {
111
99
  const byKind = new Map(episodes.map((e) => [e.kind, e]));
112
100
  const aggregateByKind = new Map(aggregates.map((a) => [a.kind, a]));
113
- const decision: SweepDecision = { escalate: [], update: [], resolve: [] };
101
+ const decision: SweepDecision = { escalate: [], resolve: [] };
114
102
 
115
103
  for (const aggregate of aggregates) {
116
104
  const episode = byKind.get(aggregate.kind);
@@ -126,19 +114,23 @@ export function decideSweep(
126
114
  // verified).
127
115
  if (!episode.escalatedAt) {
128
116
  decision.escalate.push({ aggregate, episode });
129
- continue;
130
- }
131
- // Open, delivered, still recurring — throttled evidence update only.
132
- const lastUpdate = episode.lastUpdatedAt ?? episode.escalatedAt;
133
- if (minutesBetween(lastUpdate, now) >= policy.updateAfterMinutes) {
134
- decision.update.push({ aggregate, episode });
135
117
  }
118
+ // An open, delivered episode emits NOTHING while it recurs: the open
119
+ // emission already happened and the close emission fires when the kind
120
+ // goes quiet. A cadence-driven "still recurring" repeat is exactly the
121
+ // silenced-alarm shape `emit-transitions-not-conditions` prohibits.
136
122
  }
137
123
 
138
124
  for (const episode of episodes) {
139
125
  if (episode.resolvedAt) continue;
140
126
  const aggregate = aggregateByKind.get(episode.kind);
141
- const quietReference = aggregate?.lastSeen ?? episode.lastSeenAt;
127
+ // A future-dated aggregate lastSeen (producer clock skew) cannot serve
128
+ // as the quiet reference — it would freeze quietMinutes negative and
129
+ // hold the episode open forever while the kind is actually quiet. Fall
130
+ // back to the episode's own last_seen_at, which the bounded aggregate
131
+ // wrote and is therefore never in the future.
132
+ const quietReference =
133
+ aggregate && aggregate.lastSeen.getTime() <= now.getTime() ? aggregate.lastSeen : episode.lastSeenAt;
142
134
  const quietMinutes = minutesBetween(quietReference, now);
143
135
  if (quietMinutes >= policy.resolveAfterMinutes) {
144
136
  decision.resolve.push({ episode, quietMinutes });
@@ -176,39 +168,56 @@ export function extractGowaMessageId(sendResult: unknown): string | null {
176
168
  return null;
177
169
  }
178
170
 
171
+ /**
172
+ * One sampled payload value rendered for the message's evidence line: a
173
+ * kind+count escalation is a pointer to evidence, not evidence (2026-08-26:
174
+ * the llm.provider.call.failed payloads held our own bounded-output contract
175
+ * violations while the escalation's impact line named healthy providers).
176
+ * JSON-encoded when non-string so structure survives, whitespace-collapsed
177
+ * to one line, credential shapes redacted before truncation (a hard cut can
178
+ * never expose the tail of a secret the length limit happened to spare),
179
+ * hard-truncated with an ellipsis so the reader knows the cut happened.
180
+ */
181
+ export function formatEscalationSample(value: unknown): string {
182
+ const maxChars = 200;
183
+ const redactions: ReadonlyArray<[RegExp, string]> = [
184
+ [/\b(?:sk|gsk|ghp|gho|xox[baprs])[-_][A-Za-z0-9_-]{8,}/g, "[redacted]"],
185
+ [/\b(api[_-]?key|token|password|secret|authorization)\b["'=:\s]+(?:(?:bearer|basic)\s+)?[A-Za-z0-9._~+/=-]{4,}/gi, "$1=[redacted]"],
186
+ [/Bearer\s+[A-Za-z0-9._~+/=-]{8,}/gi, "Bearer [redacted]"],
187
+ ];
188
+ const raw = typeof value === "string" ? value : (JSON.stringify(value) ?? "");
189
+ if (raw.length === 0) return "";
190
+ const redacted = redactions.reduce((acc, [pattern, replacement]) => acc.replace(pattern, replacement), raw);
191
+ const collapsed = redacted.replace(/\s+/g, " ").trim();
192
+ return collapsed.length > maxChars ? `${collapsed.slice(0, maxChars)}…` : collapsed;
193
+ }
194
+
179
195
  /**
180
196
  * The escalation message — escalation-routing's three-field contract adapted
181
- * to an automated detector: what was OBSERVED (evidence), what it MEANS
182
- * (impact), and the RECOMMENDATION. Short: the channel is a phone.
197
+ * to an automated detector: what was OBSERVED (evidence + sample), what it
198
+ * MEANS (impact), and the RECOMMENDATION. Short: the channel is a phone. The
199
+ * sample line is the triage evidence — `sample_fields` whitelisted per kind
200
+ * by the caller, formatted by formatEscalationSample; empty means the kind
201
+ * samples nothing or the latest payload carried none of its fields.
183
202
  */
184
203
  export function composeEscalationMessage(input: {
185
204
  kind: string;
186
205
  aggregate: FaultAggregate;
187
206
  impact: string;
188
207
  recommendation: string;
208
+ sample: string;
209
+ /** ISO timestamp + event id of the newest backing row, when resolved —
210
+ * cited verbatim so the reader can resolve the claim against event_log. */
211
+ lastEvent?: { occurredAt: string; eventId: string } | null;
189
212
  }): string {
190
213
  const span = formatSpanMinutes(
191
214
  (input.aggregate.lastSeen.getTime() - input.aggregate.firstSeen.getTime()) / 60_000,
192
215
  );
193
- return [
194
- `🔴 Platform fault: ${input.kind}`,
195
- `Observed: ${input.aggregate.count} errors over ${span}, still recurring.`,
196
- `Impact: ${input.impact}`,
197
- `Recommendation: ${input.recommendation}`,
198
- ].join("\n");
199
- }
200
-
201
- /** The work-item description carries the same three fields plus the evidence. */
202
- export function composeEscalationDescription(input: {
203
- aggregate: FaultAggregate;
204
- impact: string;
205
- recommendation: string;
206
- }): string {
207
- return [
208
- "Automated platform-fault escalation (platform-fault-escalation sweep).",
209
- `Observed: ${input.aggregate.count} '${input.aggregate.kind}' events in event_log from ${input.aggregate.firstSeen.toISOString()} to ${input.aggregate.lastSeen.toISOString()}, still recurring.`,
210
- `Impact: ${input.impact}`,
211
- `Recommendation: ${input.recommendation}`,
212
- "Delivery: WhatsApp via the scala PA. This task closes automatically when the fault stops for the resolve window; reply via the PA to rule otherwise.",
213
- ].join("\n");
216
+ const observed = input.lastEvent
217
+ ? `Observed: ${input.aggregate.count} errors over ${span}, last ${input.lastEvent.occurredAt} (event ${input.lastEvent.eventId.slice(0, 8)}).`
218
+ : `Observed: ${input.aggregate.count} errors over ${span}, last ${input.aggregate.lastSeen.toISOString()}.`;
219
+ const lines = [`🔴 Platform fault: ${input.kind}`, observed];
220
+ if (input.sample) lines.push(`Sample: ${input.sample}`);
221
+ lines.push(`Impact: ${input.impact}`, `Recommendation: ${input.recommendation}`);
222
+ return lines.join("\n");
214
223
  }
@@ -13,6 +13,7 @@ import {
13
13
  composeEscalationMessage,
14
14
  decideSweep,
15
15
  extractGowaMessageId,
16
+ formatEscalationSample,
16
17
  type EscalationEpisode,
17
18
  type FaultAggregate,
18
19
  type FaultEscalationPolicy,
@@ -23,7 +24,6 @@ const POLICY: FaultEscalationPolicy = {
23
24
  minCount: 10,
24
25
  minSpanMinutes: 25,
25
26
  resolveAfterMinutes: 30,
26
- updateAfterMinutes: 30,
27
27
  };
28
28
 
29
29
  function agg(kind: string, count: number, firstSeenMinutesAgo: number, lastSeenMinutesAgo: number): FaultAggregate {
@@ -101,29 +101,20 @@ describe("decideSweep — dedup (one escalation per fault, no re-fire)", () => {
101
101
  expect(decision.escalate).toHaveLength(0);
102
102
  });
103
103
 
104
- test("open delivered episode past the update window gets an UPDATE, not a message", () => {
104
+ test("open delivered episode emits NOTHING while it recurs — transitions, not cadence", () => {
105
+ // 2026-08-26: the hourly "still recurring" comment made the alarm
106
+ // skippable noise and hid a real signal. An open, delivered episode now
107
+ // stays silent until the close transition fires (see the resolve suite).
105
108
  const sustained = agg("llm.provider.call.failed", 300, 720, 0);
106
- const open = episode({
109
+ const longOpen = episode({
107
110
  kind: "llm.provider.call.failed",
108
- escalatedAt: new Date(NOW.getTime() - 90 * 60_000), // last update long ago
111
+ escalatedAt: new Date(NOW.getTime() - 90 * 60_000),
109
112
  lastUpdatedAt: new Date(NOW.getTime() - 90 * 60_000),
110
113
  });
111
- const decision = decideSweep([sustained], [open], POLICY, NOW);
112
- expect(decision.escalate).toHaveLength(0);
113
- expect(decision.update).toHaveLength(1);
114
- expect(decision.update[0]!.episode.workItemId).toBe("wi-1");
115
- });
116
-
117
- test("open episode inside the update window gets NOTHING (dedup window)", () => {
118
- const sustained = agg("llm.provider.call.failed", 300, 720, 0);
119
- const open = episode({
120
- kind: "llm.provider.call.failed",
121
- escalatedAt: new Date(NOW.getTime() - 10 * 60_000),
122
- lastUpdatedAt: new Date(NOW.getTime() - 10 * 60_000),
123
- });
124
- const decision = decideSweep([sustained], [open], POLICY, NOW);
114
+ const decision = decideSweep([sustained], [longOpen], POLICY, NOW);
125
115
  expect(decision.escalate).toHaveLength(0);
126
- expect(decision.update).toHaveLength(0);
116
+ expect(decision.resolve).toHaveLength(0);
117
+ expect(Object.keys(decision).sort()).toEqual(["escalate", "resolve"]);
127
118
  });
128
119
 
129
120
  test("a claimed-but-undelivered episode is RETRIED, not counted as escalated", () => {
@@ -167,6 +158,34 @@ describe("decideSweep — resolution + re-arm", () => {
167
158
  });
168
159
  });
169
160
 
161
+ describe("decideSweep — a future-dated lastSeen is not recurrence evidence", () => {
162
+ // 2026-08-26: an escalation sent 08:20Z cited "last at 09:04:54Z" — 44
163
+ // minutes in its own future. The rows existed (producer envelopes whose
164
+ // clock ran 70 minutes ahead), and `minutesBetween(lastSeen, now)` went
165
+ // negative, passing every recency check while the kind was actually quiet.
166
+ // A timestamp that has not happened yet cannot evidence a fault happening
167
+ // now — no matter how the aggregate was built.
168
+ test("a future-dated aggregate does not escalate", () => {
169
+ const phantom = agg("guard.fail.open", 40, 720, -44);
170
+ const decision = decideSweep([phantom], [], POLICY, NOW);
171
+ expect(decision.escalate).toHaveLength(0);
172
+ });
173
+
174
+ test("a future-dated aggregate does not hold an open episode open past the quiet window", () => {
175
+ // lastSeen 44 min in the future means the kind's real events are quiet:
176
+ // the episode's own last_seen_at (honest) drives the resolve decision,
177
+ // and the phantom aggregate must not veto it.
178
+ const quietEpisode = episode({
179
+ kind: "guard.fail.open",
180
+ lastSeenAt: new Date(NOW.getTime() - 60 * 60_000),
181
+ });
182
+ const phantom = agg("guard.fail.open", 40, 720, -44);
183
+ const decision = decideSweep([phantom], [quietEpisode], POLICY, NOW);
184
+ expect(decision.escalate).toHaveLength(0);
185
+ expect(decision.resolve).toHaveLength(1);
186
+ });
187
+ });
188
+
170
189
  describe("extractGowaMessageId — delivery verification", () => {
171
190
  test("the canonical GOWA SendResponse shape yields results.message_id", () => {
172
191
  expect(extractGowaMessageId({ code: "SUCCESS", results: { message_id: "3EB089B9D6ADD58153C561", status: "sent" } })).toBe(
@@ -194,6 +213,8 @@ describe("composeEscalationMessage", () => {
194
213
  aggregate: agg("host-command-bus.command.failed", 490, 720, 0),
195
214
  impact: "scala-tools verbs are failing host-wide",
196
215
  recommendation: "check oxmgr logs for the bus consumer",
216
+ sample: "",
217
+ lastEvent: null,
197
218
  });
198
219
  expect(message).toContain("Platform fault: host-command-bus.command.failed");
199
220
  expect(message).toContain("Observed: 490 errors over 12 h");
@@ -201,6 +222,74 @@ describe("composeEscalationMessage", () => {
201
222
  expect(message).toContain("Recommendation: check oxmgr logs for the bus consumer");
202
223
  expect(message.split("\n")).toHaveLength(4);
203
224
  });
225
+
226
+ // 2026-08-26: an escalation sent 08:20Z cited "last at 09:04:54Z", a
227
+ // timestamp 44 minutes in its own future matching no event the operator
228
+ // could find. The Observed line now cites the newest backing row's own
229
+ // occurred_at and id, so "last at T" resolves to a real event_log row.
230
+ test("cites the newest backing row's timestamp and event id — resolvable evidence", () => {
231
+ const message = composeEscalationMessage({
232
+ kind: "guard.fail.open",
233
+ aggregate: agg("guard.fail.open", 490, 720, 0),
234
+ impact: "guard enforcement is failing open",
235
+ recommendation: "check the guard daemon",
236
+ sample: "",
237
+ lastEvent: { occurredAt: "2026-08-26T08:20:29Z", eventId: "01a03b79-c124-748f-a79f-a9a436729544" },
238
+ });
239
+ expect(message).toContain("last 2026-08-26T08:20:29Z (event 01a03b79)");
240
+ });
241
+
242
+ // 2026-08-26: three escalations reached the orchestrator seat carrying
243
+ // kind, count and duration only — triaging each required a hand event_log
244
+ // query, and the llm.provider.call.failed impact line pointed AWAY from
245
+ // the defect (payloads held our own 'Invalid length: Expected <=1200 but
246
+ // received 1255' contract violations recorded against healthy providers).
247
+ // The sample line carries the evidence in the first read.
248
+ test("carries the sampled payload evidence as its own line", () => {
249
+ const message = composeEscalationMessage({
250
+ kind: "llm.provider.call.failed",
251
+ aggregate: agg("llm.provider.call.failed", 26, 560, 0),
252
+ sample: 'status=429; message=Usage limit reached for 5 hour. Your limit will reset at 2026-08-26 01:19:03',
253
+ impact: "AI chat/completion calls are failing",
254
+ recommendation: "check provider quota",
255
+ lastEvent: null,
256
+ });
257
+ const lines = message.split("\n");
258
+ expect(lines).toHaveLength(5);
259
+ expect(lines[2]).toContain("Sample: status=429");
260
+ expect(lines[2]).toContain("reset at 2026-08-26 01:19:03");
261
+ });
262
+ });
263
+
264
+ describe("formatEscalationSample — evidence truncated hard, secrets never shipped", () => {
265
+ test("non-string values are JSON-encoded so structure survives the phone channel", () => {
266
+ expect(formatEscalationSample(429)).toBe("429");
267
+ expect(formatEscalationSample(true)).toBe("true");
268
+ expect(formatEscalationSample({ expected: 1200, received: 1255 })).toBe('{"expected":1200,"received":1255}');
269
+ });
270
+
271
+ test("truncates hard at the bound with an ellipsis so the reader knows the cut happened", () => {
272
+ const formatted = formatEscalationSample("x".repeat(500));
273
+ expect(formatted.length).toBe(201); // 200 chars + ellipsis
274
+ expect(formatted.endsWith("…")).toBe(true);
275
+ });
276
+
277
+ test("redacts credential shapes before truncation — a cut can never expose a secret tail", () => {
278
+ expect(formatEscalationSample("key sk-abcdefghijklmnopqrstuvwxyz failed")).toBe("key [redacted] failed");
279
+ expect(formatEscalationSample("Authorization: Bearer abcdefghijklmnop")).toBe(
280
+ "Authorization=[redacted]",
281
+ );
282
+ expect(formatEscalationSample("api_key=abcdefgh1234 rejected")).toBe("api_key=[redacted] rejected");
283
+ });
284
+
285
+ test("collapses whitespace to one line — the sample is one message line", () => {
286
+ expect(formatEscalationSample("line one\nline two\t tabbed")).toBe("line one line two tabbed");
287
+ });
288
+
289
+ test("empty input yields an empty sample (no Sample line)", () => {
290
+ expect(formatEscalationSample(undefined)).toBe("");
291
+ expect(formatEscalationSample("")).toBe("");
292
+ });
204
293
  });
205
294
 
206
295
  describe("decideSweep — qualification asks 'is this happening', not 'did this happen'", () => {
@@ -3,45 +3,10 @@
3
3
  * @status handwritten @indivisible-unit
4
4
  * @edit edit directly
5
5
  *
6
- * The `platform-fault-escalation` pg-boss handler — closes the "receiving is
7
- * decoupled from consequence" gap measured 2026-08-15 (760 error events over
8
- * 12 hours, zero escalations: host-command-bus.command.failed (then spelled
9
- * host-command-bus.error) 490, llm.provider.call.failed (then spelled
10
- * llm.provider.error) 231, guard.fail.open 16, scale-to-zero.wake_failed 12,
11
- * listen.connect_failed 11). Emission works and observability-is-passive is satisfied; nothing
12
- * acted. This sweep makes RECEIVING have CONSEQUENCE: sustained platform
13
- * faults escalate to the OPERATOR over the existing PA path (WhatsApp, the
14
- * scala PA operator-relay bot — escalation-routing's channel; no second
15
- * channel invented), with the durable record as a work_items task the PA can
16
- * read back and the operator can answer on.
17
- *
18
- * THIS IS NOT A DRIFT-DETECTOR PATROLLING INSTANCES. It reacts only to
19
- * components reporting THEIR OWN failure — event_log kinds that already
20
- * assert a fault about their emitter. Thresholds (duration + recurrence,
21
- * never a single event), the watched-kind patterns, the resolve window and
22
- * the per-kind impact/recommendation copy are all in the
23
- * `config/platform-fault-escalation` registry row — changing policy is a row
24
- * edit, not a redeploy. Episode dedup lives in the
25
- * `platform_fault_escalations` table (migration 501): one escalation per
26
- * distinct fault kind per episode; an already-escalated-and-unresolved fault
27
- * only appends throttled work-item comments, never a second message.
28
- *
29
- * DELIVERY IS VERIFIED, NOT QUEUED. The WhatsApp send goes through the
30
- * central `/mcp` loopback (`api_gowa_message` sendMessage — the single
31
- * sealed GOWA-credential surface, same as whatsapp-liveness) and an episode
32
- * only carries `escalated_at` once the send returned a WhatsApp message id;
33
- * before that the row is a CLAIM the next sweep retries. Success is recorded
34
- * only after readback of the delivery facts (message id + work item id).
35
- *
36
- * WHY A TICK EXISTS AT ALL, given observability-is-passive and
37
- * no-polling-for-pushable-state: the trigger is DURATION + RECURRENCE of
38
- * events already durably logged — a fact about the PATTERN of past events,
39
- * not a state transition any single event can announce. The sweep reads one
40
- * server-side aggregate (GROUP BY kind) per run against indexed event_log;
41
- * while the platform is healthy it returns zero rows and costs nothing
42
- * beyond the tick (idle-costs-nothing).
43
- *
44
- * Registered under slug `@teamscala/service-boot:platform-fault-escalation`.
6
+ * The `platform-fault-escalation` pg-boss handler. Full contract in curated
7
+ * `apps/escalation.md` — episodes EMIT ON TRANSITIONS ONLY; evidence bounded
8
+ * `occurred_at <= now()` and cited by resolvable event id; delivery verified
9
+ * by WhatsApp message id before `escalated_at` is recorded.
45
10
  */
46
11
  import type { ExecutionContext } from "@teamscala/os/contracts/mcp";
47
12
  import type { ScheduledJobConfig } from "@teamscala/db-validation/registry-schemas/scheduled-job";
@@ -52,10 +17,10 @@ import { recordEvent } from "@teamscala/event-log/record-event";
52
17
  import { getAppLogger } from "@teamscala/logger/app-loggers";
53
18
  import { pgBossHandlerRegistry } from "../pg-boss-handler-registry.ts";
54
19
  import {
55
- composeEscalationDescription,
56
20
  composeEscalationMessage,
57
21
  decideSweep,
58
22
  extractGowaMessageId,
23
+ formatEscalationSample,
59
24
  type EscalationEpisode,
60
25
  type FaultAggregate,
61
26
  type KindThresholdOverride,
@@ -81,7 +46,6 @@ const FaultEscalationConfigSchema = v.object({
81
46
  ),
82
47
  ),
83
48
  resolve_after_minutes: v.number(),
84
- update_after_minutes: v.number(),
85
49
  /** The orchestrator's work item — escalations are COMMENTS on it, never
86
50
  * task rows (the fleet-task-placement CHECK constraint refuses a system
87
51
  * task without session identity, and an escalation is a finding, not an
@@ -90,6 +54,10 @@ const FaultEscalationConfigSchema = v.object({
90
54
  /** Per-kind copy for the message (impact + recommendation). Falls back to default_copy. */
91
55
  copy: v.record(v.string(), v.object({ impact: v.string(), recommendation: v.string() })),
92
56
  default_copy: v.object({ impact: v.string(), recommendation: v.string() }),
57
+ /** Per-kind payload-field whitelist sampled into the message's evidence
58
+ * line — named fields ONLY, never the whole payload: some kinds embed
59
+ * request context, and the escalation channel is WhatsApp. */
60
+ sample_fields: v.optional(v.record(v.string(), v.array(v.string()))),
93
61
  });
94
62
 
95
63
  type FaultEscalationConfig = v.InferOutput<typeof FaultEscalationConfigSchema>;
@@ -128,8 +96,9 @@ async function getSystemIdentity(): Promise<{ userId: string | undefined; organi
128
96
  * BACKEND service, where the page-server client is never registered — the
129
97
  * empty-router fallback returns a client with no model namespaces (the
130
98
  * TypeError this replaced). Per `no-direct-prisma-outside-orpc`, every WRITE
131
- * in this handler routes through this client; the single raw read (the
132
- * event_log recurrence aggregate, a GROUP BY no generated CRUD expresses) uses
99
+ * in this handler routes through this client; the two raw reads (the
100
+ * event_log recurrence aggregate and the latest-payload sample, shapes no
101
+ * generated CRUD expresses) use
133
102
  * the assigned-client form with $queryRawUnsafe. Timestamps are ISO strings,
134
103
  * never Date instances (the generated valibot schemas require it).
135
104
  */
@@ -184,9 +153,6 @@ interface OrpcSurface {
184
153
  messaging_bots: {
185
154
  findFirst(args: Record<string, unknown>): Promise<Record<string, unknown> | null>;
186
155
  };
187
- work_items: {
188
- create(args: { data: Record<string, unknown>; select?: Record<string, unknown> }): Promise<{ id: string }>;
189
- };
190
156
  comments: {
191
157
  create(args: { data: Record<string, unknown> }): Promise<{ id: string }>;
192
158
  };
@@ -201,6 +167,11 @@ async function fetchAggregates(patterns: string[], windowMinutes: number): Promi
201
167
  // Patterns come from the valibot-validated registry row and are regex-only
202
168
  // by construction below; embedding them as one alternation keeps the query
203
169
  // parameter-free on the pattern side while the window is a real parameter.
170
+ // The window is bounded on BOTH sides by the database's own statement
171
+ // clock: `occurred_at` is producer-claimed, and an unbounded top edge
172
+ // admits rows stamped in the future by a skewed or mocked producer clock —
173
+ // which then fabricate "still recurring" for a quiet fault (measured
174
+ // 2026-08-26). A future row is not evidence of anything happening now.
204
175
  const regexClass = /^[A-Za-z0-9_\\^$.*+?()[\]{}|-]+$/;
205
176
  for (const pattern of patterns) {
206
177
  if (!regexClass.test(pattern)) {
@@ -210,10 +181,11 @@ async function fetchAggregates(patterns: string[], windowMinutes: number): Promi
210
181
  const combined = `(${patterns.join("|")})`;
211
182
  const rows = await prisma.$queryRawUnsafe<EventLogAggregateRow[]>(
212
183
  `SELECT kind, count(*)::bigint AS n, min(occurred_at) AS first_seen, max(occurred_at) AS last_seen
213
- FROM event_log
214
- WHERE occurred_at > now() - ($1::int * interval '1 minute')
215
- AND kind ~ $2::text
216
- GROUP BY kind`,
184
+ FROM event_log
185
+ WHERE occurred_at > now() - ($1::int * interval '1 minute')
186
+ AND occurred_at <= now()
187
+ AND kind ~ $2::text
188
+ GROUP BY kind`,
217
189
  windowMinutes,
218
190
  combined,
219
191
  );
@@ -225,6 +197,70 @@ async function fetchAggregates(patterns: string[], windowMinutes: number): Promi
225
197
  }));
226
198
  }
227
199
 
200
+ /**
201
+ * The message's evidence: the whitelisted sample fields AND the row identity
202
+ * (id + occurred_at) of the newest payload for the kind. Field names come
203
+ * from the valibot-validated config row and are constrained to bare
204
+ * identifiers below, so the JSON path cannot break out of the string. The
205
+ * `occurred_at <= now()` bound matches the aggregate's top edge: the cited
206
+ * "last" row must be a real, non-future event whose id rides in the message
207
+ * so the claim resolves against event_log.
208
+ */
209
+ async function fetchLatestEvidence(
210
+ kind: string,
211
+ fields: string[],
212
+ ): Promise<{ sample: string; lastEvent: { occurredAt: string; eventId: string } | null }> {
213
+ const prisma = getPrisma();
214
+ if (fields.length === 0) {
215
+ const rows = await prisma.$queryRawUnsafe<Array<{ id: unknown; occurred_at: Date }>>(
216
+ `SELECT id::text, occurred_at
217
+ FROM event_log
218
+ WHERE kind = $1::text AND occurred_at <= now()
219
+ ORDER BY occurred_at DESC
220
+ LIMIT 1`,
221
+ kind,
222
+ );
223
+ const top = rows[0];
224
+ return { sample: "", lastEvent: top ? { occurredAt: isoSeconds(top.occurred_at), eventId: String(top.id) } : null };
225
+ }
226
+ for (const field of fields) {
227
+ if (!/^[A-Za-z0-9_]+$/.test(field)) {
228
+ throw new Error(`config/${CONFIG_SLUG}: sample_fields entry is not a bare field name: ${field}`);
229
+ }
230
+ }
231
+ const selection = fields.map((field) => `payload_json->'${field}' AS "${field}"`).join(", ");
232
+ const rows = await prisma.$queryRawUnsafe<Array<Record<string, unknown>>>(
233
+ `SELECT id::text, occurred_at, ${selection}
234
+ FROM event_log
235
+ WHERE kind = $1::text AND occurred_at <= now()
236
+ ORDER BY occurred_at DESC
237
+ LIMIT 1`,
238
+ kind,
239
+ );
240
+ const row = rows[0];
241
+ if (!row) return { sample: "", lastEvent: null };
242
+ const parts: string[] = [];
243
+ for (const field of fields) {
244
+ const value = row[field];
245
+ if (value === null || value === undefined) continue;
246
+ const formatted = formatEscalationSample(value);
247
+ if (formatted) parts.push(`${field}=${formatted}`);
248
+ }
249
+ return {
250
+ sample: parts.join("; "),
251
+ lastEvent: {
252
+ occurredAt: isoSeconds(row.occurred_at as string | Date),
253
+ eventId: String(row.id),
254
+ },
255
+ };
256
+ }
257
+
258
+ /** ISO-8601 to whole seconds — the same precision event_log stores, so the
259
+ * cited string round-trips a `WHERE occurred_at = '<cited>'` query. */
260
+ function isoSeconds(value: Date | string): string {
261
+ return new Date(value).toISOString().replace(/\.\d{3}Z$/, "Z");
262
+ }
263
+
228
264
  async function resolveOperatorRelay(): Promise<OperatorRelayBot> {
229
265
  const bot = (await (await orpc()).messaging_bots.findFirst({
230
266
  where: {
@@ -284,9 +320,12 @@ async function deliverEscalation(
284
320
  const bot = await resolveOperatorRelay();
285
321
  const operatorPhone = (bot.flow_config as { operator_phone?: string }).operator_phone!;
286
322
  const copy = config.copy[kind] ?? config.default_copy;
323
+ const { sample, lastEvent } = await fetchLatestEvidence(kind, config.sample_fields?.[kind] ?? []);
287
324
  const message = composeEscalationMessage({
288
325
  kind,
289
326
  aggregate,
327
+ sample,
328
+ lastEvent,
290
329
  impact: copy.impact,
291
330
  recommendation: copy.recommendation,
292
331
  });
@@ -325,12 +364,7 @@ async function deliverEscalation(
325
364
  const comment = await (await orpc()).comments.create({
326
365
  data: {
327
366
  work_item_id: config.orchestrator_work_item_id,
328
- content: composeEscalationMessage({
329
- kind,
330
- aggregate,
331
- impact: copy.impact,
332
- recommendation: copy.recommendation,
333
- }),
367
+ content: message,
334
368
  source_cli_session_id: null,
335
369
  },
336
370
  });
@@ -353,6 +387,9 @@ async function deliverEscalation(
353
387
  payload: {
354
388
  event_kind: kind,
355
389
  count: aggregate.count,
390
+ sample,
391
+ last_event_id: lastEvent?.eventId ?? null,
392
+ last_event_occurred_at: lastEvent?.occurredAt ?? null,
356
393
  work_item_id: config.orchestrator_work_item_id,
357
394
  delivered_comment_id: comment.id,
358
395
  delivered_message_id: messageId,
@@ -361,33 +398,6 @@ async function deliverEscalation(
361
398
  return { messageId, workItemId: config.orchestrator_work_item_id };
362
399
  }
363
400
 
364
- async function updateEpisode(aggregate: FaultAggregate, episode: EscalationEpisode): Promise<void> {
365
- const now = new Date();
366
- await (await orpc()).platform_fault_escalations.update({
367
- where: { kind: episode.kind },
368
- data: {
369
- last_seen_at: aggregate.lastSeen.toISOString(),
370
- event_count: aggregate.count,
371
- last_updated_at: now.toISOString(),
372
- updated_at: now.toISOString(),
373
- },
374
- });
375
- if (episode.workItemId) {
376
- await (await orpc()).comments.create({
377
- data: {
378
- work_item_id: episode.workItemId,
379
- content: `Still recurring: ${aggregate.count} events, last at ${aggregate.lastSeen.toISOString()}. No new escalation (already escalated and unresolved).`,
380
- sender_role: "system",
381
- sender_name: "platform-fault-escalation",
382
- },
383
- });
384
- }
385
- recordEvent({
386
- kind: "platform.fault.updated",
387
- payload: { event_kind: episode.kind, count: aggregate.count, work_item_id: episode.workItemId },
388
- });
389
- }
390
-
391
401
  async function resolveEpisode(episode: EscalationEpisode, quietMinutes: number): Promise<void> {
392
402
  const now = new Date();
393
403
  await (await orpc()).platform_fault_escalations.update({
@@ -456,7 +466,6 @@ async function platformFaultEscalationSweep(ctx: { config: ScheduledJobConfig })
456
466
  minCount: config.min_count,
457
467
  minSpanMinutes: config.min_span_minutes,
458
468
  resolveAfterMinutes: config.resolve_after_minutes,
459
- updateAfterMinutes: config.update_after_minutes,
460
469
  },
461
470
  now,
462
471
  kindThresholds,
@@ -477,13 +486,6 @@ async function platformFaultEscalationSweep(ctx: { config: ScheduledJobConfig })
477
486
  recordEvent({ kind: "platform.fault.escalation_failed", payload: { event_kind: aggregate.kind, reason } });
478
487
  }
479
488
  }
480
- for (const { aggregate, episode } of decision.update) {
481
- try {
482
- await updateEpisode(aggregate, episode);
483
- } catch (err) {
484
- log.error(`[platform-fault-escalation] update for '${episode.kind}' failed: ${err instanceof Error ? err.message : String(err)}`);
485
- }
486
- }
487
489
  for (const { episode, quietMinutes } of decision.resolve) {
488
490
  try {
489
491
  await resolveEpisode(episode, quietMinutes);
@@ -499,18 +501,17 @@ async function platformFaultEscalationSweep(ctx: { config: ScheduledJobConfig })
499
501
  payload: {
500
502
  watched_kinds: aggregates.length,
501
503
  escalated: decision.escalate.length,
502
- updated: decision.update.length,
503
504
  resolved: decision.resolve.length,
504
505
  },
505
506
  });
506
- if (decision.escalate.length > 0 || decision.update.length > 0 || decision.resolve.length > 0) {
507
+ if (decision.escalate.length > 0 || decision.resolve.length > 0) {
507
508
  log.info(
508
- `[platform-fault-escalation] sweep — ${aggregates.length} kind(s) watched, ${decision.escalate.length} escalated, ${decision.update.length} updated, ${decision.resolve.length} resolved`,
509
+ `[platform-fault-escalation] sweep — ${aggregates.length} kind(s) watched, ${decision.escalate.length} escalated, ${decision.resolve.length} resolved`,
509
510
  );
510
511
  }
511
- }
512
+ }
512
513
 
513
- pgBossHandlerRegistry.register(
514
+ pgBossHandlerRegistry.register(
514
515
  "@teamscala/service-boot:platform-fault-escalation",
515
516
  platformFaultEscalationSweep as (ctx: { config: ScheduledJobConfig }) => Promise<void>,
516
517
  );