@teamscala/service-boot 0.23.428 → 0.23.430

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -6,7 +6,7 @@
6
6
  },
7
7
  "name": "@teamscala/service-boot",
8
8
  "type": "module",
9
- "version": "0.23.428",
9
+ "version": "0.23.430",
10
10
  "sideEffects": false,
11
11
  "imports": {
12
12
  "#service-boot/*": "./src/*",
@@ -33,6 +33,12 @@ export const FaultEscalationConfigSchema = v.object({
33
33
  ),
34
34
  ),
35
35
  resolve_after_minutes: v.number(),
36
+ /** Kinds a standing agent lane answers, which therefore escalate to the
37
+ * work item WITHOUT paging the operator (operator ruling 2026-08-28: "why
38
+ * is the WhatsApp PA messaging me about stuff that you should be doing").
39
+ * The episode, the comment and the durable record are unchanged — only the
40
+ * phone is spared, so silencing a page never silences the signal. */
41
+ agent_handled_kinds: v.optional(v.array(v.string())),
36
42
  /** Per-kind volume sources closing the quiet-vs-dead gap: a fault kind
37
43
  * whose producer can DIE maps to the kind whose row VOLUME proves the
38
44
  * pipeline alive. Without a source, resolution on quiet is unchanged;
@@ -18,6 +18,7 @@ import {
18
18
  composeEscalationMessage,
19
19
  composeZeroVolumeMessage,
20
20
  extractWhatsAppMessageId,
21
+ pagesOperator,
21
22
  type EscalationEpisode,
22
23
  type FaultAggregate,
23
24
  type ZeroVolumeSource,
@@ -53,7 +54,7 @@ export async function deliverEscalation(
53
54
  kind: string,
54
55
  aggregate: FaultAggregate,
55
56
  config: FaultEscalationConfig,
56
- ): Promise<{ messageId: string; workItemId: string }> {
57
+ ): Promise<{ messageId: string | null; workItemId: string }> {
57
58
  const now = new Date();
58
59
 
59
60
  // 1. Claim the episode BEFORE sending, so a send failure leaves a durable
@@ -80,8 +81,11 @@ export async function deliverEscalation(
80
81
  // that picks the device and records the send in messaging_logs. The retired
81
82
  // api_gowa_message tool (deactivated 2026-08-26) is no longer callable;
82
83
  // naming it here left every escalation send failing with "Unknown tool".
83
- const bot = await resolveOperatorRelay();
84
- const operatorPhone = (bot.flow_config as { operator_phone?: string }).operator_phone!;
84
+
85
+ // PAGING IS PER-KIND (operator ruling 2026-08-28). Only the phone is
86
+ // skipped; episode, comment and record are identical, so a silenced page
87
+ // never becomes a silenced signal.
88
+ const paging = pagesOperator(kind, config.agent_handled_kinds);
85
89
  const copy = config.copy[kind] ?? config.default_copy;
86
90
  const { sample, lastEvent } = await fetchLatestEvidence(kind, config.sample_fields?.[kind] ?? []);
87
91
  const message = composeEscalationMessage({
@@ -93,23 +97,31 @@ export async function deliverEscalation(
93
97
  recommendation: copy.recommendation,
94
98
  });
95
99
 
96
- const { userId, organisationId } = await getSystemIdentity();
97
- const systemCtx: ExecutionContext = { userId, organisationId } as ExecutionContext;
98
- const { forwardToolToCentralMcp } = await import("@teamscala/chat-completion/central-tool-forwarder");
99
- const sendResult = await forwardToolToCentralMcp(
100
- "messaging_whatsapp_send",
101
- { phone: operatorPhone, message },
102
- systemCtx,
103
- );
104
-
105
100
  // 3. VERIFY delivery before recording it: the send must return a WhatsApp
106
101
  // message id. No id => throw, the claim stays undelivered, next sweep retries.
107
- const messageId = extractWhatsAppMessageId(sendResult);
108
- if (!messageId) {
109
- throw new Error(
110
- `platform-fault-escalation: WhatsApp send for '${kind}' returned no message id — not recording delivery. ` +
111
- `Response: ${JSON.stringify(sendResult).slice(0, 400)}`,
102
+ // An agent-handled kind sends nothing, so there is no id to verify and none
103
+ // is recorded — absence here means "never paged", never "paged unverified".
104
+ let messageId: string | null = null;
105
+ if (paging) {
106
+ const bot = await resolveOperatorRelay();
107
+ const operatorPhone = (bot.flow_config as { operator_phone?: string }).operator_phone!;
108
+ const { userId, organisationId } = await getSystemIdentity();
109
+ const systemCtx: ExecutionContext = { userId, organisationId } as ExecutionContext;
110
+ const { forwardToolToCentralMcp } = await import(
111
+ "@teamscala/chat-completion/central-tool-forwarder"
112
112
  );
113
+ const sendResult = await forwardToolToCentralMcp(
114
+ "messaging_whatsapp_send",
115
+ { phone: operatorPhone, message },
116
+ systemCtx,
117
+ );
118
+ messageId = extractWhatsAppMessageId(sendResult);
119
+ if (!messageId) {
120
+ throw new Error(
121
+ `platform-fault-escalation: WhatsApp send for '${kind}' returned no message id — not recording delivery. ` +
122
+ `Response: ${JSON.stringify(sendResult).slice(0, 400)}`,
123
+ );
124
+ }
113
125
  }
114
126
 
115
127
  // 4. Durable record: a COMMENT on the orchestrator's work item — an
@@ -178,6 +190,30 @@ export async function resolveEpisode(
178
190
  });
179
191
  }
180
192
 
193
+ /**
194
+ * Refresh an OPEN episode's window facts from the live aggregate.
195
+ *
196
+ * Escalation wrote the row once and never again, so its facts froze while the
197
+ * fault kept firing (2026-08-28: fifteen hours stale, 1,150 later events).
198
+ * That also corrupts RESOLUTION, since `decideSweep` falls back to
199
+ * `lastSeenAt` once a kind leaves the window. Fields mirror the escalation
200
+ * upsert. `last_updated_at` is untouched: it gates the resolve-hold dedupe.
201
+ */
202
+ export async function refreshOpenEpisode(
203
+ kind: string,
204
+ aggregate: { firstSeen: Date; lastSeen: Date; count: number },
205
+ ): Promise<void> {
206
+ await (await orpc()).platform_fault_escalations.update({
207
+ where: { kind },
208
+ data: {
209
+ first_seen_at: aggregate.firstSeen.toISOString(),
210
+ last_seen_at: aggregate.lastSeen.toISOString(),
211
+ event_count: aggregate.count,
212
+ updated_at: new Date().toISOString(),
213
+ },
214
+ });
215
+ }
216
+
181
217
  /** Send one message to the operator through the verified WhatsApp relay and
182
218
  * record it as a comment on the orchestrator's work item. Returns the
183
219
  * verified ids; throws when the send carries no message id so the caller's
@@ -229,6 +229,32 @@ export function decideSweep(
229
229
  }
230
230
 
231
231
  /** Human "N min" span, rounded — for the phone-sized escalation message. */
232
+ /**
233
+ * Do this OPEN episode's stored window facts lag the live aggregate?
234
+ *
235
+ * Both NEGATIVE answers carry weight: a quiet kind keeps its last true
236
+ * sighting (the value resolve falls back to), and an aggregate no newer than
237
+ * the row is never written, so a steady-state sweep writes nothing.
238
+ */
239
+ export function episodeNeedsRefresh(
240
+ episode: Pick<EscalationEpisode, "lastSeenAt">,
241
+ aggregate: { lastSeen: Date } | undefined,
242
+ ): boolean {
243
+ if (!aggregate) return false;
244
+ return aggregate.lastSeen.getTime() > episode.lastSeenAt.getTime();
245
+ }
246
+
247
+ /**
248
+ * Does this fault kind page the operator, or land on the work item alone?
249
+ *
250
+ * DEFAULT IS TO PAGE. An unlisted kind reaches the phone, so forgetting to
251
+ * classify a new fault fails loudly rather than silently — the opposite
252
+ * default would let a kind be silenced by omission.
253
+ */
254
+ export function pagesOperator(kind: string, agentHandledKinds?: string[]): boolean {
255
+ return !(agentHandledKinds ?? []).includes(kind);
256
+ }
257
+
232
258
  export function formatSpanMinutes(minutes: number): string {
233
259
  if (minutes < 60) return `${Math.round(minutes)} min`;
234
260
  const hours = minutes / 60;
@@ -0,0 +1,85 @@
1
+ /**
2
+ * @system server-boot
3
+ * @status handwritten
4
+ * @edit edit directly
5
+ *
6
+ * Unit tests for episode FACT CURRENCY — a separate concern from the
7
+ * threshold/dedup semantics locked by `platform-fault-escalation.test.ts`,
8
+ * and kept in its own file because that one already sits over the size
9
+ * budget.
10
+ *
11
+ * WHAT THIS LOCKS. The sweep escalated once and then never wrote the row
12
+ * again, so an open episode's window facts froze at their escalation-time
13
+ * values while the fault kept firing. Measured 2026-08-28: the
14
+ * `host-command-bus.command.failed` row read `last_seen_at` fifteen hours old
15
+ * with 1,150 further events recorded after it, and a reader — this lane
16
+ * included — concluded from the row that the fault had stopped.
17
+ *
18
+ * The stale value is not only cosmetic: `decideSweep` falls back to
19
+ * `episode.lastSeenAt` once a kind drops out of the window, so a frozen
20
+ * timestamp makes a just-gone-quiet fault look quiet for hours and closes the
21
+ * episode immediately instead of after its configured resolve window.
22
+ */
23
+ import { describe, expect, test } from "bun:test";
24
+ import { episodeNeedsRefresh, pagesOperator } from "./platform-fault-escalation-logic.ts";
25
+
26
+ const at = (iso: string) => ({ lastSeenAt: new Date(iso) });
27
+ const seen = (iso: string) => ({ lastSeen: new Date(iso) });
28
+
29
+ describe("episodeNeedsRefresh", () => {
30
+ test("a still-firing fault whose row has fallen behind is refreshed", () => {
31
+ // The measured incident, in miniature: row frozen at escalation time,
32
+ // events still arriving fifteen hours later.
33
+ expect(episodeNeedsRefresh(at("2026-08-27T12:25:03Z"), seen("2026-08-28T03:00:41Z"))).toBe(
34
+ true,
35
+ );
36
+ });
37
+
38
+ test("a kind that has gone quiet keeps its last true sighting", () => {
39
+ // No aggregate means nothing landed in the window. The stored value must
40
+ // STAND rather than advance, because it is the reference the resolve
41
+ // decision falls back to — advancing it here would hold every quiet
42
+ // episode open forever.
43
+ expect(episodeNeedsRefresh(at("2026-08-27T12:25:03Z"), undefined)).toBe(false);
44
+ });
45
+
46
+ test("an unchanged fault is not written back", () => {
47
+ // Equal timestamps must not write: a sweep over steady state performs
48
+ // zero updates, so the refresh cannot become a per-sweep write storm.
49
+ const same = "2026-08-28T03:00:41Z";
50
+ expect(episodeNeedsRefresh(at(same), seen(same))).toBe(false);
51
+ });
52
+
53
+ test("an aggregate OLDER than the row never rewinds it", () => {
54
+ // A shrinking window can report an older newest-event than the row
55
+ // already holds. Rewinding would fabricate quiet time and resolve a
56
+ // live fault early.
57
+ expect(episodeNeedsRefresh(at("2026-08-28T03:00:41Z"), seen("2026-08-28T01:00:00Z"))).toBe(
58
+ false,
59
+ );
60
+ });
61
+ });
62
+
63
+ describe("pagesOperator", () => {
64
+ // Operator ruling 2026-08-28: "why is the WhatsApp PA messaging me about
65
+ // stuff that you should be doing". A kind a standing lane answers escalates
66
+ // to the work item; the phone is spared.
67
+ const AGENT_HANDLED = ["guard.fail.open", "host-command-bus.command.failed"];
68
+
69
+ test("a kind the maintenance lane answers does not page", () => {
70
+ expect(pagesOperator("guard.fail.open", AGENT_HANDLED)).toBe(false);
71
+ expect(pagesOperator("host-command-bus.command.failed", AGENT_HANDLED)).toBe(false);
72
+ });
73
+
74
+ test("an unlisted kind still pages", () => {
75
+ expect(pagesOperator("supervisor.halt.entered", AGENT_HANDLED)).toBe(true);
76
+ });
77
+
78
+ test("DEFAULT IS TO PAGE when nothing is configured", () => {
79
+ // The load-bearing direction: a kind is silenced only by being named.
80
+ // If the list is missing or empty, every fault reaches the operator, so
81
+ // a new kind can never be silenced by omission.
82
+ expect(pagesOperator("guard.fail.open", undefined)).toBe(true);
83
+ expect(pagesOperator("guard.fail.open", [])).toBe(true);
84
+ });
85
+ });
@@ -20,6 +20,7 @@ import { getAppLogger } from "@teamscala/logger/app-loggers";
20
20
  import { pgBossHandlerRegistry } from "../pg-boss-handler-registry.ts";
21
21
  import {
22
22
  decideSweep,
23
+ episodeNeedsRefresh,
23
24
  type EscalationEpisode,
24
25
  type KindThresholdOverride,
25
26
  type ZeroVolumeInputs,
@@ -37,6 +38,7 @@ import {
37
38
  clearZeroVolumeHold,
38
39
  deliverEscalation,
39
40
  deliverZeroVolumeHold,
41
+ refreshOpenEpisode,
40
42
  resolveEpisode,
41
43
  } from "./platform-fault-escalation/delivery.ts";
42
44
 
@@ -117,7 +119,7 @@ async function platformFaultEscalationSweep(ctx: { config: ScheduledJobConfig })
117
119
  try {
118
120
  const delivered = await deliverEscalation(aggregate.kind, aggregate, config);
119
121
  log.warn(
120
- `[platform-fault-escalation] ESCALATED '${aggregate.kind}' — WhatsApp message ${delivered.messageId}, work item ${delivered.workItemId}`,
122
+ `[platform-fault-escalation] ESCALATED '${aggregate.kind}' — ${delivered.messageId ? `WhatsApp message ${delivered.messageId}` : "work item only (agent-handled kind, operator not paged)"}, work item ${delivered.workItemId}`,
121
123
  );
122
124
  } catch (err) {
123
125
  const reason = err instanceof Error ? err.message : String(err);
@@ -125,6 +127,24 @@ async function platformFaultEscalationSweep(ctx: { config: ScheduledJobConfig })
125
127
  recordEvent({ kind: "platform.fault.escalation_failed", payload: { event_kind: aggregate.kind, reason } });
126
128
  }
127
129
  }
130
+ // AN OPEN EPISODE KEEPS ITS FACTS CURRENT. Escalation is a TRANSITION and
131
+ // fires once (`emit-transitions-not-conditions`) — but the ROW is the
132
+ // durable record of the condition, and a record that stops updating starts
133
+ // lying. Refreshing is not emitting: nothing is delivered here, no comment
134
+ // is written, no message is sent. Only the stored window facts move.
135
+ const aggregateByKind = new Map(aggregates.map((aggregate) => [aggregate.kind, aggregate]));
136
+ for (const episode of episodes) {
137
+ const aggregate = aggregateByKind.get(episode.kind);
138
+ if (!episodeNeedsRefresh(episode, aggregate) || !aggregate) continue;
139
+ try {
140
+ await refreshOpenEpisode(episode.kind, aggregate);
141
+ } catch (err) {
142
+ log.error(
143
+ `[platform-fault-escalation] refresh for '${episode.kind}' failed (facts stay stale for this sweep, retried next): ${err instanceof Error ? err.message : String(err)}`,
144
+ );
145
+ }
146
+ }
147
+
128
148
  for (const { episode, quietMinutes } of decision.zeroVolumeHold) {
129
149
  const source = volumeSources[episode.kind];
130
150
  if (!source) continue;