@teamscala/service-boot 0.23.407 → 0.23.409

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -6,7 +6,7 @@
6
6
  },
7
7
  "name": "@teamscala/service-boot",
8
8
  "type": "module",
9
- "version": "0.23.407",
9
+ "version": "0.23.409",
10
10
  "sideEffects": false,
11
11
  "imports": {
12
12
  "#service-boot/*": "./src/*",
@@ -30,6 +30,7 @@ import { publicGptSessionRouter } from "./public-gpt-session-router.ts";
30
30
  import { signInFlowRouter } from "./sign-in-flow-router.ts";
31
31
  import { userPreferencesRouter } from "./user-preferences-router.ts";
32
32
  import { workItemsRouter } from "./work-items-router.ts";
33
+ import { messagingWhatsappRouter } from "./messaging-whatsapp-router.ts";
33
34
 
34
35
  export function registerFnRouters(): void {
35
36
  registerCustomRouter("fn", {
@@ -38,6 +39,7 @@ export function registerFnRouters(): void {
38
39
  auth: authRouter,
39
40
  cacheAdmin: cacheAdminRouter,
40
41
  chatSession: chatSessionRouter,
42
+ messagingWhatsappSend: messagingWhatsappRouter.messagingWhatsappSend,
41
43
  publicAgreement: publicAgreementRouter,
42
44
  publicAssessment: publicAssessmentRouter,
43
45
  publicGptSession: publicGptSessionRouter,
@@ -0,0 +1,119 @@
1
+ /**
2
+ * @system service-boot
3
+ * @status handwritten
4
+ * @edit edit directly
5
+ *
6
+ * Proves fn.messagingWhatsappSend routes through scala-messaging-service: the
7
+ * URL is DERIVED from the service_install row (never spelled), the sealed-row
8
+ * credential is passed through the decrypt mock (identity, matching the
9
+ * public-assessment-router test pattern), the request carries the internal
10
+ * auth header, and the caller receives WhatsApp's message_id — the only thing
11
+ * that makes delivery provable. Failure modes (401, blank input) fail loudly
12
+ * with named causes, never a generic send error.
13
+ */
14
+
15
+ import { createRouterClient } from "@orpc/server";
16
+ import { afterAll, describe, expect, mock, test } from "bun:test";
17
+
18
+ const SEND_CALLS: Array<{ url: string; headers: Record<string, string>; body: unknown }> = [];
19
+
20
+ mock.module("@teamscala/db/registry/load-config", () => ({
21
+ loadRegistryConfig: async (type: string, slug: string) => {
22
+ if (type === "service_install" && slug === "scala-messaging-service") {
23
+ return { deploy_target: "railway", port: 8080 };
24
+ }
25
+ if (type === "secret" && slug === "railway-scala-messaging-whatsapp-webhook-secret") {
26
+ return { token: "test-token" };
27
+ }
28
+ throw new Error(`unexpected loadRegistryConfig(${type}, ${slug})`);
29
+ },
30
+ }));
31
+
32
+ // Identity decrypt (the public-assessment-router test pattern).
33
+ mock.module("@teamscala/encryption/crypto/secret-config", () => ({
34
+ decryptSecretConfig: (raw: unknown) => raw,
35
+ }));
36
+
37
+ mock.module("@teamscala/logger/app-loggers", () => ({
38
+ getAppLogger: () => ({
39
+ info: () => {},
40
+ warn: () => {},
41
+ error: () => {},
42
+ debug: () => {},
43
+ }),
44
+ }));
45
+
46
+ const originalFetch = globalThis.fetch;
47
+ (globalThis as { fetch: unknown }).fetch = async (url: string, init: RequestInit) => {
48
+ SEND_CALLS.push({
49
+ url,
50
+ headers: init.headers as Record<string, string>,
51
+ body: JSON.parse(init.body as string),
52
+ });
53
+ const body = JSON.parse(init.body as string) as { phone?: string };
54
+ if (body.phone === "0-0-1-refuse") {
55
+ return new Response("no", { status: 401 });
56
+ }
57
+ return Response.json({ message_id: "3EB07EE8TESTID" });
58
+ };
59
+
60
+ const { messagingWhatsappRouter } = await import("./messaging-whatsapp-router.ts");
61
+
62
+ // Invoke through a real ORPC client so the authed middleware runs too —
63
+ // calling the DecoratedProcedure directly is not a round trip.
64
+ const client = createRouterClient(
65
+ { messagingWhatsappSend: messagingWhatsappRouter.messagingWhatsappSend } as never,
66
+ {
67
+ context: {
68
+ user: { id: "system", organisationId: null },
69
+ headers: new Headers(),
70
+ prisma: {},
71
+ } as never,
72
+ },
73
+ ) as { messagingWhatsappSend: (a: unknown) => Promise<unknown> };
74
+
75
+ describe("fn.messagingWhatsappSend", () => {
76
+ test("derives the URL from service_install, sends with the decrypted header value, returns the message_id", async () => {
77
+ SEND_CALLS.length = 0;
78
+ const result = await client.messagingWhatsappSend({
79
+ phone: "61428193498",
80
+ message: "migration probe",
81
+ });
82
+
83
+ expect(SEND_CALLS).toHaveLength(1);
84
+ expect(SEND_CALLS[0]?.url).toBe(
85
+ "http://scala-messaging-service.railway.internal:8080/api/messaging/whatsapp/send",
86
+ );
87
+ expect(SEND_CALLS[0]?.headers["x-webhook-secret"]).toBe("test-token");
88
+ expect(SEND_CALLS[0]?.body).toEqual({ phone: "61428193498", message: "migration probe" });
89
+ expect(result).toEqual({ ok: true, message_id: "3EB07EE8TESTID" });
90
+ });
91
+
92
+ test("a 401 from the messaging service names the credential, not a generic failure", async () => {
93
+ SEND_CALLS.length = 0;
94
+ const result = await client.messagingWhatsappSend({
95
+ phone: "0-0-1-refuse",
96
+ message: "x",
97
+ });
98
+ expect(result).toEqual({
99
+ ok: false,
100
+ error: "messaging service refused the webhook secret (401)",
101
+ });
102
+ });
103
+
104
+ test("blank phone or message is refused before any request is made", async () => {
105
+ SEND_CALLS.length = 0;
106
+ const blank = await client.messagingWhatsappSend({
107
+ phone: " ",
108
+ message: "x",
109
+ });
110
+ expect(blank).toEqual({ ok: false, error: "phone and message are required" });
111
+ expect(SEND_CALLS).toHaveLength(0);
112
+ });
113
+ });
114
+
115
+ // Restore INSIDE afterAll — a top-level restore would run before the tests
116
+ // (all top-level code executes first) and hand the router back the real fetch.
117
+ afterAll(() => {
118
+ (globalThis as { fetch: unknown }).fetch = originalFetch;
119
+ });
@@ -0,0 +1,180 @@
1
+ /**
2
+ * @system service-boot
3
+ * @status handwritten
4
+ * @edit edit directly
5
+ *
6
+ * fn.messagingWhatsappSend — the agent-facing WhatsApp send that routes through
7
+ * scala-messaging-service (the Rust owner of the channel), replacing the
8
+ * retired TS @teamscala/api-gowa SDK send path (operator ruling 2026-08-26:
9
+ * "make sure the migration is finished, i cant have half ts and half rust").
10
+ *
11
+ * The messaging service's POST /api/messaging/whatsapp/send is the single send
12
+ * surface: it resolves the sending device, returns WhatsApp's message_id for
13
+ * delivery correlation, and records the send in messaging_logs itself (Rust,
14
+ * `write_outbound_tracked`) — better than the deleted TS recorder, which only
15
+ * ran when the send happened to flow through tool-executor.
16
+ *
17
+ * Both facts this procedure needs are DERIVED at call time, never spelled:
18
+ * - the service URL comes from the service_install row (slug + port +
19
+ * deploy_target → Railway private DNS <slug>.railway.internal:<port>), the
20
+ * same derivation every *.railway.internal consumer uses;
21
+ * - the auth credential is the existing sealed secret row
22
+ * railway-scala-messaging-whatsapp-webhook-secret, decrypted per
23
+ * registry-via-helper + the agreement-send precedent.
24
+ *
25
+ * Registered flat under the `fn` namespace (two-segment procedure
26
+ * "fn.messagingWhatsappSend") because the tool-executor ORPC dispatch resolves
27
+ * "model.method" — nested routers would not be reachable as a tool procedure.
28
+ */
29
+
30
+ import { os } from "@orpc/server";
31
+ import * as v from "valibot";
32
+ import {
33
+ decryptSecretConfig,
34
+ type SecretConfigLike,
35
+ } from "@teamscala/encryption/crypto/secret-config";
36
+ import { loadRegistryConfig } from "@teamscala/db/registry/load-config";
37
+ import { getAppLogger } from "@teamscala/logger/app-loggers";
38
+
39
+ interface User {
40
+ id: string;
41
+ organisationId: string | null;
42
+ }
43
+
44
+ interface Context {
45
+ user: User | null | undefined;
46
+ headers: Headers;
47
+ resolveUser?: (headers: Headers, prisma: any) => Promise<User | null>;
48
+ prisma: any;
49
+ }
50
+
51
+ const authed = os.$context<Context>().use(async (opt) => {
52
+ const { context } = opt;
53
+ let user = context.user;
54
+ if (!user && context.headers && context.resolveUser) {
55
+ user = await context.resolveUser(context.headers, context.prisma);
56
+ }
57
+ if (!user) {
58
+ const { ORPCError } = await import("@orpc/server");
59
+ throw new ORPCError("UNAUTHORIZED", { message: "Authentication required" });
60
+ }
61
+ return await opt.next({ context: { ...context, user } });
62
+ });
63
+
64
+ /** service_install shape for deriving the messaging service's internal URL. */
65
+ const MessagingInstallSchema = v.looseObject({
66
+ port: v.optional(v.number(), 8080),
67
+ deploy_target: v.string(),
68
+ });
69
+
70
+ /** The sealed webhook-secret row shape (value is decryptSecretConfig input). */
71
+ const MessagingSecretSchema = v.looseObject({});
72
+
73
+ /** Derive the messaging service base URL from its service_install row. */
74
+ async function resolveMessagingBaseUrl(): Promise<string> {
75
+ const install = await loadRegistryConfig(
76
+ "service_install",
77
+ "scala-messaging-service",
78
+ MessagingInstallSchema,
79
+ );
80
+ if (install.deploy_target !== "railway") {
81
+ throw new Error(
82
+ `scala-messaging-service deploy_target is "${install.deploy_target}", not "railway" — ` +
83
+ "the private-DNS URL derivation only holds on Railway. Fix the service_install row.",
84
+ );
85
+ }
86
+ return `http://scala-messaging-service.railway.internal:${install.port}`;
87
+ }
88
+
89
+ const sendSchema = v.object({
90
+ /** Destination phone number in E.164 (e.g. 61428193498). */
91
+ phone: v.string(),
92
+ /** Text body to send. */
93
+ message: v.string(),
94
+ });
95
+
96
+ export const messagingWhatsappRouter = {
97
+ messagingWhatsappSend: authed
98
+ .input(sendSchema)
99
+ .handler(async ({ input }) => {
100
+ const log = getAppLogger();
101
+ const phone = input.phone.trim();
102
+ const message = input.message;
103
+ if (!phone || !message) {
104
+ return { ok: false as const, error: "phone and message are required" };
105
+ }
106
+
107
+ try {
108
+ const [baseUrl, rawSecret] = await Promise.all([
109
+ resolveMessagingBaseUrl(),
110
+ loadRegistryConfig(
111
+ "secret",
112
+ "railway-scala-messaging-whatsapp-webhook-secret",
113
+ MessagingSecretSchema,
114
+ ).catch(() => {
115
+ throw new Error(
116
+ "secret row railway-scala-messaging-whatsapp-webhook-secret is missing — " +
117
+ "the messaging service send endpoint cannot be authenticated",
118
+ );
119
+ }),
120
+ ]);
121
+ const decrypted = decryptSecretConfig(
122
+ rawSecret as SecretConfigLike,
123
+ ) as { token?: string };
124
+ if (!decrypted.token) {
125
+ return {
126
+ ok: false as const,
127
+ error: "whatsapp webhook secret row has no token",
128
+ };
129
+ }
130
+
131
+ const response = await fetch(
132
+ `${baseUrl}/api/messaging/whatsapp/send`,
133
+ {
134
+ method: "POST",
135
+ headers: {
136
+ "Content-Type": "application/json",
137
+ // The Rust handler accepts x-webhook-secret (or
138
+ // x-api-key) against WHATSAPP_WEBHOOK_SECRET.
139
+ "x-webhook-secret": decrypted.token,
140
+ },
141
+ body: JSON.stringify({ phone, message }),
142
+ signal: AbortSignal.timeout(30_000),
143
+ },
144
+ );
145
+ if (response.status === 401) {
146
+ return {
147
+ ok: false as const,
148
+ error: "messaging service refused the webhook secret (401)",
149
+ };
150
+ }
151
+ const body = (await response.json().catch(() => null)) as {
152
+ message_id?: string;
153
+ error?: string;
154
+ } | null;
155
+ if (!response.ok) {
156
+ return {
157
+ ok: false as const,
158
+ error:
159
+ body?.error ??
160
+ `messaging service send failed: ${response.status}`,
161
+ };
162
+ }
163
+ // The message_id is the only thing that makes delivery provable —
164
+ // hand it back so the caller can correlate the ack (gowa.rs
165
+ // parse_message_id contract).
166
+ log.info("[fn.messagingWhatsappSend] sent", {
167
+ phone,
168
+ message_id: body?.message_id ?? null,
169
+ });
170
+ return {
171
+ ok: true as const,
172
+ message_id: body?.message_id ?? null,
173
+ };
174
+ } catch (err) {
175
+ const errorMessage = err instanceof Error ? err.message : String(err);
176
+ log.error(`[fn.messagingWhatsappSend] failed: ${errorMessage}`);
177
+ return { ok: false as const, error: errorMessage };
178
+ }
179
+ }),
180
+ };
@@ -1,6 +1,6 @@
1
1
  /**
2
2
  * @system service-boot
3
- * @status handwritten
3
+ * @status handwritten @indivisible-unit
4
4
  * @edit edit directly
5
5
  *
6
6
  * Public ORPC procedures for DocuSign-style agreement signing, reached by the
@@ -3,25 +3,9 @@
3
3
  * @status handwritten
4
4
  * @edit edit directly
5
5
  *
6
- * Pure decision logic for the platform-fault escalation sweep
7
- * (`platform-fault-escalation` handler). Split from the I/O shell so the
8
- * threshold/dedup/episode semantics are unit-testable with zero adapter
9
- * dependencies (same shape as whatsapp-liveness-logic).
10
- *
11
- * The semantics this locks (from the 2026-08-15 "760 errors escalated to
12
- * nobody" incident brief):
13
- * - Threshold on DURATION and RECURRENCE, never a single event. A kind
14
- * qualifies as a sustained fault only when BOTH hold inside the lookback
15
- * window: at least `min_count` occurrences (recurrence) spread over at
16
- * least `min_span_minutes` (duration). One failed push is noise; the same
17
- * kind recurring for half an hour is an outage.
18
- * - One escalation per distinct fault kind per episode. An episode is open
19
- * from first qualification until `resolve_after_minutes` of quiet; while
20
- * open, the fault never re-fires — it only appends throttled updates.
21
- * - Delivery is verified, not queued: an episode row with `escalated_at`
22
- * NULL is a CLAIM, not an escalation. The sweep re-attempts delivery for
23
- * claimed-but-undelivered episodes until the WhatsApp send returns a
24
- * message id.
6
+ * Pure decision logic for the platform-fault escalation sweep — full
7
+ * contract in curated `apps/escalation.md`. Episodes emit on TRANSITIONS
8
+ * ONLY; a future-dated `lastSeen` is never recurrence evidence.
25
9
  */
26
10
 
27
11
  export interface FaultAggregate {
@@ -29,6 +13,10 @@ export interface FaultAggregate {
29
13
  count: number;
30
14
  firstSeen: Date;
31
15
  lastSeen: Date;
16
+ /** Id of the newest row backing `lastSeen`, when the caller resolved it —
17
+ * the resolvable handle the escalation cites so "last at T" is checkable
18
+ * against a real event_log row, not a computed or cached value. */
19
+ lastEventId?: string | null;
32
20
  }
33
21
 
34
22
  export interface EscalationEpisode {
@@ -50,8 +38,6 @@ export interface FaultEscalationPolicy {
50
38
  minSpanMinutes: number;
51
39
  /** Quiet period that closes an episode and re-arms escalation, minutes. */
52
40
  resolveAfterMinutes: number;
53
- /** Minimum spacing between throttled updates on an open episode, minutes. */
54
- updateAfterMinutes: number;
55
41
  }
56
42
 
57
43
  /** Per-kind threshold overrides — a low-rate-but-persistent kind (guard.fail.open
@@ -65,8 +51,6 @@ export interface KindThresholdOverride {
65
51
  export interface SweepDecision {
66
52
  /** Sustained faults needing a FIRST delivery (new episode, or a claim whose send failed). */
67
53
  escalate: Array<{ aggregate: FaultAggregate; episode: EscalationEpisode | null }>;
68
- /** Open, delivered episodes still recurring — append evidence, never re-send. */
69
- update: Array<{ aggregate: FaultAggregate; episode: EscalationEpisode }>;
70
54
  /** Open episodes whose kind went quiet past the resolve window — close them. */
71
55
  resolve: Array<{ episode: EscalationEpisode; quietMinutes: number }>;
72
56
  }
@@ -88,9 +72,13 @@ function isSustained(
88
72
  // happening. Without this the 720-minute lookback keeps a dead fault
89
73
  // qualifying for hours, and since a resolved episode reads as no episode
90
74
  // it re-escalates on every sweep until its events age out (2026-08-26).
75
+ // A future `lastSeen` (producer clock skew / mocked test clock, measured
76
+ // 2026-08-26: "still recurring" citing a timestamp 44 min in its own
77
+ // future) makes minutesBetween go NEGATIVE — refuse it as evidence.
91
78
  return (
92
79
  aggregate.count >= minCount &&
93
80
  minutesBetween(aggregate.firstSeen, aggregate.lastSeen) >= minSpan &&
81
+ aggregate.lastSeen.getTime() <= now.getTime() &&
94
82
  minutesBetween(aggregate.lastSeen, now) < policy.resolveAfterMinutes
95
83
  );
96
84
  }
@@ -110,7 +98,7 @@ export function decideSweep(
110
98
  ): SweepDecision {
111
99
  const byKind = new Map(episodes.map((e) => [e.kind, e]));
112
100
  const aggregateByKind = new Map(aggregates.map((a) => [a.kind, a]));
113
- const decision: SweepDecision = { escalate: [], update: [], resolve: [] };
101
+ const decision: SweepDecision = { escalate: [], resolve: [] };
114
102
 
115
103
  for (const aggregate of aggregates) {
116
104
  const episode = byKind.get(aggregate.kind);
@@ -126,19 +114,23 @@ export function decideSweep(
126
114
  // verified).
127
115
  if (!episode.escalatedAt) {
128
116
  decision.escalate.push({ aggregate, episode });
129
- continue;
130
- }
131
- // Open, delivered, still recurring — throttled evidence update only.
132
- const lastUpdate = episode.lastUpdatedAt ?? episode.escalatedAt;
133
- if (minutesBetween(lastUpdate, now) >= policy.updateAfterMinutes) {
134
- decision.update.push({ aggregate, episode });
135
117
  }
118
+ // An open, delivered episode emits NOTHING while it recurs: the open
119
+ // emission already happened and the close emission fires when the kind
120
+ // goes quiet. A cadence-driven "still recurring" repeat is exactly the
121
+ // silenced-alarm shape `emit-transitions-not-conditions` prohibits.
136
122
  }
137
123
 
138
124
  for (const episode of episodes) {
139
125
  if (episode.resolvedAt) continue;
140
126
  const aggregate = aggregateByKind.get(episode.kind);
141
- const quietReference = aggregate?.lastSeen ?? episode.lastSeenAt;
127
+ // A future-dated aggregate lastSeen (producer clock skew) cannot serve
128
+ // as the quiet reference — it would freeze quietMinutes negative and
129
+ // hold the episode open forever while the kind is actually quiet. Fall
130
+ // back to the episode's own last_seen_at, which the bounded aggregate
131
+ // wrote and is therefore never in the future.
132
+ const quietReference =
133
+ aggregate && aggregate.lastSeen.getTime() <= now.getTime() ? aggregate.lastSeen : episode.lastSeenAt;
142
134
  const quietMinutes = minutesBetween(quietReference, now);
143
135
  if (quietMinutes >= policy.resolveAfterMinutes) {
144
136
  decision.resolve.push({ episode, quietMinutes });
@@ -214,14 +206,17 @@ export function composeEscalationMessage(input: {
214
206
  impact: string;
215
207
  recommendation: string;
216
208
  sample: string;
209
+ /** ISO timestamp + event id of the newest backing row, when resolved —
210
+ * cited verbatim so the reader can resolve the claim against event_log. */
211
+ lastEvent?: { occurredAt: string; eventId: string } | null;
217
212
  }): string {
218
213
  const span = formatSpanMinutes(
219
214
  (input.aggregate.lastSeen.getTime() - input.aggregate.firstSeen.getTime()) / 60_000,
220
215
  );
221
- const lines = [
222
- `🔴 Platform fault: ${input.kind}`,
223
- `Observed: ${input.aggregate.count} errors over ${span}, still recurring.`,
224
- ];
216
+ const observed = input.lastEvent
217
+ ? `Observed: ${input.aggregate.count} errors over ${span}, last ${input.lastEvent.occurredAt} (event ${input.lastEvent.eventId.slice(0, 8)}).`
218
+ : `Observed: ${input.aggregate.count} errors over ${span}, last ${input.aggregate.lastSeen.toISOString()}.`;
219
+ const lines = [`🔴 Platform fault: ${input.kind}`, observed];
225
220
  if (input.sample) lines.push(`Sample: ${input.sample}`);
226
221
  lines.push(`Impact: ${input.impact}`, `Recommendation: ${input.recommendation}`);
227
222
  return lines.join("\n");
@@ -24,7 +24,6 @@ const POLICY: FaultEscalationPolicy = {
24
24
  minCount: 10,
25
25
  minSpanMinutes: 25,
26
26
  resolveAfterMinutes: 30,
27
- updateAfterMinutes: 30,
28
27
  };
29
28
 
30
29
  function agg(kind: string, count: number, firstSeenMinutesAgo: number, lastSeenMinutesAgo: number): FaultAggregate {
@@ -102,29 +101,20 @@ describe("decideSweep — dedup (one escalation per fault, no re-fire)", () => {
102
101
  expect(decision.escalate).toHaveLength(0);
103
102
  });
104
103
 
105
- test("open delivered episode past the update window gets an UPDATE, not a message", () => {
104
+ test("open delivered episode emits NOTHING while it recurs — transitions, not cadence", () => {
105
+ // 2026-08-26: the hourly "still recurring" comment made the alarm
106
+ // skippable noise and hid a real signal. An open, delivered episode now
107
+ // stays silent until the close transition fires (see the resolve suite).
106
108
  const sustained = agg("llm.provider.call.failed", 300, 720, 0);
107
- const open = episode({
109
+ const longOpen = episode({
108
110
  kind: "llm.provider.call.failed",
109
- escalatedAt: new Date(NOW.getTime() - 90 * 60_000), // last update long ago
111
+ escalatedAt: new Date(NOW.getTime() - 90 * 60_000),
110
112
  lastUpdatedAt: new Date(NOW.getTime() - 90 * 60_000),
111
113
  });
112
- const decision = decideSweep([sustained], [open], POLICY, NOW);
113
- expect(decision.escalate).toHaveLength(0);
114
- expect(decision.update).toHaveLength(1);
115
- expect(decision.update[0]!.episode.workItemId).toBe("wi-1");
116
- });
117
-
118
- test("open episode inside the update window gets NOTHING (dedup window)", () => {
119
- const sustained = agg("llm.provider.call.failed", 300, 720, 0);
120
- const open = episode({
121
- kind: "llm.provider.call.failed",
122
- escalatedAt: new Date(NOW.getTime() - 10 * 60_000),
123
- lastUpdatedAt: new Date(NOW.getTime() - 10 * 60_000),
124
- });
125
- const decision = decideSweep([sustained], [open], POLICY, NOW);
114
+ const decision = decideSweep([sustained], [longOpen], POLICY, NOW);
126
115
  expect(decision.escalate).toHaveLength(0);
127
- expect(decision.update).toHaveLength(0);
116
+ expect(decision.resolve).toHaveLength(0);
117
+ expect(Object.keys(decision).sort()).toEqual(["escalate", "resolve"]);
128
118
  });
129
119
 
130
120
  test("a claimed-but-undelivered episode is RETRIED, not counted as escalated", () => {
@@ -168,6 +158,34 @@ describe("decideSweep — resolution + re-arm", () => {
168
158
  });
169
159
  });
170
160
 
161
+ describe("decideSweep — a future-dated lastSeen is not recurrence evidence", () => {
162
+ // 2026-08-26: an escalation sent 08:20Z cited "last at 09:04:54Z" — 44
163
+ // minutes in its own future. The rows existed (producer envelopes whose
164
+ // clock ran 70 minutes ahead), and `minutesBetween(lastSeen, now)` went
165
+ // negative, passing every recency check while the kind was actually quiet.
166
+ // A timestamp that has not happened yet cannot evidence a fault happening
167
+ // now — no matter how the aggregate was built.
168
+ test("a future-dated aggregate does not escalate", () => {
169
+ const phantom = agg("guard.fail.open", 40, 720, -44);
170
+ const decision = decideSweep([phantom], [], POLICY, NOW);
171
+ expect(decision.escalate).toHaveLength(0);
172
+ });
173
+
174
+ test("a future-dated aggregate does not hold an open episode open past the quiet window", () => {
175
+ // lastSeen 44 min in the future means the kind's real events are quiet:
176
+ // the episode's own last_seen_at (honest) drives the resolve decision,
177
+ // and the phantom aggregate must not veto it.
178
+ const quietEpisode = episode({
179
+ kind: "guard.fail.open",
180
+ lastSeenAt: new Date(NOW.getTime() - 60 * 60_000),
181
+ });
182
+ const phantom = agg("guard.fail.open", 40, 720, -44);
183
+ const decision = decideSweep([phantom], [quietEpisode], POLICY, NOW);
184
+ expect(decision.escalate).toHaveLength(0);
185
+ expect(decision.resolve).toHaveLength(1);
186
+ });
187
+ });
188
+
171
189
  describe("extractGowaMessageId — delivery verification", () => {
172
190
  test("the canonical GOWA SendResponse shape yields results.message_id", () => {
173
191
  expect(extractGowaMessageId({ code: "SUCCESS", results: { message_id: "3EB089B9D6ADD58153C561", status: "sent" } })).toBe(
@@ -196,6 +214,7 @@ describe("composeEscalationMessage", () => {
196
214
  impact: "scala-tools verbs are failing host-wide",
197
215
  recommendation: "check oxmgr logs for the bus consumer",
198
216
  sample: "",
217
+ lastEvent: null,
199
218
  });
200
219
  expect(message).toContain("Platform fault: host-command-bus.command.failed");
201
220
  expect(message).toContain("Observed: 490 errors over 12 h");
@@ -204,6 +223,22 @@ describe("composeEscalationMessage", () => {
204
223
  expect(message.split("\n")).toHaveLength(4);
205
224
  });
206
225
 
226
+ // 2026-08-26: an escalation sent 08:20Z cited "last at 09:04:54Z", a
227
+ // timestamp 44 minutes in its own future matching no event the operator
228
+ // could find. The Observed line now cites the newest backing row's own
229
+ // occurred_at and id, so "last at T" resolves to a real event_log row.
230
+ test("cites the newest backing row's timestamp and event id — resolvable evidence", () => {
231
+ const message = composeEscalationMessage({
232
+ kind: "guard.fail.open",
233
+ aggregate: agg("guard.fail.open", 490, 720, 0),
234
+ impact: "guard enforcement is failing open",
235
+ recommendation: "check the guard daemon",
236
+ sample: "",
237
+ lastEvent: { occurredAt: "2026-08-26T08:20:29Z", eventId: "01a03b79-c124-748f-a79f-a9a436729544" },
238
+ });
239
+ expect(message).toContain("last 2026-08-26T08:20:29Z (event 01a03b79)");
240
+ });
241
+
207
242
  // 2026-08-26: three escalations reached the orchestrator seat carrying
208
243
  // kind, count and duration only — triaging each required a hand event_log
209
244
  // query, and the llm.provider.call.failed impact line pointed AWAY from
@@ -217,6 +252,7 @@ describe("composeEscalationMessage", () => {
217
252
  sample: 'status=429; message=Usage limit reached for 5 hour. Your limit will reset at 2026-08-26 01:19:03',
218
253
  impact: "AI chat/completion calls are failing",
219
254
  recommendation: "check provider quota",
255
+ lastEvent: null,
220
256
  });
221
257
  const lines = message.split("\n");
222
258
  expect(lines).toHaveLength(5);
@@ -3,46 +3,10 @@
3
3
  * @status handwritten @indivisible-unit
4
4
  * @edit edit directly
5
5
  *
6
- * The `platform-fault-escalation` pg-boss handler — closes the "receiving is
7
- * decoupled from consequence" gap measured 2026-08-15 (760 error events over
8
- * 12 hours, zero escalations: host-command-bus.command.failed (then spelled
9
- * host-command-bus.error) 490, llm.provider.call.failed (then spelled
10
- * llm.provider.error) 231, guard.fail.open 16, scale-to-zero.wake_failed 12,
11
- * listen.connect_failed 11). Emission works and observability-is-passive is satisfied; nothing
12
- * acted. This sweep makes RECEIVING have CONSEQUENCE: sustained platform
13
- * faults escalate to the OPERATOR over the existing PA path (WhatsApp, the
14
- * scala PA operator-relay bot — escalation-routing's channel; no second
15
- * channel invented), with the durable record as a work_items task the PA can
16
- * read back and the operator can answer on.
17
- *
18
- * THIS IS NOT A DRIFT-DETECTOR PATROLLING INSTANCES. It reacts only to
19
- * components reporting THEIR OWN failure — event_log kinds that already
20
- * assert a fault about their emitter. Thresholds (duration + recurrence,
21
- * never a single event), the watched-kind patterns, the resolve window, the
22
- * per-kind impact/recommendation copy and the per-kind payload sample-field
23
- * whitelists are all in the
24
- * `config/platform-fault-escalation` registry row — changing policy is a row
25
- * edit, not a redeploy. Episode dedup lives in the
26
- * `platform_fault_escalations` table (migration 501): one escalation per
27
- * distinct fault kind per episode; an already-escalated-and-unresolved fault
28
- * only appends throttled work-item comments, never a second message.
29
- *
30
- * DELIVERY IS VERIFIED, NOT QUEUED. The WhatsApp send goes through the
31
- * central `/mcp` loopback (`api_gowa_message` sendMessage — the single
32
- * sealed GOWA-credential surface, same as whatsapp-liveness) and an episode
33
- * only carries `escalated_at` once the send returned a WhatsApp message id;
34
- * before that the row is a CLAIM the next sweep retries. Success is recorded
35
- * only after readback of the delivery facts (message id + work item id).
36
- *
37
- * WHY A TICK EXISTS AT ALL, given observability-is-passive and
38
- * no-polling-for-pushable-state: the trigger is DURATION + RECURRENCE of
39
- * events already durably logged — a fact about the PATTERN of past events,
40
- * not a state transition any single event can announce. The sweep reads one
41
- * server-side aggregate (GROUP BY kind) per run against indexed event_log;
42
- * while the platform is healthy it returns zero rows and costs nothing
43
- * beyond the tick (idle-costs-nothing).
44
- *
45
- * Registered under slug `@teamscala/service-boot:platform-fault-escalation`.
6
+ * The `platform-fault-escalation` pg-boss handler. Full contract in curated
7
+ * `apps/escalation.md` — episodes EMIT ON TRANSITIONS ONLY; evidence bounded
8
+ * `occurred_at <= now()` and cited by resolvable event id; delivery verified
9
+ * by WhatsApp message id before `escalated_at` is recorded.
46
10
  */
47
11
  import type { ExecutionContext } from "@teamscala/os/contracts/mcp";
48
12
  import type { ScheduledJobConfig } from "@teamscala/db-validation/registry-schemas/scheduled-job";
@@ -82,7 +46,6 @@ const FaultEscalationConfigSchema = v.object({
82
46
  ),
83
47
  ),
84
48
  resolve_after_minutes: v.number(),
85
- update_after_minutes: v.number(),
86
49
  /** The orchestrator's work item — escalations are COMMENTS on it, never
87
50
  * task rows (the fleet-task-placement CHECK constraint refuses a system
88
51
  * task without session identity, and an escalation is a finding, not an
@@ -204,6 +167,11 @@ async function fetchAggregates(patterns: string[], windowMinutes: number): Promi
204
167
  // Patterns come from the valibot-validated registry row and are regex-only
205
168
  // by construction below; embedding them as one alternation keeps the query
206
169
  // parameter-free on the pattern side while the window is a real parameter.
170
+ // The window is bounded on BOTH sides by the database's own statement
171
+ // clock: `occurred_at` is producer-claimed, and an unbounded top edge
172
+ // admits rows stamped in the future by a skewed or mocked producer clock —
173
+ // which then fabricate "still recurring" for a quiet fault (measured
174
+ // 2026-08-26). A future row is not evidence of anything happening now.
207
175
  const regexClass = /^[A-Za-z0-9_\\^$.*+?()[\]{}|-]+$/;
208
176
  for (const pattern of patterns) {
209
177
  if (!regexClass.test(pattern)) {
@@ -213,10 +181,11 @@ async function fetchAggregates(patterns: string[], windowMinutes: number): Promi
213
181
  const combined = `(${patterns.join("|")})`;
214
182
  const rows = await prisma.$queryRawUnsafe<EventLogAggregateRow[]>(
215
183
  `SELECT kind, count(*)::bigint AS n, min(occurred_at) AS first_seen, max(occurred_at) AS last_seen
216
- FROM event_log
217
- WHERE occurred_at > now() - ($1::int * interval '1 minute')
218
- AND kind ~ $2::text
219
- GROUP BY kind`,
184
+ FROM event_log
185
+ WHERE occurred_at > now() - ($1::int * interval '1 minute')
186
+ AND occurred_at <= now()
187
+ AND kind ~ $2::text
188
+ GROUP BY kind`,
220
189
  windowMinutes,
221
190
  combined,
222
191
  );
@@ -229,31 +198,47 @@ async function fetchAggregates(patterns: string[], windowMinutes: number): Promi
229
198
  }
230
199
 
231
200
  /**
232
- * The message's evidence line: the whitelisted sample fields from the most
233
- * recent payload for the kind. Field names come from the valibot-validated
234
- * config row and are constrained to bare identifiers below, so the JSON path
235
- * cannot break out of the string; only the named keys are selected, so an
236
- * unlisted field is unreachable by construction rather than filtered later.
201
+ * The message's evidence: the whitelisted sample fields AND the row identity
202
+ * (id + occurred_at) of the newest payload for the kind. Field names come
203
+ * from the valibot-validated config row and are constrained to bare
204
+ * identifiers below, so the JSON path cannot break out of the string. The
205
+ * `occurred_at <= now()` bound matches the aggregate's top edge: the cited
206
+ * "last" row must be a real, non-future event whose id rides in the message
207
+ * so the claim resolves against event_log.
237
208
  */
238
- async function fetchLatestPayloadSample(kind: string, fields: string[]): Promise<string> {
239
- if (fields.length === 0) return "";
209
+ async function fetchLatestEvidence(
210
+ kind: string,
211
+ fields: string[],
212
+ ): Promise<{ sample: string; lastEvent: { occurredAt: string; eventId: string } | null }> {
213
+ const prisma = getPrisma();
214
+ if (fields.length === 0) {
215
+ const rows = await prisma.$queryRawUnsafe<Array<{ id: unknown; occurred_at: Date }>>(
216
+ `SELECT id::text, occurred_at
217
+ FROM event_log
218
+ WHERE kind = $1::text AND occurred_at <= now()
219
+ ORDER BY occurred_at DESC
220
+ LIMIT 1`,
221
+ kind,
222
+ );
223
+ const top = rows[0];
224
+ return { sample: "", lastEvent: top ? { occurredAt: isoSeconds(top.occurred_at), eventId: String(top.id) } : null };
225
+ }
240
226
  for (const field of fields) {
241
227
  if (!/^[A-Za-z0-9_]+$/.test(field)) {
242
228
  throw new Error(`config/${CONFIG_SLUG}: sample_fields entry is not a bare field name: ${field}`);
243
229
  }
244
230
  }
245
231
  const selection = fields.map((field) => `payload_json->'${field}' AS "${field}"`).join(", ");
246
- const prisma = getPrisma();
247
232
  const rows = await prisma.$queryRawUnsafe<Array<Record<string, unknown>>>(
248
- `SELECT ${selection}
249
- FROM event_log
250
- WHERE kind = $1::text
251
- ORDER BY occurred_at DESC
252
- LIMIT 1`,
233
+ `SELECT id::text, occurred_at, ${selection}
234
+ FROM event_log
235
+ WHERE kind = $1::text AND occurred_at <= now()
236
+ ORDER BY occurred_at DESC
237
+ LIMIT 1`,
253
238
  kind,
254
239
  );
255
240
  const row = rows[0];
256
- if (!row) return "";
241
+ if (!row) return { sample: "", lastEvent: null };
257
242
  const parts: string[] = [];
258
243
  for (const field of fields) {
259
244
  const value = row[field];
@@ -261,7 +246,19 @@ async function fetchLatestPayloadSample(kind: string, fields: string[]): Promise
261
246
  const formatted = formatEscalationSample(value);
262
247
  if (formatted) parts.push(`${field}=${formatted}`);
263
248
  }
264
- return parts.join("; ");
249
+ return {
250
+ sample: parts.join("; "),
251
+ lastEvent: {
252
+ occurredAt: isoSeconds(row.occurred_at as string | Date),
253
+ eventId: String(row.id),
254
+ },
255
+ };
256
+ }
257
+
258
+ /** ISO-8601 to whole seconds — the same precision event_log stores, so the
259
+ * cited string round-trips a `WHERE occurred_at = '<cited>'` query. */
260
+ function isoSeconds(value: Date | string): string {
261
+ return new Date(value).toISOString().replace(/\.\d{3}Z$/, "Z");
265
262
  }
266
263
 
267
264
  async function resolveOperatorRelay(): Promise<OperatorRelayBot> {
@@ -323,11 +320,12 @@ async function deliverEscalation(
323
320
  const bot = await resolveOperatorRelay();
324
321
  const operatorPhone = (bot.flow_config as { operator_phone?: string }).operator_phone!;
325
322
  const copy = config.copy[kind] ?? config.default_copy;
326
- const sample = await fetchLatestPayloadSample(kind, config.sample_fields?.[kind] ?? []);
323
+ const { sample, lastEvent } = await fetchLatestEvidence(kind, config.sample_fields?.[kind] ?? []);
327
324
  const message = composeEscalationMessage({
328
325
  kind,
329
326
  aggregate,
330
327
  sample,
328
+ lastEvent,
331
329
  impact: copy.impact,
332
330
  recommendation: copy.recommendation,
333
331
  });
@@ -390,6 +388,8 @@ async function deliverEscalation(
390
388
  event_kind: kind,
391
389
  count: aggregate.count,
392
390
  sample,
391
+ last_event_id: lastEvent?.eventId ?? null,
392
+ last_event_occurred_at: lastEvent?.occurredAt ?? null,
393
393
  work_item_id: config.orchestrator_work_item_id,
394
394
  delivered_comment_id: comment.id,
395
395
  delivered_message_id: messageId,
@@ -398,33 +398,6 @@ async function deliverEscalation(
398
398
  return { messageId, workItemId: config.orchestrator_work_item_id };
399
399
  }
400
400
 
401
- async function updateEpisode(aggregate: FaultAggregate, episode: EscalationEpisode): Promise<void> {
402
- const now = new Date();
403
- await (await orpc()).platform_fault_escalations.update({
404
- where: { kind: episode.kind },
405
- data: {
406
- last_seen_at: aggregate.lastSeen.toISOString(),
407
- event_count: aggregate.count,
408
- last_updated_at: now.toISOString(),
409
- updated_at: now.toISOString(),
410
- },
411
- });
412
- if (episode.workItemId) {
413
- await (await orpc()).comments.create({
414
- data: {
415
- work_item_id: episode.workItemId,
416
- content: `Still recurring: ${aggregate.count} events, last at ${aggregate.lastSeen.toISOString()}. No new escalation (already escalated and unresolved).`,
417
- sender_role: "system",
418
- sender_name: "platform-fault-escalation",
419
- },
420
- });
421
- }
422
- recordEvent({
423
- kind: "platform.fault.updated",
424
- payload: { event_kind: episode.kind, count: aggregate.count, work_item_id: episode.workItemId },
425
- });
426
- }
427
-
428
401
  async function resolveEpisode(episode: EscalationEpisode, quietMinutes: number): Promise<void> {
429
402
  const now = new Date();
430
403
  await (await orpc()).platform_fault_escalations.update({
@@ -493,7 +466,6 @@ async function platformFaultEscalationSweep(ctx: { config: ScheduledJobConfig })
493
466
  minCount: config.min_count,
494
467
  minSpanMinutes: config.min_span_minutes,
495
468
  resolveAfterMinutes: config.resolve_after_minutes,
496
- updateAfterMinutes: config.update_after_minutes,
497
469
  },
498
470
  now,
499
471
  kindThresholds,
@@ -514,13 +486,6 @@ async function platformFaultEscalationSweep(ctx: { config: ScheduledJobConfig })
514
486
  recordEvent({ kind: "platform.fault.escalation_failed", payload: { event_kind: aggregate.kind, reason } });
515
487
  }
516
488
  }
517
- for (const { aggregate, episode } of decision.update) {
518
- try {
519
- await updateEpisode(aggregate, episode);
520
- } catch (err) {
521
- log.error(`[platform-fault-escalation] update for '${episode.kind}' failed: ${err instanceof Error ? err.message : String(err)}`);
522
- }
523
- }
524
489
  for (const { episode, quietMinutes } of decision.resolve) {
525
490
  try {
526
491
  await resolveEpisode(episode, quietMinutes);
@@ -536,18 +501,17 @@ async function platformFaultEscalationSweep(ctx: { config: ScheduledJobConfig })
536
501
  payload: {
537
502
  watched_kinds: aggregates.length,
538
503
  escalated: decision.escalate.length,
539
- updated: decision.update.length,
540
504
  resolved: decision.resolve.length,
541
505
  },
542
506
  });
543
- if (decision.escalate.length > 0 || decision.update.length > 0 || decision.resolve.length > 0) {
507
+ if (decision.escalate.length > 0 || decision.resolve.length > 0) {
544
508
  log.info(
545
- `[platform-fault-escalation] sweep — ${aggregates.length} kind(s) watched, ${decision.escalate.length} escalated, ${decision.update.length} updated, ${decision.resolve.length} resolved`,
509
+ `[platform-fault-escalation] sweep — ${aggregates.length} kind(s) watched, ${decision.escalate.length} escalated, ${decision.resolve.length} resolved`,
546
510
  );
547
511
  }
548
- }
512
+ }
549
513
 
550
- pgBossHandlerRegistry.register(
514
+ pgBossHandlerRegistry.register(
551
515
  "@teamscala/service-boot:platform-fault-escalation",
552
516
  platformFaultEscalationSweep as (ctx: { config: ScheduledJobConfig }) => Promise<void>,
553
517
  );
@@ -87,6 +87,14 @@ export async function mountOrpcRouter(
87
87
  configureToolExecutor({
88
88
  getAppRouter: () => getAppRouter(),
89
89
  getPrisma: () => getConfiguredPrisma(),
90
+ // Central MCP gateway URL — sdk:* tool calls from any non-host service
91
+ // forward here (scala-mcp owns every @teamscala/api-* adapter per
92
+ // operator ruling 2026-08-26). The HTTP layer reads from the
93
+ // per-request config surface (the same place getAppRouter / getPrisma
94
+ // are resolved from), not the ambient environment, to honour
95
+ // `configured-primitives`.
96
+ getMcpGatewayUrl: () =>
97
+ (ctx as { config?: { MCP_GATEWAY_URL?: string } }).config?.MCP_GATEWAY_URL,
90
98
  });
91
99
 
92
100
  const resolveUser = async (headers: Headers) => {