@teamscala/service-boot 0.23.358 → 0.23.361

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -6,7 +6,7 @@
6
6
  },
7
7
  "name": "@teamscala/service-boot",
8
8
  "type": "module",
9
- "version": "0.23.358",
9
+ "version": "0.23.361",
10
10
  "sideEffects": false,
11
11
  "imports": {
12
12
  "#service-boot/*": "./src/*",
@@ -21,6 +21,7 @@
21
21
  "@teamscala/auth": "*",
22
22
  "@teamscala/cache": "*",
23
23
  "@teamscala/channel": "*",
24
+ "@teamscala/chat-completion": "*",
24
25
  "@teamscala/db": "*",
25
26
  "@teamscala/db-validation": "*",
26
27
  "@teamscala/document-signing": "*",
@@ -229,6 +230,14 @@
229
230
  "types": "./src/content-page-floor.d.ts",
230
231
  "default": "./src/content-page-floor.ts"
231
232
  },
233
+ "./csrf-protection": {
234
+ "types": "./src/csrf-protection.d.ts",
235
+ "default": "./src/csrf-protection.ts"
236
+ },
237
+ "./csrf-protection.ts": {
238
+ "types": "./src/csrf-protection.d.ts",
239
+ "default": "./src/csrf-protection.ts"
240
+ },
232
241
  "./discover-blocks": {
233
242
  "types": "./src/discover-blocks.d.ts",
234
243
  "default": "./src/discover-blocks.ts"
@@ -417,6 +426,22 @@
417
426
  "types": "./src/handlers/pane-outbox-drain.d.ts",
418
427
  "default": "./src/handlers/pane-outbox-drain.ts"
419
428
  },
429
+ "./handlers/platform-fault-escalation": {
430
+ "types": "./src/handlers/platform-fault-escalation.d.ts",
431
+ "default": "./src/handlers/platform-fault-escalation.ts"
432
+ },
433
+ "./handlers/platform-fault-escalation.ts": {
434
+ "types": "./src/handlers/platform-fault-escalation.d.ts",
435
+ "default": "./src/handlers/platform-fault-escalation.ts"
436
+ },
437
+ "./handlers/platform-fault-escalation-logic": {
438
+ "types": "./src/handlers/platform-fault-escalation-logic.d.ts",
439
+ "default": "./src/handlers/platform-fault-escalation-logic.ts"
440
+ },
441
+ "./handlers/platform-fault-escalation-logic.ts": {
442
+ "types": "./src/handlers/platform-fault-escalation-logic.d.ts",
443
+ "default": "./src/handlers/platform-fault-escalation-logic.ts"
444
+ },
420
445
  "./handlers/projects-digest": {
421
446
  "types": "./src/handlers/projects-digest.d.ts",
422
447
  "default": "./src/handlers/projects-digest.ts"
@@ -62,6 +62,16 @@ export async function attachTransfersUpload(app: Hono): Promise<void> {
62
62
  const userId = session?.user?.id;
63
63
  if (!userId) return c.json({ error: "Not authenticated" }, 401);
64
64
 
65
+ // Reject on Content-Length BEFORE formData() buffers the body — a
66
+ // file.size check after buffering is a report, not a limit (an attacker
67
+ // sends the bytes either way). The global 30MB boot-level bodyLimit
68
+ // remains the real bound for lying/absent Content-Length; this early
69
+ // return bounds the honest case without parsing a megabyte stream.
70
+ const declaredLength = Number(c.req.header("content-length") ?? 0);
71
+ if (declaredLength > MAX_FILE_SIZE) {
72
+ return c.json({ error: "file too large (max 25MB)" }, 413);
73
+ }
74
+
65
75
  const form = await c.req.formData();
66
76
  const file = form.get("file");
67
77
  const type = String(form.get("type") ?? "uploads");
@@ -0,0 +1,177 @@
1
+ /**
2
+ * @system service-boot
3
+ * @status handwritten
4
+ * @edit edit directly
5
+ *
6
+ * Behavior proofs for the two boot-chain security invariants:
7
+ * - security-csrf-protection: a state-changing form-element POST from an
8
+ * untrusted Origin is refused 403; the managed-base-domain Origin (the
9
+ * derived trusted set) is allowed; same-origin sec-fetch-site is allowed.
10
+ * - security-body-limit: a body over the global limit is refused 413 BEFORE
11
+ * any handler runs, and the transfers upload rejects an over-sized
12
+ * Content-Length without buffering the body.
13
+ * Proxy-is-not-behaviour guards: these exercise the real middleware through
14
+ * real Hono requests.
15
+ */
16
+ import { describe, expect, it } from "bun:test";
17
+ import { Hono } from "hono";
18
+ import { bodyLimit } from "hono/body-limit";
19
+ import { csrf } from "hono/csrf";
20
+ import { csrfProtection } from "./csrf-protection.ts";
21
+
22
+ type CacheGlobal = typeof globalThis & {
23
+ __registryCache?: Map<string, { type: string; slug: string; config: unknown }>;
24
+ };
25
+
26
+ function seedPortalBaseDomain(base: string): void {
27
+ const g = globalThis as CacheGlobal;
28
+ if (!g.__registryCache) g.__registryCache = new Map();
29
+ g.__registryCache.set("config:portal-domains", {
30
+ type: "config",
31
+ slug: "portal-domains",
32
+ config: { base },
33
+ });
34
+ }
35
+
36
+ describe("csrfProtection", () => {
37
+ it("refuses a cross-site form POST from an untrusted origin (403)", async () => {
38
+ seedPortalBaseDomain("scala.business");
39
+ const app = new Hono();
40
+ app.use("*", csrfProtection());
41
+ app.post("/api/settings", (c) => c.text("ok"));
42
+ const res = await app.request("https://portal.scala.business/api/settings", {
43
+ method: "POST",
44
+ headers: {
45
+ "content-type": "application/x-www-form-urlencoded",
46
+ origin: "https://evil.example",
47
+ },
48
+ body: "x=1",
49
+ });
50
+ expect(res.status).toBe(403);
51
+ });
52
+
53
+ it("allows a form POST whose origin is under the managed base domain", async () => {
54
+ seedPortalBaseDomain("scala.business");
55
+ const app = new Hono();
56
+ app.use("*", csrfProtection());
57
+ app.post("/api/settings", (c) => c.text("ok"));
58
+ const res = await app.request("https://portal.scala.business/api/settings", {
59
+ method: "POST",
60
+ headers: {
61
+ "content-type": "application/x-www-form-urlencoded",
62
+ // cross-SERVICE, same platform: the proposals → sign redirect-chain shape
63
+ origin: "https://sign.scala.business",
64
+ },
65
+ body: "x=1",
66
+ });
67
+ expect(res.status).toBe(200);
68
+ });
69
+
70
+ it("allows a same-origin browser post via sec-fetch-site", async () => {
71
+ seedPortalBaseDomain("scala.business");
72
+ const app = new Hono();
73
+ app.use("*", csrfProtection());
74
+ app.post("/api/settings", (c) => c.text("ok"));
75
+ const res = await app.request("https://portal.scala.business/api/settings", {
76
+ method: "POST",
77
+ headers: {
78
+ "content-type": "application/x-www-form-urlencoded",
79
+ "sec-fetch-site": "same-origin",
80
+ },
81
+ body: "x=1",
82
+ });
83
+ expect(res.status).toBe(200);
84
+ });
85
+
86
+ it("does not arm on application/json (cross-site JSON is not formable)", async () => {
87
+ seedPortalBaseDomain("scala.business");
88
+ const app = new Hono();
89
+ app.use("*", csrfProtection());
90
+ app.post("/api/rpc/x", (c) => c.text("ok"));
91
+ const res = await app.request("https://portal.scala.business/api/rpc/x", {
92
+ method: "POST",
93
+ headers: { "content-type": "application/json", origin: "https://evil.example" },
94
+ body: JSON.stringify({ x: 1 }),
95
+ });
96
+ expect(res.status).toBe(200);
97
+ });
98
+ });
99
+
100
+ describe("global body limit (security-body-limit)", () => {
101
+ const limit = 1024;
102
+
103
+ function appWithLimit(): Hono {
104
+ const app = new Hono();
105
+ let handlerRan = false;
106
+ app.use(
107
+ "*",
108
+ bodyLimit({
109
+ maxSize: limit,
110
+ onError: (c) => c.text("Payload Too Large", 413),
111
+ }),
112
+ );
113
+ app.post("/api/upload", async (c) => {
114
+ handlerRan = true;
115
+ return c.text("ok");
116
+ });
117
+ // expose the flag after the request via a closure read below
118
+ (app as unknown as Record<string, unknown>).__handlerRan = () => handlerRan;
119
+ return app;
120
+ }
121
+
122
+ it("refuses an oversized body with 413 before the handler runs", async () => {
123
+ const app = appWithLimit();
124
+ const res = await app.request("https://host.example/api/upload", {
125
+ method: "POST",
126
+ headers: { "content-type": "application/octet-stream" },
127
+ body: "x".repeat(limit + 1),
128
+ });
129
+ expect(res.status).toBe(413);
130
+ expect(
131
+ (app as unknown as { __handlerRan: () => boolean }).__handlerRan(),
132
+ ).toBe(false);
133
+ });
134
+
135
+ it("allows a body within the limit", async () => {
136
+ const app = appWithLimit();
137
+ const res = await app.request("https://host.example/api/upload", {
138
+ method: "POST",
139
+ headers: { "content-type": "application/octet-stream" },
140
+ body: "x".repeat(limit - 1),
141
+ });
142
+ expect(res.status).toBe(200);
143
+ });
144
+
145
+ it("a route-level smaller limit still refuses what the global limit allows", async () => {
146
+ const app = new Hono();
147
+ app.use(
148
+ "*",
149
+ bodyLimit({
150
+ maxSize: 4 * limit,
151
+ onError: (c) => c.text("Payload Too Large", 413),
152
+ }),
153
+ );
154
+ app.post(
155
+ "/api/rpc/*",
156
+ bodyLimit({
157
+ maxSize: limit,
158
+ onError: (c) => c.text("Request body too large", 413),
159
+ }),
160
+ (c) => c.text("ok"),
161
+ );
162
+ const res = await app.request("https://host.example/api/rpc/x", {
163
+ method: "POST",
164
+ headers: { "content-type": "application/json" },
165
+ body: "x".repeat(2 * limit),
166
+ });
167
+ expect(res.status).toBe(413);
168
+ });
169
+ });
170
+
171
+ describe("csrf import wiring", () => {
172
+ it("csrfProtection() composes hono csrf without options drift", () => {
173
+ const mw = csrfProtection();
174
+ expect(typeof mw).toBe("function");
175
+ expect(typeof csrf).toBe("function");
176
+ });
177
+ });
@@ -0,0 +1,100 @@
1
+ /**
2
+ * @system server-boot
3
+ * @status handwritten
4
+ * @edit edit directly
5
+ *
6
+ * csrf-protection.ts — platform CSRF middleware (`security-csrf-protection`).
7
+ *
8
+ * Hono's csrf() with a DERIVED trusted-origin set, never a hand-authored
9
+ * allowlist: every trusted origin comes from ONE source per
10
+ * `offered-is-derived` —
11
+ * - the managed base domain (registry portal-domains) covers the platform's
12
+ * own `*.scala.business` origins, INCLUDING the cross-service form POST
13
+ * redirect chains CSP form-action allowlists (proposals → sign etc.), so
14
+ * the CSRF origin set and the CSP form-action set cannot disagree about
15
+ * what "one of ours" means;
16
+ * - customer custom domains resolve through `isTrustedAuthOrigin`
17
+ * (builda_websites lookup) — the SAME lookup better-auth's trustedOrigins
18
+ * uses in @teamscala/auth, so a new custom domain is trusted everywhere at
19
+ * once or nowhere.
20
+ *
21
+ * hono csrf() arms only on form-element content types (urlencoded, multipart,
22
+ * text/plain) — browsers cannot produce a cross-site application/json request
23
+ * from a form, so the JSON API surface (ORPC) is protected by cookie
24
+ * SameSite=Lax + this middleware's text/plain coverage.
25
+ *
26
+ * Custom-domain lookups are cached per-hostname with a TTL: a POSITIVE result
27
+ * is cached for the full TTL (a domain that stopped being ours keeps working
28
+ * for at most the TTL), a NEGATIVE result for a short window (a brand-new
29
+ * custom domain becomes trusted within seconds). The cache is a request
30
+ * accelerator over a DB fact, never the source of truth.
31
+ */
32
+
33
+ import { getManagedBaseDomain } from "@teamscala/auth/auth-env-resolution";
34
+ import { isTrustedAuthOrigin } from "@teamscala/auth/is-trusted-auth-origin";
35
+ import { csrf } from "hono/csrf";
36
+ import type { Context, MiddlewareHandler } from "hono";
37
+
38
+ const POSITIVE_TTL_MS = 5 * 60_000;
39
+ const NEGATIVE_TTL_MS = 10_000;
40
+
41
+ type CacheEntry = { trusted: boolean; expiresAt: number };
42
+ const customDomainCache = new Map<string, CacheEntry>();
43
+
44
+ function isPlatformOrigin(origin: string, baseDomain: string): boolean {
45
+ let host: string;
46
+ try {
47
+ host = new URL(origin).hostname.toLowerCase();
48
+ } catch {
49
+ return false;
50
+ }
51
+ if (!host) return false;
52
+ return host === baseDomain || host.endsWith(`.${baseDomain}`);
53
+ }
54
+
55
+ async function isTrustedCustomOrigin(origin: string): Promise<boolean> {
56
+ let host: string;
57
+ try {
58
+ host = new URL(origin).hostname.toLowerCase();
59
+ } catch {
60
+ return false;
61
+ }
62
+ if (!host) return false;
63
+
64
+ const now = Date.now();
65
+ const cached = customDomainCache.get(host);
66
+ if (cached && cached.expiresAt > now) return cached.trusted;
67
+
68
+ const trusted = await (async () => {
69
+ try {
70
+ const { getPrisma } = await import("@teamscala/db/prisma-registry");
71
+ const prisma = getPrisma();
72
+ return await isTrustedAuthOrigin(
73
+ prisma as unknown as Parameters<typeof isTrustedAuthOrigin>[0],
74
+ host,
75
+ );
76
+ } catch {
77
+ // A lookup that cannot run is a lookup that trusts nothing: an
78
+ // unavailable DB degrades to platform-origin-only CSRF, never to
79
+ // allow-everything.
80
+ return false;
81
+ }
82
+ })();
83
+
84
+ customDomainCache.set(host, {
85
+ trusted,
86
+ expiresAt: now + (trusted ? POSITIVE_TTL_MS : NEGATIVE_TTL_MS),
87
+ });
88
+ if (customDomainCache.size > 512) customDomainCache.clear();
89
+ return trusted;
90
+ }
91
+
92
+ export function csrfProtection(): MiddlewareHandler {
93
+ return csrf({
94
+ origin: async (origin: string, _c: Context) => {
95
+ const baseDomain = getManagedBaseDomain();
96
+ if (isPlatformOrigin(origin, baseDomain)) return true;
97
+ return isTrustedCustomOrigin(origin);
98
+ },
99
+ });
100
+ }
@@ -0,0 +1,185 @@
1
+ /**
2
+ * @system server-boot
3
+ * @status handwritten
4
+ * @edit edit directly
5
+ *
6
+ * Pure decision logic for the platform-fault escalation sweep
7
+ * (`platform-fault-escalation` handler). Split from the I/O shell so the
8
+ * threshold/dedup/episode semantics are unit-testable with zero adapter
9
+ * dependencies (same shape as whatsapp-liveness-logic).
10
+ *
11
+ * The semantics this locks (from the 2026-08-15 "760 errors escalated to
12
+ * nobody" incident brief):
13
+ * - Threshold on DURATION and RECURRENCE, never a single event. A kind
14
+ * qualifies as a sustained fault only when BOTH hold inside the lookback
15
+ * window: at least `min_count` occurrences (recurrence) spread over at
16
+ * least `min_span_minutes` (duration). One failed push is noise; the same
17
+ * kind recurring for half an hour is an outage.
18
+ * - One escalation per distinct fault kind per episode. An episode is open
19
+ * from first qualification until `resolve_after_minutes` of quiet; while
20
+ * open, the fault never re-fires — it only appends throttled updates.
21
+ * - Delivery is verified, not queued: an episode row with `escalated_at`
22
+ * NULL is a CLAIM, not an escalation. The sweep re-attempts delivery for
23
+ * claimed-but-undelivered episodes until the WhatsApp send returns a
24
+ * message id.
25
+ */
26
+
27
+ export interface FaultAggregate {
28
+ kind: string;
29
+ count: number;
30
+ firstSeen: Date;
31
+ lastSeen: Date;
32
+ }
33
+
34
+ export interface EscalationEpisode {
35
+ kind: string;
36
+ firstSeenAt: Date;
37
+ lastSeenAt: Date;
38
+ eventCount: number;
39
+ escalatedAt: Date | null;
40
+ workItemId: string | null;
41
+ resolvedAt: Date | null;
42
+ lastUpdatedAt: Date | null;
43
+ }
44
+
45
+ export interface FaultEscalationPolicy {
46
+ /** Recurrence floor: minimum event count inside the window. */
47
+ minCount: number;
48
+ /** Duration floor: minimum first→last spread inside the window, minutes. */
49
+ minSpanMinutes: number;
50
+ /** Quiet period that closes an episode and re-arms escalation, minutes. */
51
+ resolveAfterMinutes: number;
52
+ /** Minimum spacing between throttled updates on an open episode, minutes. */
53
+ updateAfterMinutes: number;
54
+ }
55
+
56
+ /** Per-kind threshold overrides — a low-rate-but-persistent kind (guard.fail.open
57
+ * at ~1/hour over 12h) has different noise characteristics than a bursty one, so
58
+ * the recurrence/duration floors are per-kind policy in the config row. */
59
+ export interface KindThresholdOverride {
60
+ minCount?: number;
61
+ minSpanMinutes?: number;
62
+ }
63
+
64
+ export interface SweepDecision {
65
+ /** Sustained faults needing a FIRST delivery (new episode, or a claim whose send failed). */
66
+ escalate: Array<{ aggregate: FaultAggregate; episode: EscalationEpisode | null }>;
67
+ /** Open, delivered episodes still recurring — append evidence, never re-send. */
68
+ update: Array<{ aggregate: FaultAggregate; episode: EscalationEpisode }>;
69
+ /** Open episodes whose kind went quiet past the resolve window — close them. */
70
+ resolve: Array<{ episode: EscalationEpisode; quietMinutes: number }>;
71
+ }
72
+
73
+ function minutesBetween(a: Date, b: Date): number {
74
+ return (b.getTime() - a.getTime()) / 60_000;
75
+ }
76
+
77
+ function isSustained(
78
+ aggregate: FaultAggregate,
79
+ policy: FaultEscalationPolicy,
80
+ override: KindThresholdOverride | undefined,
81
+ ): boolean {
82
+ const minCount = override?.minCount ?? policy.minCount;
83
+ const minSpan = override?.minSpanMinutes ?? policy.minSpanMinutes;
84
+ return (
85
+ aggregate.count >= minCount &&
86
+ minutesBetween(aggregate.firstSeen, aggregate.lastSeen) >= minSpan
87
+ );
88
+ }
89
+
90
+ /**
91
+ * Classify the current sweep. `aggregates` are the per-kind error aggregates
92
+ * from the lookback window; `episodes` are the existing episode rows for those
93
+ * kinds PLUS every still-open episode (an open episode whose kind no longer
94
+ * appears in the window at all is the strongest resolve candidate).
95
+ */
96
+ export function decideSweep(
97
+ aggregates: FaultAggregate[],
98
+ episodes: EscalationEpisode[],
99
+ policy: FaultEscalationPolicy,
100
+ now: Date,
101
+ kindThresholds: Record<string, KindThresholdOverride> = {},
102
+ ): SweepDecision {
103
+ const byKind = new Map(episodes.map((e) => [e.kind, e]));
104
+ const aggregateByKind = new Map(aggregates.map((a) => [a.kind, a]));
105
+ const decision: SweepDecision = { escalate: [], update: [], resolve: [] };
106
+
107
+ for (const aggregate of aggregates) {
108
+ const episode = byKind.get(aggregate.kind);
109
+ if (!isSustained(aggregate, policy, kindThresholds[aggregate.kind])) continue;
110
+ // No episode, or the previous episode closed (quiet past resolve window):
111
+ // a NEW episode begins and must be delivered.
112
+ if (!episode || episode.resolvedAt) {
113
+ decision.escalate.push({ aggregate, episode: episode ?? null });
114
+ continue;
115
+ }
116
+ // Open episode with no verified delivery — the claim is retried, never
117
+ // counted as escalated (a component may not report a state it has not
118
+ // verified).
119
+ if (!episode.escalatedAt) {
120
+ decision.escalate.push({ aggregate, episode });
121
+ continue;
122
+ }
123
+ // Open, delivered, still recurring — throttled evidence update only.
124
+ const lastUpdate = episode.lastUpdatedAt ?? episode.escalatedAt;
125
+ if (minutesBetween(lastUpdate, now) >= policy.updateAfterMinutes) {
126
+ decision.update.push({ aggregate, episode });
127
+ }
128
+ }
129
+
130
+ for (const episode of episodes) {
131
+ if (episode.resolvedAt) continue;
132
+ const aggregate = aggregateByKind.get(episode.kind);
133
+ const quietReference = aggregate?.lastSeen ?? episode.lastSeenAt;
134
+ const quietMinutes = minutesBetween(quietReference, now);
135
+ if (quietMinutes >= policy.resolveAfterMinutes) {
136
+ decision.resolve.push({ episode, quietMinutes });
137
+ }
138
+ }
139
+
140
+ return decision;
141
+ }
142
+
143
+ /** Human "N min" span, rounded — for the phone-sized escalation message. */
144
+ export function formatSpanMinutes(minutes: number): string {
145
+ if (minutes < 60) return `${Math.round(minutes)} min`;
146
+ const hours = minutes / 60;
147
+ return `${hours >= 10 ? Math.round(hours) : Math.round(hours * 10) / 10} h`;
148
+ }
149
+
150
+ /**
151
+ * The escalation message — escalation-routing's three-field contract adapted
152
+ * to an automated detector: what was OBSERVED (evidence), what it MEANS
153
+ * (impact), and the RECOMMENDATION. Short: the channel is a phone.
154
+ */
155
+ export function composeEscalationMessage(input: {
156
+ kind: string;
157
+ aggregate: FaultAggregate;
158
+ impact: string;
159
+ recommendation: string;
160
+ }): string {
161
+ const span = formatSpanMinutes(
162
+ (input.aggregate.lastSeen.getTime() - input.aggregate.firstSeen.getTime()) / 60_000,
163
+ );
164
+ return [
165
+ `🔴 Platform fault: ${input.kind}`,
166
+ `Observed: ${input.aggregate.count} errors over ${span}, still recurring.`,
167
+ `Impact: ${input.impact}`,
168
+ `Recommendation: ${input.recommendation}`,
169
+ ].join("\n");
170
+ }
171
+
172
+ /** The work-item description carries the same three fields plus the evidence. */
173
+ export function composeEscalationDescription(input: {
174
+ aggregate: FaultAggregate;
175
+ impact: string;
176
+ recommendation: string;
177
+ }): string {
178
+ return [
179
+ "Automated platform-fault escalation (platform-fault-escalation sweep).",
180
+ `Observed: ${input.aggregate.count} '${input.aggregate.kind}' events in event_log from ${input.aggregate.firstSeen.toISOString()} to ${input.aggregate.lastSeen.toISOString()}, still recurring.`,
181
+ `Impact: ${input.impact}`,
182
+ `Recommendation: ${input.recommendation}`,
183
+ "Delivery: WhatsApp via the scala PA. This task closes automatically when the fault stops for the resolve window; reply via the PA to rule otherwise.",
184
+ ].join("\n");
185
+ }
@@ -0,0 +1,183 @@
1
+ /**
2
+ * @system server-boot
3
+ * @status handwritten
4
+ * @edit edit directly
5
+ *
6
+ * Unit tests for the pure decision core of the platform-fault escalation
7
+ * sweep (`platform-fault-escalation` handler). The handler's DB/WhatsApp I/O
8
+ * is runtime-verified by the scheduled_job firing; this locks the threshold,
9
+ * dedup and episode semantics from the 2026-08-15 incident brief.
10
+ */
11
+ import { describe, expect, test } from "bun:test";
12
+ import {
13
+ composeEscalationMessage,
14
+ decideSweep,
15
+ type EscalationEpisode,
16
+ type FaultAggregate,
17
+ type FaultEscalationPolicy,
18
+ } from "./platform-fault-escalation-logic.ts";
19
+
20
+ const NOW = new Date("2026-08-15T23:45:00+10:00");
21
+ const POLICY: FaultEscalationPolicy = {
22
+ minCount: 10,
23
+ minSpanMinutes: 25,
24
+ resolveAfterMinutes: 30,
25
+ updateAfterMinutes: 30,
26
+ };
27
+
28
+ function agg(kind: string, count: number, firstSeenMinutesAgo: number, lastSeenMinutesAgo: number): FaultAggregate {
29
+ return {
30
+ kind,
31
+ count,
32
+ firstSeen: new Date(NOW.getTime() - firstSeenMinutesAgo * 60_000),
33
+ lastSeen: new Date(NOW.getTime() - lastSeenMinutesAgo * 60_000),
34
+ };
35
+ }
36
+
37
+ function episode(overrides: Partial<EscalationEpisode> & { kind: string }): EscalationEpisode {
38
+ return {
39
+ firstSeenAt: NOW,
40
+ lastSeenAt: NOW,
41
+ eventCount: 10,
42
+ escalatedAt: new Date(NOW.getTime() - 60 * 60_000),
43
+ workItemId: "wi-1",
44
+ resolvedAt: null,
45
+ lastUpdatedAt: null,
46
+ ...overrides,
47
+ };
48
+ }
49
+
50
+ describe("decideSweep — thresholds", () => {
51
+ test("a single event is noise — never escalates", () => {
52
+ const decision = decideSweep([agg("push.error", 1, 2, 2)], [], POLICY, NOW);
53
+ expect(decision.escalate).toHaveLength(0);
54
+ });
55
+
56
+ test("recurrence without duration does not escalate (10 events in 5 min)", () => {
57
+ const decision = decideSweep([agg("burst.error", 10, 5, 0)], [], POLICY, NOW);
58
+ expect(decision.escalate).toHaveLength(0);
59
+ });
60
+
61
+ test("duration without recurrence does not escalate (3 events over 40 min)", () => {
62
+ const decision = decideSweep([agg("sparse.error", 3, 40, 0)], [], POLICY, NOW);
63
+ expect(decision.escalate).toHaveLength(0);
64
+ });
65
+
66
+ test("the 490-over-12h shape escalates (duration + recurrence)", () => {
67
+ const sustained = agg("host-command-bus.error", 490, 720, 0);
68
+ const decision = decideSweep([sustained], [], POLICY, NOW);
69
+ expect(decision.escalate).toHaveLength(1);
70
+ expect(decision.escalate[0]!.aggregate.kind).toBe("host-command-bus.error");
71
+ expect(decision.escalate[0]!.episode).toBeNull();
72
+ });
73
+
74
+ test("per-kind threshold override lets a low-rate persistent kind through", () => {
75
+ // guard.fail.open: 8 events over 12h — below the default recurrence floor
76
+ // of 10, but persistent. The override lowers recurrence to 4 over 60 min.
77
+ const drip = agg("guard.fail.open", 8, 720, 0);
78
+ const withoutOverride = decideSweep([drip], [], POLICY, NOW);
79
+ expect(withoutOverride.escalate).toHaveLength(0);
80
+ const withOverride = decideSweep([drip], [], POLICY, NOW, {
81
+ "guard.fail.open": { minCount: 4, minSpanMinutes: 60 },
82
+ });
83
+ expect(withOverride.escalate).toHaveLength(1);
84
+ });
85
+
86
+ test("override raises a floor too — bursty kinds can be tightened", () => {
87
+ const sustained = agg("llm.provider.error", 40, 60, 0); // passes default
88
+ const decision = decideSweep([sustained], [], POLICY, NOW, {
89
+ "llm.provider.error": { minCount: 100 },
90
+ });
91
+ expect(decision.escalate).toHaveLength(0);
92
+ });
93
+ });
94
+
95
+ describe("decideSweep — dedup (one escalation per fault, no re-fire)", () => {
96
+ test("already escalated and unresolved does NOT re-escalate", () => {
97
+ const sustained = agg("host-command-bus.error", 500, 720, 0);
98
+ const open = episode({ kind: "host-command-bus.error", escalatedAt: new Date(NOW.getTime() - 3_600_000) });
99
+ const decision = decideSweep([sustained], [open], POLICY, NOW);
100
+ expect(decision.escalate).toHaveLength(0);
101
+ });
102
+
103
+ test("open delivered episode past the update window gets an UPDATE, not a message", () => {
104
+ const sustained = agg("llm.provider.error", 300, 720, 0);
105
+ const open = episode({
106
+ kind: "llm.provider.error",
107
+ escalatedAt: new Date(NOW.getTime() - 90 * 60_000), // last update long ago
108
+ lastUpdatedAt: new Date(NOW.getTime() - 90 * 60_000),
109
+ });
110
+ const decision = decideSweep([sustained], [open], POLICY, NOW);
111
+ expect(decision.escalate).toHaveLength(0);
112
+ expect(decision.update).toHaveLength(1);
113
+ expect(decision.update[0]!.episode.workItemId).toBe("wi-1");
114
+ });
115
+
116
+ test("open episode inside the update window gets NOTHING (dedup window)", () => {
117
+ const sustained = agg("llm.provider.error", 300, 720, 0);
118
+ const open = episode({
119
+ kind: "llm.provider.error",
120
+ escalatedAt: new Date(NOW.getTime() - 10 * 60_000),
121
+ lastUpdatedAt: new Date(NOW.getTime() - 10 * 60_000),
122
+ });
123
+ const decision = decideSweep([sustained], [open], POLICY, NOW);
124
+ expect(decision.escalate).toHaveLength(0);
125
+ expect(decision.update).toHaveLength(0);
126
+ });
127
+
128
+ test("a claimed-but-undelivered episode is RETRIED, not counted as escalated", () => {
129
+ const sustained = agg("guard.fail.open", 40, 720, 0);
130
+ const claim = episode({ kind: "guard.fail.open", escalatedAt: null, workItemId: null });
131
+ const decision = decideSweep([sustained], [claim], POLICY, NOW);
132
+ expect(decision.escalate).toHaveLength(1);
133
+ expect(decision.escalate[0]!.episode).toBe(claim);
134
+ });
135
+ });
136
+
137
+ describe("decideSweep — resolution + re-arm", () => {
138
+ test("quiet past the resolve window closes the episode", () => {
139
+ const gone = episode({ kind: "wake.failed", lastSeenAt: new Date(NOW.getTime() - 60 * 60_000) });
140
+ const decision = decideSweep([], [gone], POLICY, NOW);
141
+ expect(decision.resolve).toHaveLength(1);
142
+ expect(decision.resolve[0]!.quietMinutes).toBeGreaterThanOrEqual(30);
143
+ });
144
+
145
+ test("an absent kind resolves by its episode's last_seen, not the sweep clock", () => {
146
+ const gone = episode({ kind: "listen.connect_failed", lastSeenAt: new Date(NOW.getTime() - 45 * 60_000) });
147
+ const decision = decideSweep([], [gone], POLICY, NOW);
148
+ expect(decision.resolve).toHaveLength(1);
149
+ });
150
+
151
+ test("after resolution the same fault escalates again (new episode)", () => {
152
+ const recurring = agg("host-command-bus.error", 50, 40, 0);
153
+ const resolved = episode({
154
+ kind: "host-command-bus.error",
155
+ resolvedAt: new Date(NOW.getTime() - 35 * 60_000),
156
+ });
157
+ const decision = decideSweep([recurring], [resolved], POLICY, NOW);
158
+ expect(decision.escalate).toHaveLength(1);
159
+ });
160
+
161
+ test("still-active kind within the resolve window is not resolved even if below threshold", () => {
162
+ const active = agg("slow.error", 3, 20, 0); // below threshold, but recent
163
+ const open = episode({ kind: "slow.error", lastSeenAt: active.lastSeen });
164
+ const decision = decideSweep([active], [open], POLICY, NOW);
165
+ expect(decision.resolve).toHaveLength(0);
166
+ });
167
+ });
168
+
169
+ describe("composeEscalationMessage", () => {
170
+ test("carries the three escalation-routing fields and stays phone-sized", () => {
171
+ const message = composeEscalationMessage({
172
+ kind: "host-command-bus.error",
173
+ aggregate: agg("host-command-bus.error", 490, 720, 0),
174
+ impact: "scala-tools verbs are failing host-wide",
175
+ recommendation: "check oxmgr logs for the bus consumer",
176
+ });
177
+ expect(message).toContain("Platform fault: host-command-bus.error");
178
+ expect(message).toContain("Observed: 490 errors over 12 h");
179
+ expect(message).toContain("Impact: scala-tools verbs are failing host-wide");
180
+ expect(message).toContain("Recommendation: check oxmgr logs for the bus consumer");
181
+ expect(message.split("\n")).toHaveLength(4);
182
+ });
183
+ });
@@ -0,0 +1,447 @@
1
+ /**
2
+ * @system server-boot
3
+ * @status handwritten
4
+ * @edit edit directly
5
+ *
6
+ * The `platform-fault-escalation` pg-boss handler — closes the "receiving is
7
+ * decoupled from consequence" gap measured 2026-08-15 (760 error events over
8
+ * 12 hours, zero escalations: host-command-bus.error 490, llm.provider.error
9
+ * 231, guard.fail.open 16, scale-to-zero.wake_failed 12, listen.connect_failed
10
+ * 11). Emission works and observability-is-passive is satisfied; nothing
11
+ * acted. This sweep makes RECEIVING have CONSEQUENCE: sustained platform
12
+ * faults escalate to the OPERATOR over the existing PA path (WhatsApp, the
13
+ * scala PA operator-relay bot — escalation-routing's channel; no second
14
+ * channel invented), with the durable record as a work_items task the PA can
15
+ * read back and the operator can answer on.
16
+ *
17
+ * THIS IS NOT A DRIFT-DETECTOR PATROLLING INSTANCES. It reacts only to
18
+ * components reporting THEIR OWN failure — event_log kinds that already
19
+ * assert a fault about their emitter. Thresholds (duration + recurrence,
20
+ * never a single event), the watched-kind patterns, the resolve window and
21
+ * the per-kind impact/recommendation copy are all in the
22
+ * `config/platform-fault-escalation` registry row — changing policy is a row
23
+ * edit, not a redeploy. Episode dedup lives in the
24
+ * `platform_fault_escalations` table (migration 501): one escalation per
25
+ * distinct fault kind per episode; an already-escalated-and-unresolved fault
26
+ * only appends throttled work-item comments, never a second message.
27
+ *
28
+ * DELIVERY IS VERIFIED, NOT QUEUED. The WhatsApp send goes through the
29
+ * central `/mcp` loopback (`api_gowa_message` sendMessage — the single
30
+ * sealed GOWA-credential surface, same as whatsapp-liveness) and an episode
31
+ * only carries `escalated_at` once the send returned a WhatsApp message id;
32
+ * before that the row is a CLAIM the next sweep retries. Success is recorded
33
+ * only after readback of the delivery facts (message id + work item id).
34
+ *
35
+ * WHY A TICK EXISTS AT ALL, given observability-is-passive and
36
+ * no-polling-for-pushable-state: the trigger is DURATION + RECURRENCE of
37
+ * events already durably logged — a fact about the PATTERN of past events,
38
+ * not a state transition any single event can announce. The sweep reads one
39
+ * server-side aggregate (GROUP BY kind) per run against indexed event_log;
40
+ * while the platform is healthy it returns zero rows and costs nothing
41
+ * beyond the tick (idle-costs-nothing). Same justification
42
+ * lane-attention-sweep carries on its row as `poll_justification`.
43
+ *
44
+ * Registered under slug `@teamscala/service-boot:platform-fault-escalation`.
45
+ */
46
+ import type { ExecutionContext } from "@teamscala/os/contracts/mcp";
47
+ import type { ScheduledJobConfig } from "@teamscala/db-validation/registry-schemas/scheduled-job";
48
+ import * as v from "valibot";
49
+ import { getPrisma } from "@teamscala/db/prisma-registry";
50
+ import { loadRegistryConfig } from "@teamscala/db/registry/load-config";
51
+ import { recordEvent } from "@teamscala/event-log/record-event";
52
+ import { getAppLogger } from "@teamscala/logger/app-loggers";
53
+ import { pgBossHandlerRegistry } from "../pg-boss-handler-registry.ts";
54
+ import {
55
+ composeEscalationDescription,
56
+ composeEscalationMessage,
57
+ decideSweep,
58
+ type EscalationEpisode,
59
+ type FaultAggregate,
60
+ type KindThresholdOverride,
61
+ } from "./platform-fault-escalation-logic.ts";
62
+
63
+ const CONFIG_SLUG = "platform-fault-escalation";
64
+
65
+ const FaultEscalationConfigSchema = v.object({
66
+ /** Postgres regex patterns selecting watched event_log kinds. */
67
+ watched_kind_patterns: v.array(v.string()),
68
+ /** Lookback window for the recurrence/duration aggregate, minutes. */
69
+ window_minutes: v.number(),
70
+ min_count: v.number(),
71
+ min_span_minutes: v.number(),
72
+ /** Per-kind recurrence/duration overrides — low-rate-but-persistent kinds. */
73
+ kind_thresholds: v.optional(
74
+ v.record(
75
+ v.string(),
76
+ v.object({
77
+ min_count: v.optional(v.number()),
78
+ min_span_minutes: v.optional(v.number()),
79
+ }),
80
+ ),
81
+ ),
82
+ resolve_after_minutes: v.number(),
83
+ update_after_minutes: v.number(),
84
+ /** The developer-session project escalations are parented to. */
85
+ parent_project_id: v.string(),
86
+ /** Per-kind copy for the message (impact + recommendation). Falls back to default_copy. */
87
+ copy: v.record(v.string(), v.object({ impact: v.string(), recommendation: v.string() })),
88
+ default_copy: v.object({ impact: v.string(), recommendation: v.string() }),
89
+ });
90
+
91
+ type FaultEscalationConfig = v.InferOutput<typeof FaultEscalationConfigSchema>;
92
+
93
+ interface EventLogAggregateRow {
94
+ kind: string;
95
+ n: bigint;
96
+ first_seen: Date;
97
+ last_seen: Date;
98
+ }
99
+
100
+ interface OperatorRelayBot {
101
+ id: string;
102
+ organisation_id: string;
103
+ external_id: string | null;
104
+ flow_config: { operator_phone?: string } | null;
105
+ }
106
+
107
+ let cachedSystemIdentity: { userId: string | undefined; organisationId: string | undefined } | undefined;
108
+ async function getSystemIdentity(): Promise<{ userId: string | undefined; organisationId: string | undefined }> {
109
+ if (cachedSystemIdentity) return cachedSystemIdentity;
110
+ const row = await loadRegistryConfig("config", "system-identity", v.looseObject({}));
111
+ cachedSystemIdentity = {
112
+ userId: row?.system_user_id as string | undefined,
113
+ organisationId: row?.system_org_id as string | undefined,
114
+ };
115
+ return cachedSystemIdentity;
116
+ }
117
+
118
+ async function loadConfig(): Promise<FaultEscalationConfig> {
119
+ return loadRegistryConfig("config", CONFIG_SLUG, FaultEscalationConfigSchema);
120
+ }
121
+
122
+ async function fetchAggregates(patterns: string[], windowMinutes: number): Promise<FaultAggregate[]> {
123
+ const prisma = getPrisma();
124
+ // Patterns come from the valibot-validated registry row and are regex-only
125
+ // by construction below; embedding them as one alternation keeps the query
126
+ // parameter-free on the pattern side while the window is a real parameter.
127
+ const regexClass = /^[A-Za-z0-9_\\^$.*+?()[\]{}|-]+$/;
128
+ for (const pattern of patterns) {
129
+ if (!regexClass.test(pattern)) {
130
+ throw new Error(`config/${CONFIG_SLUG}: watched_kind_pattern is not regex-only: ${pattern}`);
131
+ }
132
+ }
133
+ const combined = `(${patterns.join("|")})`;
134
+ const rows = await prisma.$queryRawUnsafe<EventLogAggregateRow[]>(
135
+ `SELECT kind, count(*)::bigint AS n, min(occurred_at) AS first_seen, max(occurred_at) AS last_seen
136
+ FROM event_log
137
+ WHERE occurred_at > now() - ($1::int * interval '1 minute')
138
+ AND kind ~ $2::text
139
+ GROUP BY kind`,
140
+ windowMinutes,
141
+ combined,
142
+ );
143
+ return rows.map((row) => ({
144
+ kind: row.kind,
145
+ count: Number(row.n),
146
+ firstSeen: new Date(row.first_seen),
147
+ lastSeen: new Date(row.last_seen),
148
+ }));
149
+ }
150
+
151
+ async function resolveOperatorRelay(): Promise<OperatorRelayBot> {
152
+ const prisma = getPrisma();
153
+ const bot = await prisma.messaging_bots.findFirst({
154
+ where: {
155
+ is_active: true,
156
+ platform: "whatsapp",
157
+ flow_config: { path: ["mode"], equals: "operator-relay" },
158
+ },
159
+ select: { id: true, organisation_id: true, external_id: true, flow_config: true },
160
+ });
161
+ if (!bot) throw new Error("platform-fault-escalation: no active operator-relay WhatsApp bot row");
162
+ const phone = (bot.flow_config as { operator_phone?: string } | null)?.operator_phone;
163
+ if (!phone) throw new Error("platform-fault-escalation: operator-relay bot has no operator_phone");
164
+ return bot as OperatorRelayBot;
165
+ }
166
+
167
+ /** GOWA device whose JID matches the PA number, so the escalation is sent BY the PA. */
168
+ async function resolvePaDeviceId(paPhoneDigits: string): Promise<string | undefined> {
169
+ const { userId, organisationId } = await getSystemIdentity();
170
+ const systemCtx: ExecutionContext = { userId, organisationId } as ExecutionContext;
171
+ const { forwardToolToCentralMcp } = await import("@teamscala/chat-completion/central-tool-forwarder");
172
+ const res = (await forwardToolToCentralMcp(
173
+ "api_gowa_devices",
174
+ { action: "listDevices" },
175
+ systemCtx,
176
+ )) as { results?: Array<{ id: string; jid?: string }> } | undefined;
177
+ const match = (res?.results ?? []).find((device) => (device.jid ?? "").includes(paPhoneDigits));
178
+ return match?.id;
179
+ }
180
+
181
+ function extractMessageId(sendResult: unknown): string | null {
182
+ const results = (sendResult as { results?: unknown } | undefined)?.results;
183
+ const id =
184
+ typeof results === "string"
185
+ ? results
186
+ : Array.isArray(results)
187
+ ? ((results[0] as { id?: unknown } | undefined)?.id as string | undefined)
188
+ : ((results as { id?: unknown } | undefined)?.id as string | undefined);
189
+ return typeof id === "string" && id.length > 0 ? id : null;
190
+ }
191
+
192
+ async function deliverEscalation(
193
+ kind: string,
194
+ aggregate: FaultAggregate,
195
+ config: FaultEscalationConfig,
196
+ ): Promise<{ messageId: string; workItemId: string }> {
197
+ const log = getAppLogger();
198
+ const prisma = getPrisma();
199
+ const now = new Date();
200
+
201
+ // 1. Claim the episode BEFORE sending, so a send failure leaves a durable
202
+ // claim the next sweep retries instead of a lost escalation.
203
+ await prisma.platform_fault_escalations.upsert({
204
+ where: { kind },
205
+ create: {
206
+ kind,
207
+ first_seen_at: aggregate.firstSeen,
208
+ last_seen_at: aggregate.lastSeen,
209
+ event_count: aggregate.count,
210
+ },
211
+ update: {
212
+ first_seen_at: aggregate.firstSeen,
213
+ last_seen_at: aggregate.lastSeen,
214
+ event_count: aggregate.count,
215
+ resolved_at: null,
216
+ },
217
+ });
218
+
219
+ // 2. Resolve the PA path (bot → operator phone → PA device) and send.
220
+ const bot = await resolveOperatorRelay();
221
+ const operatorPhone = (bot.flow_config as { operator_phone?: string }).operator_phone!;
222
+ const copy = config.copy[kind] ?? config.default_copy;
223
+ const message = composeEscalationMessage({
224
+ kind,
225
+ aggregate,
226
+ impact: copy.impact,
227
+ recommendation: copy.recommendation,
228
+ });
229
+
230
+ const { userId, organisationId } = await getSystemIdentity();
231
+ const systemCtx: ExecutionContext = { userId, organisationId } as ExecutionContext;
232
+ const { forwardToolToCentralMcp } = await import("@teamscala/chat-completion/central-tool-forwarder");
233
+ const paDigits = (bot.external_id ?? "").replace(/\D/g, "");
234
+ const deviceId = paDigits ? await resolvePaDeviceId(paDigits) : undefined;
235
+ if (paDigits && !deviceId) {
236
+ log.warn("[platform-fault-escalation] no GOWA device matched the PA number — sending via default device");
237
+ }
238
+ const sendResult = await forwardToolToCentralMcp(
239
+ "api_gowa_message",
240
+ {
241
+ action: "sendMessage",
242
+ params: { phone: operatorPhone, message, ...(deviceId ? { device_id: deviceId } : {}) },
243
+ },
244
+ systemCtx,
245
+ );
246
+
247
+ // 3. VERIFY delivery before recording it: the send must return a WhatsApp
248
+ // message id. No id => throw, the claim stays undelivered, next sweep retries.
249
+ const messageId = extractMessageId(sendResult);
250
+ if (!messageId) {
251
+ throw new Error(
252
+ `platform-fault-escalation: WhatsApp send for '${kind}' returned no message id — not recording delivery`,
253
+ );
254
+ }
255
+
256
+ // 4. Durable record: the work_items task the PA reads back / the operator answers on.
257
+ const workItem = await prisma.work_items.create({
258
+ data: {
259
+ kind: "task",
260
+ parent_id: config.parent_project_id,
261
+ organisation_id: bot.organisation_id,
262
+ title: `Fault: ${kind}`.slice(0, 50),
263
+ description: composeEscalationDescription({ aggregate, impact: copy.impact, recommendation: copy.recommendation }),
264
+ status: "pending",
265
+ kind_data: {
266
+ platform_fault: true,
267
+ event_kind: kind,
268
+ event_count: aggregate.count,
269
+ first_seen_at: aggregate.firstSeen.toISOString(),
270
+ last_seen_at: aggregate.lastSeen.toISOString(),
271
+ delivered_message_id: messageId,
272
+ escalated_at: now.toISOString(),
273
+ },
274
+ },
275
+ select: { id: true },
276
+ });
277
+
278
+ // 5. Readback-correct the episode: delivered + linked in one update.
279
+ await prisma.platform_fault_escalations.update({
280
+ where: { kind },
281
+ data: {
282
+ escalated_at: now,
283
+ work_item_id: workItem.id,
284
+ delivered_message_id: messageId,
285
+ last_updated_at: now,
286
+ updated_at: now,
287
+ },
288
+ });
289
+
290
+ recordEvent({
291
+ kind: "platform.fault.escalated",
292
+ payload: {
293
+ event_kind: kind,
294
+ count: aggregate.count,
295
+ work_item_id: workItem.id,
296
+ delivered_message_id: messageId,
297
+ },
298
+ });
299
+ return { messageId, workItemId: workItem.id };
300
+ }
301
+
302
+ async function updateEpisode(aggregate: FaultAggregate, episode: EscalationEpisode): Promise<void> {
303
+ const prisma = getPrisma();
304
+ const now = new Date();
305
+ await prisma.platform_fault_escalations.update({
306
+ where: { kind: episode.kind },
307
+ data: {
308
+ last_seen_at: aggregate.lastSeen,
309
+ event_count: aggregate.count,
310
+ last_updated_at: now,
311
+ updated_at: now,
312
+ },
313
+ });
314
+ if (episode.workItemId) {
315
+ await prisma.comments.create({
316
+ data: {
317
+ work_item_id: episode.workItemId,
318
+ content: `Still recurring: ${aggregate.count} events, last at ${aggregate.lastSeen.toISOString()}. No new escalation (already escalated and unresolved).`,
319
+ sender_role: "system",
320
+ sender_name: "platform-fault-escalation",
321
+ },
322
+ });
323
+ }
324
+ recordEvent({
325
+ kind: "platform.fault.updated",
326
+ payload: { event_kind: episode.kind, count: aggregate.count, work_item_id: episode.workItemId },
327
+ });
328
+ }
329
+
330
+ async function resolveEpisode(episode: EscalationEpisode, quietMinutes: number): Promise<void> {
331
+ const prisma = getPrisma();
332
+ const now = new Date();
333
+ await prisma.platform_fault_escalations.update({
334
+ where: { kind: episode.kind },
335
+ data: { resolved_at: now, updated_at: now },
336
+ });
337
+ if (episode.workItemId) {
338
+ await prisma.comments.create({
339
+ data: {
340
+ work_item_id: episode.workItemId,
341
+ content: `Fault stopped — no '${episode.kind}' events for ${Math.round(quietMinutes)} min. Episode closed; the fault will re-escalate if it recurs.`,
342
+ sender_role: "system",
343
+ sender_name: "platform-fault-escalation",
344
+ },
345
+ });
346
+ }
347
+ recordEvent({
348
+ kind: "platform.fault.resolved",
349
+ payload: { event_kind: episode.kind, quiet_minutes: Math.round(quietMinutes), work_item_id: episode.workItemId },
350
+ });
351
+ }
352
+
353
+ async function platformFaultEscalationSweep(ctx: { config: ScheduledJobConfig }): Promise<void> {
354
+ const log = getAppLogger();
355
+ const config = await loadConfig();
356
+ const now = new Date();
357
+
358
+ const aggregates = await fetchAggregates(config.watched_kind_patterns, config.window_minutes);
359
+ // Only OPEN episodes matter: a resolved episode and no episode at all have
360
+ // the same meaning to decideSweep (a recurring sustained fault opens a NEW
361
+ // episode and escalates again).
362
+ const openEpisodes = await getPrisma().platform_fault_escalations.findMany({
363
+ where: { resolved_at: null },
364
+ });
365
+ const episodes: EscalationEpisode[] = openEpisodes.map((row) => ({
366
+ kind: row.kind,
367
+ firstSeenAt: new Date(row.first_seen_at),
368
+ lastSeenAt: new Date(row.last_seen_at),
369
+ eventCount: row.event_count,
370
+ escalatedAt: row.escalated_at ? new Date(row.escalated_at) : null,
371
+ workItemId: row.work_item_id,
372
+ resolvedAt: row.resolved_at ? new Date(row.resolved_at) : null,
373
+ lastUpdatedAt: row.last_updated_at ? new Date(row.last_updated_at) : null,
374
+ }));
375
+
376
+ const kindThresholds: Record<string, KindThresholdOverride> = {};
377
+ for (const [kind, override] of Object.entries(config.kind_thresholds ?? {})) {
378
+ kindThresholds[kind] = {
379
+ minCount: override?.min_count,
380
+ minSpanMinutes: override?.min_span_minutes,
381
+ };
382
+ }
383
+ const decision = decideSweep(
384
+ aggregates,
385
+ episodes,
386
+ {
387
+ minCount: config.min_count,
388
+ minSpanMinutes: config.min_span_minutes,
389
+ resolveAfterMinutes: config.resolve_after_minutes,
390
+ updateAfterMinutes: config.update_after_minutes,
391
+ },
392
+ now,
393
+ kindThresholds,
394
+ );
395
+
396
+ // Per-kind isolation: one fault's delivery failure must never block the
397
+ // others. Failures are logged + emitted so the escalator itself is
398
+ // observable (the sweep that silently stops sweeping is the original bug).
399
+ for (const { aggregate } of decision.escalate) {
400
+ try {
401
+ const delivered = await deliverEscalation(aggregate.kind, aggregate, config);
402
+ log.warn(
403
+ `[platform-fault-escalation] ESCALATED '${aggregate.kind}' — WhatsApp message ${delivered.messageId}, work item ${delivered.workItemId}`,
404
+ );
405
+ } catch (err) {
406
+ const reason = err instanceof Error ? err.message : String(err);
407
+ log.error(`[platform-fault-escalation] delivery for '${aggregate.kind}' failed (claim retained, retried next sweep): ${reason}`);
408
+ recordEvent({ kind: "platform.fault.escalation_failed", payload: { event_kind: aggregate.kind, reason } });
409
+ }
410
+ }
411
+ for (const { aggregate, episode } of decision.update) {
412
+ try {
413
+ await updateEpisode(aggregate, episode);
414
+ } catch (err) {
415
+ log.error(`[platform-fault-escalation] update for '${episode.kind}' failed: ${err instanceof Error ? err.message : String(err)}`);
416
+ }
417
+ }
418
+ for (const { episode, quietMinutes } of decision.resolve) {
419
+ try {
420
+ await resolveEpisode(episode, quietMinutes);
421
+ } catch (err) {
422
+ log.error(`[platform-fault-escalation] resolve for '${episode.kind}' failed: ${err instanceof Error ? err.message : String(err)}`);
423
+ }
424
+ }
425
+
426
+ // Emitted every run, even when nothing fired — a sweep that only reports
427
+ // when it escalated is indistinguishable from one that stopped running.
428
+ recordEvent({
429
+ kind: "platform.fault.swept",
430
+ payload: {
431
+ watched_kinds: aggregates.length,
432
+ escalated: decision.escalate.length,
433
+ updated: decision.update.length,
434
+ resolved: decision.resolve.length,
435
+ },
436
+ });
437
+ if (decision.escalate.length > 0 || decision.update.length > 0 || decision.resolve.length > 0) {
438
+ log.info(
439
+ `[platform-fault-escalation] sweep — ${aggregates.length} kind(s) watched, ${decision.escalate.length} escalated, ${decision.update.length} updated, ${decision.resolve.length} resolved`,
440
+ );
441
+ }
442
+ }
443
+
444
+ pgBossHandlerRegistry.register(
445
+ "@teamscala/service-boot:platform-fault-escalation",
446
+ platformFaultEscalationSweep as (ctx: { config: ScheduledJobConfig }) => Promise<void>,
447
+ );
@@ -35,6 +35,7 @@ import "./handlers/singleton-tools-currency-watch.ts";
35
35
  import "./handlers/registry-stale-inactive-sweep.ts";
36
36
  import "./handlers/pane-outbox-drain.ts";
37
37
  import "./handlers/lane-attention-sweep.ts";
38
+ import "./handlers/platform-fault-escalation.ts";
38
39
 
39
40
  /**
40
41
  * Module-level reference to the mounted PgBoss instance — set by mountPgBoss on
@@ -0,0 +1,82 @@
1
+ /**
2
+ * @system service-boot
3
+ * @status handwritten
4
+ * @edit edit directly
5
+ *
6
+ * Behavior proof for the `security-content-type-nosniff` +
7
+ * `security-permissions-policy` invariants: exercise injectSecurityHeaders
8
+ * through a real Hono request and assert the headers on the RESPONSE —
9
+ * a registry row naming a header is not evidence the header is served.
10
+ * Covers both modes: the seeded-registry path (config row applied, CSP
11
+ * {{nonce}} substituted) and the DB-less fallback (secureHeaders defaults).
12
+ */
13
+ import { describe, expect, it } from "bun:test";
14
+ import { Hono } from "hono";
15
+ import { injectSecurityHeaders } from "./security-headers.ts";
16
+
17
+ type CacheGlobal = typeof globalThis & {
18
+ __registryCache?: Map<string, { type: string; slug: string; config: unknown }>;
19
+ };
20
+
21
+ function seedRegistryHeaders(config: Record<string, string>): void {
22
+ const g = globalThis as CacheGlobal;
23
+ if (!g.__registryCache) g.__registryCache = new Map();
24
+ g.__registryCache.set("config:security-headers", {
25
+ type: "config",
26
+ slug: "security-headers",
27
+ config,
28
+ });
29
+ }
30
+
31
+ function clearRegistryHeaders(): void {
32
+ (globalThis as CacheGlobal).__registryCache?.delete("config:security-headers");
33
+ }
34
+
35
+ function appWithMiddleware(): Hono {
36
+ const app = new Hono();
37
+ app.use("*", injectSecurityHeaders);
38
+ app.get("/", (c) => c.text("ok"));
39
+ return app;
40
+ }
41
+
42
+ describe("injectSecurityHeaders (registry mode)", () => {
43
+ it("serves X-Content-Type-Options: nosniff and the configured Permissions-Policy", async () => {
44
+ seedRegistryHeaders({
45
+ "X-Content-Type-Options": "nosniff",
46
+ "Permissions-Policy": "camera=(), microphone=(self), geolocation=()",
47
+ "Content-Security-Policy": "script-src 'nonce-{{nonce}}' 'strict-dynamic'",
48
+ });
49
+ try {
50
+ const res = await appWithMiddleware().request("https://host.example/");
51
+ expect(res.headers.get("X-Content-Type-Options")).toBe("nosniff");
52
+ expect(res.headers.get("Permissions-Policy")).toBe(
53
+ "camera=(), microphone=(self), geolocation=()",
54
+ );
55
+ } finally {
56
+ clearRegistryHeaders();
57
+ }
58
+ });
59
+
60
+ it("substitutes the CSP {{nonce}} placeholder — the served policy carries a real nonce", async () => {
61
+ seedRegistryHeaders({
62
+ "Content-Security-Policy": "script-src 'nonce-{{nonce}}' 'strict-dynamic'",
63
+ });
64
+ try {
65
+ const res = await appWithMiddleware().request("https://host.example/");
66
+ const csp = res.headers.get("Content-Security-Policy") ?? "";
67
+ expect(csp).not.toContain("{{nonce}}");
68
+ expect(csp).toMatch(/'nonce-[^']+'/);
69
+ } finally {
70
+ clearRegistryHeaders();
71
+ }
72
+ });
73
+ });
74
+
75
+ describe("injectSecurityHeaders (fallback mode, no registry)", () => {
76
+ it("still serves X-Content-Type-Options: nosniff from secureHeaders defaults", async () => {
77
+ clearRegistryHeaders();
78
+ const res = await appWithMiddleware().request("https://host.example/");
79
+ expect(res.headers.get("X-Content-Type-Options")).toBe("nosniff");
80
+ expect((res.headers.get("Content-Security-Policy") ?? "")).not.toContain("{{nonce}}");
81
+ });
82
+ });
@@ -18,6 +18,7 @@ import { buildWakingPageHtml } from "@teamscala/os/brand/waking-page-html";
18
18
  import { BootFatalError } from "@teamscala/os/errors/boot-error";
19
19
  import { createTimer } from "@teamscala/timing/create-timer";
20
20
  import { Hono, type Context } from "hono";
21
+ import { bodyLimit } from "hono/body-limit";
21
22
  import { compress } from "hono/compress";
22
23
  import { cors } from "hono/cors";
23
24
  import { timing } from "hono/timing";
@@ -33,6 +34,7 @@ import { attachProposalMagic } from "./attach-proposal-magic.ts";
33
34
  import { attachPortalMembers } from "./attach-portal-members.ts";
34
35
  import { attachTransfersUpload } from "./attach-transfers-upload.ts";
35
36
  import { attachAuthGate } from "./auth-gate.ts";
37
+ import { csrfProtection } from "./csrf-protection.ts";
36
38
  import { attachRequestLogger } from "./install-logging.ts";
37
39
  import type { ServiceInstallConfig } from "./load-service-install-row.ts";
38
40
  import { mountOrpcRouter } from "./mount-orpc-router.ts";
@@ -276,6 +278,26 @@ export async function buildMinimalApp(
276
278
  );
277
279
  app.use("*", compress());
278
280
  app.use("*", timing());
281
+ // CSRF (`security-csrf-protection`): state-changing form-element requests
282
+ // must carry a trusted Origin — derived set (managed base domain +
283
+ // builda_websites custom domains via isTrustedAuthOrigin), never an
284
+ // allowlist beside better-auth's. Registered after security headers, before
285
+ // every route + gate.
286
+ app.use("*", csrfProtection());
287
+ // GLOBAL body limit (`security-body-limit`): every request body is bounded
288
+ // BEFORE any handler buffers it — a per-route file.size check AFTER
289
+ // formData() has buffered is a report, not a limit. 30MB sits above the
290
+ // 25MB transfers-upload maximum; routes wanting a smaller bound mount their
291
+ // own bodyLimit BEFORE this one (Hono runs matching middleware in
292
+ // registration order, and /api/rpc registers its 10MB limit in
293
+ // mount-orpc-router).
294
+ app.use(
295
+ "*",
296
+ bodyLimit({
297
+ maxSize: 30 * 1024 * 1024,
298
+ onError: (c) => c.text("Payload Too Large", 413),
299
+ }),
300
+ );
279
301
  attachRequestLogger(app, serviceSlug);
280
302
 
281
303
  // Fire independent I/O + compilation NOW, overlapping with the gate chain