@teamscala/service-boot 0.23.358 → 0.23.361
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +26 -1
- package/src/attach-transfers-upload.ts +10 -0
- package/src/csrf-body-limit.test.ts +177 -0
- package/src/csrf-protection.ts +100 -0
- package/src/handlers/platform-fault-escalation-logic.ts +185 -0
- package/src/handlers/platform-fault-escalation.test.ts +183 -0
- package/src/handlers/platform-fault-escalation.ts +447 -0
- package/src/pg-boss-mount.ts +1 -0
- package/src/security-headers.test.ts +82 -0
- package/src/two-phase-boot.ts +22 -0
package/package.json
CHANGED
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
},
|
|
7
7
|
"name": "@teamscala/service-boot",
|
|
8
8
|
"type": "module",
|
|
9
|
-
"version": "0.23.
|
|
9
|
+
"version": "0.23.361",
|
|
10
10
|
"sideEffects": false,
|
|
11
11
|
"imports": {
|
|
12
12
|
"#service-boot/*": "./src/*",
|
|
@@ -21,6 +21,7 @@
|
|
|
21
21
|
"@teamscala/auth": "*",
|
|
22
22
|
"@teamscala/cache": "*",
|
|
23
23
|
"@teamscala/channel": "*",
|
|
24
|
+
"@teamscala/chat-completion": "*",
|
|
24
25
|
"@teamscala/db": "*",
|
|
25
26
|
"@teamscala/db-validation": "*",
|
|
26
27
|
"@teamscala/document-signing": "*",
|
|
@@ -229,6 +230,14 @@
|
|
|
229
230
|
"types": "./src/content-page-floor.d.ts",
|
|
230
231
|
"default": "./src/content-page-floor.ts"
|
|
231
232
|
},
|
|
233
|
+
"./csrf-protection": {
|
|
234
|
+
"types": "./src/csrf-protection.d.ts",
|
|
235
|
+
"default": "./src/csrf-protection.ts"
|
|
236
|
+
},
|
|
237
|
+
"./csrf-protection.ts": {
|
|
238
|
+
"types": "./src/csrf-protection.d.ts",
|
|
239
|
+
"default": "./src/csrf-protection.ts"
|
|
240
|
+
},
|
|
232
241
|
"./discover-blocks": {
|
|
233
242
|
"types": "./src/discover-blocks.d.ts",
|
|
234
243
|
"default": "./src/discover-blocks.ts"
|
|
@@ -417,6 +426,22 @@
|
|
|
417
426
|
"types": "./src/handlers/pane-outbox-drain.d.ts",
|
|
418
427
|
"default": "./src/handlers/pane-outbox-drain.ts"
|
|
419
428
|
},
|
|
429
|
+
"./handlers/platform-fault-escalation": {
|
|
430
|
+
"types": "./src/handlers/platform-fault-escalation.d.ts",
|
|
431
|
+
"default": "./src/handlers/platform-fault-escalation.ts"
|
|
432
|
+
},
|
|
433
|
+
"./handlers/platform-fault-escalation.ts": {
|
|
434
|
+
"types": "./src/handlers/platform-fault-escalation.d.ts",
|
|
435
|
+
"default": "./src/handlers/platform-fault-escalation.ts"
|
|
436
|
+
},
|
|
437
|
+
"./handlers/platform-fault-escalation-logic": {
|
|
438
|
+
"types": "./src/handlers/platform-fault-escalation-logic.d.ts",
|
|
439
|
+
"default": "./src/handlers/platform-fault-escalation-logic.ts"
|
|
440
|
+
},
|
|
441
|
+
"./handlers/platform-fault-escalation-logic.ts": {
|
|
442
|
+
"types": "./src/handlers/platform-fault-escalation-logic.d.ts",
|
|
443
|
+
"default": "./src/handlers/platform-fault-escalation-logic.ts"
|
|
444
|
+
},
|
|
420
445
|
"./handlers/projects-digest": {
|
|
421
446
|
"types": "./src/handlers/projects-digest.d.ts",
|
|
422
447
|
"default": "./src/handlers/projects-digest.ts"
|
|
@@ -62,6 +62,16 @@ export async function attachTransfersUpload(app: Hono): Promise<void> {
|
|
|
62
62
|
const userId = session?.user?.id;
|
|
63
63
|
if (!userId) return c.json({ error: "Not authenticated" }, 401);
|
|
64
64
|
|
|
65
|
+
// Reject on Content-Length BEFORE formData() buffers the body — a
|
|
66
|
+
// file.size check after buffering is a report, not a limit (an attacker
|
|
67
|
+
// sends the bytes either way). The global 30MB boot-level bodyLimit
|
|
68
|
+
// remains the real bound for lying/absent Content-Length; this early
|
|
69
|
+
// return bounds the honest case without parsing a megabyte stream.
|
|
70
|
+
const declaredLength = Number(c.req.header("content-length") ?? 0);
|
|
71
|
+
if (declaredLength > MAX_FILE_SIZE) {
|
|
72
|
+
return c.json({ error: "file too large (max 25MB)" }, 413);
|
|
73
|
+
}
|
|
74
|
+
|
|
65
75
|
const form = await c.req.formData();
|
|
66
76
|
const file = form.get("file");
|
|
67
77
|
const type = String(form.get("type") ?? "uploads");
|
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @system service-boot
|
|
3
|
+
* @status handwritten
|
|
4
|
+
* @edit edit directly
|
|
5
|
+
*
|
|
6
|
+
* Behavior proofs for the two boot-chain security invariants:
|
|
7
|
+
* - security-csrf-protection: a state-changing form-element POST from an
|
|
8
|
+
* untrusted Origin is refused 403; the managed-base-domain Origin (the
|
|
9
|
+
* derived trusted set) is allowed; same-origin sec-fetch-site is allowed.
|
|
10
|
+
* - security-body-limit: a body over the global limit is refused 413 BEFORE
|
|
11
|
+
* any handler runs, and the transfers upload rejects an over-sized
|
|
12
|
+
* Content-Length without buffering the body.
|
|
13
|
+
* Proxy-is-not-behaviour guards: these exercise the real middleware through
|
|
14
|
+
* real Hono requests.
|
|
15
|
+
*/
|
|
16
|
+
import { describe, expect, it } from "bun:test";
|
|
17
|
+
import { Hono } from "hono";
|
|
18
|
+
import { bodyLimit } from "hono/body-limit";
|
|
19
|
+
import { csrf } from "hono/csrf";
|
|
20
|
+
import { csrfProtection } from "./csrf-protection.ts";
|
|
21
|
+
|
|
22
|
+
type CacheGlobal = typeof globalThis & {
|
|
23
|
+
__registryCache?: Map<string, { type: string; slug: string; config: unknown }>;
|
|
24
|
+
};
|
|
25
|
+
|
|
26
|
+
function seedPortalBaseDomain(base: string): void {
|
|
27
|
+
const g = globalThis as CacheGlobal;
|
|
28
|
+
if (!g.__registryCache) g.__registryCache = new Map();
|
|
29
|
+
g.__registryCache.set("config:portal-domains", {
|
|
30
|
+
type: "config",
|
|
31
|
+
slug: "portal-domains",
|
|
32
|
+
config: { base },
|
|
33
|
+
});
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
describe("csrfProtection", () => {
|
|
37
|
+
it("refuses a cross-site form POST from an untrusted origin (403)", async () => {
|
|
38
|
+
seedPortalBaseDomain("scala.business");
|
|
39
|
+
const app = new Hono();
|
|
40
|
+
app.use("*", csrfProtection());
|
|
41
|
+
app.post("/api/settings", (c) => c.text("ok"));
|
|
42
|
+
const res = await app.request("https://portal.scala.business/api/settings", {
|
|
43
|
+
method: "POST",
|
|
44
|
+
headers: {
|
|
45
|
+
"content-type": "application/x-www-form-urlencoded",
|
|
46
|
+
origin: "https://evil.example",
|
|
47
|
+
},
|
|
48
|
+
body: "x=1",
|
|
49
|
+
});
|
|
50
|
+
expect(res.status).toBe(403);
|
|
51
|
+
});
|
|
52
|
+
|
|
53
|
+
it("allows a form POST whose origin is under the managed base domain", async () => {
|
|
54
|
+
seedPortalBaseDomain("scala.business");
|
|
55
|
+
const app = new Hono();
|
|
56
|
+
app.use("*", csrfProtection());
|
|
57
|
+
app.post("/api/settings", (c) => c.text("ok"));
|
|
58
|
+
const res = await app.request("https://portal.scala.business/api/settings", {
|
|
59
|
+
method: "POST",
|
|
60
|
+
headers: {
|
|
61
|
+
"content-type": "application/x-www-form-urlencoded",
|
|
62
|
+
// cross-SERVICE, same platform: the proposals → sign redirect-chain shape
|
|
63
|
+
origin: "https://sign.scala.business",
|
|
64
|
+
},
|
|
65
|
+
body: "x=1",
|
|
66
|
+
});
|
|
67
|
+
expect(res.status).toBe(200);
|
|
68
|
+
});
|
|
69
|
+
|
|
70
|
+
it("allows a same-origin browser post via sec-fetch-site", async () => {
|
|
71
|
+
seedPortalBaseDomain("scala.business");
|
|
72
|
+
const app = new Hono();
|
|
73
|
+
app.use("*", csrfProtection());
|
|
74
|
+
app.post("/api/settings", (c) => c.text("ok"));
|
|
75
|
+
const res = await app.request("https://portal.scala.business/api/settings", {
|
|
76
|
+
method: "POST",
|
|
77
|
+
headers: {
|
|
78
|
+
"content-type": "application/x-www-form-urlencoded",
|
|
79
|
+
"sec-fetch-site": "same-origin",
|
|
80
|
+
},
|
|
81
|
+
body: "x=1",
|
|
82
|
+
});
|
|
83
|
+
expect(res.status).toBe(200);
|
|
84
|
+
});
|
|
85
|
+
|
|
86
|
+
it("does not arm on application/json (cross-site JSON is not formable)", async () => {
|
|
87
|
+
seedPortalBaseDomain("scala.business");
|
|
88
|
+
const app = new Hono();
|
|
89
|
+
app.use("*", csrfProtection());
|
|
90
|
+
app.post("/api/rpc/x", (c) => c.text("ok"));
|
|
91
|
+
const res = await app.request("https://portal.scala.business/api/rpc/x", {
|
|
92
|
+
method: "POST",
|
|
93
|
+
headers: { "content-type": "application/json", origin: "https://evil.example" },
|
|
94
|
+
body: JSON.stringify({ x: 1 }),
|
|
95
|
+
});
|
|
96
|
+
expect(res.status).toBe(200);
|
|
97
|
+
});
|
|
98
|
+
});
|
|
99
|
+
|
|
100
|
+
describe("global body limit (security-body-limit)", () => {
|
|
101
|
+
const limit = 1024;
|
|
102
|
+
|
|
103
|
+
function appWithLimit(): Hono {
|
|
104
|
+
const app = new Hono();
|
|
105
|
+
let handlerRan = false;
|
|
106
|
+
app.use(
|
|
107
|
+
"*",
|
|
108
|
+
bodyLimit({
|
|
109
|
+
maxSize: limit,
|
|
110
|
+
onError: (c) => c.text("Payload Too Large", 413),
|
|
111
|
+
}),
|
|
112
|
+
);
|
|
113
|
+
app.post("/api/upload", async (c) => {
|
|
114
|
+
handlerRan = true;
|
|
115
|
+
return c.text("ok");
|
|
116
|
+
});
|
|
117
|
+
// expose the flag after the request via a closure read below
|
|
118
|
+
(app as unknown as Record<string, unknown>).__handlerRan = () => handlerRan;
|
|
119
|
+
return app;
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
it("refuses an oversized body with 413 before the handler runs", async () => {
|
|
123
|
+
const app = appWithLimit();
|
|
124
|
+
const res = await app.request("https://host.example/api/upload", {
|
|
125
|
+
method: "POST",
|
|
126
|
+
headers: { "content-type": "application/octet-stream" },
|
|
127
|
+
body: "x".repeat(limit + 1),
|
|
128
|
+
});
|
|
129
|
+
expect(res.status).toBe(413);
|
|
130
|
+
expect(
|
|
131
|
+
(app as unknown as { __handlerRan: () => boolean }).__handlerRan(),
|
|
132
|
+
).toBe(false);
|
|
133
|
+
});
|
|
134
|
+
|
|
135
|
+
it("allows a body within the limit", async () => {
|
|
136
|
+
const app = appWithLimit();
|
|
137
|
+
const res = await app.request("https://host.example/api/upload", {
|
|
138
|
+
method: "POST",
|
|
139
|
+
headers: { "content-type": "application/octet-stream" },
|
|
140
|
+
body: "x".repeat(limit - 1),
|
|
141
|
+
});
|
|
142
|
+
expect(res.status).toBe(200);
|
|
143
|
+
});
|
|
144
|
+
|
|
145
|
+
it("a route-level smaller limit still refuses what the global limit allows", async () => {
|
|
146
|
+
const app = new Hono();
|
|
147
|
+
app.use(
|
|
148
|
+
"*",
|
|
149
|
+
bodyLimit({
|
|
150
|
+
maxSize: 4 * limit,
|
|
151
|
+
onError: (c) => c.text("Payload Too Large", 413),
|
|
152
|
+
}),
|
|
153
|
+
);
|
|
154
|
+
app.post(
|
|
155
|
+
"/api/rpc/*",
|
|
156
|
+
bodyLimit({
|
|
157
|
+
maxSize: limit,
|
|
158
|
+
onError: (c) => c.text("Request body too large", 413),
|
|
159
|
+
}),
|
|
160
|
+
(c) => c.text("ok"),
|
|
161
|
+
);
|
|
162
|
+
const res = await app.request("https://host.example/api/rpc/x", {
|
|
163
|
+
method: "POST",
|
|
164
|
+
headers: { "content-type": "application/json" },
|
|
165
|
+
body: "x".repeat(2 * limit),
|
|
166
|
+
});
|
|
167
|
+
expect(res.status).toBe(413);
|
|
168
|
+
});
|
|
169
|
+
});
|
|
170
|
+
|
|
171
|
+
describe("csrf import wiring", () => {
|
|
172
|
+
it("csrfProtection() composes hono csrf without options drift", () => {
|
|
173
|
+
const mw = csrfProtection();
|
|
174
|
+
expect(typeof mw).toBe("function");
|
|
175
|
+
expect(typeof csrf).toBe("function");
|
|
176
|
+
});
|
|
177
|
+
});
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @system server-boot
|
|
3
|
+
* @status handwritten
|
|
4
|
+
* @edit edit directly
|
|
5
|
+
*
|
|
6
|
+
* csrf-protection.ts — platform CSRF middleware (`security-csrf-protection`).
|
|
7
|
+
*
|
|
8
|
+
* Hono's csrf() with a DERIVED trusted-origin set, never a hand-authored
|
|
9
|
+
* allowlist: every trusted origin comes from ONE source per
|
|
10
|
+
* `offered-is-derived` —
|
|
11
|
+
* - the managed base domain (registry portal-domains) covers the platform's
|
|
12
|
+
* own `*.scala.business` origins, INCLUDING the cross-service form POST
|
|
13
|
+
* redirect chains CSP form-action allowlists (proposals → sign etc.), so
|
|
14
|
+
* the CSRF origin set and the CSP form-action set cannot disagree about
|
|
15
|
+
* what "one of ours" means;
|
|
16
|
+
* - customer custom domains resolve through `isTrustedAuthOrigin`
|
|
17
|
+
* (builda_websites lookup) — the SAME lookup better-auth's trustedOrigins
|
|
18
|
+
* uses in @teamscala/auth, so a new custom domain is trusted everywhere at
|
|
19
|
+
* once or nowhere.
|
|
20
|
+
*
|
|
21
|
+
* hono csrf() arms only on form-element content types (urlencoded, multipart,
|
|
22
|
+
* text/plain) — browsers cannot produce a cross-site application/json request
|
|
23
|
+
* from a form, so the JSON API surface (ORPC) is protected by cookie
|
|
24
|
+
* SameSite=Lax + this middleware's text/plain coverage.
|
|
25
|
+
*
|
|
26
|
+
* Custom-domain lookups are cached per-hostname with a TTL: a POSITIVE result
|
|
27
|
+
* is cached for the full TTL (a domain that stopped being ours keeps working
|
|
28
|
+
* for at most the TTL), a NEGATIVE result for a short window (a brand-new
|
|
29
|
+
* custom domain becomes trusted within seconds). The cache is a request
|
|
30
|
+
* accelerator over a DB fact, never the source of truth.
|
|
31
|
+
*/
|
|
32
|
+
|
|
33
|
+
import { getManagedBaseDomain } from "@teamscala/auth/auth-env-resolution";
|
|
34
|
+
import { isTrustedAuthOrigin } from "@teamscala/auth/is-trusted-auth-origin";
|
|
35
|
+
import { csrf } from "hono/csrf";
|
|
36
|
+
import type { Context, MiddlewareHandler } from "hono";
|
|
37
|
+
|
|
38
|
+
const POSITIVE_TTL_MS = 5 * 60_000;
|
|
39
|
+
const NEGATIVE_TTL_MS = 10_000;
|
|
40
|
+
|
|
41
|
+
type CacheEntry = { trusted: boolean; expiresAt: number };
|
|
42
|
+
const customDomainCache = new Map<string, CacheEntry>();
|
|
43
|
+
|
|
44
|
+
function isPlatformOrigin(origin: string, baseDomain: string): boolean {
|
|
45
|
+
let host: string;
|
|
46
|
+
try {
|
|
47
|
+
host = new URL(origin).hostname.toLowerCase();
|
|
48
|
+
} catch {
|
|
49
|
+
return false;
|
|
50
|
+
}
|
|
51
|
+
if (!host) return false;
|
|
52
|
+
return host === baseDomain || host.endsWith(`.${baseDomain}`);
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
async function isTrustedCustomOrigin(origin: string): Promise<boolean> {
|
|
56
|
+
let host: string;
|
|
57
|
+
try {
|
|
58
|
+
host = new URL(origin).hostname.toLowerCase();
|
|
59
|
+
} catch {
|
|
60
|
+
return false;
|
|
61
|
+
}
|
|
62
|
+
if (!host) return false;
|
|
63
|
+
|
|
64
|
+
const now = Date.now();
|
|
65
|
+
const cached = customDomainCache.get(host);
|
|
66
|
+
if (cached && cached.expiresAt > now) return cached.trusted;
|
|
67
|
+
|
|
68
|
+
const trusted = await (async () => {
|
|
69
|
+
try {
|
|
70
|
+
const { getPrisma } = await import("@teamscala/db/prisma-registry");
|
|
71
|
+
const prisma = getPrisma();
|
|
72
|
+
return await isTrustedAuthOrigin(
|
|
73
|
+
prisma as unknown as Parameters<typeof isTrustedAuthOrigin>[0],
|
|
74
|
+
host,
|
|
75
|
+
);
|
|
76
|
+
} catch {
|
|
77
|
+
// A lookup that cannot run is a lookup that trusts nothing: an
|
|
78
|
+
// unavailable DB degrades to platform-origin-only CSRF, never to
|
|
79
|
+
// allow-everything.
|
|
80
|
+
return false;
|
|
81
|
+
}
|
|
82
|
+
})();
|
|
83
|
+
|
|
84
|
+
customDomainCache.set(host, {
|
|
85
|
+
trusted,
|
|
86
|
+
expiresAt: now + (trusted ? POSITIVE_TTL_MS : NEGATIVE_TTL_MS),
|
|
87
|
+
});
|
|
88
|
+
if (customDomainCache.size > 512) customDomainCache.clear();
|
|
89
|
+
return trusted;
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
export function csrfProtection(): MiddlewareHandler {
|
|
93
|
+
return csrf({
|
|
94
|
+
origin: async (origin: string, _c: Context) => {
|
|
95
|
+
const baseDomain = getManagedBaseDomain();
|
|
96
|
+
if (isPlatformOrigin(origin, baseDomain)) return true;
|
|
97
|
+
return isTrustedCustomOrigin(origin);
|
|
98
|
+
},
|
|
99
|
+
});
|
|
100
|
+
}
|
|
@@ -0,0 +1,185 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @system server-boot
|
|
3
|
+
* @status handwritten
|
|
4
|
+
* @edit edit directly
|
|
5
|
+
*
|
|
6
|
+
* Pure decision logic for the platform-fault escalation sweep
|
|
7
|
+
* (`platform-fault-escalation` handler). Split from the I/O shell so the
|
|
8
|
+
* threshold/dedup/episode semantics are unit-testable with zero adapter
|
|
9
|
+
* dependencies (same shape as whatsapp-liveness-logic).
|
|
10
|
+
*
|
|
11
|
+
* The semantics this locks (from the 2026-08-15 "760 errors escalated to
|
|
12
|
+
* nobody" incident brief):
|
|
13
|
+
* - Threshold on DURATION and RECURRENCE, never a single event. A kind
|
|
14
|
+
* qualifies as a sustained fault only when BOTH hold inside the lookback
|
|
15
|
+
* window: at least `min_count` occurrences (recurrence) spread over at
|
|
16
|
+
* least `min_span_minutes` (duration). One failed push is noise; the same
|
|
17
|
+
* kind recurring for half an hour is an outage.
|
|
18
|
+
* - One escalation per distinct fault kind per episode. An episode is open
|
|
19
|
+
* from first qualification until `resolve_after_minutes` of quiet; while
|
|
20
|
+
* open, the fault never re-fires — it only appends throttled updates.
|
|
21
|
+
* - Delivery is verified, not queued: an episode row with `escalated_at`
|
|
22
|
+
* NULL is a CLAIM, not an escalation. The sweep re-attempts delivery for
|
|
23
|
+
* claimed-but-undelivered episodes until the WhatsApp send returns a
|
|
24
|
+
* message id.
|
|
25
|
+
*/
|
|
26
|
+
|
|
27
|
+
export interface FaultAggregate {
|
|
28
|
+
kind: string;
|
|
29
|
+
count: number;
|
|
30
|
+
firstSeen: Date;
|
|
31
|
+
lastSeen: Date;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
export interface EscalationEpisode {
|
|
35
|
+
kind: string;
|
|
36
|
+
firstSeenAt: Date;
|
|
37
|
+
lastSeenAt: Date;
|
|
38
|
+
eventCount: number;
|
|
39
|
+
escalatedAt: Date | null;
|
|
40
|
+
workItemId: string | null;
|
|
41
|
+
resolvedAt: Date | null;
|
|
42
|
+
lastUpdatedAt: Date | null;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
export interface FaultEscalationPolicy {
|
|
46
|
+
/** Recurrence floor: minimum event count inside the window. */
|
|
47
|
+
minCount: number;
|
|
48
|
+
/** Duration floor: minimum first→last spread inside the window, minutes. */
|
|
49
|
+
minSpanMinutes: number;
|
|
50
|
+
/** Quiet period that closes an episode and re-arms escalation, minutes. */
|
|
51
|
+
resolveAfterMinutes: number;
|
|
52
|
+
/** Minimum spacing between throttled updates on an open episode, minutes. */
|
|
53
|
+
updateAfterMinutes: number;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/** Per-kind threshold overrides — a low-rate-but-persistent kind (guard.fail.open
|
|
57
|
+
* at ~1/hour over 12h) has different noise characteristics than a bursty one, so
|
|
58
|
+
* the recurrence/duration floors are per-kind policy in the config row. */
|
|
59
|
+
export interface KindThresholdOverride {
|
|
60
|
+
minCount?: number;
|
|
61
|
+
minSpanMinutes?: number;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
export interface SweepDecision {
|
|
65
|
+
/** Sustained faults needing a FIRST delivery (new episode, or a claim whose send failed). */
|
|
66
|
+
escalate: Array<{ aggregate: FaultAggregate; episode: EscalationEpisode | null }>;
|
|
67
|
+
/** Open, delivered episodes still recurring — append evidence, never re-send. */
|
|
68
|
+
update: Array<{ aggregate: FaultAggregate; episode: EscalationEpisode }>;
|
|
69
|
+
/** Open episodes whose kind went quiet past the resolve window — close them. */
|
|
70
|
+
resolve: Array<{ episode: EscalationEpisode; quietMinutes: number }>;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
function minutesBetween(a: Date, b: Date): number {
|
|
74
|
+
return (b.getTime() - a.getTime()) / 60_000;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
function isSustained(
|
|
78
|
+
aggregate: FaultAggregate,
|
|
79
|
+
policy: FaultEscalationPolicy,
|
|
80
|
+
override: KindThresholdOverride | undefined,
|
|
81
|
+
): boolean {
|
|
82
|
+
const minCount = override?.minCount ?? policy.minCount;
|
|
83
|
+
const minSpan = override?.minSpanMinutes ?? policy.minSpanMinutes;
|
|
84
|
+
return (
|
|
85
|
+
aggregate.count >= minCount &&
|
|
86
|
+
minutesBetween(aggregate.firstSeen, aggregate.lastSeen) >= minSpan
|
|
87
|
+
);
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/**
|
|
91
|
+
* Classify the current sweep. `aggregates` are the per-kind error aggregates
|
|
92
|
+
* from the lookback window; `episodes` are the existing episode rows for those
|
|
93
|
+
* kinds PLUS every still-open episode (an open episode whose kind no longer
|
|
94
|
+
* appears in the window at all is the strongest resolve candidate).
|
|
95
|
+
*/
|
|
96
|
+
export function decideSweep(
|
|
97
|
+
aggregates: FaultAggregate[],
|
|
98
|
+
episodes: EscalationEpisode[],
|
|
99
|
+
policy: FaultEscalationPolicy,
|
|
100
|
+
now: Date,
|
|
101
|
+
kindThresholds: Record<string, KindThresholdOverride> = {},
|
|
102
|
+
): SweepDecision {
|
|
103
|
+
const byKind = new Map(episodes.map((e) => [e.kind, e]));
|
|
104
|
+
const aggregateByKind = new Map(aggregates.map((a) => [a.kind, a]));
|
|
105
|
+
const decision: SweepDecision = { escalate: [], update: [], resolve: [] };
|
|
106
|
+
|
|
107
|
+
for (const aggregate of aggregates) {
|
|
108
|
+
const episode = byKind.get(aggregate.kind);
|
|
109
|
+
if (!isSustained(aggregate, policy, kindThresholds[aggregate.kind])) continue;
|
|
110
|
+
// No episode, or the previous episode closed (quiet past resolve window):
|
|
111
|
+
// a NEW episode begins and must be delivered.
|
|
112
|
+
if (!episode || episode.resolvedAt) {
|
|
113
|
+
decision.escalate.push({ aggregate, episode: episode ?? null });
|
|
114
|
+
continue;
|
|
115
|
+
}
|
|
116
|
+
// Open episode with no verified delivery — the claim is retried, never
|
|
117
|
+
// counted as escalated (a component may not report a state it has not
|
|
118
|
+
// verified).
|
|
119
|
+
if (!episode.escalatedAt) {
|
|
120
|
+
decision.escalate.push({ aggregate, episode });
|
|
121
|
+
continue;
|
|
122
|
+
}
|
|
123
|
+
// Open, delivered, still recurring — throttled evidence update only.
|
|
124
|
+
const lastUpdate = episode.lastUpdatedAt ?? episode.escalatedAt;
|
|
125
|
+
if (minutesBetween(lastUpdate, now) >= policy.updateAfterMinutes) {
|
|
126
|
+
decision.update.push({ aggregate, episode });
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
for (const episode of episodes) {
|
|
131
|
+
if (episode.resolvedAt) continue;
|
|
132
|
+
const aggregate = aggregateByKind.get(episode.kind);
|
|
133
|
+
const quietReference = aggregate?.lastSeen ?? episode.lastSeenAt;
|
|
134
|
+
const quietMinutes = minutesBetween(quietReference, now);
|
|
135
|
+
if (quietMinutes >= policy.resolveAfterMinutes) {
|
|
136
|
+
decision.resolve.push({ episode, quietMinutes });
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
return decision;
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
/** Human "N min" span, rounded — for the phone-sized escalation message. */
|
|
144
|
+
export function formatSpanMinutes(minutes: number): string {
|
|
145
|
+
if (minutes < 60) return `${Math.round(minutes)} min`;
|
|
146
|
+
const hours = minutes / 60;
|
|
147
|
+
return `${hours >= 10 ? Math.round(hours) : Math.round(hours * 10) / 10} h`;
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
/**
|
|
151
|
+
* The escalation message — escalation-routing's three-field contract adapted
|
|
152
|
+
* to an automated detector: what was OBSERVED (evidence), what it MEANS
|
|
153
|
+
* (impact), and the RECOMMENDATION. Short: the channel is a phone.
|
|
154
|
+
*/
|
|
155
|
+
export function composeEscalationMessage(input: {
|
|
156
|
+
kind: string;
|
|
157
|
+
aggregate: FaultAggregate;
|
|
158
|
+
impact: string;
|
|
159
|
+
recommendation: string;
|
|
160
|
+
}): string {
|
|
161
|
+
const span = formatSpanMinutes(
|
|
162
|
+
(input.aggregate.lastSeen.getTime() - input.aggregate.firstSeen.getTime()) / 60_000,
|
|
163
|
+
);
|
|
164
|
+
return [
|
|
165
|
+
`🔴 Platform fault: ${input.kind}`,
|
|
166
|
+
`Observed: ${input.aggregate.count} errors over ${span}, still recurring.`,
|
|
167
|
+
`Impact: ${input.impact}`,
|
|
168
|
+
`Recommendation: ${input.recommendation}`,
|
|
169
|
+
].join("\n");
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
/** The work-item description carries the same three fields plus the evidence. */
|
|
173
|
+
export function composeEscalationDescription(input: {
|
|
174
|
+
aggregate: FaultAggregate;
|
|
175
|
+
impact: string;
|
|
176
|
+
recommendation: string;
|
|
177
|
+
}): string {
|
|
178
|
+
return [
|
|
179
|
+
"Automated platform-fault escalation (platform-fault-escalation sweep).",
|
|
180
|
+
`Observed: ${input.aggregate.count} '${input.aggregate.kind}' events in event_log from ${input.aggregate.firstSeen.toISOString()} to ${input.aggregate.lastSeen.toISOString()}, still recurring.`,
|
|
181
|
+
`Impact: ${input.impact}`,
|
|
182
|
+
`Recommendation: ${input.recommendation}`,
|
|
183
|
+
"Delivery: WhatsApp via the scala PA. This task closes automatically when the fault stops for the resolve window; reply via the PA to rule otherwise.",
|
|
184
|
+
].join("\n");
|
|
185
|
+
}
|
|
@@ -0,0 +1,183 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @system server-boot
|
|
3
|
+
* @status handwritten
|
|
4
|
+
* @edit edit directly
|
|
5
|
+
*
|
|
6
|
+
* Unit tests for the pure decision core of the platform-fault escalation
|
|
7
|
+
* sweep (`platform-fault-escalation` handler). The handler's DB/WhatsApp I/O
|
|
8
|
+
* is runtime-verified by the scheduled_job firing; this locks the threshold,
|
|
9
|
+
* dedup and episode semantics from the 2026-08-15 incident brief.
|
|
10
|
+
*/
|
|
11
|
+
import { describe, expect, test } from "bun:test";
|
|
12
|
+
import {
|
|
13
|
+
composeEscalationMessage,
|
|
14
|
+
decideSweep,
|
|
15
|
+
type EscalationEpisode,
|
|
16
|
+
type FaultAggregate,
|
|
17
|
+
type FaultEscalationPolicy,
|
|
18
|
+
} from "./platform-fault-escalation-logic.ts";
|
|
19
|
+
|
|
20
|
+
const NOW = new Date("2026-08-15T23:45:00+10:00");
|
|
21
|
+
const POLICY: FaultEscalationPolicy = {
|
|
22
|
+
minCount: 10,
|
|
23
|
+
minSpanMinutes: 25,
|
|
24
|
+
resolveAfterMinutes: 30,
|
|
25
|
+
updateAfterMinutes: 30,
|
|
26
|
+
};
|
|
27
|
+
|
|
28
|
+
function agg(kind: string, count: number, firstSeenMinutesAgo: number, lastSeenMinutesAgo: number): FaultAggregate {
|
|
29
|
+
return {
|
|
30
|
+
kind,
|
|
31
|
+
count,
|
|
32
|
+
firstSeen: new Date(NOW.getTime() - firstSeenMinutesAgo * 60_000),
|
|
33
|
+
lastSeen: new Date(NOW.getTime() - lastSeenMinutesAgo * 60_000),
|
|
34
|
+
};
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
function episode(overrides: Partial<EscalationEpisode> & { kind: string }): EscalationEpisode {
|
|
38
|
+
return {
|
|
39
|
+
firstSeenAt: NOW,
|
|
40
|
+
lastSeenAt: NOW,
|
|
41
|
+
eventCount: 10,
|
|
42
|
+
escalatedAt: new Date(NOW.getTime() - 60 * 60_000),
|
|
43
|
+
workItemId: "wi-1",
|
|
44
|
+
resolvedAt: null,
|
|
45
|
+
lastUpdatedAt: null,
|
|
46
|
+
...overrides,
|
|
47
|
+
};
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
describe("decideSweep — thresholds", () => {
|
|
51
|
+
test("a single event is noise — never escalates", () => {
|
|
52
|
+
const decision = decideSweep([agg("push.error", 1, 2, 2)], [], POLICY, NOW);
|
|
53
|
+
expect(decision.escalate).toHaveLength(0);
|
|
54
|
+
});
|
|
55
|
+
|
|
56
|
+
test("recurrence without duration does not escalate (10 events in 5 min)", () => {
|
|
57
|
+
const decision = decideSweep([agg("burst.error", 10, 5, 0)], [], POLICY, NOW);
|
|
58
|
+
expect(decision.escalate).toHaveLength(0);
|
|
59
|
+
});
|
|
60
|
+
|
|
61
|
+
test("duration without recurrence does not escalate (3 events over 40 min)", () => {
|
|
62
|
+
const decision = decideSweep([agg("sparse.error", 3, 40, 0)], [], POLICY, NOW);
|
|
63
|
+
expect(decision.escalate).toHaveLength(0);
|
|
64
|
+
});
|
|
65
|
+
|
|
66
|
+
test("the 490-over-12h shape escalates (duration + recurrence)", () => {
|
|
67
|
+
const sustained = agg("host-command-bus.error", 490, 720, 0);
|
|
68
|
+
const decision = decideSweep([sustained], [], POLICY, NOW);
|
|
69
|
+
expect(decision.escalate).toHaveLength(1);
|
|
70
|
+
expect(decision.escalate[0]!.aggregate.kind).toBe("host-command-bus.error");
|
|
71
|
+
expect(decision.escalate[0]!.episode).toBeNull();
|
|
72
|
+
});
|
|
73
|
+
|
|
74
|
+
test("per-kind threshold override lets a low-rate persistent kind through", () => {
|
|
75
|
+
// guard.fail.open: 8 events over 12h — below the default recurrence floor
|
|
76
|
+
// of 10, but persistent. The override lowers recurrence to 4 over 60 min.
|
|
77
|
+
const drip = agg("guard.fail.open", 8, 720, 0);
|
|
78
|
+
const withoutOverride = decideSweep([drip], [], POLICY, NOW);
|
|
79
|
+
expect(withoutOverride.escalate).toHaveLength(0);
|
|
80
|
+
const withOverride = decideSweep([drip], [], POLICY, NOW, {
|
|
81
|
+
"guard.fail.open": { minCount: 4, minSpanMinutes: 60 },
|
|
82
|
+
});
|
|
83
|
+
expect(withOverride.escalate).toHaveLength(1);
|
|
84
|
+
});
|
|
85
|
+
|
|
86
|
+
test("override raises a floor too — bursty kinds can be tightened", () => {
|
|
87
|
+
const sustained = agg("llm.provider.error", 40, 60, 0); // passes default
|
|
88
|
+
const decision = decideSweep([sustained], [], POLICY, NOW, {
|
|
89
|
+
"llm.provider.error": { minCount: 100 },
|
|
90
|
+
});
|
|
91
|
+
expect(decision.escalate).toHaveLength(0);
|
|
92
|
+
});
|
|
93
|
+
});
|
|
94
|
+
|
|
95
|
+
describe("decideSweep — dedup (one escalation per fault, no re-fire)", () => {
|
|
96
|
+
test("already escalated and unresolved does NOT re-escalate", () => {
|
|
97
|
+
const sustained = agg("host-command-bus.error", 500, 720, 0);
|
|
98
|
+
const open = episode({ kind: "host-command-bus.error", escalatedAt: new Date(NOW.getTime() - 3_600_000) });
|
|
99
|
+
const decision = decideSweep([sustained], [open], POLICY, NOW);
|
|
100
|
+
expect(decision.escalate).toHaveLength(0);
|
|
101
|
+
});
|
|
102
|
+
|
|
103
|
+
test("open delivered episode past the update window gets an UPDATE, not a message", () => {
|
|
104
|
+
const sustained = agg("llm.provider.error", 300, 720, 0);
|
|
105
|
+
const open = episode({
|
|
106
|
+
kind: "llm.provider.error",
|
|
107
|
+
escalatedAt: new Date(NOW.getTime() - 90 * 60_000), // last update long ago
|
|
108
|
+
lastUpdatedAt: new Date(NOW.getTime() - 90 * 60_000),
|
|
109
|
+
});
|
|
110
|
+
const decision = decideSweep([sustained], [open], POLICY, NOW);
|
|
111
|
+
expect(decision.escalate).toHaveLength(0);
|
|
112
|
+
expect(decision.update).toHaveLength(1);
|
|
113
|
+
expect(decision.update[0]!.episode.workItemId).toBe("wi-1");
|
|
114
|
+
});
|
|
115
|
+
|
|
116
|
+
test("open episode inside the update window gets NOTHING (dedup window)", () => {
|
|
117
|
+
const sustained = agg("llm.provider.error", 300, 720, 0);
|
|
118
|
+
const open = episode({
|
|
119
|
+
kind: "llm.provider.error",
|
|
120
|
+
escalatedAt: new Date(NOW.getTime() - 10 * 60_000),
|
|
121
|
+
lastUpdatedAt: new Date(NOW.getTime() - 10 * 60_000),
|
|
122
|
+
});
|
|
123
|
+
const decision = decideSweep([sustained], [open], POLICY, NOW);
|
|
124
|
+
expect(decision.escalate).toHaveLength(0);
|
|
125
|
+
expect(decision.update).toHaveLength(0);
|
|
126
|
+
});
|
|
127
|
+
|
|
128
|
+
test("a claimed-but-undelivered episode is RETRIED, not counted as escalated", () => {
|
|
129
|
+
const sustained = agg("guard.fail.open", 40, 720, 0);
|
|
130
|
+
const claim = episode({ kind: "guard.fail.open", escalatedAt: null, workItemId: null });
|
|
131
|
+
const decision = decideSweep([sustained], [claim], POLICY, NOW);
|
|
132
|
+
expect(decision.escalate).toHaveLength(1);
|
|
133
|
+
expect(decision.escalate[0]!.episode).toBe(claim);
|
|
134
|
+
});
|
|
135
|
+
});
|
|
136
|
+
|
|
137
|
+
describe("decideSweep — resolution + re-arm", () => {
|
|
138
|
+
test("quiet past the resolve window closes the episode", () => {
|
|
139
|
+
const gone = episode({ kind: "wake.failed", lastSeenAt: new Date(NOW.getTime() - 60 * 60_000) });
|
|
140
|
+
const decision = decideSweep([], [gone], POLICY, NOW);
|
|
141
|
+
expect(decision.resolve).toHaveLength(1);
|
|
142
|
+
expect(decision.resolve[0]!.quietMinutes).toBeGreaterThanOrEqual(30);
|
|
143
|
+
});
|
|
144
|
+
|
|
145
|
+
test("an absent kind resolves by its episode's last_seen, not the sweep clock", () => {
|
|
146
|
+
const gone = episode({ kind: "listen.connect_failed", lastSeenAt: new Date(NOW.getTime() - 45 * 60_000) });
|
|
147
|
+
const decision = decideSweep([], [gone], POLICY, NOW);
|
|
148
|
+
expect(decision.resolve).toHaveLength(1);
|
|
149
|
+
});
|
|
150
|
+
|
|
151
|
+
test("after resolution the same fault escalates again (new episode)", () => {
|
|
152
|
+
const recurring = agg("host-command-bus.error", 50, 40, 0);
|
|
153
|
+
const resolved = episode({
|
|
154
|
+
kind: "host-command-bus.error",
|
|
155
|
+
resolvedAt: new Date(NOW.getTime() - 35 * 60_000),
|
|
156
|
+
});
|
|
157
|
+
const decision = decideSweep([recurring], [resolved], POLICY, NOW);
|
|
158
|
+
expect(decision.escalate).toHaveLength(1);
|
|
159
|
+
});
|
|
160
|
+
|
|
161
|
+
test("still-active kind within the resolve window is not resolved even if below threshold", () => {
|
|
162
|
+
const active = agg("slow.error", 3, 20, 0); // below threshold, but recent
|
|
163
|
+
const open = episode({ kind: "slow.error", lastSeenAt: active.lastSeen });
|
|
164
|
+
const decision = decideSweep([active], [open], POLICY, NOW);
|
|
165
|
+
expect(decision.resolve).toHaveLength(0);
|
|
166
|
+
});
|
|
167
|
+
});
|
|
168
|
+
|
|
169
|
+
describe("composeEscalationMessage", () => {
|
|
170
|
+
test("carries the three escalation-routing fields and stays phone-sized", () => {
|
|
171
|
+
const message = composeEscalationMessage({
|
|
172
|
+
kind: "host-command-bus.error",
|
|
173
|
+
aggregate: agg("host-command-bus.error", 490, 720, 0),
|
|
174
|
+
impact: "scala-tools verbs are failing host-wide",
|
|
175
|
+
recommendation: "check oxmgr logs for the bus consumer",
|
|
176
|
+
});
|
|
177
|
+
expect(message).toContain("Platform fault: host-command-bus.error");
|
|
178
|
+
expect(message).toContain("Observed: 490 errors over 12 h");
|
|
179
|
+
expect(message).toContain("Impact: scala-tools verbs are failing host-wide");
|
|
180
|
+
expect(message).toContain("Recommendation: check oxmgr logs for the bus consumer");
|
|
181
|
+
expect(message.split("\n")).toHaveLength(4);
|
|
182
|
+
});
|
|
183
|
+
});
|
|
@@ -0,0 +1,447 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @system server-boot
|
|
3
|
+
* @status handwritten
|
|
4
|
+
* @edit edit directly
|
|
5
|
+
*
|
|
6
|
+
* The `platform-fault-escalation` pg-boss handler — closes the "receiving is
|
|
7
|
+
* decoupled from consequence" gap measured 2026-08-15 (760 error events over
|
|
8
|
+
* 12 hours, zero escalations: host-command-bus.error 490, llm.provider.error
|
|
9
|
+
* 231, guard.fail.open 16, scale-to-zero.wake_failed 12, listen.connect_failed
|
|
10
|
+
* 11). Emission works and observability-is-passive is satisfied; nothing
|
|
11
|
+
* acted. This sweep makes RECEIVING have CONSEQUENCE: sustained platform
|
|
12
|
+
* faults escalate to the OPERATOR over the existing PA path (WhatsApp, the
|
|
13
|
+
* scala PA operator-relay bot — escalation-routing's channel; no second
|
|
14
|
+
* channel invented), with the durable record as a work_items task the PA can
|
|
15
|
+
* read back and the operator can answer on.
|
|
16
|
+
*
|
|
17
|
+
* THIS IS NOT A DRIFT-DETECTOR PATROLLING INSTANCES. It reacts only to
|
|
18
|
+
* components reporting THEIR OWN failure — event_log kinds that already
|
|
19
|
+
* assert a fault about their emitter. Thresholds (duration + recurrence,
|
|
20
|
+
* never a single event), the watched-kind patterns, the resolve window and
|
|
21
|
+
* the per-kind impact/recommendation copy are all in the
|
|
22
|
+
* `config/platform-fault-escalation` registry row — changing policy is a row
|
|
23
|
+
* edit, not a redeploy. Episode dedup lives in the
|
|
24
|
+
* `platform_fault_escalations` table (migration 501): one escalation per
|
|
25
|
+
* distinct fault kind per episode; an already-escalated-and-unresolved fault
|
|
26
|
+
* only appends throttled work-item comments, never a second message.
|
|
27
|
+
*
|
|
28
|
+
* DELIVERY IS VERIFIED, NOT QUEUED. The WhatsApp send goes through the
|
|
29
|
+
* central `/mcp` loopback (`api_gowa_message` sendMessage — the single
|
|
30
|
+
* sealed GOWA-credential surface, same as whatsapp-liveness) and an episode
|
|
31
|
+
* only carries `escalated_at` once the send returned a WhatsApp message id;
|
|
32
|
+
* before that the row is a CLAIM the next sweep retries. Success is recorded
|
|
33
|
+
* only after readback of the delivery facts (message id + work item id).
|
|
34
|
+
*
|
|
35
|
+
* WHY A TICK EXISTS AT ALL, given observability-is-passive and
|
|
36
|
+
* no-polling-for-pushable-state: the trigger is DURATION + RECURRENCE of
|
|
37
|
+
* events already durably logged — a fact about the PATTERN of past events,
|
|
38
|
+
* not a state transition any single event can announce. The sweep reads one
|
|
39
|
+
* server-side aggregate (GROUP BY kind) per run against indexed event_log;
|
|
40
|
+
* while the platform is healthy it returns zero rows and costs nothing
|
|
41
|
+
* beyond the tick (idle-costs-nothing). Same justification
|
|
42
|
+
* lane-attention-sweep carries on its row as `poll_justification`.
|
|
43
|
+
*
|
|
44
|
+
* Registered under slug `@teamscala/service-boot:platform-fault-escalation`.
|
|
45
|
+
*/
|
|
46
|
+
import type { ExecutionContext } from "@teamscala/os/contracts/mcp";
|
|
47
|
+
import type { ScheduledJobConfig } from "@teamscala/db-validation/registry-schemas/scheduled-job";
|
|
48
|
+
import * as v from "valibot";
|
|
49
|
+
import { getPrisma } from "@teamscala/db/prisma-registry";
|
|
50
|
+
import { loadRegistryConfig } from "@teamscala/db/registry/load-config";
|
|
51
|
+
import { recordEvent } from "@teamscala/event-log/record-event";
|
|
52
|
+
import { getAppLogger } from "@teamscala/logger/app-loggers";
|
|
53
|
+
import { pgBossHandlerRegistry } from "../pg-boss-handler-registry.ts";
|
|
54
|
+
import {
|
|
55
|
+
composeEscalationDescription,
|
|
56
|
+
composeEscalationMessage,
|
|
57
|
+
decideSweep,
|
|
58
|
+
type EscalationEpisode,
|
|
59
|
+
type FaultAggregate,
|
|
60
|
+
type KindThresholdOverride,
|
|
61
|
+
} from "./platform-fault-escalation-logic.ts";
|
|
62
|
+
|
|
63
|
+
const CONFIG_SLUG = "platform-fault-escalation";
|
|
64
|
+
|
|
65
|
+
const FaultEscalationConfigSchema = v.object({
|
|
66
|
+
/** Postgres regex patterns selecting watched event_log kinds. */
|
|
67
|
+
watched_kind_patterns: v.array(v.string()),
|
|
68
|
+
/** Lookback window for the recurrence/duration aggregate, minutes. */
|
|
69
|
+
window_minutes: v.number(),
|
|
70
|
+
min_count: v.number(),
|
|
71
|
+
min_span_minutes: v.number(),
|
|
72
|
+
/** Per-kind recurrence/duration overrides — low-rate-but-persistent kinds. */
|
|
73
|
+
kind_thresholds: v.optional(
|
|
74
|
+
v.record(
|
|
75
|
+
v.string(),
|
|
76
|
+
v.object({
|
|
77
|
+
min_count: v.optional(v.number()),
|
|
78
|
+
min_span_minutes: v.optional(v.number()),
|
|
79
|
+
}),
|
|
80
|
+
),
|
|
81
|
+
),
|
|
82
|
+
resolve_after_minutes: v.number(),
|
|
83
|
+
update_after_minutes: v.number(),
|
|
84
|
+
/** The developer-session project escalations are parented to. */
|
|
85
|
+
parent_project_id: v.string(),
|
|
86
|
+
/** Per-kind copy for the message (impact + recommendation). Falls back to default_copy. */
|
|
87
|
+
copy: v.record(v.string(), v.object({ impact: v.string(), recommendation: v.string() })),
|
|
88
|
+
default_copy: v.object({ impact: v.string(), recommendation: v.string() }),
|
|
89
|
+
});
|
|
90
|
+
|
|
91
|
+
type FaultEscalationConfig = v.InferOutput<typeof FaultEscalationConfigSchema>;
|
|
92
|
+
|
|
93
|
+
interface EventLogAggregateRow {
|
|
94
|
+
kind: string;
|
|
95
|
+
n: bigint;
|
|
96
|
+
first_seen: Date;
|
|
97
|
+
last_seen: Date;
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
interface OperatorRelayBot {
|
|
101
|
+
id: string;
|
|
102
|
+
organisation_id: string;
|
|
103
|
+
external_id: string | null;
|
|
104
|
+
flow_config: { operator_phone?: string } | null;
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
let cachedSystemIdentity: { userId: string | undefined; organisationId: string | undefined } | undefined;
|
|
108
|
+
async function getSystemIdentity(): Promise<{ userId: string | undefined; organisationId: string | undefined }> {
|
|
109
|
+
if (cachedSystemIdentity) return cachedSystemIdentity;
|
|
110
|
+
const row = await loadRegistryConfig("config", "system-identity", v.looseObject({}));
|
|
111
|
+
cachedSystemIdentity = {
|
|
112
|
+
userId: row?.system_user_id as string | undefined,
|
|
113
|
+
organisationId: row?.system_org_id as string | undefined,
|
|
114
|
+
};
|
|
115
|
+
return cachedSystemIdentity;
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
async function loadConfig(): Promise<FaultEscalationConfig> {
|
|
119
|
+
return loadRegistryConfig("config", CONFIG_SLUG, FaultEscalationConfigSchema);
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
async function fetchAggregates(patterns: string[], windowMinutes: number): Promise<FaultAggregate[]> {
|
|
123
|
+
const prisma = getPrisma();
|
|
124
|
+
// Patterns come from the valibot-validated registry row and are regex-only
|
|
125
|
+
// by construction below; embedding them as one alternation keeps the query
|
|
126
|
+
// parameter-free on the pattern side while the window is a real parameter.
|
|
127
|
+
const regexClass = /^[A-Za-z0-9_\\^$.*+?()[\]{}|-]+$/;
|
|
128
|
+
for (const pattern of patterns) {
|
|
129
|
+
if (!regexClass.test(pattern)) {
|
|
130
|
+
throw new Error(`config/${CONFIG_SLUG}: watched_kind_pattern is not regex-only: ${pattern}`);
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
const combined = `(${patterns.join("|")})`;
|
|
134
|
+
const rows = await prisma.$queryRawUnsafe<EventLogAggregateRow[]>(
|
|
135
|
+
`SELECT kind, count(*)::bigint AS n, min(occurred_at) AS first_seen, max(occurred_at) AS last_seen
|
|
136
|
+
FROM event_log
|
|
137
|
+
WHERE occurred_at > now() - ($1::int * interval '1 minute')
|
|
138
|
+
AND kind ~ $2::text
|
|
139
|
+
GROUP BY kind`,
|
|
140
|
+
windowMinutes,
|
|
141
|
+
combined,
|
|
142
|
+
);
|
|
143
|
+
return rows.map((row) => ({
|
|
144
|
+
kind: row.kind,
|
|
145
|
+
count: Number(row.n),
|
|
146
|
+
firstSeen: new Date(row.first_seen),
|
|
147
|
+
lastSeen: new Date(row.last_seen),
|
|
148
|
+
}));
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
async function resolveOperatorRelay(): Promise<OperatorRelayBot> {
|
|
152
|
+
const prisma = getPrisma();
|
|
153
|
+
const bot = await prisma.messaging_bots.findFirst({
|
|
154
|
+
where: {
|
|
155
|
+
is_active: true,
|
|
156
|
+
platform: "whatsapp",
|
|
157
|
+
flow_config: { path: ["mode"], equals: "operator-relay" },
|
|
158
|
+
},
|
|
159
|
+
select: { id: true, organisation_id: true, external_id: true, flow_config: true },
|
|
160
|
+
});
|
|
161
|
+
if (!bot) throw new Error("platform-fault-escalation: no active operator-relay WhatsApp bot row");
|
|
162
|
+
const phone = (bot.flow_config as { operator_phone?: string } | null)?.operator_phone;
|
|
163
|
+
if (!phone) throw new Error("platform-fault-escalation: operator-relay bot has no operator_phone");
|
|
164
|
+
return bot as OperatorRelayBot;
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
/** GOWA device whose JID matches the PA number, so the escalation is sent BY the PA. */
|
|
168
|
+
async function resolvePaDeviceId(paPhoneDigits: string): Promise<string | undefined> {
|
|
169
|
+
const { userId, organisationId } = await getSystemIdentity();
|
|
170
|
+
const systemCtx: ExecutionContext = { userId, organisationId } as ExecutionContext;
|
|
171
|
+
const { forwardToolToCentralMcp } = await import("@teamscala/chat-completion/central-tool-forwarder");
|
|
172
|
+
const res = (await forwardToolToCentralMcp(
|
|
173
|
+
"api_gowa_devices",
|
|
174
|
+
{ action: "listDevices" },
|
|
175
|
+
systemCtx,
|
|
176
|
+
)) as { results?: Array<{ id: string; jid?: string }> } | undefined;
|
|
177
|
+
const match = (res?.results ?? []).find((device) => (device.jid ?? "").includes(paPhoneDigits));
|
|
178
|
+
return match?.id;
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
function extractMessageId(sendResult: unknown): string | null {
|
|
182
|
+
const results = (sendResult as { results?: unknown } | undefined)?.results;
|
|
183
|
+
const id =
|
|
184
|
+
typeof results === "string"
|
|
185
|
+
? results
|
|
186
|
+
: Array.isArray(results)
|
|
187
|
+
? ((results[0] as { id?: unknown } | undefined)?.id as string | undefined)
|
|
188
|
+
: ((results as { id?: unknown } | undefined)?.id as string | undefined);
|
|
189
|
+
return typeof id === "string" && id.length > 0 ? id : null;
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
async function deliverEscalation(
|
|
193
|
+
kind: string,
|
|
194
|
+
aggregate: FaultAggregate,
|
|
195
|
+
config: FaultEscalationConfig,
|
|
196
|
+
): Promise<{ messageId: string; workItemId: string }> {
|
|
197
|
+
const log = getAppLogger();
|
|
198
|
+
const prisma = getPrisma();
|
|
199
|
+
const now = new Date();
|
|
200
|
+
|
|
201
|
+
// 1. Claim the episode BEFORE sending, so a send failure leaves a durable
|
|
202
|
+
// claim the next sweep retries instead of a lost escalation.
|
|
203
|
+
await prisma.platform_fault_escalations.upsert({
|
|
204
|
+
where: { kind },
|
|
205
|
+
create: {
|
|
206
|
+
kind,
|
|
207
|
+
first_seen_at: aggregate.firstSeen,
|
|
208
|
+
last_seen_at: aggregate.lastSeen,
|
|
209
|
+
event_count: aggregate.count,
|
|
210
|
+
},
|
|
211
|
+
update: {
|
|
212
|
+
first_seen_at: aggregate.firstSeen,
|
|
213
|
+
last_seen_at: aggregate.lastSeen,
|
|
214
|
+
event_count: aggregate.count,
|
|
215
|
+
resolved_at: null,
|
|
216
|
+
},
|
|
217
|
+
});
|
|
218
|
+
|
|
219
|
+
// 2. Resolve the PA path (bot → operator phone → PA device) and send.
|
|
220
|
+
const bot = await resolveOperatorRelay();
|
|
221
|
+
const operatorPhone = (bot.flow_config as { operator_phone?: string }).operator_phone!;
|
|
222
|
+
const copy = config.copy[kind] ?? config.default_copy;
|
|
223
|
+
const message = composeEscalationMessage({
|
|
224
|
+
kind,
|
|
225
|
+
aggregate,
|
|
226
|
+
impact: copy.impact,
|
|
227
|
+
recommendation: copy.recommendation,
|
|
228
|
+
});
|
|
229
|
+
|
|
230
|
+
const { userId, organisationId } = await getSystemIdentity();
|
|
231
|
+
const systemCtx: ExecutionContext = { userId, organisationId } as ExecutionContext;
|
|
232
|
+
const { forwardToolToCentralMcp } = await import("@teamscala/chat-completion/central-tool-forwarder");
|
|
233
|
+
const paDigits = (bot.external_id ?? "").replace(/\D/g, "");
|
|
234
|
+
const deviceId = paDigits ? await resolvePaDeviceId(paDigits) : undefined;
|
|
235
|
+
if (paDigits && !deviceId) {
|
|
236
|
+
log.warn("[platform-fault-escalation] no GOWA device matched the PA number — sending via default device");
|
|
237
|
+
}
|
|
238
|
+
const sendResult = await forwardToolToCentralMcp(
|
|
239
|
+
"api_gowa_message",
|
|
240
|
+
{
|
|
241
|
+
action: "sendMessage",
|
|
242
|
+
params: { phone: operatorPhone, message, ...(deviceId ? { device_id: deviceId } : {}) },
|
|
243
|
+
},
|
|
244
|
+
systemCtx,
|
|
245
|
+
);
|
|
246
|
+
|
|
247
|
+
// 3. VERIFY delivery before recording it: the send must return a WhatsApp
|
|
248
|
+
// message id. No id => throw, the claim stays undelivered, next sweep retries.
|
|
249
|
+
const messageId = extractMessageId(sendResult);
|
|
250
|
+
if (!messageId) {
|
|
251
|
+
throw new Error(
|
|
252
|
+
`platform-fault-escalation: WhatsApp send for '${kind}' returned no message id — not recording delivery`,
|
|
253
|
+
);
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
// 4. Durable record: the work_items task the PA reads back / the operator answers on.
|
|
257
|
+
const workItem = await prisma.work_items.create({
|
|
258
|
+
data: {
|
|
259
|
+
kind: "task",
|
|
260
|
+
parent_id: config.parent_project_id,
|
|
261
|
+
organisation_id: bot.organisation_id,
|
|
262
|
+
title: `Fault: ${kind}`.slice(0, 50),
|
|
263
|
+
description: composeEscalationDescription({ aggregate, impact: copy.impact, recommendation: copy.recommendation }),
|
|
264
|
+
status: "pending",
|
|
265
|
+
kind_data: {
|
|
266
|
+
platform_fault: true,
|
|
267
|
+
event_kind: kind,
|
|
268
|
+
event_count: aggregate.count,
|
|
269
|
+
first_seen_at: aggregate.firstSeen.toISOString(),
|
|
270
|
+
last_seen_at: aggregate.lastSeen.toISOString(),
|
|
271
|
+
delivered_message_id: messageId,
|
|
272
|
+
escalated_at: now.toISOString(),
|
|
273
|
+
},
|
|
274
|
+
},
|
|
275
|
+
select: { id: true },
|
|
276
|
+
});
|
|
277
|
+
|
|
278
|
+
// 5. Readback-correct the episode: delivered + linked in one update.
|
|
279
|
+
await prisma.platform_fault_escalations.update({
|
|
280
|
+
where: { kind },
|
|
281
|
+
data: {
|
|
282
|
+
escalated_at: now,
|
|
283
|
+
work_item_id: workItem.id,
|
|
284
|
+
delivered_message_id: messageId,
|
|
285
|
+
last_updated_at: now,
|
|
286
|
+
updated_at: now,
|
|
287
|
+
},
|
|
288
|
+
});
|
|
289
|
+
|
|
290
|
+
recordEvent({
|
|
291
|
+
kind: "platform.fault.escalated",
|
|
292
|
+
payload: {
|
|
293
|
+
event_kind: kind,
|
|
294
|
+
count: aggregate.count,
|
|
295
|
+
work_item_id: workItem.id,
|
|
296
|
+
delivered_message_id: messageId,
|
|
297
|
+
},
|
|
298
|
+
});
|
|
299
|
+
return { messageId, workItemId: workItem.id };
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
async function updateEpisode(aggregate: FaultAggregate, episode: EscalationEpisode): Promise<void> {
|
|
303
|
+
const prisma = getPrisma();
|
|
304
|
+
const now = new Date();
|
|
305
|
+
await prisma.platform_fault_escalations.update({
|
|
306
|
+
where: { kind: episode.kind },
|
|
307
|
+
data: {
|
|
308
|
+
last_seen_at: aggregate.lastSeen,
|
|
309
|
+
event_count: aggregate.count,
|
|
310
|
+
last_updated_at: now,
|
|
311
|
+
updated_at: now,
|
|
312
|
+
},
|
|
313
|
+
});
|
|
314
|
+
if (episode.workItemId) {
|
|
315
|
+
await prisma.comments.create({
|
|
316
|
+
data: {
|
|
317
|
+
work_item_id: episode.workItemId,
|
|
318
|
+
content: `Still recurring: ${aggregate.count} events, last at ${aggregate.lastSeen.toISOString()}. No new escalation (already escalated and unresolved).`,
|
|
319
|
+
sender_role: "system",
|
|
320
|
+
sender_name: "platform-fault-escalation",
|
|
321
|
+
},
|
|
322
|
+
});
|
|
323
|
+
}
|
|
324
|
+
recordEvent({
|
|
325
|
+
kind: "platform.fault.updated",
|
|
326
|
+
payload: { event_kind: episode.kind, count: aggregate.count, work_item_id: episode.workItemId },
|
|
327
|
+
});
|
|
328
|
+
}
|
|
329
|
+
|
|
330
|
+
async function resolveEpisode(episode: EscalationEpisode, quietMinutes: number): Promise<void> {
|
|
331
|
+
const prisma = getPrisma();
|
|
332
|
+
const now = new Date();
|
|
333
|
+
await prisma.platform_fault_escalations.update({
|
|
334
|
+
where: { kind: episode.kind },
|
|
335
|
+
data: { resolved_at: now, updated_at: now },
|
|
336
|
+
});
|
|
337
|
+
if (episode.workItemId) {
|
|
338
|
+
await prisma.comments.create({
|
|
339
|
+
data: {
|
|
340
|
+
work_item_id: episode.workItemId,
|
|
341
|
+
content: `Fault stopped — no '${episode.kind}' events for ${Math.round(quietMinutes)} min. Episode closed; the fault will re-escalate if it recurs.`,
|
|
342
|
+
sender_role: "system",
|
|
343
|
+
sender_name: "platform-fault-escalation",
|
|
344
|
+
},
|
|
345
|
+
});
|
|
346
|
+
}
|
|
347
|
+
recordEvent({
|
|
348
|
+
kind: "platform.fault.resolved",
|
|
349
|
+
payload: { event_kind: episode.kind, quiet_minutes: Math.round(quietMinutes), work_item_id: episode.workItemId },
|
|
350
|
+
});
|
|
351
|
+
}
|
|
352
|
+
|
|
353
|
+
async function platformFaultEscalationSweep(ctx: { config: ScheduledJobConfig }): Promise<void> {
|
|
354
|
+
const log = getAppLogger();
|
|
355
|
+
const config = await loadConfig();
|
|
356
|
+
const now = new Date();
|
|
357
|
+
|
|
358
|
+
const aggregates = await fetchAggregates(config.watched_kind_patterns, config.window_minutes);
|
|
359
|
+
// Only OPEN episodes matter: a resolved episode and no episode at all have
|
|
360
|
+
// the same meaning to decideSweep (a recurring sustained fault opens a NEW
|
|
361
|
+
// episode and escalates again).
|
|
362
|
+
const openEpisodes = await getPrisma().platform_fault_escalations.findMany({
|
|
363
|
+
where: { resolved_at: null },
|
|
364
|
+
});
|
|
365
|
+
const episodes: EscalationEpisode[] = openEpisodes.map((row) => ({
|
|
366
|
+
kind: row.kind,
|
|
367
|
+
firstSeenAt: new Date(row.first_seen_at),
|
|
368
|
+
lastSeenAt: new Date(row.last_seen_at),
|
|
369
|
+
eventCount: row.event_count,
|
|
370
|
+
escalatedAt: row.escalated_at ? new Date(row.escalated_at) : null,
|
|
371
|
+
workItemId: row.work_item_id,
|
|
372
|
+
resolvedAt: row.resolved_at ? new Date(row.resolved_at) : null,
|
|
373
|
+
lastUpdatedAt: row.last_updated_at ? new Date(row.last_updated_at) : null,
|
|
374
|
+
}));
|
|
375
|
+
|
|
376
|
+
const kindThresholds: Record<string, KindThresholdOverride> = {};
|
|
377
|
+
for (const [kind, override] of Object.entries(config.kind_thresholds ?? {})) {
|
|
378
|
+
kindThresholds[kind] = {
|
|
379
|
+
minCount: override?.min_count,
|
|
380
|
+
minSpanMinutes: override?.min_span_minutes,
|
|
381
|
+
};
|
|
382
|
+
}
|
|
383
|
+
const decision = decideSweep(
|
|
384
|
+
aggregates,
|
|
385
|
+
episodes,
|
|
386
|
+
{
|
|
387
|
+
minCount: config.min_count,
|
|
388
|
+
minSpanMinutes: config.min_span_minutes,
|
|
389
|
+
resolveAfterMinutes: config.resolve_after_minutes,
|
|
390
|
+
updateAfterMinutes: config.update_after_minutes,
|
|
391
|
+
},
|
|
392
|
+
now,
|
|
393
|
+
kindThresholds,
|
|
394
|
+
);
|
|
395
|
+
|
|
396
|
+
// Per-kind isolation: one fault's delivery failure must never block the
|
|
397
|
+
// others. Failures are logged + emitted so the escalator itself is
|
|
398
|
+
// observable (the sweep that silently stops sweeping is the original bug).
|
|
399
|
+
for (const { aggregate } of decision.escalate) {
|
|
400
|
+
try {
|
|
401
|
+
const delivered = await deliverEscalation(aggregate.kind, aggregate, config);
|
|
402
|
+
log.warn(
|
|
403
|
+
`[platform-fault-escalation] ESCALATED '${aggregate.kind}' — WhatsApp message ${delivered.messageId}, work item ${delivered.workItemId}`,
|
|
404
|
+
);
|
|
405
|
+
} catch (err) {
|
|
406
|
+
const reason = err instanceof Error ? err.message : String(err);
|
|
407
|
+
log.error(`[platform-fault-escalation] delivery for '${aggregate.kind}' failed (claim retained, retried next sweep): ${reason}`);
|
|
408
|
+
recordEvent({ kind: "platform.fault.escalation_failed", payload: { event_kind: aggregate.kind, reason } });
|
|
409
|
+
}
|
|
410
|
+
}
|
|
411
|
+
for (const { aggregate, episode } of decision.update) {
|
|
412
|
+
try {
|
|
413
|
+
await updateEpisode(aggregate, episode);
|
|
414
|
+
} catch (err) {
|
|
415
|
+
log.error(`[platform-fault-escalation] update for '${episode.kind}' failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
416
|
+
}
|
|
417
|
+
}
|
|
418
|
+
for (const { episode, quietMinutes } of decision.resolve) {
|
|
419
|
+
try {
|
|
420
|
+
await resolveEpisode(episode, quietMinutes);
|
|
421
|
+
} catch (err) {
|
|
422
|
+
log.error(`[platform-fault-escalation] resolve for '${episode.kind}' failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
423
|
+
}
|
|
424
|
+
}
|
|
425
|
+
|
|
426
|
+
// Emitted every run, even when nothing fired — a sweep that only reports
|
|
427
|
+
// when it escalated is indistinguishable from one that stopped running.
|
|
428
|
+
recordEvent({
|
|
429
|
+
kind: "platform.fault.swept",
|
|
430
|
+
payload: {
|
|
431
|
+
watched_kinds: aggregates.length,
|
|
432
|
+
escalated: decision.escalate.length,
|
|
433
|
+
updated: decision.update.length,
|
|
434
|
+
resolved: decision.resolve.length,
|
|
435
|
+
},
|
|
436
|
+
});
|
|
437
|
+
if (decision.escalate.length > 0 || decision.update.length > 0 || decision.resolve.length > 0) {
|
|
438
|
+
log.info(
|
|
439
|
+
`[platform-fault-escalation] sweep — ${aggregates.length} kind(s) watched, ${decision.escalate.length} escalated, ${decision.update.length} updated, ${decision.resolve.length} resolved`,
|
|
440
|
+
);
|
|
441
|
+
}
|
|
442
|
+
}
|
|
443
|
+
|
|
444
|
+
pgBossHandlerRegistry.register(
|
|
445
|
+
"@teamscala/service-boot:platform-fault-escalation",
|
|
446
|
+
platformFaultEscalationSweep as (ctx: { config: ScheduledJobConfig }) => Promise<void>,
|
|
447
|
+
);
|
package/src/pg-boss-mount.ts
CHANGED
|
@@ -35,6 +35,7 @@ import "./handlers/singleton-tools-currency-watch.ts";
|
|
|
35
35
|
import "./handlers/registry-stale-inactive-sweep.ts";
|
|
36
36
|
import "./handlers/pane-outbox-drain.ts";
|
|
37
37
|
import "./handlers/lane-attention-sweep.ts";
|
|
38
|
+
import "./handlers/platform-fault-escalation.ts";
|
|
38
39
|
|
|
39
40
|
/**
|
|
40
41
|
* Module-level reference to the mounted PgBoss instance — set by mountPgBoss on
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @system service-boot
|
|
3
|
+
* @status handwritten
|
|
4
|
+
* @edit edit directly
|
|
5
|
+
*
|
|
6
|
+
* Behavior proof for the `security-content-type-nosniff` +
|
|
7
|
+
* `security-permissions-policy` invariants: exercise injectSecurityHeaders
|
|
8
|
+
* through a real Hono request and assert the headers on the RESPONSE —
|
|
9
|
+
* a registry row naming a header is not evidence the header is served.
|
|
10
|
+
* Covers both modes: the seeded-registry path (config row applied, CSP
|
|
11
|
+
* {{nonce}} substituted) and the DB-less fallback (secureHeaders defaults).
|
|
12
|
+
*/
|
|
13
|
+
import { describe, expect, it } from "bun:test";
|
|
14
|
+
import { Hono } from "hono";
|
|
15
|
+
import { injectSecurityHeaders } from "./security-headers.ts";
|
|
16
|
+
|
|
17
|
+
type CacheGlobal = typeof globalThis & {
|
|
18
|
+
__registryCache?: Map<string, { type: string; slug: string; config: unknown }>;
|
|
19
|
+
};
|
|
20
|
+
|
|
21
|
+
function seedRegistryHeaders(config: Record<string, string>): void {
|
|
22
|
+
const g = globalThis as CacheGlobal;
|
|
23
|
+
if (!g.__registryCache) g.__registryCache = new Map();
|
|
24
|
+
g.__registryCache.set("config:security-headers", {
|
|
25
|
+
type: "config",
|
|
26
|
+
slug: "security-headers",
|
|
27
|
+
config,
|
|
28
|
+
});
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
function clearRegistryHeaders(): void {
|
|
32
|
+
(globalThis as CacheGlobal).__registryCache?.delete("config:security-headers");
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
function appWithMiddleware(): Hono {
|
|
36
|
+
const app = new Hono();
|
|
37
|
+
app.use("*", injectSecurityHeaders);
|
|
38
|
+
app.get("/", (c) => c.text("ok"));
|
|
39
|
+
return app;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
describe("injectSecurityHeaders (registry mode)", () => {
|
|
43
|
+
it("serves X-Content-Type-Options: nosniff and the configured Permissions-Policy", async () => {
|
|
44
|
+
seedRegistryHeaders({
|
|
45
|
+
"X-Content-Type-Options": "nosniff",
|
|
46
|
+
"Permissions-Policy": "camera=(), microphone=(self), geolocation=()",
|
|
47
|
+
"Content-Security-Policy": "script-src 'nonce-{{nonce}}' 'strict-dynamic'",
|
|
48
|
+
});
|
|
49
|
+
try {
|
|
50
|
+
const res = await appWithMiddleware().request("https://host.example/");
|
|
51
|
+
expect(res.headers.get("X-Content-Type-Options")).toBe("nosniff");
|
|
52
|
+
expect(res.headers.get("Permissions-Policy")).toBe(
|
|
53
|
+
"camera=(), microphone=(self), geolocation=()",
|
|
54
|
+
);
|
|
55
|
+
} finally {
|
|
56
|
+
clearRegistryHeaders();
|
|
57
|
+
}
|
|
58
|
+
});
|
|
59
|
+
|
|
60
|
+
it("substitutes the CSP {{nonce}} placeholder — the served policy carries a real nonce", async () => {
|
|
61
|
+
seedRegistryHeaders({
|
|
62
|
+
"Content-Security-Policy": "script-src 'nonce-{{nonce}}' 'strict-dynamic'",
|
|
63
|
+
});
|
|
64
|
+
try {
|
|
65
|
+
const res = await appWithMiddleware().request("https://host.example/");
|
|
66
|
+
const csp = res.headers.get("Content-Security-Policy") ?? "";
|
|
67
|
+
expect(csp).not.toContain("{{nonce}}");
|
|
68
|
+
expect(csp).toMatch(/'nonce-[^']+'/);
|
|
69
|
+
} finally {
|
|
70
|
+
clearRegistryHeaders();
|
|
71
|
+
}
|
|
72
|
+
});
|
|
73
|
+
});
|
|
74
|
+
|
|
75
|
+
describe("injectSecurityHeaders (fallback mode, no registry)", () => {
|
|
76
|
+
it("still serves X-Content-Type-Options: nosniff from secureHeaders defaults", async () => {
|
|
77
|
+
clearRegistryHeaders();
|
|
78
|
+
const res = await appWithMiddleware().request("https://host.example/");
|
|
79
|
+
expect(res.headers.get("X-Content-Type-Options")).toBe("nosniff");
|
|
80
|
+
expect((res.headers.get("Content-Security-Policy") ?? "")).not.toContain("{{nonce}}");
|
|
81
|
+
});
|
|
82
|
+
});
|
package/src/two-phase-boot.ts
CHANGED
|
@@ -18,6 +18,7 @@ import { buildWakingPageHtml } from "@teamscala/os/brand/waking-page-html";
|
|
|
18
18
|
import { BootFatalError } from "@teamscala/os/errors/boot-error";
|
|
19
19
|
import { createTimer } from "@teamscala/timing/create-timer";
|
|
20
20
|
import { Hono, type Context } from "hono";
|
|
21
|
+
import { bodyLimit } from "hono/body-limit";
|
|
21
22
|
import { compress } from "hono/compress";
|
|
22
23
|
import { cors } from "hono/cors";
|
|
23
24
|
import { timing } from "hono/timing";
|
|
@@ -33,6 +34,7 @@ import { attachProposalMagic } from "./attach-proposal-magic.ts";
|
|
|
33
34
|
import { attachPortalMembers } from "./attach-portal-members.ts";
|
|
34
35
|
import { attachTransfersUpload } from "./attach-transfers-upload.ts";
|
|
35
36
|
import { attachAuthGate } from "./auth-gate.ts";
|
|
37
|
+
import { csrfProtection } from "./csrf-protection.ts";
|
|
36
38
|
import { attachRequestLogger } from "./install-logging.ts";
|
|
37
39
|
import type { ServiceInstallConfig } from "./load-service-install-row.ts";
|
|
38
40
|
import { mountOrpcRouter } from "./mount-orpc-router.ts";
|
|
@@ -276,6 +278,26 @@ export async function buildMinimalApp(
|
|
|
276
278
|
);
|
|
277
279
|
app.use("*", compress());
|
|
278
280
|
app.use("*", timing());
|
|
281
|
+
// CSRF (`security-csrf-protection`): state-changing form-element requests
|
|
282
|
+
// must carry a trusted Origin — derived set (managed base domain +
|
|
283
|
+
// builda_websites custom domains via isTrustedAuthOrigin), never an
|
|
284
|
+
// allowlist beside better-auth's. Registered after security headers, before
|
|
285
|
+
// every route + gate.
|
|
286
|
+
app.use("*", csrfProtection());
|
|
287
|
+
// GLOBAL body limit (`security-body-limit`): every request body is bounded
|
|
288
|
+
// BEFORE any handler buffers it — a per-route file.size check AFTER
|
|
289
|
+
// formData() has buffered is a report, not a limit. 30MB sits above the
|
|
290
|
+
// 25MB transfers-upload maximum; routes wanting a smaller bound mount their
|
|
291
|
+
// own bodyLimit BEFORE this one (Hono runs matching middleware in
|
|
292
|
+
// registration order, and /api/rpc registers its 10MB limit in
|
|
293
|
+
// mount-orpc-router).
|
|
294
|
+
app.use(
|
|
295
|
+
"*",
|
|
296
|
+
bodyLimit({
|
|
297
|
+
maxSize: 30 * 1024 * 1024,
|
|
298
|
+
onError: (c) => c.text("Payload Too Large", 413),
|
|
299
|
+
}),
|
|
300
|
+
);
|
|
279
301
|
attachRequestLogger(app, serviceSlug);
|
|
280
302
|
|
|
281
303
|
// Fire independent I/O + compilation NOW, overlapping with the gate chain
|