@teamscala/service-boot 0.23.360 → 0.23.362

package/package.json

@@ -6,7 +6,7 @@
   },
   "name": "@teamscala/service-boot",
   "type": "module",
-  "version": "0.23.360",
+  "version": "0.23.362",
   "sideEffects": false,
   "imports": {
     "#service-boot/*": "./src/*",
@@ -21,6 +21,7 @@
     "@teamscala/auth": "*",
     "@teamscala/cache": "*",
     "@teamscala/channel": "*",
+    "@teamscala/chat-completion": "*",
     "@teamscala/db": "*",
     "@teamscala/db-validation": "*",
     "@teamscala/document-signing": "*",
@@ -229,6 +230,14 @@
       "types": "./src/content-page-floor.d.ts",
       "default": "./src/content-page-floor.ts"
     },
+    "./csrf-protection": {
+      "types": "./src/csrf-protection.d.ts",
+      "default": "./src/csrf-protection.ts"
+    },
+    "./csrf-protection.ts": {
+      "types": "./src/csrf-protection.d.ts",
+      "default": "./src/csrf-protection.ts"
+    },
     "./discover-blocks": {
       "types": "./src/discover-blocks.d.ts",
       "default": "./src/discover-blocks.ts"
@@ -417,6 +426,22 @@
       "types": "./src/handlers/pane-outbox-drain.d.ts",
       "default": "./src/handlers/pane-outbox-drain.ts"
     },
+    "./handlers/platform-fault-escalation": {
+      "types": "./src/handlers/platform-fault-escalation.d.ts",
+      "default": "./src/handlers/platform-fault-escalation.ts"
+    },
+    "./handlers/platform-fault-escalation.ts": {
+      "types": "./src/handlers/platform-fault-escalation.d.ts",
+      "default": "./src/handlers/platform-fault-escalation.ts"
+    },
+    "./handlers/platform-fault-escalation-logic": {
+      "types": "./src/handlers/platform-fault-escalation-logic.d.ts",
+      "default": "./src/handlers/platform-fault-escalation-logic.ts"
+    },
+    "./handlers/platform-fault-escalation-logic.ts": {
+      "types": "./src/handlers/platform-fault-escalation-logic.d.ts",
+      "default": "./src/handlers/platform-fault-escalation-logic.ts"
+    },
     "./handlers/projects-digest": {
       "types": "./src/handlers/projects-digest.d.ts",
       "default": "./src/handlers/projects-digest.ts"

package/src/handlers/platform-fault-escalation-logic.ts

@@ -0,0 +1,206 @@
+/**
+ * @system server-boot
+ * @status handwritten
+ * @edit edit directly
+ *
+ * Pure decision logic for the platform-fault escalation sweep
+ * (`platform-fault-escalation` handler). Split from the I/O shell so the
+ * threshold/dedup/episode semantics are unit-testable with zero adapter
+ * dependencies (same shape as whatsapp-liveness-logic).
+ *
+ * The semantics this locks (from the 2026-08-15 "760 errors escalated to
+ * nobody" incident brief):
+ * - Threshold on DURATION and RECURRENCE, never a single event. A kind
+ *   qualifies as a sustained fault only when BOTH hold inside the lookback
+ *   window: at least `min_count` occurrences (recurrence) spread over at
+ *   least `min_span_minutes` (duration). One failed push is noise; the same
+ *   kind recurring for half an hour is an outage.
+ * - One escalation per distinct fault kind per episode. An episode is open
+ *   from first qualification until `resolve_after_minutes` of quiet; while
+ *   open, the fault never re-fires — it only appends throttled updates.
+ * - Delivery is verified, not queued: an episode row with `escalated_at`
+ *   NULL is a CLAIM, not an escalation. The sweep re-attempts delivery for
+ *   claimed-but-undelivered episodes until the WhatsApp send returns a
+ *   message id.
+ */
+
+export interface FaultAggregate {
+	kind: string;
+	count: number;
+	firstSeen: Date;
+	lastSeen: Date;
+}
+
+export interface EscalationEpisode {
+	kind: string;
+	firstSeenAt: Date;
+	lastSeenAt: Date;
+	eventCount: number;
+	escalatedAt: Date | null;
+	workItemId: string | null;
+	resolvedAt: Date | null;
+	lastUpdatedAt: Date | null;
+}
+
+export interface FaultEscalationPolicy {
+	/** Recurrence floor: minimum event count inside the window. */
+	minCount: number;
+	/** Duration floor: minimum first→last spread inside the window, minutes. */
+	minSpanMinutes: number;
+	/** Quiet period that closes an episode and re-arms escalation, minutes. */
+	resolveAfterMinutes: number;
+	/** Minimum spacing between throttled updates on an open episode, minutes. */
+	updateAfterMinutes: number;
+}
+
+/** Per-kind threshold overrides — a low-rate-but-persistent kind (guard.fail.open
+ * at ~1/hour over 12h) has different noise characteristics than a bursty one, so
+ * the recurrence/duration floors are per-kind policy in the config row. */
+export interface KindThresholdOverride {
+	minCount?: number;
+	minSpanMinutes?: number;
+}
+
+export interface SweepDecision {
+	/** Sustained faults needing a FIRST delivery (new episode, or a claim whose send failed). */
+	escalate: Array<{ aggregate: FaultAggregate; episode: EscalationEpisode | null }>;
+	/** Open, delivered episodes still recurring — append evidence, never re-send. */
+	update: Array<{ aggregate: FaultAggregate; episode: EscalationEpisode }>;
+	/** Open episodes whose kind went quiet past the resolve window — close them. */
+	resolve: Array<{ episode: EscalationEpisode; quietMinutes: number }>;
+}
+
+function minutesBetween(a: Date, b: Date): number {
+	return (b.getTime() - a.getTime()) / 60_000;
+}
+
+function isSustained(
+	aggregate: FaultAggregate,
+	policy: FaultEscalationPolicy,
+	override: KindThresholdOverride | undefined,
+): boolean {
+	const minCount = override?.minCount ?? policy.minCount;
+	const minSpan = override?.minSpanMinutes ?? policy.minSpanMinutes;
+	return (
+		aggregate.count >= minCount &&
+		minutesBetween(aggregate.firstSeen, aggregate.lastSeen) >= minSpan
+	);
+}
+
+/**
+ * Classify the current sweep. `aggregates` are the per-kind error aggregates
+ * from the lookback window; `episodes` are the existing episode rows for those
+ * kinds PLUS every still-open episode (an open episode whose kind no longer
+ * appears in the window at all is the strongest resolve candidate).
+ */
+export function decideSweep(
+	aggregates: FaultAggregate[],
+	episodes: EscalationEpisode[],
+	policy: FaultEscalationPolicy,
+	now: Date,
+	kindThresholds: Record<string, KindThresholdOverride> = {},
+): SweepDecision {
+	const byKind = new Map(episodes.map((e) => [e.kind, e]));
+	const aggregateByKind = new Map(aggregates.map((a) => [a.kind, a]));
+	const decision: SweepDecision = { escalate: [], update: [], resolve: [] };
+
+	for (const aggregate of aggregates) {
+		const episode = byKind.get(aggregate.kind);
+		if (!isSustained(aggregate, policy, kindThresholds[aggregate.kind])) continue;
+		// No episode, or the previous episode closed (quiet past resolve window):
+		// a NEW episode begins and must be delivered.
+		if (!episode || episode.resolvedAt) {
+			decision.escalate.push({ aggregate, episode: episode ?? null });
+			continue;
+		}
+		// Open episode with no verified delivery — the claim is retried, never
+		// counted as escalated (a component may not report a state it has not
+		// verified).
+		if (!episode.escalatedAt) {
+			decision.escalate.push({ aggregate, episode });
+			continue;
+		}
+		// Open, delivered, still recurring — throttled evidence update only.
+		const lastUpdate = episode.lastUpdatedAt ?? episode.escalatedAt;
+		if (minutesBetween(lastUpdate, now) >= policy.updateAfterMinutes) {
+			decision.update.push({ aggregate, episode });
+		}
+	}
+
+	for (const episode of episodes) {
+		if (episode.resolvedAt) continue;
+		const aggregate = aggregateByKind.get(episode.kind);
+		const quietReference = aggregate?.lastSeen ?? episode.lastSeenAt;
+		const quietMinutes = minutesBetween(quietReference, now);
+		if (quietMinutes >= policy.resolveAfterMinutes) {
+			decision.resolve.push({ episode, quietMinutes });
+		}
+	}
+
+	return decision;
+}
+
+/** Human "N min" span, rounded — for the phone-sized escalation message. */
+export function formatSpanMinutes(minutes: number): string {
+	if (minutes < 60) return `${Math.round(minutes)} min`;
+	const hours = minutes / 60;
+	return `${hours >= 10 ? Math.round(hours) : Math.round(hours * 10) / 10} h`;
+}
+
+/**
+ * The WhatsApp message id from a GOWA /send/message result. The canonical
+ * shape (from the spec the api-gowa adapter is generated from,
+ * SendResponse = { code, message, results: { message_id, status } }) is
+ * `results.message_id`; the legacy shape returned the bare id string as
+ * `results`. Delivery is only VERIFIED on a real id — `"true"`, a status
+ * string, or an empty body is not a message id and must not be recorded as
+ * delivered.
+ */
+export function extractGowaMessageId(sendResult: unknown): string | null {
+	const results = (sendResult as { results?: unknown } | undefined)?.results;
+	if (typeof results === "string") {
+		return /^[A-Z0-9]{8,}$/i.test(results) ? results : null;
+	}
+	if (results && typeof results === "object") {
+		const id = (results as { message_id?: unknown }).message_id;
+		if (typeof id === "string" && id.length > 0) return id;
+	}
+	return null;
+}
+
+/**
+ * The escalation message — escalation-routing's three-field contract adapted
+ * to an automated detector: what was OBSERVED (evidence), what it MEANS
+ * (impact), and the RECOMMENDATION. Short: the channel is a phone.
+ */
+export function composeEscalationMessage(input: {
+	kind: string;
+	aggregate: FaultAggregate;
+	impact: string;
+	recommendation: string;
+}): string {
+	const span = formatSpanMinutes(
+		(input.aggregate.lastSeen.getTime() - input.aggregate.firstSeen.getTime()) / 60_000,
+	);
+	return [
+		`🔴 Platform fault: ${input.kind}`,
+		`Observed: ${input.aggregate.count} errors over ${span}, still recurring.`,
+		`Impact: ${input.impact}`,
+		`Recommendation: ${input.recommendation}`,
+	].join("\n");
+}
+
+/** The work-item description carries the same three fields plus the evidence. */
+export function composeEscalationDescription(input: {
+	aggregate: FaultAggregate;
+	impact: string;
+	recommendation: string;
+}): string {
+	return [
+		"Automated platform-fault escalation (platform-fault-escalation sweep).",
+		`Observed: ${input.aggregate.count} '${input.aggregate.kind}' events in event_log from ${input.aggregate.firstSeen.toISOString()} to ${input.aggregate.lastSeen.toISOString()}, still recurring.`,
+		`Impact: ${input.impact}`,
+		`Recommendation: ${input.recommendation}`,
+		"Delivery: WhatsApp via the scala PA. This task closes automatically when the fault stops for the resolve window; reply via the PA to rule otherwise.",
+	].join("\n");
+}

package/src/handlers/platform-fault-escalation.test.ts

@@ -0,0 +1,204 @@
+/**
+ * @system server-boot
+ * @status handwritten
+ * @edit edit directly
+ *
+ * Unit tests for the pure decision core of the platform-fault escalation
+ * sweep (`platform-fault-escalation` handler). The handler's DB/WhatsApp I/O
+ * is runtime-verified by the scheduled_job firing; this locks the threshold,
+ * dedup and episode semantics from the 2026-08-15 incident brief.
+ */
+import { describe, expect, test } from "bun:test";
+import {
+	composeEscalationMessage,
+	decideSweep,
+	extractGowaMessageId,
+	type EscalationEpisode,
+	type FaultAggregate,
+	type FaultEscalationPolicy,
+} from "./platform-fault-escalation-logic.ts";
+
+const NOW = new Date("2026-08-15T23:45:00+10:00");
+const POLICY: FaultEscalationPolicy = {
+	minCount: 10,
+	minSpanMinutes: 25,
+	resolveAfterMinutes: 30,
+	updateAfterMinutes: 30,
+};
+
+function agg(kind: string, count: number, firstSeenMinutesAgo: number, lastSeenMinutesAgo: number): FaultAggregate {
+	return {
+		kind,
+		count,
+		firstSeen: new Date(NOW.getTime() - firstSeenMinutesAgo * 60_000),
+		lastSeen: new Date(NOW.getTime() - lastSeenMinutesAgo * 60_000),
+	};
+}
+
+function episode(overrides: Partial<EscalationEpisode> & { kind: string }): EscalationEpisode {
+	return {
+		firstSeenAt: NOW,
+		lastSeenAt: NOW,
+		eventCount: 10,
+		escalatedAt: new Date(NOW.getTime() - 60 * 60_000),
+		workItemId: "wi-1",
+		resolvedAt: null,
+		lastUpdatedAt: null,
+		...overrides,
+	};
+}
+
+describe("decideSweep — thresholds", () => {
+	test("a single event is noise — never escalates", () => {
+		const decision = decideSweep([agg("push.error", 1, 2, 2)], [], POLICY, NOW);
+		expect(decision.escalate).toHaveLength(0);
+	});
+
+	test("recurrence without duration does not escalate (10 events in 5 min)", () => {
+		const decision = decideSweep([agg("burst.error", 10, 5, 0)], [], POLICY, NOW);
+		expect(decision.escalate).toHaveLength(0);
+	});
+
+	test("duration without recurrence does not escalate (3 events over 40 min)", () => {
+		const decision = decideSweep([agg("sparse.error", 3, 40, 0)], [], POLICY, NOW);
+		expect(decision.escalate).toHaveLength(0);
+	});
+
+	test("the 490-over-12h shape escalates (duration + recurrence)", () => {
+		const sustained = agg("host-command-bus.error", 490, 720, 0);
+		const decision = decideSweep([sustained], [], POLICY, NOW);
+		expect(decision.escalate).toHaveLength(1);
+		expect(decision.escalate[0]!.aggregate.kind).toBe("host-command-bus.error");
+		expect(decision.escalate[0]!.episode).toBeNull();
+	});
+
+	test("per-kind threshold override lets a low-rate persistent kind through", () => {
+		// guard.fail.open: 8 events over 12h — below the default recurrence floor
+		// of 10, but persistent. The override lowers recurrence to 4 over 60 min.
+		const drip = agg("guard.fail.open", 8, 720, 0);
+		const withoutOverride = decideSweep([drip], [], POLICY, NOW);
+		expect(withoutOverride.escalate).toHaveLength(0);
+		const withOverride = decideSweep([drip], [], POLICY, NOW, {
+			"guard.fail.open": { minCount: 4, minSpanMinutes: 60 },
+		});
+		expect(withOverride.escalate).toHaveLength(1);
+	});
+
+	test("override raises a floor too — bursty kinds can be tightened", () => {
+		const sustained = agg("llm.provider.error", 40, 60, 0); // passes default
+		const decision = decideSweep([sustained], [], POLICY, NOW, {
+			"llm.provider.error": { minCount: 100 },
+		});
+		expect(decision.escalate).toHaveLength(0);
+	});
+});
+
+describe("decideSweep — dedup (one escalation per fault, no re-fire)", () => {
+	test("already escalated and unresolved does NOT re-escalate", () => {
+		const sustained = agg("host-command-bus.error", 500, 720, 0);
+		const open = episode({ kind: "host-command-bus.error", escalatedAt: new Date(NOW.getTime() - 3_600_000) });
+		const decision = decideSweep([sustained], [open], POLICY, NOW);
+		expect(decision.escalate).toHaveLength(0);
+	});
+
+	test("open delivered episode past the update window gets an UPDATE, not a message", () => {
+		const sustained = agg("llm.provider.error", 300, 720, 0);
+		const open = episode({
+			kind: "llm.provider.error",
+			escalatedAt: new Date(NOW.getTime() - 90 * 60_000), // last update long ago
+			lastUpdatedAt: new Date(NOW.getTime() - 90 * 60_000),
+		});
+		const decision = decideSweep([sustained], [open], POLICY, NOW);
+		expect(decision.escalate).toHaveLength(0);
+		expect(decision.update).toHaveLength(1);
+		expect(decision.update[0]!.episode.workItemId).toBe("wi-1");
+	});
+
+	test("open episode inside the update window gets NOTHING (dedup window)", () => {
+		const sustained = agg("llm.provider.error", 300, 720, 0);
+		const open = episode({
+			kind: "llm.provider.error",
+			escalatedAt: new Date(NOW.getTime() - 10 * 60_000),
+			lastUpdatedAt: new Date(NOW.getTime() - 10 * 60_000),
+		});
+		const decision = decideSweep([sustained], [open], POLICY, NOW);
+		expect(decision.escalate).toHaveLength(0);
+		expect(decision.update).toHaveLength(0);
+	});
+
+	test("a claimed-but-undelivered episode is RETRIED, not counted as escalated", () => {
+		const sustained = agg("guard.fail.open", 40, 720, 0);
+		const claim = episode({ kind: "guard.fail.open", escalatedAt: null, workItemId: null });
+		const decision = decideSweep([sustained], [claim], POLICY, NOW);
+		expect(decision.escalate).toHaveLength(1);
+		expect(decision.escalate[0]!.episode).toBe(claim);
+	});
+});
+
+describe("decideSweep — resolution + re-arm", () => {
+	test("quiet past the resolve window closes the episode", () => {
+		const gone = episode({ kind: "wake.failed", lastSeenAt: new Date(NOW.getTime() - 60 * 60_000) });
+		const decision = decideSweep([], [gone], POLICY, NOW);
+		expect(decision.resolve).toHaveLength(1);
+		expect(decision.resolve[0]!.quietMinutes).toBeGreaterThanOrEqual(30);
+	});
+
+	test("an absent kind resolves by its episode's last_seen, not the sweep clock", () => {
+		const gone = episode({ kind: "listen.connect_failed", lastSeenAt: new Date(NOW.getTime() - 45 * 60_000) });
+		const decision = decideSweep([], [gone], POLICY, NOW);
+		expect(decision.resolve).toHaveLength(1);
+	});
+
+	test("after resolution the same fault escalates again (new episode)", () => {
+		const recurring = agg("host-command-bus.error", 50, 40, 0);
+		const resolved = episode({
+			kind: "host-command-bus.error",
+			resolvedAt: new Date(NOW.getTime() - 35 * 60_000),
+		});
+		const decision = decideSweep([recurring], [resolved], POLICY, NOW);
+		expect(decision.escalate).toHaveLength(1);
+	});
+
+	test("still-active kind within the resolve window is not resolved even if below threshold", () => {
+		const active = agg("slow.error", 3, 20, 0); // below threshold, but recent
+		const open = episode({ kind: "slow.error", lastSeenAt: active.lastSeen });
+		const decision = decideSweep([active], [open], POLICY, NOW);
+		expect(decision.resolve).toHaveLength(0);
+	});
+});
+
+describe("extractGowaMessageId — delivery verification", () => {
+	test("the canonical GOWA SendResponse shape yields results.message_id", () => {
+		expect(extractGowaMessageId({ code: "SUCCESS", results: { message_id: "3EB089B9D6ADD58153C561", status: "sent" } })).toBe(
+			"3EB089B9D6ADD58153C561",
+		);
+	});
+
+	test("the legacy bare-string results shape yields the id", () => {
+		expect(extractGowaMessageId({ results: "3EB089B9D6ADD58153C561" })).toBe("3EB089B9D6ADD58153C561");
+	});
+
+	test("a 'true' ack is NOT a message id — delivery stays unverified", () => {
+		// A boolean-ish ack string must never be recorded as a delivered id:
+		// that would mark the episode delivered with an unfollowable handle.
+		expect(extractGowaMessageId({ results: "true" })).toBeNull();
+		expect(extractGowaMessageId({ code: "SUCCESS", message: "sent", results: { status: "sent" } })).toBeNull();
+		expect(extractGowaMessageId({ code: "SUCCESS" })).toBeNull();
+	});
+});
+
+describe("composeEscalationMessage", () => {
+	test("carries the three escalation-routing fields and stays phone-sized", () => {
+		const message = composeEscalationMessage({
+			kind: "host-command-bus.error",
+			aggregate: agg("host-command-bus.error", 490, 720, 0),
+			impact: "scala-tools verbs are failing host-wide",
+			recommendation: "check oxmgr logs for the bus consumer",
+		});
+		expect(message).toContain("Platform fault: host-command-bus.error");
+		expect(message).toContain("Observed: 490 errors over 12 h");
+		expect(message).toContain("Impact: scala-tools verbs are failing host-wide");
+		expect(message).toContain("Recommendation: check oxmgr logs for the bus consumer");
+		expect(message.split("\n")).toHaveLength(4);
+	});
+});

package/src/handlers/platform-fault-escalation.ts

@@ -0,0 +1,437 @@
+/**
+ * @system server-boot
+ * @status handwritten
+ * @edit edit directly
+ *
+ * The `platform-fault-escalation` pg-boss handler — closes the "receiving is
+ * decoupled from consequence" gap measured 2026-08-15 (760 error events over
+ * 12 hours, zero escalations: host-command-bus.error 490, llm.provider.error
+ * 231, guard.fail.open 16, scale-to-zero.wake_failed 12, listen.connect_failed
+ * 11). Emission works and observability-is-passive is satisfied; nothing
+ * acted. This sweep makes RECEIVING have CONSEQUENCE: sustained platform
+ * faults escalate to the OPERATOR over the existing PA path (WhatsApp, the
+ * scala PA operator-relay bot — escalation-routing's channel; no second
+ * channel invented), with the durable record as a work_items task the PA can
+ * read back and the operator can answer on.
+ *
+ * THIS IS NOT A DRIFT-DETECTOR PATROLLING INSTANCES. It reacts only to
+ * components reporting THEIR OWN failure — event_log kinds that already
+ * assert a fault about their emitter. Thresholds (duration + recurrence,
+ * never a single event), the watched-kind patterns, the resolve window and
+ * the per-kind impact/recommendation copy are all in the
+ * `config/platform-fault-escalation` registry row — changing policy is a row
+ * edit, not a redeploy. Episode dedup lives in the
+ * `platform_fault_escalations` table (migration 501): one escalation per
+ * distinct fault kind per episode; an already-escalated-and-unresolved fault
+ * only appends throttled work-item comments, never a second message.
+ *
+ * DELIVERY IS VERIFIED, NOT QUEUED. The WhatsApp send goes through the
+ * central `/mcp` loopback (`api_gowa_message` sendMessage — the single
+ * sealed GOWA-credential surface, same as whatsapp-liveness) and an episode
+ * only carries `escalated_at` once the send returned a WhatsApp message id;
+ * before that the row is a CLAIM the next sweep retries. Success is recorded
+ * only after readback of the delivery facts (message id + work item id).
+ *
+ * WHY A TICK EXISTS AT ALL, given observability-is-passive and
+ * no-polling-for-pushable-state: the trigger is DURATION + RECURRENCE of
+ * events already durably logged — a fact about the PATTERN of past events,
+ * not a state transition any single event can announce. The sweep reads one
+ * server-side aggregate (GROUP BY kind) per run against indexed event_log;
+ * while the platform is healthy it returns zero rows and costs nothing
+ * beyond the tick (idle-costs-nothing). Same justification
+ * lane-attention-sweep carries on its row as `poll_justification`.
+ *
+ * Registered under slug `@teamscala/service-boot:platform-fault-escalation`.
+ */
+import type { ExecutionContext } from "@teamscala/os/contracts/mcp";
+import type { ScheduledJobConfig } from "@teamscala/db-validation/registry-schemas/scheduled-job";
+import * as v from "valibot";
+import { getPrisma } from "@teamscala/db/prisma-registry";
+import { loadRegistryConfig } from "@teamscala/db/registry/load-config";
+import { recordEvent } from "@teamscala/event-log/record-event";
+import { getAppLogger } from "@teamscala/logger/app-loggers";
+import { pgBossHandlerRegistry } from "../pg-boss-handler-registry.ts";
+import {
+	composeEscalationDescription,
+	composeEscalationMessage,
+	decideSweep,
+	extractGowaMessageId,
+	type EscalationEpisode,
+	type FaultAggregate,
+	type KindThresholdOverride,
+} from "./platform-fault-escalation-logic.ts";
+
+const CONFIG_SLUG = "platform-fault-escalation";
+
+const FaultEscalationConfigSchema = v.object({
+	/** Postgres regex patterns selecting watched event_log kinds. */
+	watched_kind_patterns: v.array(v.string()),
+	/** Lookback window for the recurrence/duration aggregate, minutes. */
+	window_minutes: v.number(),
+	min_count: v.number(),
+	min_span_minutes: v.number(),
+	/** Per-kind recurrence/duration overrides — low-rate-but-persistent kinds. */
+	kind_thresholds: v.optional(
+		v.record(
+			v.string(),
+			v.object({
+				min_count: v.optional(v.number()),
+				min_span_minutes: v.optional(v.number()),
+			}),
+		),
+	),
+	resolve_after_minutes: v.number(),
+	update_after_minutes: v.number(),
+	/** The developer-session project escalations are parented to. */
+	parent_project_id: v.string(),
+	/** Per-kind copy for the message (impact + recommendation). Falls back to default_copy. */
+	copy: v.record(v.string(), v.object({ impact: v.string(), recommendation: v.string() })),
+	default_copy: v.object({ impact: v.string(), recommendation: v.string() }),
+});
+
+type FaultEscalationConfig = v.InferOutput<typeof FaultEscalationConfigSchema>;
+
+interface EventLogAggregateRow {
+	kind: string;
+	n: bigint;
+	first_seen: Date;
+	last_seen: Date;
+}
+
+interface OperatorRelayBot {
+	id: string;
+	organisation_id: string;
+	external_id: string | null;
+	flow_config: { operator_phone?: string } | null;
+}
+
+let cachedSystemIdentity: { userId: string | undefined; organisationId: string | undefined } | undefined;
+async function getSystemIdentity(): Promise<{ userId: string | undefined; organisationId: string | undefined }> {
+	if (cachedSystemIdentity) return cachedSystemIdentity;
+	const row = await loadRegistryConfig("config", "system-identity", v.looseObject({}));
+	cachedSystemIdentity = {
+		userId: row?.system_user_id as string | undefined,
+		organisationId: row?.system_org_id as string | undefined,
+	};
+	return cachedSystemIdentity;
+}
+
+async function loadConfig(): Promise<FaultEscalationConfig> {
+	return loadRegistryConfig("config", CONFIG_SLUG, FaultEscalationConfigSchema);
+}
+
+async function fetchAggregates(patterns: string[], windowMinutes: number): Promise<FaultAggregate[]> {
+	const prisma = getPrisma();
+	// Patterns come from the valibot-validated registry row and are regex-only
+	// by construction below; embedding them as one alternation keeps the query
+	// parameter-free on the pattern side while the window is a real parameter.
+	const regexClass = /^[A-Za-z0-9_\\^$.*+?()[\]{}|-]+$/;
+	for (const pattern of patterns) {
+		if (!regexClass.test(pattern)) {
+			throw new Error(`config/${CONFIG_SLUG}: watched_kind_pattern is not regex-only: ${pattern}`);
+		}
+	}
+	const combined = `(${patterns.join("|")})`;
+	const rows = await prisma.$queryRawUnsafe<EventLogAggregateRow[]>(
+		`SELECT kind, count(*)::bigint AS n, min(occurred_at) AS first_seen, max(occurred_at) AS last_seen
+		 FROM event_log
+		 WHERE occurred_at > now() - ($1::int * interval '1 minute')
+		   AND kind ~ $2::text
+		 GROUP BY kind`,
+		windowMinutes,
+		combined,
+	);
+	return rows.map((row) => ({
+		kind: row.kind,
+		count: Number(row.n),
+		firstSeen: new Date(row.first_seen),
+		lastSeen: new Date(row.last_seen),
+	}));
+}
+
+async function resolveOperatorRelay(): Promise<OperatorRelayBot> {
+	const prisma = getPrisma();
+	const bot = await prisma.messaging_bots.findFirst({
+		where: {
+			is_active: true,
+			platform: "whatsapp",
+			flow_config: { path: ["mode"], equals: "operator-relay" },
+		},
+		select: { id: true, organisation_id: true, external_id: true, flow_config: true },
+	});
+	if (!bot) throw new Error("platform-fault-escalation: no active operator-relay WhatsApp bot row");
+	const phone = (bot.flow_config as { operator_phone?: string } | null)?.operator_phone;
+	if (!phone) throw new Error("platform-fault-escalation: operator-relay bot has no operator_phone");
+	return bot as OperatorRelayBot;
+}
+
+/** GOWA device whose JID matches the PA number, so the escalation is sent BY the PA. */
+async function resolvePaDeviceId(paPhoneDigits: string): Promise<string | undefined> {
+	const { userId, organisationId } = await getSystemIdentity();
+	const systemCtx: ExecutionContext = { userId, organisationId } as ExecutionContext;
+	const { forwardToolToCentralMcp } = await import("@teamscala/chat-completion/central-tool-forwarder");
+	const res = (await forwardToolToCentralMcp(
+		"api_gowa_devices",
+		{ action: "listDevices" },
+		systemCtx,
+	)) as { results?: Array<{ id: string; jid?: string }> } | undefined;
+	const match = (res?.results ?? []).find((device) => (device.jid ?? "").includes(paPhoneDigits));
+	return match?.id;
+}
+
+async function deliverEscalation(
+	kind: string,
+	aggregate: FaultAggregate,
+	config: FaultEscalationConfig,
+): Promise<{ messageId: string; workItemId: string }> {
+	const log = getAppLogger();
+	const prisma = getPrisma();
+	const now = new Date();
+
+	// 1. Claim the episode BEFORE sending, so a send failure leaves a durable
+	// claim the next sweep retries instead of a lost escalation.
+	await prisma.platform_fault_escalations.upsert({
+		where: { kind },
+		create: {
+			kind,
+			first_seen_at: aggregate.firstSeen,
+			last_seen_at: aggregate.lastSeen,
+			event_count: aggregate.count,
+		},
+		update: {
+			first_seen_at: aggregate.firstSeen,
+			last_seen_at: aggregate.lastSeen,
+			event_count: aggregate.count,
+			resolved_at: null,
+		},
+	});
+
+	// 2. Resolve the PA path (bot → operator phone → PA device) and send.
+	const bot = await resolveOperatorRelay();
+	const operatorPhone = (bot.flow_config as { operator_phone?: string }).operator_phone!;
+	const copy = config.copy[kind] ?? config.default_copy;
+	const message = composeEscalationMessage({
+		kind,
+		aggregate,
+		impact: copy.impact,
+		recommendation: copy.recommendation,
+	});
+
+	const { userId, organisationId } = await getSystemIdentity();
+	const systemCtx: ExecutionContext = { userId, organisationId } as ExecutionContext;
+	const { forwardToolToCentralMcp } = await import("@teamscala/chat-completion/central-tool-forwarder");
+	const paDigits = (bot.external_id ?? "").replace(/\D/g, "");
+	const deviceId = paDigits ? await resolvePaDeviceId(paDigits) : undefined;
+	if (paDigits && !deviceId) {
+		log.warn("[platform-fault-escalation] no GOWA device matched the PA number — sending via default device");
+	}
+	const sendResult = await forwardToolToCentralMcp(
+		"api_gowa_message",
+		{
+			action: "sendMessage",
+			params: { phone: operatorPhone, message, ...(deviceId ? { device_id: deviceId } : {}) },
+		},
+		systemCtx,
+	);
+
+	// 3. VERIFY delivery before recording it: the send must return a WhatsApp
+	// message id. No id => throw, the claim stays undelivered, next sweep retries.
+	const messageId = extractGowaMessageId(sendResult);
+	if (!messageId) {
+		throw new Error(
+			`platform-fault-escalation: WhatsApp send for '${kind}' returned no message id — not recording delivery`,
+		);
+	}
+
+	// 4. Durable record: the work_items task the PA reads back / the operator answers on.
+	const workItem = await prisma.work_items.create({
+		data: {
+			kind: "task",
+			parent_id: config.parent_project_id,
+			organisation_id: bot.organisation_id,
+			title: `Fault: ${kind}`.slice(0, 50),
+			description: composeEscalationDescription({ aggregate, impact: copy.impact, recommendation: copy.recommendation }),
+			status: "pending",
+			kind_data: {
+				platform_fault: true,
+				event_kind: kind,
+				event_count: aggregate.count,
+				first_seen_at: aggregate.firstSeen.toISOString(),
+				last_seen_at: aggregate.lastSeen.toISOString(),
+				delivered_message_id: messageId,
+				escalated_at: now.toISOString(),
+			},
+		},
+		select: { id: true },
+	});
+
+	// 5. Readback-correct the episode: delivered + linked in one update.
+	await prisma.platform_fault_escalations.update({
+		where: { kind },
+		data: {
+			escalated_at: now,
+			work_item_id: workItem.id,
+			delivered_message_id: messageId,
+			last_updated_at: now,
+			updated_at: now,
+		},
+	});
+
+	recordEvent({
+		kind: "platform.fault.escalated",
+		payload: {
+			event_kind: kind,
+			count: aggregate.count,
+			work_item_id: workItem.id,
+			delivered_message_id: messageId,
+		},
+	});
+	return { messageId, workItemId: workItem.id };
+}
+
+async function updateEpisode(aggregate: FaultAggregate, episode: EscalationEpisode): Promise<void> {
+	const prisma = getPrisma();
+	const now = new Date();
+	await prisma.platform_fault_escalations.update({
+		where: { kind: episode.kind },
+		data: {
+			last_seen_at: aggregate.lastSeen,
+			event_count: aggregate.count,
+			last_updated_at: now,
+			updated_at: now,
+		},
+	});
+	if (episode.workItemId) {
+		await prisma.comments.create({
+			data: {
+				work_item_id: episode.workItemId,
+				content: `Still recurring: ${aggregate.count} events, last at ${aggregate.lastSeen.toISOString()}. No new escalation (already escalated and unresolved).`,
+				sender_role: "system",
+				sender_name: "platform-fault-escalation",
+			},
+		});
+	}
+	recordEvent({
+		kind: "platform.fault.updated",
+		payload: { event_kind: episode.kind, count: aggregate.count, work_item_id: episode.workItemId },
+	});
+}
+
+async function resolveEpisode(episode: EscalationEpisode, quietMinutes: number): Promise<void> {
+	const prisma = getPrisma();
+	const now = new Date();
+	await prisma.platform_fault_escalations.update({
+		where: { kind: episode.kind },
+		data: { resolved_at: now, updated_at: now },
+	});
+	if (episode.workItemId) {
+		await prisma.comments.create({
+			data: {
+				work_item_id: episode.workItemId,
+				content: `Fault stopped — no '${episode.kind}' events for ${Math.round(quietMinutes)} min. Episode closed; the fault will re-escalate if it recurs.`,
+				sender_role: "system",
+				sender_name: "platform-fault-escalation",
+			},
+		});
+	}
+	recordEvent({
+		kind: "platform.fault.resolved",
+		payload: { event_kind: episode.kind, quiet_minutes: Math.round(quietMinutes), work_item_id: episode.workItemId },
+	});
+}
+
+async function platformFaultEscalationSweep(ctx: { config: ScheduledJobConfig }): Promise<void> {
+	const log = getAppLogger();
+	const config = await loadConfig();
+	const now = new Date();
+
+	const aggregates = await fetchAggregates(config.watched_kind_patterns, config.window_minutes);
+	// Only OPEN episodes matter: a resolved episode and no episode at all have
+	// the same meaning to decideSweep (a recurring sustained fault opens a NEW
+	// episode and escalates again).
+	const openEpisodes = await getPrisma().platform_fault_escalations.findMany({
+		where: { resolved_at: null },
+	});
+	const episodes: EscalationEpisode[] = openEpisodes.map((row) => ({
+		kind: row.kind,
+		firstSeenAt: new Date(row.first_seen_at),
+		lastSeenAt: new Date(row.last_seen_at),
+		eventCount: row.event_count,
+		escalatedAt: row.escalated_at ? new Date(row.escalated_at) : null,
+		workItemId: row.work_item_id,
+		resolvedAt: row.resolved_at ? new Date(row.resolved_at) : null,
+		lastUpdatedAt: row.last_updated_at ? new Date(row.last_updated_at) : null,
+	}));
+
+	const kindThresholds: Record<string, KindThresholdOverride> = {};
+	for (const [kind, override] of Object.entries(config.kind_thresholds ?? {})) {
+		kindThresholds[kind] = {
+			minCount: override?.min_count,
+			minSpanMinutes: override?.min_span_minutes,
+		};
+	}
+	const decision = decideSweep(
+		aggregates,
+		episodes,
+		{
+			minCount: config.min_count,
+			minSpanMinutes: config.min_span_minutes,
+			resolveAfterMinutes: config.resolve_after_minutes,
+			updateAfterMinutes: config.update_after_minutes,
+		},
+		now,
+		kindThresholds,
+	);
+
+	// Per-kind isolation: one fault's delivery failure must never block the
+	// others. Failures are logged + emitted so the escalator itself is
+	// observable (the sweep that silently stops sweeping is the original bug).
+	for (const { aggregate } of decision.escalate) {
+		try {
+			const delivered = await deliverEscalation(aggregate.kind, aggregate, config);
+			log.warn(
+				`[platform-fault-escalation] ESCALATED '${aggregate.kind}' — WhatsApp message ${delivered.messageId}, work item ${delivered.workItemId}`,
+			);
+		} catch (err) {
+			const reason = err instanceof Error ? err.message : String(err);
+			log.error(`[platform-fault-escalation] delivery for '${aggregate.kind}' failed (claim retained, retried next sweep): ${reason}`);
+			recordEvent({ kind: "platform.fault.escalation_failed", payload: { event_kind: aggregate.kind, reason } });
+		}
+	}
+	for (const { aggregate, episode } of decision.update) {
+		try {
+			await updateEpisode(aggregate, episode);
+		} catch (err) {
+			log.error(`[platform-fault-escalation] update for '${episode.kind}' failed: ${err instanceof Error ? err.message : String(err)}`);
+		}
+	}
+	for (const { episode, quietMinutes } of decision.resolve) {
+		try {
+			await resolveEpisode(episode, quietMinutes);
+		} catch (err) {
+			log.error(`[platform-fault-escalation] resolve for '${episode.kind}' failed: ${err instanceof Error ? err.message : String(err)}`);
+		}
+	}
+
+	// Emitted every run, even when nothing fired — a sweep that only reports
+	// when it escalated is indistinguishable from one that stopped running.
+	recordEvent({
+		kind: "platform.fault.swept",
+		payload: {
+			watched_kinds: aggregates.length,
+			escalated: decision.escalate.length,
+			updated: decision.update.length,
+			resolved: decision.resolve.length,
+		},
+	});
+	if (decision.escalate.length > 0 || decision.update.length > 0 || decision.resolve.length > 0) {
+		log.info(
+			`[platform-fault-escalation] sweep — ${aggregates.length} kind(s) watched, ${decision.escalate.length} escalated, ${decision.update.length} updated, ${decision.resolve.length} resolved`,
+		);
+	}
+}
+
+pgBossHandlerRegistry.register(
+	"@teamscala/service-boot:platform-fault-escalation",
+	platformFaultEscalationSweep as (ctx: { config: ScheduledJobConfig }) => Promise<void>,
+);

package/src/pg-boss-mount.ts

@@ -35,6 +35,7 @@ import "./handlers/singleton-tools-currency-watch.ts";
 import "./handlers/registry-stale-inactive-sweep.ts";
 import "./handlers/pane-outbox-drain.ts";
 import "./handlers/lane-attention-sweep.ts";
+import "./handlers/platform-fault-escalation.ts";
 
 /**
  * Module-level reference to the mounted PgBoss instance — set by mountPgBoss on