@teamscala/service-boot 0.23.405 → 0.23.407

package/package.json

@@ -6,7 +6,7 @@
   },
   "name": "@teamscala/service-boot",
   "type": "module",
-  "version": "0.23.405",
+  "version": "0.23.407",
   "sideEffects": false,
   "imports": {
     "#service-boot/*": "./src/*",

package/src/handlers/platform-fault-escalation-logic.ts

@@ -79,12 +79,19 @@ function isSustained(
 	aggregate: FaultAggregate,
 	policy: FaultEscalationPolicy,
 	override: KindThresholdOverride | undefined,
+	now: Date,
 ): boolean {
 	const minCount = override?.minCount ?? policy.minCount;
 	const minSpan = override?.minSpanMinutes ?? policy.minSpanMinutes;
+	// Recency, not merely presence in the lookback window. Count and span
+	// answer "did this happen"; a fault qualifies only while it is STILL
+	// happening. Without this the 720-minute lookback keeps a dead fault
+	// qualifying for hours, and since a resolved episode reads as no episode
+	// it re-escalates on every sweep until its events age out (2026-08-26).
 	return (
 		aggregate.count >= minCount &&
-		minutesBetween(aggregate.firstSeen, aggregate.lastSeen) >= minSpan
+		minutesBetween(aggregate.firstSeen, aggregate.lastSeen) >= minSpan &&
+		minutesBetween(aggregate.lastSeen, now) < policy.resolveAfterMinutes
 	);
 }
 
@@ -107,7 +114,7 @@ export function decideSweep(
 
 	for (const aggregate of aggregates) {
 		const episode = byKind.get(aggregate.kind);
-		if (!isSustained(aggregate, policy, kindThresholds[aggregate.kind])) continue;
+		if (!isSustained(aggregate, policy, kindThresholds[aggregate.kind], now)) continue;
 		// No episode, or the previous episode closed (quiet past resolve window):
 		// a NEW episode begins and must be delivered.
 		if (!episode || episode.resolvedAt) {
@@ -169,39 +176,53 @@ export function extractGowaMessageId(sendResult: unknown): string | null {
 	return null;
 }
 
+/**
+ * One sampled payload value rendered for the message's evidence line: a
+ * kind+count escalation is a pointer to evidence, not evidence (2026-08-26:
+ * the llm.provider.call.failed payloads held our own bounded-output contract
+ * violations while the escalation's impact line named healthy providers).
+ * JSON-encoded when non-string so structure survives, whitespace-collapsed
+ * to one line, credential shapes redacted before truncation (a hard cut can
+ * never expose the tail of a secret the length limit happened to spare),
+ * hard-truncated with an ellipsis so the reader knows the cut happened.
+ */
+export function formatEscalationSample(value: unknown): string {
+	const maxChars = 200;
+	const redactions: ReadonlyArray<[RegExp, string]> = [
+		[/\b(?:sk|gsk|ghp|gho|xox[baprs])[-_][A-Za-z0-9_-]{8,}/g, "[redacted]"],
+		[/\b(api[_-]?key|token|password|secret|authorization)\b["'=:\s]+(?:(?:bearer|basic)\s+)?[A-Za-z0-9._~+/=-]{4,}/gi, "$1=[redacted]"],
+		[/Bearer\s+[A-Za-z0-9._~+/=-]{8,}/gi, "Bearer [redacted]"],
+	];
+	const raw = typeof value === "string" ? value : (JSON.stringify(value) ?? "");
+	if (raw.length === 0) return "";
+	const redacted = redactions.reduce((acc, [pattern, replacement]) => acc.replace(pattern, replacement), raw);
+	const collapsed = redacted.replace(/\s+/g, " ").trim();
+	return collapsed.length > maxChars ? `${collapsed.slice(0, maxChars)}…` : collapsed;
+}
+
 /**
  * The escalation message — escalation-routing's three-field contract adapted
- * to an automated detector: what was OBSERVED (evidence), what it MEANS
- * (impact), and the RECOMMENDATION. Short: the channel is a phone.
+ * to an automated detector: what was OBSERVED (evidence + sample), what it
+ * MEANS (impact), and the RECOMMENDATION. Short: the channel is a phone. The
+ * sample line is the triage evidence — `sample_fields` whitelisted per kind
+ * by the caller, formatted by formatEscalationSample; empty means the kind
+ * samples nothing or the latest payload carried none of its fields.
  */
 export function composeEscalationMessage(input: {
 	kind: string;
 	aggregate: FaultAggregate;
 	impact: string;
 	recommendation: string;
+	sample: string;
 }): string {
 	const span = formatSpanMinutes(
 		(input.aggregate.lastSeen.getTime() - input.aggregate.firstSeen.getTime()) / 60_000,
 	);
-	return [
+	const lines = [
 		`🔴 Platform fault: ${input.kind}`,
 		`Observed: ${input.aggregate.count} errors over ${span}, still recurring.`,
-		`Impact: ${input.impact}`,
-		`Recommendation: ${input.recommendation}`,
-	].join("\n");
-}
-
-/** The work-item description carries the same three fields plus the evidence. */
-export function composeEscalationDescription(input: {
-	aggregate: FaultAggregate;
-	impact: string;
-	recommendation: string;
-}): string {
-	return [
-		"Automated platform-fault escalation (platform-fault-escalation sweep).",
-		`Observed: ${input.aggregate.count} '${input.aggregate.kind}' events in event_log from ${input.aggregate.firstSeen.toISOString()} to ${input.aggregate.lastSeen.toISOString()}, still recurring.`,
-		`Impact: ${input.impact}`,
-		`Recommendation: ${input.recommendation}`,
-		"Delivery: WhatsApp via the scala PA. This task closes automatically when the fault stops for the resolve window; reply via the PA to rule otherwise.",
-	].join("\n");
+	];
+	if (input.sample) lines.push(`Sample: ${input.sample}`);
+	lines.push(`Impact: ${input.impact}`, `Recommendation: ${input.recommendation}`);
+	return lines.join("\n");
 }

package/src/handlers/platform-fault-escalation.test.ts

@@ -13,6 +13,7 @@ import {
 	composeEscalationMessage,
 	decideSweep,
 	extractGowaMessageId,
+	formatEscalationSample,
 	type EscalationEpisode,
 	type FaultAggregate,
 	type FaultEscalationPolicy,
@@ -194,6 +195,7 @@ describe("composeEscalationMessage", () => {
 			aggregate: agg("host-command-bus.command.failed", 490, 720, 0),
 			impact: "scala-tools verbs are failing host-wide",
 			recommendation: "check oxmgr logs for the bus consumer",
+			sample: "",
 		});
 		expect(message).toContain("Platform fault: host-command-bus.command.failed");
 		expect(message).toContain("Observed: 490 errors over 12 h");
@@ -201,4 +203,99 @@ describe("composeEscalationMessage", () => {
 		expect(message).toContain("Recommendation: check oxmgr logs for the bus consumer");
 		expect(message.split("\n")).toHaveLength(4);
 	});
+
+	// 2026-08-26: three escalations reached the orchestrator seat carrying
+	// kind, count and duration only — triaging each required a hand event_log
+	// query, and the llm.provider.call.failed impact line pointed AWAY from
+	// the defect (payloads held our own 'Invalid length: Expected <=1200 but
+	// received 1255' contract violations recorded against healthy providers).
+	// The sample line carries the evidence in the first read.
+	test("carries the sampled payload evidence as its own line", () => {
+		const message = composeEscalationMessage({
+			kind: "llm.provider.call.failed",
+			aggregate: agg("llm.provider.call.failed", 26, 560, 0),
+			sample: 'status=429; message=Usage limit reached for 5 hour. Your limit will reset at 2026-08-26 01:19:03',
+			impact: "AI chat/completion calls are failing",
+			recommendation: "check provider quota",
+		});
+		const lines = message.split("\n");
+		expect(lines).toHaveLength(5);
+		expect(lines[2]).toContain("Sample: status=429");
+		expect(lines[2]).toContain("reset at 2026-08-26 01:19:03");
+	});
+});
+
+describe("formatEscalationSample — evidence truncated hard, secrets never shipped", () => {
+	test("non-string values are JSON-encoded so structure survives the phone channel", () => {
+		expect(formatEscalationSample(429)).toBe("429");
+		expect(formatEscalationSample(true)).toBe("true");
+		expect(formatEscalationSample({ expected: 1200, received: 1255 })).toBe('{"expected":1200,"received":1255}');
+	});
+
+	test("truncates hard at the bound with an ellipsis so the reader knows the cut happened", () => {
+		const formatted = formatEscalationSample("x".repeat(500));
+		expect(formatted.length).toBe(201); // 200 chars + ellipsis
+		expect(formatted.endsWith("…")).toBe(true);
+	});
+
+	test("redacts credential shapes before truncation — a cut can never expose a secret tail", () => {
+		expect(formatEscalationSample("key sk-abcdefghijklmnopqrstuvwxyz failed")).toBe("key [redacted] failed");
+		expect(formatEscalationSample("Authorization: Bearer abcdefghijklmnop")).toBe(
+			"Authorization=[redacted]",
+		);
+		expect(formatEscalationSample("api_key=abcdefgh1234 rejected")).toBe("api_key=[redacted] rejected");
+	});
+
+	test("collapses whitespace to one line — the sample is one message line", () => {
+		expect(formatEscalationSample("line one\nline two\t tabbed")).toBe("line one line two tabbed");
+	});
+
+	test("empty input yields an empty sample (no Sample line)", () => {
+		expect(formatEscalationSample(undefined)).toBe("");
+		expect(formatEscalationSample("")).toBe("");
+	});
+});
+
+describe("decideSweep — qualification asks 'is this happening', not 'did this happen'", () => {
+	// 2026-08-26: llm.provider.call.failed re-escalated every ~10 minutes with
+	// lastSeen frozen at 16:54 — escalated 19:40 on a fault three hours dead,
+	// resolved 19:45, repeat. The tell was the closure counter climbing
+	// (141/151/171 min): quiet measured from one frozen aggregate lastSeen.
+	// isSustained checked count and span but never recency, so a dead fault
+	// kept qualifying for the rest of the 720-minute lookback while a resolved
+	// episode read as "no episode" and re-escalated it.
+	const resolved = (kind: string) =>
+		episode({ kind, resolvedAt: new Date(NOW.getTime() - 5 * 60_000) });
+
+	test("a fault quiet past the resolve window does not re-escalate after resolution", () => {
+		const decision = decideSweep(
+			[agg("stale.error", 30, 400, 170)],
+			[resolved("stale.error")],
+			POLICY,
+			NOW,
+		);
+		expect(decision.escalate).toHaveLength(0);
+	});
+
+	test("a fault quiet past the resolve window does not escalate with no episode at all", () => {
+		const decision = decideSweep([agg("stale.error", 30, 400, 170)], [], POLICY, NOW);
+		expect(decision.escalate).toHaveLength(0);
+	});
+
+	// The other direction, and the one that protects live escalations: the fix
+	// must not silence a fault that is still emitting.
+	test("a fault still emitting escalates", () => {
+		const decision = decideSweep([agg("live.error", 30, 400, 2)], [], POLICY, NOW);
+		expect(decision.escalate).toHaveLength(1);
+	});
+
+	test("a recurring fault opens a NEW episode after its previous one resolved", () => {
+		const decision = decideSweep(
+			[agg("live.error", 30, 400, 2)],
+			[resolved("live.error")],
+			POLICY,
+			NOW,
+		);
+		expect(decision.escalate).toHaveLength(1);
+	});
 });

package/src/handlers/platform-fault-escalation.ts

@@ -18,8 +18,9 @@
  * THIS IS NOT A DRIFT-DETECTOR PATROLLING INSTANCES. It reacts only to
  * components reporting THEIR OWN failure — event_log kinds that already
  * assert a fault about their emitter. Thresholds (duration + recurrence,
- * never a single event), the watched-kind patterns, the resolve window and
- * the per-kind impact/recommendation copy are all in the
+ * never a single event), the watched-kind patterns, the resolve window, the
+ * per-kind impact/recommendation copy and the per-kind payload sample-field
+ * whitelists are all in the
  * `config/platform-fault-escalation` registry row — changing policy is a row
  * edit, not a redeploy. Episode dedup lives in the
  * `platform_fault_escalations` table (migration 501): one escalation per
@@ -52,10 +53,10 @@ import { recordEvent } from "@teamscala/event-log/record-event";
 import { getAppLogger } from "@teamscala/logger/app-loggers";
 import { pgBossHandlerRegistry } from "../pg-boss-handler-registry.ts";
 import {
-	composeEscalationDescription,
 	composeEscalationMessage,
 	decideSweep,
 	extractGowaMessageId,
+	formatEscalationSample,
 	type EscalationEpisode,
 	type FaultAggregate,
 	type KindThresholdOverride,
@@ -90,6 +91,10 @@ const FaultEscalationConfigSchema = v.object({
 	/** Per-kind copy for the message (impact + recommendation). Falls back to default_copy. */
 	copy: v.record(v.string(), v.object({ impact: v.string(), recommendation: v.string() })),
 	default_copy: v.object({ impact: v.string(), recommendation: v.string() }),
+	/** Per-kind payload-field whitelist sampled into the message's evidence
+	 * line — named fields ONLY, never the whole payload: some kinds embed
+	 * request context, and the escalation channel is WhatsApp. */
+	sample_fields: v.optional(v.record(v.string(), v.array(v.string()))),
 });
 
 type FaultEscalationConfig = v.InferOutput<typeof FaultEscalationConfigSchema>;
@@ -128,8 +133,9 @@ async function getSystemIdentity(): Promise<{ userId: string | undefined; organi
  * BACKEND service, where the page-server client is never registered — the
  * empty-router fallback returns a client with no model namespaces (the
  * TypeError this replaced). Per `no-direct-prisma-outside-orpc`, every WRITE
- * in this handler routes through this client; the single raw read (the
- * event_log recurrence aggregate, a GROUP BY no generated CRUD expresses) uses
+ * in this handler routes through this client; the two raw reads (the
+ * event_log recurrence aggregate and the latest-payload sample, shapes no
+ * generated CRUD expresses) use
  * the assigned-client form with $queryRawUnsafe. Timestamps are ISO strings,
  * never Date instances (the generated valibot schemas require it).
  */
@@ -184,9 +190,6 @@ interface OrpcSurface {
 	messaging_bots: {
 		findFirst(args: Record<string, unknown>): Promise<Record<string, unknown> | null>;
 	};
-	work_items: {
-		create(args: { data: Record<string, unknown>; select?: Record<string, unknown> }): Promise<{ id: string }>;
-	};
 	comments: {
 		create(args: { data: Record<string, unknown> }): Promise<{ id: string }>;
 	};
@@ -225,6 +228,42 @@ async function fetchAggregates(patterns: string[], windowMinutes: number): Promi
 	}));
 }
 
+/**
+ * The message's evidence line: the whitelisted sample fields from the most
+ * recent payload for the kind. Field names come from the valibot-validated
+ * config row and are constrained to bare identifiers below, so the JSON path
+ * cannot break out of the string; only the named keys are selected, so an
+ * unlisted field is unreachable by construction rather than filtered later.
+ */
+async function fetchLatestPayloadSample(kind: string, fields: string[]): Promise<string> {
+	if (fields.length === 0) return "";
+	for (const field of fields) {
+		if (!/^[A-Za-z0-9_]+$/.test(field)) {
+			throw new Error(`config/${CONFIG_SLUG}: sample_fields entry is not a bare field name: ${field}`);
+		}
+	}
+	const selection = fields.map((field) => `payload_json->'${field}' AS "${field}"`).join(", ");
+	const prisma = getPrisma();
+	const rows = await prisma.$queryRawUnsafe<Array<Record<string, unknown>>>(
+		`SELECT ${selection}
+			 FROM event_log
+			 WHERE kind = $1::text
+			 ORDER BY occurred_at DESC
+			 LIMIT 1`,
+		kind,
+	);
+	const row = rows[0];
+	if (!row) return "";
+	const parts: string[] = [];
+	for (const field of fields) {
+		const value = row[field];
+		if (value === null || value === undefined) continue;
+		const formatted = formatEscalationSample(value);
+		if (formatted) parts.push(`${field}=${formatted}`);
+	}
+	return parts.join("; ");
+}
+
 async function resolveOperatorRelay(): Promise<OperatorRelayBot> {
 	const bot = (await (await orpc()).messaging_bots.findFirst({
 		where: {
@@ -284,9 +323,11 @@ async function deliverEscalation(
 	const bot = await resolveOperatorRelay();
 	const operatorPhone = (bot.flow_config as { operator_phone?: string }).operator_phone!;
 	const copy = config.copy[kind] ?? config.default_copy;
+	const sample = await fetchLatestPayloadSample(kind, config.sample_fields?.[kind] ?? []);
 	const message = composeEscalationMessage({
 		kind,
 		aggregate,
+		sample,
 		impact: copy.impact,
 		recommendation: copy.recommendation,
 	});
@@ -325,12 +366,7 @@ async function deliverEscalation(
 	const comment = await (await orpc()).comments.create({
 		data: {
 			work_item_id: config.orchestrator_work_item_id,
-			content: composeEscalationMessage({
-				kind,
-				aggregate,
-				impact: copy.impact,
-				recommendation: copy.recommendation,
-			}),
+			content: message,
 			source_cli_session_id: null,
 		},
 	});
@@ -353,6 +389,7 @@ async function deliverEscalation(
 		payload: {
 			event_kind: kind,
 			count: aggregate.count,
+			sample,
 			work_item_id: config.orchestrator_work_item_id,
 			delivered_comment_id: comment.id,
 			delivered_message_id: messageId,