@teamscala/service-boot 0.23.430 → 0.23.431

package/package.json

@@ -6,7 +6,7 @@
   },
   "name": "@teamscala/service-boot",
   "type": "module",
-  "version": "0.23.430",
+  "version": "0.23.431",
   "sideEffects": false,
   "imports": {
     "#service-boot/*": "./src/*",

package/src/handlers/platform-fault-escalation/delivery.ts

@@ -72,6 +72,12 @@ export async function deliverEscalation(
 			last_seen_at: aggregate.lastSeen.toISOString(),
 			event_count: aggregate.count,
 			resolved_at: null,
+			// CLEARING THIS IS WHAT STOPS AN ESCALATION STORM. A re-armed row
+			// keeps resolved_at NULL, so it still loads as open, and
+			// `escalate` fires for any episode that is resolved OR re-armed.
+			// Leaving rearmed_at set would therefore re-escalate the same kind
+			// on EVERY sweep — the firing state must be consumed by the firing.
+			rearmed_at: null,
 		},
 	});
 
@@ -165,6 +171,36 @@ export async function deliverEscalation(
 	return { messageId, workItemId: config.orchestrator_work_item_id };
 }
 
+/**
+ * Re-arm an episode whose kind went quiet but whose condition NOBODY CHECKED.
+ * The alarm becomes audible again and no claim is made that the fault
+ * stopped, so there is deliberately NO operator comment: "Fault stopped" on
+ * the strength of silence is the sentence this change exists to stop writing.
+ * The event is still recorded — re-arming is a state change. See migration 811.
+ */
+export async function rearmEpisode(
+	episode: EscalationEpisode,
+	quietMinutes: number,
+): Promise<void> {
+	const now = new Date();
+	await (await orpc()).platform_fault_escalations.update({
+		where: { kind: episode.kind },
+		data: { rearmed_at: now.toISOString(), updated_at: now.toISOString() },
+	});
+	recordEvent({
+		kind: "platform.fault.rearmed",
+		payload: {
+			event_kind: episode.kind,
+			quiet_minutes: Math.round(quietMinutes),
+			work_item_id: episode.workItemId,
+			// Named so a reader never mistakes this for a resolution: no probe
+			// exists for this kind, so nothing verified the condition ended.
+			verified: false,
+			reason: "no capability probe declared for this kind",
+		},
+	});
+}
+
 export async function resolveEpisode(
 	episode: EscalationEpisode,
 	quietMinutes: number,

package/src/handlers/platform-fault-escalation-logic.ts

@@ -26,7 +26,13 @@ export interface EscalationEpisode {
 	eventCount: number;
 	escalatedAt: Date | null;
 	workItemId: string | null;
+	/** VERIFIED over — a capability probe answered OK, or a declared recovery
+	 * transition arrived. Never written from silence. */
 	resolvedAt: Date | null;
+	/** The kind went quiet and the alarm is ARMED AGAIN. This is all that
+	 * quiet ever honestly supported, and it is deliberately weaker than
+	 * `resolvedAt`: it claims nothing about the condition. */
+	rearmedAt: Date | null;
 	lastUpdatedAt: Date | null;
 	deliveredCommentId?: string | null;
 }
@@ -51,8 +57,18 @@ export interface KindThresholdOverride {
 export interface SweepDecision {
 	/** Sustained faults needing a FIRST delivery (new episode, or a claim whose send failed). */
 	escalate: Array<{ aggregate: FaultAggregate; episode: EscalationEpisode | null }>;
-	/** Open episodes whose kind went quiet past the resolve window — close them. */
+	/** Open episodes whose kind went quiet past the resolve window — close them.
+	 * ONLY reachable from the probe gate: quiet alone never populates this. */
 	resolve: Array<{ episode: EscalationEpisode; quietMinutes: number; clearZeroVolumeFlag: boolean }>;
+	/** Open episodes quiet past the resolve window whose kind declares NO
+	 * capability probe: the alarm is RE-ARMED (the next occurrence escalates
+	 * again) and nothing is claimed about the condition. Quiet used to write
+	 * `resolved_at` and post "Fault stopped", which is the unsound close
+	 * `emit-transitions-not-conditions` names. It is re-armed rather than
+	 * merely held open because `escalate` fires only for an absent, resolved
+	 * or re-armed episode — holding it open would silence the alarm for good.
+	 * Rationale + measurements: migration 811. */
+	rearm: Array<{ episode: EscalationEpisode; quietMinutes: number; clearZeroVolumeFlag: boolean }>;
 	/** Open episodes quiet past the resolve window whose producer ALSO went
 	 * silent: zero failure rows AND zero call volume that would produce them.
 	 * Quiet is NOT evidence of health here (measured 2026-08-27: llm-router
@@ -150,14 +166,17 @@ export function decideSweep(
 	const byKind = new Map(episodes.map((e) => [e.kind, e]));
 	const aggregateByKind = new Map(aggregates.map((a) => [a.kind, a]));
 	const decision: SweepDecision =
-		{ escalate: [], resolve: [], resolveProbeGate: [], zeroVolumeHold: [] };
+		{ escalate: [], resolve: [], rearm: [], resolveProbeGate: [], zeroVolumeHold: [] };
 
 	for (const aggregate of aggregates) {
 		const episode = byKind.get(aggregate.kind);
 		if (!isSustained(aggregate, policy, kindThresholds[aggregate.kind], now)) continue;
 		// No episode, or the previous episode closed (quiet past resolve window):
 		// a NEW episode begins and must be delivered.
-		if (!episode || episode.resolvedAt) {
+		// A RE-ARMED episode starts a new one exactly as a resolved one does.
+		// This is the half that keeps the alarm audible: without it, dropping
+		// the quiet-close would leave every probe-less kind permanently silent.
+		if (!episode || episode.resolvedAt || episode.rearmedAt) {
 			decision.escalate.push({ aggregate, episode: episode ?? null });
 			continue;
 		}
@@ -174,7 +193,7 @@ export function decideSweep(
 	}
 
 	for (const episode of episodes) {
-		if (episode.resolvedAt) continue;
+		if (episode.resolvedAt || episode.rearmedAt) continue;
 		const aggregate = aggregateByKind.get(episode.kind);
 		// A future-dated aggregate lastSeen (producer clock skew) cannot serve
 		// as the quiet reference — it would freeze quietMinutes negative and
@@ -218,7 +237,9 @@ export function decideSweep(
 			});
 			continue;
 		}
-		decision.resolve.push({
+		// NO PROBE DECLARED: re-arm, never resolve. Quiet re-arms the alarm and
+		// says nothing about the condition — see the `rearm` field's comment.
+		decision.rearm.push({
 			episode,
 			quietMinutes,
 			clearZeroVolumeFlag: Boolean(source && zeroVolume?.flagActiveByKind[episode.kind]),

package/src/handlers/platform-fault-escalation.test.ts

@@ -44,6 +44,7 @@ function episode(overrides: Partial<EscalationEpisode> & { kind: string }): Esca
 		escalatedAt: new Date(NOW.getTime() - 60 * 60_000),
 		workItemId: "wi-1",
 		resolvedAt: null,
+		rearmedAt: null,
 		lastUpdatedAt: null,
 		...overrides,
 	};
@@ -117,6 +118,7 @@ describe("decideSweep — dedup (one escalation per fault, no re-fire)", () => {
 		expect(decision.resolve).toHaveLength(0);
 		expect(Object.keys(decision).sort()).toEqual([
 			"escalate",
+			"rearm",
 			"resolve",
 			"resolveProbeGate",
 			"zeroVolumeHold",
@@ -133,17 +135,60 @@ describe("decideSweep — dedup (one escalation per fault, no re-fire)", () => {
 });
 
 describe("decideSweep — resolution + re-arm", () => {
-	test("quiet past the resolve window closes the episode", () => {
+	// QUIET RE-ARMS, IT DOES NOT RESOLVE. These two asserted `resolve` until
+	// 2026-08-29, which is the unsound close: only a probe can say a condition
+	// ended, and no probe is declared for these kinds.
+	test("quiet past the resolve window RE-ARMS the episode, and claims nothing", () => {
 		const gone = episode({ kind: "wake.failed", lastSeenAt: new Date(NOW.getTime() - 60 * 60_000) });
 		const decision = decideSweep([], [gone], POLICY, NOW);
-		expect(decision.resolve).toHaveLength(1);
-		expect(decision.resolve[0]!.quietMinutes).toBeGreaterThanOrEqual(30);
+		expect(decision.rearm).toHaveLength(1);
+		expect(decision.rearm[0]!.quietMinutes).toBeGreaterThanOrEqual(30);
+		expect(decision.resolve).toHaveLength(0);
 	});
 
-	test("an absent kind resolves by its episode's last_seen, not the sweep clock", () => {
+	test("an absent kind re-arms by its episode's last_seen, not the sweep clock", () => {
 		const gone = episode({ kind: "listen.connect_failed", lastSeenAt: new Date(NOW.getTime() - 45 * 60_000) });
 		const decision = decideSweep([], [gone], POLICY, NOW);
-		expect(decision.resolve).toHaveLength(1);
+		expect(decision.rearm).toHaveLength(1);
+		expect(decision.resolve).toHaveLength(0);
+	});
+
+	// THE INVARIANT THAT MAKES RE-ARMING SAFE. If a re-armed episode did not
+	// escalate again, dropping the quiet-close would have left every
+	// probe-less alarm permanently silent — worse than the false close.
+	test("a RE-ARMED episode escalates again when its fault recurs", () => {
+		const recurring = agg("guard.fail.open", 50, 40, 0);
+		const rearmed = episode({
+			kind: "guard.fail.open",
+			rearmedAt: new Date(NOW.getTime() - 35 * 60_000),
+		});
+		const decision = decideSweep([recurring], [rearmed], POLICY, NOW);
+		expect(decision.escalate).toHaveLength(1);
+	});
+
+	// A re-armed episode is already armed; re-arming it every sweep would emit
+	// a transition per sweep, which is the repeating-condition shape itself.
+	test("an already re-armed episode is not re-armed again while it stays quiet", () => {
+		const rearmed = episode({
+			kind: "guard.fail.open",
+			lastSeenAt: new Date(NOW.getTime() - 60 * 60_000),
+			rearmedAt: new Date(NOW.getTime() - 35 * 60_000),
+		});
+		const decision = decideSweep([], [rearmed], POLICY, NOW);
+		expect(decision.rearm).toHaveLength(0);
+		expect(decision.resolve).toHaveLength(0);
+	});
+
+	// A kind WITH a probe keeps the stronger path: it is never re-armed on
+	// quiet, because something can actually be asked about it.
+	test("a kind declaring a capability probe goes to the probe gate, never to re-arm", () => {
+		const gone = episode({ kind: "llm.provider.call.failed", lastSeenAt: new Date(NOW.getTime() - 60 * 60_000) });
+		const decision = decideSweep([], [gone], POLICY, NOW, {}, undefined, {
+			"llm.provider.call.failed": { url: "http://127.0.0.1:9/health" },
+		});
+		expect(decision.resolveProbeGate).toHaveLength(1);
+		expect(decision.rearm).toHaveLength(0);
+		expect(decision.resolve).toHaveLength(0);
 	});
 
 	test("after resolution the same fault escalates again (new episode)", () => {
@@ -161,6 +206,7 @@ describe("decideSweep — resolution + re-arm", () => {
 		const open = episode({ kind: "slow.error", lastSeenAt: active.lastSeen });
 		const decision = decideSweep([active], [open], POLICY, NOW);
 		expect(decision.resolve).toHaveLength(0);
+		expect(decision.rearm).toHaveLength(0);
 	});
 });
 
@@ -188,7 +234,9 @@ describe("decideSweep — a future-dated lastSeen is not recurrence evidence", (
 		const phantom = agg("guard.fail.open", 40, 720, -44);
 		const decision = decideSweep([phantom], [quietEpisode], POLICY, NOW);
 		expect(decision.escalate).toHaveLength(0);
-		expect(decision.resolve).toHaveLength(1);
+		// Re-armed rather than resolved: no probe is declared for this kind.
+		expect(decision.rearm).toHaveLength(1);
+		expect(decision.resolve).toHaveLength(0);
 	});
 });
 
@@ -354,19 +402,21 @@ describe("decideSweep — zero-volume gate (quiet vs dead producer)", () => {
 		expect(decision.zeroVolumeHold[0].volumeCount).toBe(0);
 	});
 
-	test("quiet WITH healthy call volume still resolves (the normal close)", () => {
+	test("quiet WITH healthy call volume re-arms (the normal path for a probe-less kind)", () => {
 		const episodes = [episode({ kind: "llm.provider.call.failed", lastSeenAt: new Date(NOW.getTime() - 45 * 60_000) })];
 		const inputs = { ...VOLUME_BACKED, countByKind: { "llm.provider.call.failed": 120 } };
 		const decision = decideSweep([], episodes, POLICY, NOW, {}, inputs);
 		expect(decision.zeroVolumeHold).toHaveLength(0);
-		expect(decision.resolve).toHaveLength(1);
-		expect(decision.resolve[0].clearZeroVolumeFlag).toBe(false);
+		expect(decision.rearm).toHaveLength(1);
+		expect(decision.resolve).toHaveLength(0);
+		expect(decision.rearm[0]!.clearZeroVolumeFlag).toBe(false);
 	});
 
-	test("a sourceless kind resolves on quiet exactly as before", () => {
+	test("a sourceless kind re-arms on quiet", () => {
 		const episodes = [episode({ kind: "guard.fail.open", lastSeenAt: new Date(NOW.getTime() - 45 * 60_000) })];
 		const decision = decideSweep([], episodes, POLICY, NOW, {}, { ...VOLUME_BACKED, countByKind: {} });
-		expect(decision.resolve).toHaveLength(1);
+		expect(decision.rearm).toHaveLength(1);
+		expect(decision.resolve).toHaveLength(0);
 		expect(decision.zeroVolumeHold).toHaveLength(0);
 	});
 
@@ -378,13 +428,14 @@ describe("decideSweep — zero-volume gate (quiet vs dead producer)", () => {
 			flagActiveByKind: { "llm.provider.call.failed": true },
 		};
 		const decision = decideSweep([], episodes, POLICY, NOW, {}, inputs);
-		expect(decision.resolve[0].clearZeroVolumeFlag).toBe(true);
+		expect(decision.rearm[0]!.clearZeroVolumeFlag).toBe(true);
 	});
 
 	test("without a sixth argument the legacy shape is untouched", () => {
 		const episodes = [episode({ kind: "llm.provider.call.failed", lastSeenAt: new Date(NOW.getTime() - 45 * 60_000) })];
 		const decision = decideSweep([], episodes, POLICY, NOW);
-		expect(decision.resolve).toHaveLength(1);
+		expect(decision.rearm).toHaveLength(1);
+		expect(decision.resolve).toHaveLength(0);
 	});
 });
 
@@ -422,10 +473,11 @@ describe("decideSweep — probe-gated resolution (operator ruling 2026-08-28)",
 		expect(decision.resolveProbeGate[0]!.quietMinutes).toBeGreaterThanOrEqual(30);
 	});
 
-	test("a kind without a declared probe resolves on quiet exactly as before", () => {
+	test("a kind without a declared probe RE-ARMS on quiet — it is never closed unverified", () => {
 		const gone = episode({ kind: "wake.failed", lastSeenAt: new Date(NOW.getTime() - 60 * 60_000) });
 		const decision = decideSweep([agg("wake.failed", 10, 700, 60)], [gone], POLICY, NOW, {}, undefined, PROBE_SOURCES);
-		expect(decision.resolve).toHaveLength(1);
+		expect(decision.rearm).toHaveLength(1);
+		expect(decision.resolve).toHaveLength(0);
 		expect(decision.resolveProbeGate).toHaveLength(0);
 	});
 

package/src/handlers/platform-fault-escalation.ts

@@ -39,6 +39,7 @@ import {
 	deliverEscalation,
 	deliverZeroVolumeHold,
 	refreshOpenEpisode,
+	rearmEpisode,
 	resolveEpisode,
 } from "./platform-fault-escalation/delivery.ts";
 
@@ -61,6 +62,7 @@ async function platformFaultEscalationSweep(ctx: { config: ScheduledJobConfig })
 		escalated_at: string | Date | null;
 		work_item_id: string | null;
 		resolved_at: string | Date | null;
+		rearmed_at: string | Date | null;
 		last_updated_at: string | Date | null;
 	}>;
 	const episodes: EscalationEpisode[] = openEpisodes.map((row) => ({
@@ -71,6 +73,7 @@ async function platformFaultEscalationSweep(ctx: { config: ScheduledJobConfig })
 		escalatedAt: row.escalated_at ? new Date(row.escalated_at) : null,
 		workItemId: row.work_item_id,
 		resolvedAt: row.resolved_at ? new Date(row.resolved_at) : null,
+		rearmedAt: row.rearmed_at ? new Date(row.rearmed_at) : null,
 		lastUpdatedAt: row.last_updated_at ? new Date(row.last_updated_at) : null,
 	}));
 
@@ -204,6 +207,19 @@ async function platformFaultEscalationSweep(ctx: { config: ScheduledJobConfig })
 			log.error(`[platform-fault-escalation] resolve for '${episode.kind}' failed: ${err instanceof Error ? err.message : String(err)}`);
 		}
 	}
+	// RE-ARM, NOT RESOLVE (2026-08-29). A kind with no declared capability
+	// probe went quiet: the alarm is armed for the next occurrence and NOTHING
+	// is claimed about the condition. No operator comment is written here on
+	// purpose — see rearmEpisode.
+	for (const { episode, quietMinutes, clearZeroVolumeFlag } of decision.rearm) {
+		try {
+			await rearmEpisode(episode, quietMinutes);
+			if (clearZeroVolumeFlag) await clearZeroVolumeHold(episode, zeroVolume.countByKind[episode.kind] ?? 0);
+			log.info(`[platform-fault-escalation] RE-ARMED '${episode.kind}' after ${Math.round(quietMinutes)} quiet min — no probe declared, so the condition is UNVERIFIED and no close was claimed`);
+		} catch (err) {
+			log.error(`[platform-fault-escalation] re-arm for '${episode.kind}' failed (retried next sweep): ${err instanceof Error ? err.message : String(err)}`);
+		}
+	}
 
 	// Emitted every run, even when nothing fired — a sweep that only reports
 	// when it escalated is indistinguishable from one that stopped running.