@teamscala/service-boot 0.23.384 → 0.23.386

package/package.json

@@ -6,7 +6,7 @@
   },
   "name": "@teamscala/service-boot",
   "type": "module",
-  "version": "0.23.384",
+  "version": "0.23.386",
   "sideEffects": false,
   "imports": {
     "#service-boot/*": "./src/*",

package/src/handlers/messaging-send-recording-logic.test.ts

@@ -0,0 +1,86 @@
+/**
+ * @system server-boot
+ * @status handwritten
+ * @edit edit directly
+ *
+ * Unit tests for the pure detection logic of the messaging send-recording
+ * watch: healthy shape (zero rows) yields null; the failure shape yields the
+ * aggregate with ordered bounds; the sample keeps the MOST RECENT rows.
+ */
+import { describe, expect, test } from "bun:test";
+import {
+	buildUnrecordedPayload,
+	detectUnrecordedSends,
+	type UnrecordedSendRow,
+} from "./messaging-send-recording-logic.ts";
+
+const T = (iso: string) => new Date(iso);
+
+function row(id: string, iso: string, sendKind: string | null = null): UnrecordedSendRow {
+	return { id, created_at: T(iso), external_chat_id: "447@s.whatsapp.net", send_kind: sendKind };
+}
+
+describe("detectUnrecordedSends", () => {
+	test("zero rows is healthy — no finding, no event", () => {
+		expect(detectUnrecordedSends([], 5)).toBeNull();
+	});
+
+	test("one row yields the finding with matching bounds", () => {
+		const finding = detectUnrecordedSends([row("a", "2026-08-20T05:54:37.715Z")], 5);
+		expect(finding).not.toBeNull();
+		expect(finding!.unrecordedCount).toBe(1);
+		expect(finding!.firstAt).toEqual(T("2026-08-20T05:54:37.715Z"));
+		expect(finding!.lastAt).toEqual(T("2026-08-20T05:54:37.715Z"));
+	});
+
+	test("unordered rows are ordered into first/last bounds", () => {
+		// The live pre-fix rows, newest first as a LIMIT-descending read might return.
+		const finding = detectUnrecordedSends(
+			[
+				row("c", "2026-08-20T08:37:15.667Z"),
+				row("a", "2026-08-20T05:54:37.715Z"),
+				row("b", "2026-08-20T08:36:49.149Z"),
+			],
+			5,
+		);
+		expect(finding!.unrecordedCount).toBe(3);
+		expect(finding!.firstAt).toEqual(T("2026-08-20T05:54:37.715Z"));
+		expect(finding!.lastAt).toEqual(T("2026-08-20T08:37:15.667Z"));
+	});
+
+	test("the sample keeps the MOST RECENT rows up to the limit", () => {
+		const rows = [
+			row("a", "2026-08-20T05:54:37.715Z"),
+			row("b", "2026-08-20T08:36:49.149Z"),
+			row("c", "2026-08-20T08:37:15.667Z"),
+		];
+		const finding = detectUnrecordedSends(rows, 2);
+		expect(finding!.sample.map((r) => r.id)).toEqual(["b", "c"]);
+		// The count still reports every row, not just the sample.
+		expect(finding!.unrecordedCount).toBe(3);
+	});
+});
+
+describe("buildUnrecordedPayload", () => {
+	test("carries count, baseline, bounds, and ISO-string sample rows", () => {
+		const finding = detectUnrecordedSends(
+			[row("a", "2026-08-20T08:36:49.149Z", "relay-notify")],
+			5,
+		)!;
+		const payload = buildUnrecordedPayload(finding, "2026-08-20T10:19:00Z");
+		expect(payload).toEqual({
+			unrecordedCount: 1,
+			recordedSince: "2026-08-20T10:19:00Z",
+			firstAt: "2026-08-20T08:36:49.149Z",
+			lastAt: "2026-08-20T08:36:49.149Z",
+			sample: [
+				{
+					id: "a",
+					createdAt: "2026-08-20T08:36:49.149Z",
+					chatId: "447@s.whatsapp.net",
+					sendKind: "relay-notify",
+				},
+			],
+		});
+	});
+});

package/src/handlers/messaging-send-recording-logic.ts

@@ -0,0 +1,68 @@
+/**
+ * @system server-boot
+ * @status handwritten
+ * @edit edit directly
+ *
+ * Pure detection logic for the messaging send-recording watch
+ * (`messaging-send-recording-watch` handler). Split from the DB-coupled handler
+ * so the row-shape decision is unit-testable with zero database dependencies,
+ * the same shape as whatsapp-liveness-logic.
+ */
+
+/** One outbound messaging_logs row that never recorded its provider message id. */
+export interface UnrecordedSendRow {
+	id: string;
+	created_at: Date;
+	external_chat_id: string | null;
+	send_kind: string | null;
+}
+
+/** The aggregate the handler emits while the failure shape persists. */
+export interface UnrecordedSendFinding {
+	unrecordedCount: number;
+	firstAt: Date;
+	lastAt: Date;
+	sample: UnrecordedSendRow[];
+}
+
+/**
+ * Classify freshly-read unrecorded outbound rows. Zero rows is the healthy
+ * state (every send recorded its provider id) and yields null — no event,
+ * idle-costs-nothing. Any row at/after the baseline is the failure signature
+ * handed over 2026-08-20: a send whose provider id was never recorded can
+ * never correlate its delivery receipt, so "did it reach the handset" is
+ * unanswerable for exactly those sends.
+ */
+export function detectUnrecordedSends(
+	rows: UnrecordedSendRow[],
+	sampleLimit: number,
+): UnrecordedSendFinding | null {
+	if (rows.length === 0) return null;
+	const sorted = [...rows].sort((a, b) => a.created_at.getTime() - b.created_at.getTime());
+	return {
+		unrecordedCount: rows.length,
+		firstAt: sorted[0]!.created_at,
+		lastAt: sorted[sorted.length - 1]!.created_at,
+		// The most recent rows are the ones a responder acts on first.
+		sample: sorted.slice(-sampleLimit),
+	};
+}
+
+/** The durable-event payload — ISO strings + the baseline it was read against. */
+export function buildUnrecordedPayload(
+	finding: UnrecordedSendFinding,
+	recordedSince: string,
+): Record<string, unknown> {
+	return {
+		unrecordedCount: finding.unrecordedCount,
+		recordedSince,
+		firstAt: finding.firstAt.toISOString(),
+		lastAt: finding.lastAt.toISOString(),
+		sample: finding.sample.map((row) => ({
+			id: row.id,
+			createdAt: row.created_at.toISOString(),
+			chatId: row.external_chat_id,
+			sendKind: row.send_kind,
+		})),
+	};
+}

package/src/handlers/messaging-send-recording-watch.test.ts

@@ -0,0 +1,93 @@
+/**
+ * @system server-boot
+ * @status handwritten
+ * @edit edit directly
+ *
+ * Handler-level integration test for `messagingSendRecordingWatch`: mocks the
+ * Prisma registry + recordEvent, drives the two observed live shapes (the
+ * three pre-fix rows measured 2026-08-20, and the healthy all-recorded state),
+ * and asserts the `messaging.send.unrecorded` durable event fires for exactly
+ * the failure shape — without touching the real database.
+ */
+import { beforeEach, expect, mock, test } from "bun:test";
+
+const recordEventCalls: Array<{ kind: string; payload: Record<string, unknown> }> = [];
+
+// The live unrecorded rows the query returns — mutated per test.
+let mockRows: Array<{
+	id: string;
+	created_at: Date;
+	external_chat_id: string | null;
+	send_kind: string | null;
+}>;
+let queryArgs: unknown[] = [];
+
+mock.module("@teamscala/db/prisma-registry", () => ({
+	getPrisma: () => ({
+		$queryRawUnsafe: async (_sql: string, ...args: unknown[]) => {
+			queryArgs = args;
+			return mockRows;
+		},
+	}),
+}));
+mock.module("@teamscala/event-log/record-event", () => ({
+	recordEvent: (e: { kind: string; payload: Record<string, unknown> }) => {
+		recordEventCalls.push(e);
+	},
+}));
+// Self-contained in any suite order: another handler test file installs a
+// partial process-global app-loggers mock that leaks into this one, so this
+// file pins the logger shape it needs rather than depending on run order.
+mock.module("@teamscala/logger/app-loggers", () => ({
+	getAppLogger: () => ({
+		info: () => {},
+		warn: () => {},
+		error: () => {},
+	}),
+}));
+
+const { messagingSendRecordingWatch } = await import("./messaging-send-recording-watch.ts");
+
+const CONFIG = {
+	recorded_since: "2026-08-01T00:00:00Z",
+	sample_limit: 2,
+} as never;
+
+beforeEach(() => {
+	recordEventCalls.length = 0;
+	queryArgs = [];
+});
+
+test("unrecorded rows at/after the baseline emit one messaging.send.unrecorded event", async () => {
+	mockRows = [
+		{ id: "a", created_at: new Date("2026-08-20T05:54:37.715Z"), external_chat_id: "447@s.whatsapp.net", send_kind: null },
+		{ id: "b", created_at: new Date("2026-08-20T08:36:49.149Z"), external_chat_id: "447@s.whatsapp.net", send_kind: null },
+		{ id: "c", created_at: new Date("2026-08-20T08:37:15.667Z"), external_chat_id: "447@s.whatsapp.net", send_kind: null },
+	];
+	await messagingSendRecordingWatch({ config: CONFIG });
+
+	expect(recordEventCalls).toHaveLength(1);
+	expect(recordEventCalls[0]!.kind).toBe("messaging.send.unrecorded");
+	expect(recordEventCalls[0]!.payload).toMatchObject({
+		unrecordedCount: 3,
+		recordedSince: "2026-08-01T00:00:00Z",
+	});
+	// The query reads the baseline as a bound — the predicate's timestamp is
+	// part of the failure signature, not decoration.
+	expect(queryArgs).toEqual(["2026-08-01T00:00:00Z"]);
+});
+
+test("healthy shape (every send recorded) emits nothing", async () => {
+	mockRows = [];
+	await messagingSendRecordingWatch({ config: CONFIG });
+	expect(recordEventCalls).toEqual([]);
+});
+
+test("missing recorded_since baseline skips the query rather than sweeping pre-fix history", async () => {
+	mockRows = [
+		{ id: "historic", created_at: new Date("2026-08-01T00:00:00Z"), external_chat_id: null, send_kind: null },
+	];
+	await messagingSendRecordingWatch({ config: {} as never });
+	expect(recordEventCalls).toEqual([]);
+	expect(queryArgs).toEqual([]);
+});

package/src/handlers/messaging-send-recording-watch.ts

@@ -0,0 +1,86 @@
+/**
+ * @system server-boot
+ * @status handwritten
+ * @edit edit directly
+ *
+ * The `messaging-send-recording-watch` pg-boss handler — watches the durable
+ * row shape that makes delivery acks correlatable: every outbound
+ * messaging_logs row created at/after `recorded_since` must carry the provider
+ * message id (`external_msg_id`). A row without it is a send whose
+ * `message.ack` receipt can never correlate — the regression fixed
+ * 2026-08-20T10:19Z (the single send path in messaging-service outbound.rs),
+ * whose silent return is otherwise invisible: the ack-time "cannot correlate"
+ * warning fires only when a live webhook consumer receives an ack, so a dead
+ * consumer or a recording break both read as healthy silence. This watch
+ * reads the shared platform database directly — independent of the webhook
+ * consumer, GOWA, and any scale-to-zero surface — and emits one
+ * `messaging.send.unrecorded` durable event per tick while the shape
+ * persists. Thresholding (duration + recurrence), episode dedup, and operator
+ * delivery belong to the platform-fault escalator, not here.
+ *
+ * Registered under `@teamscala/service-boot:messaging-send-recording-watch`.
+ */
+import type { ScheduledJobConfig } from "@teamscala/db-validation/registry-schemas/scheduled-job";
+import { getPrisma } from "@teamscala/db/prisma-registry";
+import { recordEvent } from "@teamscala/event-log/record-event";
+import { getAppLogger } from "@teamscala/logger/app-loggers";
+import { pgBossHandlerRegistry } from "../pg-boss-handler-registry.ts";
+import {
+	buildUnrecordedPayload,
+	detectUnrecordedSends,
+	type UnrecordedSendRow,
+} from "./messaging-send-recording-logic.ts";
+
+const DEFAULT_SAMPLE_LIMIT = 5;
+/** Bounds the per-tick read while a break runs for weeks; far above daily volume. */
+const QUERY_ROW_CAP = 1000;
+
+export async function messagingSendRecordingWatch({
+	config,
+}: {
+	config: ScheduledJobConfig;
+}): Promise<void> {
+	const log = getAppLogger();
+
+	// Fail explicit, not historic: without a baseline the query would sweep
+	// pre-fix history and alert on rows that were never promised an id.
+	const recordedSince = config.recorded_since;
+	if (!recordedSince) {
+		log.error(
+			"[messaging-send-recording-watch] scheduled_job row has no recorded_since baseline — skipping (a baseline-less watch would fire on pre-fix history)",
+		);
+		return;
+	}
+
+	const prisma = getPrisma();
+	const rows = await prisma.$queryRawUnsafe<UnrecordedSendRow[]>(
+		`SELECT id, created_at, external_chat_id, payload->>'send_kind' AS send_kind
+		 FROM messaging_logs
+		 WHERE direction = 'outbound'
+			 AND external_msg_id IS NULL
+			 AND created_at >= $1::timestamptz
+		 ORDER BY created_at ASC
+		 LIMIT ${QUERY_ROW_CAP}`,
+		recordedSince,
+	);
+
+	const finding = detectUnrecordedSends(rows, config.sample_limit ?? DEFAULT_SAMPLE_LIMIT);
+	if (!finding) {
+		log.info(
+			`[messaging-send-recording-watch] poll complete — every outbound send at/after ${recordedSince} recorded its provider id`,
+		);
+		return;
+	}
+
+	const payload = buildUnrecordedPayload(finding, recordedSince);
+	recordEvent({ kind: "messaging.send.unrecorded", payload });
+	log.warn(
+		`[messaging-send-recording-watch] ${finding.unrecordedCount} outbound send(s) at/after ${recordedSince} recorded no provider message id — their delivery receipts can never correlate`,
+		payload,
+	);
+}
+
+pgBossHandlerRegistry.register(
+	"@teamscala/service-boot:messaging-send-recording-watch",
+	messagingSendRecordingWatch,
+);

package/src/page-server-orpc/server-client.test.ts

@@ -19,7 +19,9 @@
 import { afterEach, describe, expect, test } from "bun:test";
 import {
 	getServerClient,
+	getServerClientCapabilityFault,
 	isServerClientRegistered,
+	recordServerClientCapability,
 	setServerClient,
 } from "./server-client.ts";
 
@@ -66,3 +68,45 @@ describe("page-server server client", () => {
 		expect(isServerClientRegistered()).toBe(true);
 	});
 });
+
+describe("declared-vs-present capability probe (/health)", () => {
+	test("no fault when every declared model is servable", () => {
+		setServerClient({
+			assessments: { findFirst: async () => null },
+			assessment_responses: { findFirst: async () => null },
+		} as never);
+		recordServerClientCapability(["assessments", "assessment_responses"]);
+		expect(getServerClientCapabilityFault()).toBeNull();
+	});
+
+	test("names the models the client cannot serve", () => {
+		// THE OUTAGE, AS A HEALTH ANSWER. A client was registered and carried only
+		// the platform routers, so every process-existence signal was green while
+		// every data-backed page 500d. /health must be able to say this.
+		setServerClient({ puckPages: {}, userPreferences: {} } as never);
+		recordServerClientCapability(["assessments", "assessment_responses"]);
+		const fault = getServerClientCapabilityFault();
+		expect(fault).toContain("assessments");
+		expect(fault).toContain("assessment_responses");
+	});
+
+	test("a service declaring no models is never faulted", () => {
+		// Daemon/portal shells own no DB schema and are healthy with an empty client.
+		setServerClient({ puckPages: {} } as never);
+		recordServerClientCapability([]);
+		expect(getServerClientCapabilityFault()).toBeNull();
+	});
+
+	test("an unregistered client is itself the fault", () => {
+		setServerClient(null as never);
+		recordServerClientCapability(["assessments"]);
+		expect(getServerClientCapabilityFault()).toContain("no ORPC server client registered");
+	});
+
+	test("a model present without findFirst still counts as missing", () => {
+		// The generated router shape is what matters, not mere key presence.
+		setServerClient({ assessments: {} } as never);
+		recordServerClientCapability(["assessments"]);
+		expect(getServerClientCapabilityFault()).toContain("assessments");
+	});
+});

package/src/page-server-orpc/server-client.ts

@@ -55,3 +55,36 @@ export function setServerClient(
 ): void {
 	serverClient = client;
 }
+
+/**
+ * Declared-vs-present capability, recorded once at boot by
+ * registerPageServerClient and read by /health.
+ *
+ * The outage this exists for: a client WAS registered, so every
+ * process-existence signal was green, and it carried no CRUD models. Liveness
+ * must be a capability probe, never a registration check
+ * (`liveness-by-capability-not-process-existence`).
+ */
+let capabilityFault: string | null = null;
+
+/**
+ * Record whether the registered client can actually serve the models the
+ * service declares. Property lookups only — no I/O — so /health stays cheap.
+ */
+export function recordServerClientCapability(declaredModels: readonly string[]): void {
+	if (!serverClient) {
+		capabilityFault = "no ORPC server client registered";
+		return;
+	}
+	const client = serverClient as unknown as Record<string, { findFirst?: unknown } | undefined>;
+	const missing = declaredModels.filter((m) => typeof client[m]?.findFirst !== "function");
+	capabilityFault =
+		missing.length === 0
+			? null
+			: `generated client exposes no findFirst for ${missing.length} declared model(s): ${missing.slice(0, 5).join(", ")}${missing.length > 5 ? ", …" : ""}`;
+}
+
+/** The capability fault, or null when the service can serve its declared models. */
+export function getServerClientCapabilityFault(): string | null {
+	return capabilityFault;
+}

package/src/pg-boss-mount.ts

@@ -31,6 +31,7 @@ import "./handlers/sweep-event-log.ts";
 // bound. No TypeScript copy may exist beside it.
 import "./handlers/bun-stable-watch.ts";
 import "./handlers/whatsapp-liveness.ts";
+import "./handlers/messaging-send-recording-watch.ts";
 import "./handlers/upstream-issue-sync.ts";
 import "./handlers/scala-tools-currency-watch.ts";
 import "./handlers/singleton-tools-currency-watch.ts";

package/src/register-page-server-client.ts

@@ -148,6 +148,9 @@ export async function registerPageServerClient(
 			} as never,
 		);
 		mod.setServerClient(client);
+		// Record declared-vs-present capability for /health. Property lookups only.
+		(mod as unknown as { recordServerClientCapability?: (m: readonly string[]) => void })
+			.recordServerClientCapability?.(declaredModels);
 		// State the model count. "registered" alone was true in the outage and told
 		// nobody anything; a zero here is now visible in the boot log.
 		log.info(