@teamscala/service-boot 0.23.418 → 0.23.419

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -6,7 +6,7 @@
6
6
  },
7
7
  "name": "@teamscala/service-boot",
8
8
  "type": "module",
9
- "version": "0.23.418",
9
+ "version": "0.23.419",
10
10
  "sideEffects": false,
11
11
  "imports": {
12
12
  "#service-boot/*": "./src/*",
@@ -0,0 +1,54 @@
1
+ /**
2
+ * @system server-boot
3
+ * @status handwritten
4
+ * @edit edit directly
5
+ *
6
+ * The config row of the platform-fault-escalation pg-boss handler
7
+ * (config/platform-fault-escalation).
8
+ *
9
+ * Split out of handlers/platform-fault-escalation.ts (500 lines) —
10
+ * byte-for-byte move; import paths adjusted only.
11
+ */
12
+
13
+ import { loadRegistryConfig } from "@teamscala/db/registry/load-config";
14
+ import * as v from "valibot";
15
+
16
+ export const CONFIG_SLUG = "platform-fault-escalation";
17
+
18
+ export const FaultEscalationConfigSchema = v.object({
19
+ /** Postgres regex patterns selecting watched event_log kinds. */
20
+ watched_kind_patterns: v.array(v.string()),
21
+ /** Lookback window for the recurrence/duration aggregate, minutes. */
22
+ window_minutes: v.number(),
23
+ min_count: v.number(),
24
+ min_span_minutes: v.number(),
25
+ /** Per-kind recurrence/duration overrides — low-rate-but-persistent kinds. */
26
+ kind_thresholds: v.optional(
27
+ v.record(
28
+ v.string(),
29
+ v.object({
30
+ min_count: v.optional(v.number()),
31
+ min_span_minutes: v.optional(v.number()),
32
+ }),
33
+ ),
34
+ ),
35
+ resolve_after_minutes: v.number(),
36
+ /** The orchestrator's work item — escalations are COMMENTS on it, never
37
+ * task rows (the fleet-task-placement CHECK constraint refuses a system
38
+ * task without session identity, and an escalation is a finding, not an
39
+ * assigned unit of work). */
40
+ orchestrator_work_item_id: v.string(),
41
+ /** Per-kind copy for the message (impact + recommendation). Falls back to default_copy. */
42
+ copy: v.record(v.string(), v.object({ impact: v.string(), recommendation: v.string() })),
43
+ default_copy: v.object({ impact: v.string(), recommendation: v.string() }),
44
+ /** Per-kind payload-field whitelist sampled into the message's evidence
45
+ * line — named fields ONLY, never the whole payload: some kinds embed
46
+ * request context, and the escalation channel is WhatsApp. */
47
+ sample_fields: v.optional(v.record(v.string(), v.array(v.string()))),
48
+ });
49
+
50
+ export type FaultEscalationConfig = v.InferOutput<typeof FaultEscalationConfigSchema>;
51
+
52
+ export function loadFaultEscalationConfig(): Promise<FaultEscalationConfig> {
53
+ return loadRegistryConfig("config", CONFIG_SLUG, FaultEscalationConfigSchema);
54
+ }
@@ -0,0 +1,173 @@
1
+ /**
2
+ * @system server-boot
3
+ * @status handwritten
4
+ * @edit edit directly
5
+ *
6
+ * Escalation delivery of the platform-fault-escalation handler: claim the
7
+ * episode, resolve the operator-relay WhatsApp bot, send via
8
+ * messaging_whatsapp_send, verify the message id, record the comment +
9
+ * readback-corrected episode — and close a quiet episode.
10
+ *
11
+ * Split out of handlers/platform-fault-escalation.ts (500 lines) —
12
+ * byte-for-byte move; import paths adjusted only.
13
+ */
14
+
15
+ import type { ExecutionContext } from "@teamscala/os/contracts/mcp";
16
+ import { recordEvent } from "@teamscala/event-log/record-event";
17
+ import {
18
+ composeEscalationMessage,
19
+ extractWhatsAppMessageId,
20
+ type EscalationEpisode,
21
+ type FaultAggregate,
22
+ } from "../platform-fault-escalation-logic.ts";
23
+ import type { FaultEscalationConfig } from "./config.ts";
24
+ import { fetchLatestEvidence } from "./event-log-queries.ts";
25
+ import { getSystemIdentity, orpc } from "./orpc-client.ts";
26
+
27
+ interface OperatorRelayBot {
28
+ id: string;
29
+ organisation_id: string;
30
+ external_id: string | null;
31
+ flow_config: { operator_phone?: string } | null;
32
+ }
33
+
34
+ async function resolveOperatorRelay(): Promise<OperatorRelayBot> {
35
+ const bot = (await (await orpc()).messaging_bots.findFirst({
36
+ where: {
37
+ is_active: true,
38
+ platform: "whatsapp",
39
+ flow_config: { path: ["mode"], equals: "operator-relay" },
40
+ },
41
+ select: { id: true, organisation_id: true, external_id: true, flow_config: true },
42
+ })) as unknown as OperatorRelayBot | null;
43
+ if (!bot) throw new Error("platform-fault-escalation: no active operator-relay WhatsApp bot row");
44
+ const phone = (bot.flow_config as { operator_phone?: string } | null)?.operator_phone;
45
+ if (!phone) throw new Error("platform-fault-escalation: operator-relay bot has no operator_phone");
46
+ return bot as OperatorRelayBot;
47
+ }
48
+
49
+ export async function deliverEscalation(
50
+ kind: string,
51
+ aggregate: FaultAggregate,
52
+ config: FaultEscalationConfig,
53
+ ): Promise<{ messageId: string; workItemId: string }> {
54
+ const now = new Date();
55
+
56
+ // 1. Claim the episode BEFORE sending, so a send failure leaves a durable
57
+ // claim the next sweep retries instead of a lost escalation.
58
+ await (await orpc()).platform_fault_escalations.upsert({
59
+ where: { kind },
60
+ create: {
61
+ kind,
62
+ first_seen_at: aggregate.firstSeen.toISOString(),
63
+ last_seen_at: aggregate.lastSeen.toISOString(),
64
+ event_count: aggregate.count,
65
+ },
66
+ update: {
67
+ first_seen_at: aggregate.firstSeen.toISOString(),
68
+ last_seen_at: aggregate.lastSeen.toISOString(),
69
+ event_count: aggregate.count,
70
+ resolved_at: null,
71
+ },
72
+ });
73
+
74
+ // 2. Resolve the operator relay and send. The send surface is the
75
+ // `messaging_whatsapp_send` intrinsic (scala-mcp / messaging-mcp-boot),
76
+ // which delegates to the Rust scala-messaging-service — the channel owner
77
+ // that picks the device and records the send in messaging_logs. The retired
78
+ // api_gowa_message tool (deactivated 2026-08-26) is no longer callable;
79
+ // naming it here left every escalation send failing with "Unknown tool".
80
+ const bot = await resolveOperatorRelay();
81
+ const operatorPhone = (bot.flow_config as { operator_phone?: string }).operator_phone!;
82
+ const copy = config.copy[kind] ?? config.default_copy;
83
+ const { sample, lastEvent } = await fetchLatestEvidence(kind, config.sample_fields?.[kind] ?? []);
84
+ const message = composeEscalationMessage({
85
+ kind,
86
+ aggregate,
87
+ sample,
88
+ lastEvent,
89
+ impact: copy.impact,
90
+ recommendation: copy.recommendation,
91
+ });
92
+
93
+ const { userId, organisationId } = await getSystemIdentity();
94
+ const systemCtx: ExecutionContext = { userId, organisationId } as ExecutionContext;
95
+ const { forwardToolToCentralMcp } = await import("@teamscala/chat-completion/central-tool-forwarder");
96
+ const sendResult = await forwardToolToCentralMcp(
97
+ "messaging_whatsapp_send",
98
+ { phone: operatorPhone, message },
99
+ systemCtx,
100
+ );
101
+
102
+ // 3. VERIFY delivery before recording it: the send must return a WhatsApp
103
+ // message id. No id => throw, the claim stays undelivered, next sweep retries.
104
+ const messageId = extractWhatsAppMessageId(sendResult);
105
+ if (!messageId) {
106
+ throw new Error(
107
+ `platform-fault-escalation: WhatsApp send for '${kind}' returned no message id — not recording delivery. ` +
108
+ `Response: ${JSON.stringify(sendResult).slice(0, 400)}`,
109
+ );
110
+ }
111
+
112
+ // 4. Durable record: a COMMENT on the orchestrator's work item — an
113
+ // escalation is a finding the orchestrator/operator triages, not an
114
+ // assigned task (the fleet-task-placement constraint refuses task rows
115
+ // without session identity by design).
116
+ const comment = await (await orpc()).comments.create({
117
+ data: {
118
+ work_item_id: config.orchestrator_work_item_id,
119
+ content: message,
120
+ source_cli_session_id: null,
121
+ },
122
+ });
123
+
124
+ // 5. Readback-correct the episode: delivered + linked in one update.
125
+ await (await orpc()).platform_fault_escalations.update({
126
+ where: { kind },
127
+ data: {
128
+ escalated_at: now.toISOString(),
129
+ work_item_id: config.orchestrator_work_item_id,
130
+ delivered_comment_id: comment.id,
131
+ delivered_message_id: messageId,
132
+ last_updated_at: now.toISOString(),
133
+ updated_at: now.toISOString(),
134
+ },
135
+ });
136
+
137
+ recordEvent({
138
+ kind: "platform.fault.escalated",
139
+ payload: {
140
+ event_kind: kind,
141
+ count: aggregate.count,
142
+ sample,
143
+ last_event_id: lastEvent?.eventId ?? null,
144
+ last_event_occurred_at: lastEvent?.occurredAt ?? null,
145
+ work_item_id: config.orchestrator_work_item_id,
146
+ delivered_comment_id: comment.id,
147
+ delivered_message_id: messageId,
148
+ },
149
+ });
150
+ return { messageId, workItemId: config.orchestrator_work_item_id };
151
+ }
152
+
153
+ export async function resolveEpisode(episode: EscalationEpisode, quietMinutes: number): Promise<void> {
154
+ const now = new Date();
155
+ await (await orpc()).platform_fault_escalations.update({
156
+ where: { kind: episode.kind },
157
+ data: { resolved_at: now.toISOString(), updated_at: now.toISOString() },
158
+ });
159
+ if (episode.workItemId) {
160
+ await (await orpc()).comments.create({
161
+ data: {
162
+ work_item_id: episode.workItemId,
163
+ content: `Fault stopped — no '${episode.kind}' events for ${Math.round(quietMinutes)} min. Episode closed; the fault will re-escalate if it recurs.`,
164
+ sender_role: "system",
165
+ sender_name: "platform-fault-escalation",
166
+ },
167
+ });
168
+ }
169
+ recordEvent({
170
+ kind: "platform.fault.resolved",
171
+ payload: { event_kind: episode.kind, quiet_minutes: Math.round(quietMinutes), work_item_id: episode.workItemId },
172
+ });
173
+ }
@@ -0,0 +1,122 @@
1
+ /**
2
+ * @system server-boot
3
+ * @status handwritten
4
+ * @edit edit directly
5
+ *
6
+ * The bounded event_log reads of the platform-fault-escalation handler: the
7
+ * recurrence/duration aggregate and the latest-payload evidence sample, both
8
+ * with the `occurred_at <= now()` database-clock top edge.
9
+ *
10
+ * Split out of handlers/platform-fault-escalation.ts (500 lines) —
11
+ * byte-for-byte move; import paths adjusted only.
12
+ */
13
+
14
+ import { getPrisma } from "@teamscala/db/prisma-registry";
15
+ import { formatEscalationSample, type FaultAggregate } from "../platform-fault-escalation-logic.ts";
16
+ import { CONFIG_SLUG } from "./config.ts";
17
+
18
+ interface EventLogAggregateRow {
19
+ kind: string;
20
+ n: bigint;
21
+ first_seen: Date;
22
+ last_seen: Date;
23
+ }
24
+
25
+ export async function fetchAggregates(patterns: string[], windowMinutes: number): Promise<FaultAggregate[]> {
26
+ const prisma = getPrisma();
27
+ // Patterns come from the valibot-validated registry row and are regex-only
28
+ // by construction below; embedding them as one alternation keeps the query
29
+ // parameter-free on the pattern side while the window is a real parameter.
30
+ // The window is bounded on BOTH sides by the database's own statement
31
+ // clock: `occurred_at` is producer-claimed, and an unbounded top edge
32
+ // admits rows stamped in the future by a skewed or mocked producer clock —
33
+ // which then fabricate "still recurring" for a quiet fault (measured
34
+ // 2026-08-26). A future row is not evidence of anything happening now.
35
+ const regexClass = /^[A-Za-z0-9_\\^$.*+?()[\]{}|-]+$/;
36
+ for (const pattern of patterns) {
37
+ if (!regexClass.test(pattern)) {
38
+ throw new Error(`config/${CONFIG_SLUG}: watched_kind_pattern is not regex-only: ${pattern}`);
39
+ }
40
+ }
41
+ const combined = `(${patterns.join("|")})`;
42
+ const rows = await prisma.$queryRawUnsafe<EventLogAggregateRow[]>(
43
+ `SELECT kind, count(*)::bigint AS n, min(occurred_at) AS first_seen, max(occurred_at) AS last_seen
44
+ FROM event_log
45
+ WHERE occurred_at > now() - ($1::int * interval '1 minute')
46
+ AND occurred_at <= now()
47
+ AND kind ~ $2::text
48
+ GROUP BY kind`,
49
+ windowMinutes,
50
+ combined,
51
+ );
52
+ return rows.map((row) => ({
53
+ kind: row.kind,
54
+ count: Number(row.n),
55
+ firstSeen: new Date(row.first_seen),
56
+ lastSeen: new Date(row.last_seen),
57
+ }));
58
+ }
59
+
60
+ /**
61
+ * The message's evidence: the whitelisted sample fields AND the row identity
62
+ * (id + occurred_at) of the newest payload for the kind. Field names come
63
+ * from the valibot-validated config row and are constrained to bare
64
+ * identifiers below, so the JSON path cannot break out of the string. The
65
+ * `occurred_at <= now()` bound matches the aggregate's top edge: the cited
66
+ * "last" row must be a real, non-future event whose id rides in the message
67
+ * so the claim resolves against event_log.
68
+ */
69
+ export async function fetchLatestEvidence(
70
+ kind: string,
71
+ fields: string[],
72
+ ): Promise<{ sample: string; lastEvent: { occurredAt: string; eventId: string } | null }> {
73
+ const prisma = getPrisma();
74
+ if (fields.length === 0) {
75
+ const rows = await prisma.$queryRawUnsafe<Array<{ id: unknown; occurred_at: Date }>>(
76
+ `SELECT id::text, occurred_at
77
+ FROM event_log
78
+ WHERE kind = $1::text AND occurred_at <= now()
79
+ ORDER BY occurred_at DESC
80
+ LIMIT 1`,
81
+ kind,
82
+ );
83
+ const top = rows[0];
84
+ return { sample: "", lastEvent: top ? { occurredAt: isoSeconds(top.occurred_at), eventId: String(top.id) } : null };
85
+ }
86
+ for (const field of fields) {
87
+ if (!/^[A-Za-z0-9_]+$/.test(field)) {
88
+ throw new Error(`config/${CONFIG_SLUG}: sample_fields entry is not a bare field name: ${field}`);
89
+ }
90
+ }
91
+ const selection = fields.map((field) => `payload_json->'${field}' AS "${field}"`).join(", ");
92
+ const rows = await prisma.$queryRawUnsafe<Array<Record<string, unknown>>>(
93
+ `SELECT id::text, occurred_at, ${selection}
94
+ FROM event_log
95
+ WHERE kind = $1::text AND occurred_at <= now()
96
+ ORDER BY occurred_at DESC
97
+ LIMIT 1`,
98
+ kind,
99
+ );
100
+ const row = rows[0];
101
+ if (!row) return { sample: "", lastEvent: null };
102
+ const parts: string[] = [];
103
+ for (const field of fields) {
104
+ const value = row[field];
105
+ if (value === null || value === undefined) continue;
106
+ const formatted = formatEscalationSample(value);
107
+ if (formatted) parts.push(`${field}=${formatted}`);
108
+ }
109
+ return {
110
+ sample: parts.join("; "),
111
+ lastEvent: {
112
+ occurredAt: isoSeconds(row.occurred_at as string | Date),
113
+ eventId: String(row.id),
114
+ },
115
+ };
116
+ }
117
+
118
+ /** ISO-8601 to whole seconds — the same precision event_log stores, so the
119
+ * cited string round-trips a `WHERE occurred_at = '<cited>'` query. */
120
+ function isoSeconds(value: Date | string): string {
121
+ return new Date(value).toISOString().replace(/\.\d{3}Z$/, "Z");
122
+ }
@@ -0,0 +1,97 @@
1
+ /**
2
+ * @system server-boot
3
+ * @status handwritten
4
+ * @edit edit directly
5
+ *
6
+ * The system-authed in-process ORPC client of the platform-fault-escalation
7
+ * handler (lazy, built from the composed appRouter) and the system identity
8
+ * it falls back to.
9
+ *
10
+ * Split out of handlers/platform-fault-escalation.ts (500 lines) —
11
+ * byte-for-byte move; import paths adjusted only.
12
+ */
13
+
14
+ import { loadRegistryConfig } from "@teamscala/db/registry/load-config";
15
+ import * as v from "valibot";
16
+
17
+ let cachedSystemIdentity: { userId: string | undefined; organisationId: string | undefined } | undefined;
18
+ export async function getSystemIdentity(): Promise<{ userId: string | undefined; organisationId: string | undefined }> {
19
+ if (cachedSystemIdentity) return cachedSystemIdentity;
20
+ const row = await loadRegistryConfig("config", "system-identity", v.looseObject({}));
21
+ cachedSystemIdentity = {
22
+ userId: row?.system_user_id as string | undefined,
23
+ organisationId: row?.system_org_id as string | undefined,
24
+ };
25
+ return cachedSystemIdentity;
26
+ }
27
+
28
+ /**
29
+ * The system-authed in-process ORPC client, built LAZILY on first use from the
30
+ * SAME composed appRouter the /api/rpc mount uses (getAppRouter — never a raw
31
+ * re-import of the generated routers, per register-page-server-client's
32
+ * documented module-identity trap). Built here rather than read from
33
+ * page-server-orpc/server-client because a pg-boss handler can fire on a
34
+ * BACKEND service, where the page-server client is never registered — the
35
+ * empty-router fallback returns a client with no model namespaces (the
36
+ * TypeError this replaced). Per `no-direct-prisma-outside-orpc`, every WRITE
37
+ * in this handler routes through this client; the two raw reads (the
38
+ * event_log recurrence aggregate and the latest-payload sample, shapes no
39
+ * generated CRUD expresses) use
40
+ * the assigned-client form with $queryRawUnsafe. Timestamps are ISO strings,
41
+ * never Date instances (the generated valibot schemas require it).
42
+ */
43
+ let cachedOrpc: OrpcSurface | null = null;
44
+ export async function orpc(): Promise<OrpcSurface> {
45
+ if (cachedOrpc) return cachedOrpc;
46
+ const { createRouterClient } = await import("@orpc/server");
47
+ const { getAppRouter } = await import("@teamscala/orpc/app-router");
48
+ const appRouter = (await getAppRouter()) as Record<string, unknown> | null;
49
+ if (!appRouter) {
50
+ throw new Error(
51
+ "platform-fault-escalation: no ORPC appRouter — the service has no generated routers to write through",
52
+ );
53
+ }
54
+ const prismaRegistry = await import("@teamscala/db/prisma-registry");
55
+ const registryLoader = await import("@teamscala/db/registry/load-config");
56
+ const { PlatformIdentitySchema } = await import(
57
+ "@teamscala/db-validation/registry-schemas/platform-identity"
58
+ );
59
+ const platformIdentity = await registryLoader.loadRegistryConfig(
60
+ "config",
61
+ "platform-identity",
62
+ PlatformIdentitySchema,
63
+ );
64
+ cachedOrpc = createRouterClient(
65
+ { ...appRouter } as never,
66
+ {
67
+ context: {
68
+ prisma: prismaRegistry.getPrisma(),
69
+ user: {
70
+ id: "system",
71
+ isSuperAdmin: true,
72
+ organisationId: platformIdentity.platformOrganisationId,
73
+ },
74
+ },
75
+ } as never,
76
+ ) as unknown as OrpcSurface;
77
+ return cachedOrpc;
78
+ }
79
+
80
+ /** The router subset this handler uses — mirrors the generated procedure args. */
81
+ export interface OrpcSurface {
82
+ platform_fault_escalations: {
83
+ upsert(args: {
84
+ where: { kind: string };
85
+ create: Record<string, unknown>;
86
+ update: Record<string, unknown>;
87
+ }): Promise<unknown>;
88
+ update(args: { where: { kind: string }; data: Record<string, unknown> }): Promise<unknown>;
89
+ findMany(args?: { where?: Record<string, unknown> }): Promise<Array<Record<string, unknown>>>;
90
+ };
91
+ messaging_bots: {
92
+ findFirst(args: Record<string, unknown>): Promise<Record<string, unknown> | null>;
93
+ };
94
+ comments: {
95
+ create(args: { data: Record<string, unknown> }): Promise<{ id: string }>;
96
+ };
97
+ }
@@ -1,411 +1,39 @@
1
1
  /**
2
2
  * @system server-boot
3
- * @status handwritten @indivisible-unit
3
+ * @status handwritten
4
4
  * @edit edit directly
5
5
  *
6
6
  * The `platform-fault-escalation` pg-boss handler. Full contract in curated
7
7
  * `apps/escalation.md` — episodes EMIT ON TRANSITIONS ONLY; evidence bounded
8
8
  * `occurred_at <= now()` and cited by resolvable event id; delivery verified
9
9
  * by WhatsApp message id before `escalated_at` is recorded.
10
+ *
11
+ * This module is the SWEEP unit (load config → aggregate → decide → deliver/
12
+ * resolve per kind, isolated). The machinery lives as siblings under
13
+ * ./platform-fault-escalation/ (config, event-log-queries, orpc-client,
14
+ * delivery) — split when the parent held them all at 500 lines; the pure
15
+ * decision logic was already platform-fault-escalation-logic.ts.
10
16
  */
11
- import type { ExecutionContext } from "@teamscala/os/contracts/mcp";
12
17
  import type { ScheduledJobConfig } from "@teamscala/db-validation/registry-schemas/scheduled-job";
13
- import * as v from "valibot";
14
- import { getPrisma } from "@teamscala/db/prisma-registry";
15
- import { loadRegistryConfig } from "@teamscala/db/registry/load-config";
16
18
  import { recordEvent } from "@teamscala/event-log/record-event";
17
19
  import { getAppLogger } from "@teamscala/logger/app-loggers";
18
20
  import { pgBossHandlerRegistry } from "../pg-boss-handler-registry.ts";
19
21
  import {
20
- composeEscalationMessage,
21
22
  decideSweep,
22
- extractWhatsAppMessageId,
23
- formatEscalationSample,
24
23
  type EscalationEpisode,
25
- type FaultAggregate,
26
24
  type KindThresholdOverride,
27
25
  } from "./platform-fault-escalation-logic.ts";
28
-
29
- const CONFIG_SLUG = "platform-fault-escalation";
30
-
31
- const FaultEscalationConfigSchema = v.object({
32
- /** Postgres regex patterns selecting watched event_log kinds. */
33
- watched_kind_patterns: v.array(v.string()),
34
- /** Lookback window for the recurrence/duration aggregate, minutes. */
35
- window_minutes: v.number(),
36
- min_count: v.number(),
37
- min_span_minutes: v.number(),
38
- /** Per-kind recurrence/duration overrides — low-rate-but-persistent kinds. */
39
- kind_thresholds: v.optional(
40
- v.record(
41
- v.string(),
42
- v.object({
43
- min_count: v.optional(v.number()),
44
- min_span_minutes: v.optional(v.number()),
45
- }),
46
- ),
47
- ),
48
- resolve_after_minutes: v.number(),
49
- /** The orchestrator's work item — escalations are COMMENTS on it, never
50
- * task rows (the fleet-task-placement CHECK constraint refuses a system
51
- * task without session identity, and an escalation is a finding, not an
52
- * assigned unit of work). */
53
- orchestrator_work_item_id: v.string(),
54
- /** Per-kind copy for the message (impact + recommendation). Falls back to default_copy. */
55
- copy: v.record(v.string(), v.object({ impact: v.string(), recommendation: v.string() })),
56
- default_copy: v.object({ impact: v.string(), recommendation: v.string() }),
57
- /** Per-kind payload-field whitelist sampled into the message's evidence
58
- * line — named fields ONLY, never the whole payload: some kinds embed
59
- * request context, and the escalation channel is WhatsApp. */
60
- sample_fields: v.optional(v.record(v.string(), v.array(v.string()))),
61
- });
62
-
63
- type FaultEscalationConfig = v.InferOutput<typeof FaultEscalationConfigSchema>;
64
-
65
- interface EventLogAggregateRow {
66
- kind: string;
67
- n: bigint;
68
- first_seen: Date;
69
- last_seen: Date;
70
- }
71
-
72
- interface OperatorRelayBot {
73
- id: string;
74
- organisation_id: string;
75
- external_id: string | null;
76
- flow_config: { operator_phone?: string } | null;
77
- }
78
-
79
- let cachedSystemIdentity: { userId: string | undefined; organisationId: string | undefined } | undefined;
80
- async function getSystemIdentity(): Promise<{ userId: string | undefined; organisationId: string | undefined }> {
81
- if (cachedSystemIdentity) return cachedSystemIdentity;
82
- const row = await loadRegistryConfig("config", "system-identity", v.looseObject({}));
83
- cachedSystemIdentity = {
84
- userId: row?.system_user_id as string | undefined,
85
- organisationId: row?.system_org_id as string | undefined,
86
- };
87
- return cachedSystemIdentity;
88
- }
89
-
90
- /**
91
- * The system-authed in-process ORPC client, built LAZILY on first use from the
92
- * SAME composed appRouter the /api/rpc mount uses (getAppRouter — never a raw
93
- * re-import of the generated routers, per register-page-server-client's
94
- * documented module-identity trap). Built here rather than read from
95
- * page-server-orpc/server-client because a pg-boss handler can fire on a
96
- * BACKEND service, where the page-server client is never registered — the
97
- * empty-router fallback returns a client with no model namespaces (the
98
- * TypeError this replaced). Per `no-direct-prisma-outside-orpc`, every WRITE
99
- * in this handler routes through this client; the two raw reads (the
100
- * event_log recurrence aggregate and the latest-payload sample, shapes no
101
- * generated CRUD expresses) use
102
- * the assigned-client form with $queryRawUnsafe. Timestamps are ISO strings,
103
- * never Date instances (the generated valibot schemas require it).
104
- */
105
- let cachedOrpc: OrpcSurface | null = null;
106
- async function orpc(): Promise<OrpcSurface> {
107
- if (cachedOrpc) return cachedOrpc;
108
- const { createRouterClient } = await import("@orpc/server");
109
- const { getAppRouter } = await import("@teamscala/orpc/app-router");
110
- const appRouter = (await getAppRouter()) as Record<string, unknown> | null;
111
- if (!appRouter) {
112
- throw new Error(
113
- "platform-fault-escalation: no ORPC appRouter — the service has no generated routers to write through",
114
- );
115
- }
116
- const prismaRegistry = await import("@teamscala/db/prisma-registry");
117
- const registryLoader = await import("@teamscala/db/registry/load-config");
118
- const { PlatformIdentitySchema } = await import(
119
- "@teamscala/db-validation/registry-schemas/platform-identity"
120
- );
121
- const platformIdentity = await registryLoader.loadRegistryConfig(
122
- "config",
123
- "platform-identity",
124
- PlatformIdentitySchema,
125
- );
126
- cachedOrpc = createRouterClient(
127
- { ...appRouter } as never,
128
- {
129
- context: {
130
- prisma: prismaRegistry.getPrisma(),
131
- user: {
132
- id: "system",
133
- isSuperAdmin: true,
134
- organisationId: platformIdentity.platformOrganisationId,
135
- },
136
- },
137
- } as never,
138
- ) as unknown as OrpcSurface;
139
- return cachedOrpc;
140
- }
141
-
142
- /** The router subset this handler uses — mirrors the generated procedure args. */
143
- interface OrpcSurface {
144
- platform_fault_escalations: {
145
- upsert(args: {
146
- where: { kind: string };
147
- create: Record<string, unknown>;
148
- update: Record<string, unknown>;
149
- }): Promise<unknown>;
150
- update(args: { where: { kind: string }; data: Record<string, unknown> }): Promise<unknown>;
151
- findMany(args?: { where?: Record<string, unknown> }): Promise<Array<Record<string, unknown>>>;
152
- };
153
- messaging_bots: {
154
- findFirst(args: Record<string, unknown>): Promise<Record<string, unknown> | null>;
155
- };
156
- comments: {
157
- create(args: { data: Record<string, unknown> }): Promise<{ id: string }>;
158
- };
159
- }
160
-
161
- async function loadConfig(): Promise<FaultEscalationConfig> {
162
- return loadRegistryConfig("config", CONFIG_SLUG, FaultEscalationConfigSchema);
163
- }
164
-
165
- async function fetchAggregates(patterns: string[], windowMinutes: number): Promise<FaultAggregate[]> {
166
- const prisma = getPrisma();
167
- // Patterns come from the valibot-validated registry row and are regex-only
168
- // by construction below; embedding them as one alternation keeps the query
169
- // parameter-free on the pattern side while the window is a real parameter.
170
- // The window is bounded on BOTH sides by the database's own statement
171
- // clock: `occurred_at` is producer-claimed, and an unbounded top edge
172
- // admits rows stamped in the future by a skewed or mocked producer clock —
173
- // which then fabricate "still recurring" for a quiet fault (measured
174
- // 2026-08-26). A future row is not evidence of anything happening now.
175
- const regexClass = /^[A-Za-z0-9_\\^$.*+?()[\]{}|-]+$/;
176
- for (const pattern of patterns) {
177
- if (!regexClass.test(pattern)) {
178
- throw new Error(`config/${CONFIG_SLUG}: watched_kind_pattern is not regex-only: ${pattern}`);
179
- }
180
- }
181
- const combined = `(${patterns.join("|")})`;
182
- const rows = await prisma.$queryRawUnsafe<EventLogAggregateRow[]>(
183
- `SELECT kind, count(*)::bigint AS n, min(occurred_at) AS first_seen, max(occurred_at) AS last_seen
184
- FROM event_log
185
- WHERE occurred_at > now() - ($1::int * interval '1 minute')
186
- AND occurred_at <= now()
187
- AND kind ~ $2::text
188
- GROUP BY kind`,
189
- windowMinutes,
190
- combined,
191
- );
192
- return rows.map((row) => ({
193
- kind: row.kind,
194
- count: Number(row.n),
195
- firstSeen: new Date(row.first_seen),
196
- lastSeen: new Date(row.last_seen),
197
- }));
198
- }
199
-
200
- /**
201
- * The message's evidence: the whitelisted sample fields AND the row identity
202
- * (id + occurred_at) of the newest payload for the kind. Field names come
203
- * from the valibot-validated config row and are constrained to bare
204
- * identifiers below, so the JSON path cannot break out of the string. The
205
- * `occurred_at <= now()` bound matches the aggregate's top edge: the cited
206
- * "last" row must be a real, non-future event whose id rides in the message
207
- * so the claim resolves against event_log.
208
- */
209
- async function fetchLatestEvidence(
210
- kind: string,
211
- fields: string[],
212
- ): Promise<{ sample: string; lastEvent: { occurredAt: string; eventId: string } | null }> {
213
- const prisma = getPrisma();
214
- if (fields.length === 0) {
215
- const rows = await prisma.$queryRawUnsafe<Array<{ id: unknown; occurred_at: Date }>>(
216
- `SELECT id::text, occurred_at
217
- FROM event_log
218
- WHERE kind = $1::text AND occurred_at <= now()
219
- ORDER BY occurred_at DESC
220
- LIMIT 1`,
221
- kind,
222
- );
223
- const top = rows[0];
224
- return { sample: "", lastEvent: top ? { occurredAt: isoSeconds(top.occurred_at), eventId: String(top.id) } : null };
225
- }
226
- for (const field of fields) {
227
- if (!/^[A-Za-z0-9_]+$/.test(field)) {
228
- throw new Error(`config/${CONFIG_SLUG}: sample_fields entry is not a bare field name: ${field}`);
229
- }
230
- }
231
- const selection = fields.map((field) => `payload_json->'${field}' AS "${field}"`).join(", ");
232
- const rows = await prisma.$queryRawUnsafe<Array<Record<string, unknown>>>(
233
- `SELECT id::text, occurred_at, ${selection}
234
- FROM event_log
235
- WHERE kind = $1::text AND occurred_at <= now()
236
- ORDER BY occurred_at DESC
237
- LIMIT 1`,
238
- kind,
239
- );
240
- const row = rows[0];
241
- if (!row) return { sample: "", lastEvent: null };
242
- const parts: string[] = [];
243
- for (const field of fields) {
244
- const value = row[field];
245
- if (value === null || value === undefined) continue;
246
- const formatted = formatEscalationSample(value);
247
- if (formatted) parts.push(`${field}=${formatted}`);
248
- }
249
- return {
250
- sample: parts.join("; "),
251
- lastEvent: {
252
- occurredAt: isoSeconds(row.occurred_at as string | Date),
253
- eventId: String(row.id),
254
- },
255
- };
256
- }
257
-
258
- /** ISO-8601 to whole seconds — the same precision event_log stores, so the
259
- * cited string round-trips a `WHERE occurred_at = '<cited>'` query. */
260
- function isoSeconds(value: Date | string): string {
261
- return new Date(value).toISOString().replace(/\.\d{3}Z$/, "Z");
262
- }
263
-
264
- async function resolveOperatorRelay(): Promise<OperatorRelayBot> {
265
- const bot = (await (await orpc()).messaging_bots.findFirst({
266
- where: {
267
- is_active: true,
268
- platform: "whatsapp",
269
- flow_config: { path: ["mode"], equals: "operator-relay" },
270
- },
271
- select: { id: true, organisation_id: true, external_id: true, flow_config: true },
272
- })) as unknown as OperatorRelayBot | null;
273
- if (!bot) throw new Error("platform-fault-escalation: no active operator-relay WhatsApp bot row");
274
- const phone = (bot.flow_config as { operator_phone?: string } | null)?.operator_phone;
275
- if (!phone) throw new Error("platform-fault-escalation: operator-relay bot has no operator_phone");
276
- return bot as OperatorRelayBot;
277
- }
278
-
279
- async function deliverEscalation(
280
- kind: string,
281
- aggregate: FaultAggregate,
282
- config: FaultEscalationConfig,
283
- ): Promise<{ messageId: string; workItemId: string }> {
284
- const log = getAppLogger();
285
- const now = new Date();
286
-
287
- // 1. Claim the episode BEFORE sending, so a send failure leaves a durable
288
- // claim the next sweep retries instead of a lost escalation.
289
- await (await orpc()).platform_fault_escalations.upsert({
290
- where: { kind },
291
- create: {
292
- kind,
293
- first_seen_at: aggregate.firstSeen.toISOString(),
294
- last_seen_at: aggregate.lastSeen.toISOString(),
295
- event_count: aggregate.count,
296
- },
297
- update: {
298
- first_seen_at: aggregate.firstSeen.toISOString(),
299
- last_seen_at: aggregate.lastSeen.toISOString(),
300
- event_count: aggregate.count,
301
- resolved_at: null,
302
- },
303
- });
304
-
305
- // 2. Resolve the operator relay and send. The send surface is the
306
- // `messaging_whatsapp_send` intrinsic (scala-mcp / messaging-mcp-boot),
307
- // which delegates to the Rust scala-messaging-service — the channel owner
308
- // that picks the device and records the send in messaging_logs. The retired
309
- // api_gowa_message tool (deactivated 2026-08-26) is no longer callable;
310
- // naming it here left every escalation send failing with "Unknown tool".
311
- const bot = await resolveOperatorRelay();
312
- const operatorPhone = (bot.flow_config as { operator_phone?: string }).operator_phone!;
313
- const copy = config.copy[kind] ?? config.default_copy;
314
- const { sample, lastEvent } = await fetchLatestEvidence(kind, config.sample_fields?.[kind] ?? []);
315
- const message = composeEscalationMessage({
316
- kind,
317
- aggregate,
318
- sample,
319
- lastEvent,
320
- impact: copy.impact,
321
- recommendation: copy.recommendation,
322
- });
323
-
324
- const { userId, organisationId } = await getSystemIdentity();
325
- const systemCtx: ExecutionContext = { userId, organisationId } as ExecutionContext;
326
- const { forwardToolToCentralMcp } = await import("@teamscala/chat-completion/central-tool-forwarder");
327
- const sendResult = await forwardToolToCentralMcp(
328
- "messaging_whatsapp_send",
329
- { phone: operatorPhone, message },
330
- systemCtx,
331
- );
332
-
333
- // 3. VERIFY delivery before recording it: the send must return a WhatsApp
334
- // message id. No id => throw, the claim stays undelivered, next sweep retries.
335
- const messageId = extractWhatsAppMessageId(sendResult);
336
- if (!messageId) {
337
- throw new Error(
338
- `platform-fault-escalation: WhatsApp send for '${kind}' returned no message id — not recording delivery. ` +
339
- `Response: ${JSON.stringify(sendResult).slice(0, 400)}`,
340
- );
341
- }
342
-
343
- // 4. Durable record: a COMMENT on the orchestrator's work item — an
344
- // escalation is a finding the orchestrator/operator triages, not an
345
- // assigned task (the fleet-task-placement constraint refuses task rows
346
- // without session identity by design).
347
- const comment = await (await orpc()).comments.create({
348
- data: {
349
- work_item_id: config.orchestrator_work_item_id,
350
- content: message,
351
- source_cli_session_id: null,
352
- },
353
- });
354
-
355
- // 5. Readback-correct the episode: delivered + linked in one update.
356
- await (await orpc()).platform_fault_escalations.update({
357
- where: { kind },
358
- data: {
359
- escalated_at: now.toISOString(),
360
- work_item_id: config.orchestrator_work_item_id,
361
- delivered_comment_id: comment.id,
362
- delivered_message_id: messageId,
363
- last_updated_at: now.toISOString(),
364
- updated_at: now.toISOString(),
365
- },
366
- });
367
-
368
- recordEvent({
369
- kind: "platform.fault.escalated",
370
- payload: {
371
- event_kind: kind,
372
- count: aggregate.count,
373
- sample,
374
- last_event_id: lastEvent?.eventId ?? null,
375
- last_event_occurred_at: lastEvent?.occurredAt ?? null,
376
- work_item_id: config.orchestrator_work_item_id,
377
- delivered_comment_id: comment.id,
378
- delivered_message_id: messageId,
379
- },
380
- });
381
- return { messageId, workItemId: config.orchestrator_work_item_id };
382
- }
383
-
384
- async function resolveEpisode(episode: EscalationEpisode, quietMinutes: number): Promise<void> {
385
- const now = new Date();
386
- await (await orpc()).platform_fault_escalations.update({
387
- where: { kind: episode.kind },
388
- data: { resolved_at: now.toISOString(), updated_at: now.toISOString() },
389
- });
390
- if (episode.workItemId) {
391
- await (await orpc()).comments.create({
392
- data: {
393
- work_item_id: episode.workItemId,
394
- content: `Fault stopped — no '${episode.kind}' events for ${Math.round(quietMinutes)} min. Episode closed; the fault will re-escalate if it recurs.`,
395
- sender_role: "system",
396
- sender_name: "platform-fault-escalation",
397
- },
398
- });
399
- }
400
- recordEvent({
401
- kind: "platform.fault.resolved",
402
- payload: { event_kind: episode.kind, quiet_minutes: Math.round(quietMinutes), work_item_id: episode.workItemId },
403
- });
404
- }
26
+ import { loadFaultEscalationConfig } from "./platform-fault-escalation/config.ts";
27
+ import { fetchAggregates } from "./platform-fault-escalation/event-log-queries.ts";
28
+ import { orpc } from "./platform-fault-escalation/orpc-client.ts";
29
+ import {
30
+ deliverEscalation,
31
+ resolveEpisode,
32
+ } from "./platform-fault-escalation/delivery.ts";
405
33
 
406
34
  async function platformFaultEscalationSweep(ctx: { config: ScheduledJobConfig }): Promise<void> {
407
35
  const log = getAppLogger();
408
- const config = await loadConfig();
36
+ const config = await loadFaultEscalationConfig();
409
37
  const now = new Date();
410
38
 
411
39
  const aggregates = await fetchAggregates(config.watched_kind_patterns, config.window_minutes);