@teamscala/service-boot 0.23.418 → 0.23.419
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/handlers/platform-fault-escalation/config.ts +54 -0
- package/src/handlers/platform-fault-escalation/delivery.ts +173 -0
- package/src/handlers/platform-fault-escalation/event-log-queries.ts +122 -0
- package/src/handlers/platform-fault-escalation/orpc-client.ts +97 -0
- package/src/handlers/platform-fault-escalation.ts +15 -387
package/package.json
CHANGED
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @system server-boot
|
|
3
|
+
* @status handwritten
|
|
4
|
+
* @edit edit directly
|
|
5
|
+
*
|
|
6
|
+
* The config row of the platform-fault-escalation pg-boss handler
|
|
7
|
+
* (config/platform-fault-escalation).
|
|
8
|
+
*
|
|
9
|
+
* Split out of handlers/platform-fault-escalation.ts (500 lines) —
|
|
10
|
+
* byte-for-byte move; import paths adjusted only.
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
import { loadRegistryConfig } from "@teamscala/db/registry/load-config";
|
|
14
|
+
import * as v from "valibot";
|
|
15
|
+
|
|
16
|
+
export const CONFIG_SLUG = "platform-fault-escalation";
|
|
17
|
+
|
|
18
|
+
export const FaultEscalationConfigSchema = v.object({
|
|
19
|
+
/** Postgres regex patterns selecting watched event_log kinds. */
|
|
20
|
+
watched_kind_patterns: v.array(v.string()),
|
|
21
|
+
/** Lookback window for the recurrence/duration aggregate, minutes. */
|
|
22
|
+
window_minutes: v.number(),
|
|
23
|
+
min_count: v.number(),
|
|
24
|
+
min_span_minutes: v.number(),
|
|
25
|
+
/** Per-kind recurrence/duration overrides — low-rate-but-persistent kinds. */
|
|
26
|
+
kind_thresholds: v.optional(
|
|
27
|
+
v.record(
|
|
28
|
+
v.string(),
|
|
29
|
+
v.object({
|
|
30
|
+
min_count: v.optional(v.number()),
|
|
31
|
+
min_span_minutes: v.optional(v.number()),
|
|
32
|
+
}),
|
|
33
|
+
),
|
|
34
|
+
),
|
|
35
|
+
resolve_after_minutes: v.number(),
|
|
36
|
+
/** The orchestrator's work item — escalations are COMMENTS on it, never
|
|
37
|
+
* task rows (the fleet-task-placement CHECK constraint refuses a system
|
|
38
|
+
* task without session identity, and an escalation is a finding, not an
|
|
39
|
+
* assigned unit of work). */
|
|
40
|
+
orchestrator_work_item_id: v.string(),
|
|
41
|
+
/** Per-kind copy for the message (impact + recommendation). Falls back to default_copy. */
|
|
42
|
+
copy: v.record(v.string(), v.object({ impact: v.string(), recommendation: v.string() })),
|
|
43
|
+
default_copy: v.object({ impact: v.string(), recommendation: v.string() }),
|
|
44
|
+
/** Per-kind payload-field whitelist sampled into the message's evidence
|
|
45
|
+
* line — named fields ONLY, never the whole payload: some kinds embed
|
|
46
|
+
* request context, and the escalation channel is WhatsApp. */
|
|
47
|
+
sample_fields: v.optional(v.record(v.string(), v.array(v.string()))),
|
|
48
|
+
});
|
|
49
|
+
|
|
50
|
+
export type FaultEscalationConfig = v.InferOutput<typeof FaultEscalationConfigSchema>;
|
|
51
|
+
|
|
52
|
+
export function loadFaultEscalationConfig(): Promise<FaultEscalationConfig> {
|
|
53
|
+
return loadRegistryConfig("config", CONFIG_SLUG, FaultEscalationConfigSchema);
|
|
54
|
+
}
|
|
@@ -0,0 +1,173 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @system server-boot
|
|
3
|
+
* @status handwritten
|
|
4
|
+
* @edit edit directly
|
|
5
|
+
*
|
|
6
|
+
* Escalation delivery of the platform-fault-escalation handler: claim the
|
|
7
|
+
* episode, resolve the operator-relay WhatsApp bot, send via
|
|
8
|
+
* messaging_whatsapp_send, verify the message id, record the comment +
|
|
9
|
+
* readback-corrected episode — and close a quiet episode.
|
|
10
|
+
*
|
|
11
|
+
* Split out of handlers/platform-fault-escalation.ts (500 lines) —
|
|
12
|
+
* byte-for-byte move; import paths adjusted only.
|
|
13
|
+
*/
|
|
14
|
+
|
|
15
|
+
import type { ExecutionContext } from "@teamscala/os/contracts/mcp";
|
|
16
|
+
import { recordEvent } from "@teamscala/event-log/record-event";
|
|
17
|
+
import {
|
|
18
|
+
composeEscalationMessage,
|
|
19
|
+
extractWhatsAppMessageId,
|
|
20
|
+
type EscalationEpisode,
|
|
21
|
+
type FaultAggregate,
|
|
22
|
+
} from "../platform-fault-escalation-logic.ts";
|
|
23
|
+
import type { FaultEscalationConfig } from "./config.ts";
|
|
24
|
+
import { fetchLatestEvidence } from "./event-log-queries.ts";
|
|
25
|
+
import { getSystemIdentity, orpc } from "./orpc-client.ts";
|
|
26
|
+
|
|
27
|
+
interface OperatorRelayBot {
|
|
28
|
+
id: string;
|
|
29
|
+
organisation_id: string;
|
|
30
|
+
external_id: string | null;
|
|
31
|
+
flow_config: { operator_phone?: string } | null;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
async function resolveOperatorRelay(): Promise<OperatorRelayBot> {
|
|
35
|
+
const bot = (await (await orpc()).messaging_bots.findFirst({
|
|
36
|
+
where: {
|
|
37
|
+
is_active: true,
|
|
38
|
+
platform: "whatsapp",
|
|
39
|
+
flow_config: { path: ["mode"], equals: "operator-relay" },
|
|
40
|
+
},
|
|
41
|
+
select: { id: true, organisation_id: true, external_id: true, flow_config: true },
|
|
42
|
+
})) as unknown as OperatorRelayBot | null;
|
|
43
|
+
if (!bot) throw new Error("platform-fault-escalation: no active operator-relay WhatsApp bot row");
|
|
44
|
+
const phone = (bot.flow_config as { operator_phone?: string } | null)?.operator_phone;
|
|
45
|
+
if (!phone) throw new Error("platform-fault-escalation: operator-relay bot has no operator_phone");
|
|
46
|
+
return bot as OperatorRelayBot;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
export async function deliverEscalation(
|
|
50
|
+
kind: string,
|
|
51
|
+
aggregate: FaultAggregate,
|
|
52
|
+
config: FaultEscalationConfig,
|
|
53
|
+
): Promise<{ messageId: string; workItemId: string }> {
|
|
54
|
+
const now = new Date();
|
|
55
|
+
|
|
56
|
+
// 1. Claim the episode BEFORE sending, so a send failure leaves a durable
|
|
57
|
+
// claim the next sweep retries instead of a lost escalation.
|
|
58
|
+
await (await orpc()).platform_fault_escalations.upsert({
|
|
59
|
+
where: { kind },
|
|
60
|
+
create: {
|
|
61
|
+
kind,
|
|
62
|
+
first_seen_at: aggregate.firstSeen.toISOString(),
|
|
63
|
+
last_seen_at: aggregate.lastSeen.toISOString(),
|
|
64
|
+
event_count: aggregate.count,
|
|
65
|
+
},
|
|
66
|
+
update: {
|
|
67
|
+
first_seen_at: aggregate.firstSeen.toISOString(),
|
|
68
|
+
last_seen_at: aggregate.lastSeen.toISOString(),
|
|
69
|
+
event_count: aggregate.count,
|
|
70
|
+
resolved_at: null,
|
|
71
|
+
},
|
|
72
|
+
});
|
|
73
|
+
|
|
74
|
+
// 2. Resolve the operator relay and send. The send surface is the
|
|
75
|
+
// `messaging_whatsapp_send` intrinsic (scala-mcp / messaging-mcp-boot),
|
|
76
|
+
// which delegates to the Rust scala-messaging-service — the channel owner
|
|
77
|
+
// that picks the device and records the send in messaging_logs. The retired
|
|
78
|
+
// api_gowa_message tool (deactivated 2026-08-26) is no longer callable;
|
|
79
|
+
// naming it here left every escalation send failing with "Unknown tool".
|
|
80
|
+
const bot = await resolveOperatorRelay();
|
|
81
|
+
const operatorPhone = (bot.flow_config as { operator_phone?: string }).operator_phone!;
|
|
82
|
+
const copy = config.copy[kind] ?? config.default_copy;
|
|
83
|
+
const { sample, lastEvent } = await fetchLatestEvidence(kind, config.sample_fields?.[kind] ?? []);
|
|
84
|
+
const message = composeEscalationMessage({
|
|
85
|
+
kind,
|
|
86
|
+
aggregate,
|
|
87
|
+
sample,
|
|
88
|
+
lastEvent,
|
|
89
|
+
impact: copy.impact,
|
|
90
|
+
recommendation: copy.recommendation,
|
|
91
|
+
});
|
|
92
|
+
|
|
93
|
+
const { userId, organisationId } = await getSystemIdentity();
|
|
94
|
+
const systemCtx: ExecutionContext = { userId, organisationId } as ExecutionContext;
|
|
95
|
+
const { forwardToolToCentralMcp } = await import("@teamscala/chat-completion/central-tool-forwarder");
|
|
96
|
+
const sendResult = await forwardToolToCentralMcp(
|
|
97
|
+
"messaging_whatsapp_send",
|
|
98
|
+
{ phone: operatorPhone, message },
|
|
99
|
+
systemCtx,
|
|
100
|
+
);
|
|
101
|
+
|
|
102
|
+
// 3. VERIFY delivery before recording it: the send must return a WhatsApp
|
|
103
|
+
// message id. No id => throw, the claim stays undelivered, next sweep retries.
|
|
104
|
+
const messageId = extractWhatsAppMessageId(sendResult);
|
|
105
|
+
if (!messageId) {
|
|
106
|
+
throw new Error(
|
|
107
|
+
`platform-fault-escalation: WhatsApp send for '${kind}' returned no message id — not recording delivery. ` +
|
|
108
|
+
`Response: ${JSON.stringify(sendResult).slice(0, 400)}`,
|
|
109
|
+
);
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
// 4. Durable record: a COMMENT on the orchestrator's work item — an
|
|
113
|
+
// escalation is a finding the orchestrator/operator triages, not an
|
|
114
|
+
// assigned task (the fleet-task-placement constraint refuses task rows
|
|
115
|
+
// without session identity by design).
|
|
116
|
+
const comment = await (await orpc()).comments.create({
|
|
117
|
+
data: {
|
|
118
|
+
work_item_id: config.orchestrator_work_item_id,
|
|
119
|
+
content: message,
|
|
120
|
+
source_cli_session_id: null,
|
|
121
|
+
},
|
|
122
|
+
});
|
|
123
|
+
|
|
124
|
+
// 5. Readback-correct the episode: delivered + linked in one update.
|
|
125
|
+
await (await orpc()).platform_fault_escalations.update({
|
|
126
|
+
where: { kind },
|
|
127
|
+
data: {
|
|
128
|
+
escalated_at: now.toISOString(),
|
|
129
|
+
work_item_id: config.orchestrator_work_item_id,
|
|
130
|
+
delivered_comment_id: comment.id,
|
|
131
|
+
delivered_message_id: messageId,
|
|
132
|
+
last_updated_at: now.toISOString(),
|
|
133
|
+
updated_at: now.toISOString(),
|
|
134
|
+
},
|
|
135
|
+
});
|
|
136
|
+
|
|
137
|
+
recordEvent({
|
|
138
|
+
kind: "platform.fault.escalated",
|
|
139
|
+
payload: {
|
|
140
|
+
event_kind: kind,
|
|
141
|
+
count: aggregate.count,
|
|
142
|
+
sample,
|
|
143
|
+
last_event_id: lastEvent?.eventId ?? null,
|
|
144
|
+
last_event_occurred_at: lastEvent?.occurredAt ?? null,
|
|
145
|
+
work_item_id: config.orchestrator_work_item_id,
|
|
146
|
+
delivered_comment_id: comment.id,
|
|
147
|
+
delivered_message_id: messageId,
|
|
148
|
+
},
|
|
149
|
+
});
|
|
150
|
+
return { messageId, workItemId: config.orchestrator_work_item_id };
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
export async function resolveEpisode(episode: EscalationEpisode, quietMinutes: number): Promise<void> {
|
|
154
|
+
const now = new Date();
|
|
155
|
+
await (await orpc()).platform_fault_escalations.update({
|
|
156
|
+
where: { kind: episode.kind },
|
|
157
|
+
data: { resolved_at: now.toISOString(), updated_at: now.toISOString() },
|
|
158
|
+
});
|
|
159
|
+
if (episode.workItemId) {
|
|
160
|
+
await (await orpc()).comments.create({
|
|
161
|
+
data: {
|
|
162
|
+
work_item_id: episode.workItemId,
|
|
163
|
+
content: `Fault stopped — no '${episode.kind}' events for ${Math.round(quietMinutes)} min. Episode closed; the fault will re-escalate if it recurs.`,
|
|
164
|
+
sender_role: "system",
|
|
165
|
+
sender_name: "platform-fault-escalation",
|
|
166
|
+
},
|
|
167
|
+
});
|
|
168
|
+
}
|
|
169
|
+
recordEvent({
|
|
170
|
+
kind: "platform.fault.resolved",
|
|
171
|
+
payload: { event_kind: episode.kind, quiet_minutes: Math.round(quietMinutes), work_item_id: episode.workItemId },
|
|
172
|
+
});
|
|
173
|
+
}
|
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @system server-boot
|
|
3
|
+
* @status handwritten
|
|
4
|
+
* @edit edit directly
|
|
5
|
+
*
|
|
6
|
+
* The bounded event_log reads of the platform-fault-escalation handler: the
|
|
7
|
+
* recurrence/duration aggregate and the latest-payload evidence sample, both
|
|
8
|
+
* with the `occurred_at <= now()` database-clock top edge.
|
|
9
|
+
*
|
|
10
|
+
* Split out of handlers/platform-fault-escalation.ts (500 lines) —
|
|
11
|
+
* byte-for-byte move; import paths adjusted only.
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
import { getPrisma } from "@teamscala/db/prisma-registry";
|
|
15
|
+
import { formatEscalationSample, type FaultAggregate } from "../platform-fault-escalation-logic.ts";
|
|
16
|
+
import { CONFIG_SLUG } from "./config.ts";
|
|
17
|
+
|
|
18
|
+
interface EventLogAggregateRow {
|
|
19
|
+
kind: string;
|
|
20
|
+
n: bigint;
|
|
21
|
+
first_seen: Date;
|
|
22
|
+
last_seen: Date;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
export async function fetchAggregates(patterns: string[], windowMinutes: number): Promise<FaultAggregate[]> {
|
|
26
|
+
const prisma = getPrisma();
|
|
27
|
+
// Patterns come from the valibot-validated registry row and are regex-only
|
|
28
|
+
// by construction below; embedding them as one alternation keeps the query
|
|
29
|
+
// parameter-free on the pattern side while the window is a real parameter.
|
|
30
|
+
// The window is bounded on BOTH sides by the database's own statement
|
|
31
|
+
// clock: `occurred_at` is producer-claimed, and an unbounded top edge
|
|
32
|
+
// admits rows stamped in the future by a skewed or mocked producer clock —
|
|
33
|
+
// which then fabricate "still recurring" for a quiet fault (measured
|
|
34
|
+
// 2026-08-26). A future row is not evidence of anything happening now.
|
|
35
|
+
const regexClass = /^[A-Za-z0-9_\\^$.*+?()[\]{}|-]+$/;
|
|
36
|
+
for (const pattern of patterns) {
|
|
37
|
+
if (!regexClass.test(pattern)) {
|
|
38
|
+
throw new Error(`config/${CONFIG_SLUG}: watched_kind_pattern is not regex-only: ${pattern}`);
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
const combined = `(${patterns.join("|")})`;
|
|
42
|
+
const rows = await prisma.$queryRawUnsafe<EventLogAggregateRow[]>(
|
|
43
|
+
`SELECT kind, count(*)::bigint AS n, min(occurred_at) AS first_seen, max(occurred_at) AS last_seen
|
|
44
|
+
FROM event_log
|
|
45
|
+
WHERE occurred_at > now() - ($1::int * interval '1 minute')
|
|
46
|
+
AND occurred_at <= now()
|
|
47
|
+
AND kind ~ $2::text
|
|
48
|
+
GROUP BY kind`,
|
|
49
|
+
windowMinutes,
|
|
50
|
+
combined,
|
|
51
|
+
);
|
|
52
|
+
return rows.map((row) => ({
|
|
53
|
+
kind: row.kind,
|
|
54
|
+
count: Number(row.n),
|
|
55
|
+
firstSeen: new Date(row.first_seen),
|
|
56
|
+
lastSeen: new Date(row.last_seen),
|
|
57
|
+
}));
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* The message's evidence: the whitelisted sample fields AND the row identity
|
|
62
|
+
* (id + occurred_at) of the newest payload for the kind. Field names come
|
|
63
|
+
* from the valibot-validated config row and are constrained to bare
|
|
64
|
+
* identifiers below, so the JSON path cannot break out of the string. The
|
|
65
|
+
* `occurred_at <= now()` bound matches the aggregate's top edge: the cited
|
|
66
|
+
* "last" row must be a real, non-future event whose id rides in the message
|
|
67
|
+
* so the claim resolves against event_log.
|
|
68
|
+
*/
|
|
69
|
+
export async function fetchLatestEvidence(
|
|
70
|
+
kind: string,
|
|
71
|
+
fields: string[],
|
|
72
|
+
): Promise<{ sample: string; lastEvent: { occurredAt: string; eventId: string } | null }> {
|
|
73
|
+
const prisma = getPrisma();
|
|
74
|
+
if (fields.length === 0) {
|
|
75
|
+
const rows = await prisma.$queryRawUnsafe<Array<{ id: unknown; occurred_at: Date }>>(
|
|
76
|
+
`SELECT id::text, occurred_at
|
|
77
|
+
FROM event_log
|
|
78
|
+
WHERE kind = $1::text AND occurred_at <= now()
|
|
79
|
+
ORDER BY occurred_at DESC
|
|
80
|
+
LIMIT 1`,
|
|
81
|
+
kind,
|
|
82
|
+
);
|
|
83
|
+
const top = rows[0];
|
|
84
|
+
return { sample: "", lastEvent: top ? { occurredAt: isoSeconds(top.occurred_at), eventId: String(top.id) } : null };
|
|
85
|
+
}
|
|
86
|
+
for (const field of fields) {
|
|
87
|
+
if (!/^[A-Za-z0-9_]+$/.test(field)) {
|
|
88
|
+
throw new Error(`config/${CONFIG_SLUG}: sample_fields entry is not a bare field name: ${field}`);
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
const selection = fields.map((field) => `payload_json->'${field}' AS "${field}"`).join(", ");
|
|
92
|
+
const rows = await prisma.$queryRawUnsafe<Array<Record<string, unknown>>>(
|
|
93
|
+
`SELECT id::text, occurred_at, ${selection}
|
|
94
|
+
FROM event_log
|
|
95
|
+
WHERE kind = $1::text AND occurred_at <= now()
|
|
96
|
+
ORDER BY occurred_at DESC
|
|
97
|
+
LIMIT 1`,
|
|
98
|
+
kind,
|
|
99
|
+
);
|
|
100
|
+
const row = rows[0];
|
|
101
|
+
if (!row) return { sample: "", lastEvent: null };
|
|
102
|
+
const parts: string[] = [];
|
|
103
|
+
for (const field of fields) {
|
|
104
|
+
const value = row[field];
|
|
105
|
+
if (value === null || value === undefined) continue;
|
|
106
|
+
const formatted = formatEscalationSample(value);
|
|
107
|
+
if (formatted) parts.push(`${field}=${formatted}`);
|
|
108
|
+
}
|
|
109
|
+
return {
|
|
110
|
+
sample: parts.join("; "),
|
|
111
|
+
lastEvent: {
|
|
112
|
+
occurredAt: isoSeconds(row.occurred_at as string | Date),
|
|
113
|
+
eventId: String(row.id),
|
|
114
|
+
},
|
|
115
|
+
};
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
/** ISO-8601 to whole seconds — the same precision event_log stores, so the
|
|
119
|
+
* cited string round-trips a `WHERE occurred_at = '<cited>'` query. */
|
|
120
|
+
function isoSeconds(value: Date | string): string {
|
|
121
|
+
return new Date(value).toISOString().replace(/\.\d{3}Z$/, "Z");
|
|
122
|
+
}
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @system server-boot
|
|
3
|
+
* @status handwritten
|
|
4
|
+
* @edit edit directly
|
|
5
|
+
*
|
|
6
|
+
* The system-authed in-process ORPC client of the platform-fault-escalation
|
|
7
|
+
* handler (lazy, built from the composed appRouter) and the system identity
|
|
8
|
+
* it falls back to.
|
|
9
|
+
*
|
|
10
|
+
* Split out of handlers/platform-fault-escalation.ts (500 lines) —
|
|
11
|
+
* byte-for-byte move; import paths adjusted only.
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
import { loadRegistryConfig } from "@teamscala/db/registry/load-config";
|
|
15
|
+
import * as v from "valibot";
|
|
16
|
+
|
|
17
|
+
let cachedSystemIdentity: { userId: string | undefined; organisationId: string | undefined } | undefined;
|
|
18
|
+
export async function getSystemIdentity(): Promise<{ userId: string | undefined; organisationId: string | undefined }> {
|
|
19
|
+
if (cachedSystemIdentity) return cachedSystemIdentity;
|
|
20
|
+
const row = await loadRegistryConfig("config", "system-identity", v.looseObject({}));
|
|
21
|
+
cachedSystemIdentity = {
|
|
22
|
+
userId: row?.system_user_id as string | undefined,
|
|
23
|
+
organisationId: row?.system_org_id as string | undefined,
|
|
24
|
+
};
|
|
25
|
+
return cachedSystemIdentity;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* The system-authed in-process ORPC client, built LAZILY on first use from the
|
|
30
|
+
* SAME composed appRouter the /api/rpc mount uses (getAppRouter — never a raw
|
|
31
|
+
* re-import of the generated routers, per register-page-server-client's
|
|
32
|
+
* documented module-identity trap). Built here rather than read from
|
|
33
|
+
* page-server-orpc/server-client because a pg-boss handler can fire on a
|
|
34
|
+
* BACKEND service, where the page-server client is never registered — the
|
|
35
|
+
* empty-router fallback returns a client with no model namespaces (the
|
|
36
|
+
* TypeError this replaced). Per `no-direct-prisma-outside-orpc`, every WRITE
|
|
37
|
+
* in this handler routes through this client; the two raw reads (the
|
|
38
|
+
* event_log recurrence aggregate and the latest-payload sample, shapes no
|
|
39
|
+
* generated CRUD expresses) use
|
|
40
|
+
* the assigned-client form with $queryRawUnsafe. Timestamps are ISO strings,
|
|
41
|
+
* never Date instances (the generated valibot schemas require it).
|
|
42
|
+
*/
|
|
43
|
+
let cachedOrpc: OrpcSurface | null = null;
|
|
44
|
+
export async function orpc(): Promise<OrpcSurface> {
|
|
45
|
+
if (cachedOrpc) return cachedOrpc;
|
|
46
|
+
const { createRouterClient } = await import("@orpc/server");
|
|
47
|
+
const { getAppRouter } = await import("@teamscala/orpc/app-router");
|
|
48
|
+
const appRouter = (await getAppRouter()) as Record<string, unknown> | null;
|
|
49
|
+
if (!appRouter) {
|
|
50
|
+
throw new Error(
|
|
51
|
+
"platform-fault-escalation: no ORPC appRouter — the service has no generated routers to write through",
|
|
52
|
+
);
|
|
53
|
+
}
|
|
54
|
+
const prismaRegistry = await import("@teamscala/db/prisma-registry");
|
|
55
|
+
const registryLoader = await import("@teamscala/db/registry/load-config");
|
|
56
|
+
const { PlatformIdentitySchema } = await import(
|
|
57
|
+
"@teamscala/db-validation/registry-schemas/platform-identity"
|
|
58
|
+
);
|
|
59
|
+
const platformIdentity = await registryLoader.loadRegistryConfig(
|
|
60
|
+
"config",
|
|
61
|
+
"platform-identity",
|
|
62
|
+
PlatformIdentitySchema,
|
|
63
|
+
);
|
|
64
|
+
cachedOrpc = createRouterClient(
|
|
65
|
+
{ ...appRouter } as never,
|
|
66
|
+
{
|
|
67
|
+
context: {
|
|
68
|
+
prisma: prismaRegistry.getPrisma(),
|
|
69
|
+
user: {
|
|
70
|
+
id: "system",
|
|
71
|
+
isSuperAdmin: true,
|
|
72
|
+
organisationId: platformIdentity.platformOrganisationId,
|
|
73
|
+
},
|
|
74
|
+
},
|
|
75
|
+
} as never,
|
|
76
|
+
) as unknown as OrpcSurface;
|
|
77
|
+
return cachedOrpc;
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/** The router subset this handler uses — mirrors the generated procedure args. */
|
|
81
|
+
export interface OrpcSurface {
|
|
82
|
+
platform_fault_escalations: {
|
|
83
|
+
upsert(args: {
|
|
84
|
+
where: { kind: string };
|
|
85
|
+
create: Record<string, unknown>;
|
|
86
|
+
update: Record<string, unknown>;
|
|
87
|
+
}): Promise<unknown>;
|
|
88
|
+
update(args: { where: { kind: string }; data: Record<string, unknown> }): Promise<unknown>;
|
|
89
|
+
findMany(args?: { where?: Record<string, unknown> }): Promise<Array<Record<string, unknown>>>;
|
|
90
|
+
};
|
|
91
|
+
messaging_bots: {
|
|
92
|
+
findFirst(args: Record<string, unknown>): Promise<Record<string, unknown> | null>;
|
|
93
|
+
};
|
|
94
|
+
comments: {
|
|
95
|
+
create(args: { data: Record<string, unknown> }): Promise<{ id: string }>;
|
|
96
|
+
};
|
|
97
|
+
}
|
|
@@ -1,411 +1,39 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* @system server-boot
|
|
3
|
-
* @status handwritten
|
|
3
|
+
* @status handwritten
|
|
4
4
|
* @edit edit directly
|
|
5
5
|
*
|
|
6
6
|
* The `platform-fault-escalation` pg-boss handler. Full contract in curated
|
|
7
7
|
* `apps/escalation.md` — episodes EMIT ON TRANSITIONS ONLY; evidence bounded
|
|
8
8
|
* `occurred_at <= now()` and cited by resolvable event id; delivery verified
|
|
9
9
|
* by WhatsApp message id before `escalated_at` is recorded.
|
|
10
|
+
*
|
|
11
|
+
* This module is the SWEEP unit (load config → aggregate → decide → deliver/
|
|
12
|
+
* resolve per kind, isolated). The machinery lives as siblings under
|
|
13
|
+
* ./platform-fault-escalation/ (config, event-log-queries, orpc-client,
|
|
14
|
+
* delivery) — split when the parent held them all at 500 lines; the pure
|
|
15
|
+
* decision logic was already platform-fault-escalation-logic.ts.
|
|
10
16
|
*/
|
|
11
|
-
import type { ExecutionContext } from "@teamscala/os/contracts/mcp";
|
|
12
17
|
import type { ScheduledJobConfig } from "@teamscala/db-validation/registry-schemas/scheduled-job";
|
|
13
|
-
import * as v from "valibot";
|
|
14
|
-
import { getPrisma } from "@teamscala/db/prisma-registry";
|
|
15
|
-
import { loadRegistryConfig } from "@teamscala/db/registry/load-config";
|
|
16
18
|
import { recordEvent } from "@teamscala/event-log/record-event";
|
|
17
19
|
import { getAppLogger } from "@teamscala/logger/app-loggers";
|
|
18
20
|
import { pgBossHandlerRegistry } from "../pg-boss-handler-registry.ts";
|
|
19
21
|
import {
|
|
20
|
-
composeEscalationMessage,
|
|
21
22
|
decideSweep,
|
|
22
|
-
extractWhatsAppMessageId,
|
|
23
|
-
formatEscalationSample,
|
|
24
23
|
type EscalationEpisode,
|
|
25
|
-
type FaultAggregate,
|
|
26
24
|
type KindThresholdOverride,
|
|
27
25
|
} from "./platform-fault-escalation-logic.ts";
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
window_minutes: v.number(),
|
|
36
|
-
min_count: v.number(),
|
|
37
|
-
min_span_minutes: v.number(),
|
|
38
|
-
/** Per-kind recurrence/duration overrides — low-rate-but-persistent kinds. */
|
|
39
|
-
kind_thresholds: v.optional(
|
|
40
|
-
v.record(
|
|
41
|
-
v.string(),
|
|
42
|
-
v.object({
|
|
43
|
-
min_count: v.optional(v.number()),
|
|
44
|
-
min_span_minutes: v.optional(v.number()),
|
|
45
|
-
}),
|
|
46
|
-
),
|
|
47
|
-
),
|
|
48
|
-
resolve_after_minutes: v.number(),
|
|
49
|
-
/** The orchestrator's work item — escalations are COMMENTS on it, never
|
|
50
|
-
* task rows (the fleet-task-placement CHECK constraint refuses a system
|
|
51
|
-
* task without session identity, and an escalation is a finding, not an
|
|
52
|
-
* assigned unit of work). */
|
|
53
|
-
orchestrator_work_item_id: v.string(),
|
|
54
|
-
/** Per-kind copy for the message (impact + recommendation). Falls back to default_copy. */
|
|
55
|
-
copy: v.record(v.string(), v.object({ impact: v.string(), recommendation: v.string() })),
|
|
56
|
-
default_copy: v.object({ impact: v.string(), recommendation: v.string() }),
|
|
57
|
-
/** Per-kind payload-field whitelist sampled into the message's evidence
|
|
58
|
-
* line — named fields ONLY, never the whole payload: some kinds embed
|
|
59
|
-
* request context, and the escalation channel is WhatsApp. */
|
|
60
|
-
sample_fields: v.optional(v.record(v.string(), v.array(v.string()))),
|
|
61
|
-
});
|
|
62
|
-
|
|
63
|
-
type FaultEscalationConfig = v.InferOutput<typeof FaultEscalationConfigSchema>;
|
|
64
|
-
|
|
65
|
-
interface EventLogAggregateRow {
|
|
66
|
-
kind: string;
|
|
67
|
-
n: bigint;
|
|
68
|
-
first_seen: Date;
|
|
69
|
-
last_seen: Date;
|
|
70
|
-
}
|
|
71
|
-
|
|
72
|
-
interface OperatorRelayBot {
|
|
73
|
-
id: string;
|
|
74
|
-
organisation_id: string;
|
|
75
|
-
external_id: string | null;
|
|
76
|
-
flow_config: { operator_phone?: string } | null;
|
|
77
|
-
}
|
|
78
|
-
|
|
79
|
-
let cachedSystemIdentity: { userId: string | undefined; organisationId: string | undefined } | undefined;
|
|
80
|
-
async function getSystemIdentity(): Promise<{ userId: string | undefined; organisationId: string | undefined }> {
|
|
81
|
-
if (cachedSystemIdentity) return cachedSystemIdentity;
|
|
82
|
-
const row = await loadRegistryConfig("config", "system-identity", v.looseObject({}));
|
|
83
|
-
cachedSystemIdentity = {
|
|
84
|
-
userId: row?.system_user_id as string | undefined,
|
|
85
|
-
organisationId: row?.system_org_id as string | undefined,
|
|
86
|
-
};
|
|
87
|
-
return cachedSystemIdentity;
|
|
88
|
-
}
|
|
89
|
-
|
|
90
|
-
/**
|
|
91
|
-
* The system-authed in-process ORPC client, built LAZILY on first use from the
|
|
92
|
-
* SAME composed appRouter the /api/rpc mount uses (getAppRouter — never a raw
|
|
93
|
-
* re-import of the generated routers, per register-page-server-client's
|
|
94
|
-
* documented module-identity trap). Built here rather than read from
|
|
95
|
-
* page-server-orpc/server-client because a pg-boss handler can fire on a
|
|
96
|
-
* BACKEND service, where the page-server client is never registered — the
|
|
97
|
-
* empty-router fallback returns a client with no model namespaces (the
|
|
98
|
-
* TypeError this replaced). Per `no-direct-prisma-outside-orpc`, every WRITE
|
|
99
|
-
* in this handler routes through this client; the two raw reads (the
|
|
100
|
-
* event_log recurrence aggregate and the latest-payload sample, shapes no
|
|
101
|
-
* generated CRUD expresses) use
|
|
102
|
-
* the assigned-client form with $queryRawUnsafe. Timestamps are ISO strings,
|
|
103
|
-
* never Date instances (the generated valibot schemas require it).
|
|
104
|
-
*/
|
|
105
|
-
let cachedOrpc: OrpcSurface | null = null;
|
|
106
|
-
async function orpc(): Promise<OrpcSurface> {
|
|
107
|
-
if (cachedOrpc) return cachedOrpc;
|
|
108
|
-
const { createRouterClient } = await import("@orpc/server");
|
|
109
|
-
const { getAppRouter } = await import("@teamscala/orpc/app-router");
|
|
110
|
-
const appRouter = (await getAppRouter()) as Record<string, unknown> | null;
|
|
111
|
-
if (!appRouter) {
|
|
112
|
-
throw new Error(
|
|
113
|
-
"platform-fault-escalation: no ORPC appRouter — the service has no generated routers to write through",
|
|
114
|
-
);
|
|
115
|
-
}
|
|
116
|
-
const prismaRegistry = await import("@teamscala/db/prisma-registry");
|
|
117
|
-
const registryLoader = await import("@teamscala/db/registry/load-config");
|
|
118
|
-
const { PlatformIdentitySchema } = await import(
|
|
119
|
-
"@teamscala/db-validation/registry-schemas/platform-identity"
|
|
120
|
-
);
|
|
121
|
-
const platformIdentity = await registryLoader.loadRegistryConfig(
|
|
122
|
-
"config",
|
|
123
|
-
"platform-identity",
|
|
124
|
-
PlatformIdentitySchema,
|
|
125
|
-
);
|
|
126
|
-
cachedOrpc = createRouterClient(
|
|
127
|
-
{ ...appRouter } as never,
|
|
128
|
-
{
|
|
129
|
-
context: {
|
|
130
|
-
prisma: prismaRegistry.getPrisma(),
|
|
131
|
-
user: {
|
|
132
|
-
id: "system",
|
|
133
|
-
isSuperAdmin: true,
|
|
134
|
-
organisationId: platformIdentity.platformOrganisationId,
|
|
135
|
-
},
|
|
136
|
-
},
|
|
137
|
-
} as never,
|
|
138
|
-
) as unknown as OrpcSurface;
|
|
139
|
-
return cachedOrpc;
|
|
140
|
-
}
|
|
141
|
-
|
|
142
|
-
/** The router subset this handler uses — mirrors the generated procedure args. */
|
|
143
|
-
interface OrpcSurface {
|
|
144
|
-
platform_fault_escalations: {
|
|
145
|
-
upsert(args: {
|
|
146
|
-
where: { kind: string };
|
|
147
|
-
create: Record<string, unknown>;
|
|
148
|
-
update: Record<string, unknown>;
|
|
149
|
-
}): Promise<unknown>;
|
|
150
|
-
update(args: { where: { kind: string }; data: Record<string, unknown> }): Promise<unknown>;
|
|
151
|
-
findMany(args?: { where?: Record<string, unknown> }): Promise<Array<Record<string, unknown>>>;
|
|
152
|
-
};
|
|
153
|
-
messaging_bots: {
|
|
154
|
-
findFirst(args: Record<string, unknown>): Promise<Record<string, unknown> | null>;
|
|
155
|
-
};
|
|
156
|
-
comments: {
|
|
157
|
-
create(args: { data: Record<string, unknown> }): Promise<{ id: string }>;
|
|
158
|
-
};
|
|
159
|
-
}
|
|
160
|
-
|
|
161
|
-
async function loadConfig(): Promise<FaultEscalationConfig> {
|
|
162
|
-
return loadRegistryConfig("config", CONFIG_SLUG, FaultEscalationConfigSchema);
|
|
163
|
-
}
|
|
164
|
-
|
|
165
|
-
async function fetchAggregates(patterns: string[], windowMinutes: number): Promise<FaultAggregate[]> {
|
|
166
|
-
const prisma = getPrisma();
|
|
167
|
-
// Patterns come from the valibot-validated registry row and are regex-only
|
|
168
|
-
// by construction below; embedding them as one alternation keeps the query
|
|
169
|
-
// parameter-free on the pattern side while the window is a real parameter.
|
|
170
|
-
// The window is bounded on BOTH sides by the database's own statement
|
|
171
|
-
// clock: `occurred_at` is producer-claimed, and an unbounded top edge
|
|
172
|
-
// admits rows stamped in the future by a skewed or mocked producer clock —
|
|
173
|
-
// which then fabricate "still recurring" for a quiet fault (measured
|
|
174
|
-
// 2026-08-26). A future row is not evidence of anything happening now.
|
|
175
|
-
const regexClass = /^[A-Za-z0-9_\\^$.*+?()[\]{}|-]+$/;
|
|
176
|
-
for (const pattern of patterns) {
|
|
177
|
-
if (!regexClass.test(pattern)) {
|
|
178
|
-
throw new Error(`config/${CONFIG_SLUG}: watched_kind_pattern is not regex-only: ${pattern}`);
|
|
179
|
-
}
|
|
180
|
-
}
|
|
181
|
-
const combined = `(${patterns.join("|")})`;
|
|
182
|
-
const rows = await prisma.$queryRawUnsafe<EventLogAggregateRow[]>(
|
|
183
|
-
`SELECT kind, count(*)::bigint AS n, min(occurred_at) AS first_seen, max(occurred_at) AS last_seen
|
|
184
|
-
FROM event_log
|
|
185
|
-
WHERE occurred_at > now() - ($1::int * interval '1 minute')
|
|
186
|
-
AND occurred_at <= now()
|
|
187
|
-
AND kind ~ $2::text
|
|
188
|
-
GROUP BY kind`,
|
|
189
|
-
windowMinutes,
|
|
190
|
-
combined,
|
|
191
|
-
);
|
|
192
|
-
return rows.map((row) => ({
|
|
193
|
-
kind: row.kind,
|
|
194
|
-
count: Number(row.n),
|
|
195
|
-
firstSeen: new Date(row.first_seen),
|
|
196
|
-
lastSeen: new Date(row.last_seen),
|
|
197
|
-
}));
|
|
198
|
-
}
|
|
199
|
-
|
|
200
|
-
/**
|
|
201
|
-
* The message's evidence: the whitelisted sample fields AND the row identity
|
|
202
|
-
* (id + occurred_at) of the newest payload for the kind. Field names come
|
|
203
|
-
* from the valibot-validated config row and are constrained to bare
|
|
204
|
-
* identifiers below, so the JSON path cannot break out of the string. The
|
|
205
|
-
* `occurred_at <= now()` bound matches the aggregate's top edge: the cited
|
|
206
|
-
* "last" row must be a real, non-future event whose id rides in the message
|
|
207
|
-
* so the claim resolves against event_log.
|
|
208
|
-
*/
|
|
209
|
-
async function fetchLatestEvidence(
|
|
210
|
-
kind: string,
|
|
211
|
-
fields: string[],
|
|
212
|
-
): Promise<{ sample: string; lastEvent: { occurredAt: string; eventId: string } | null }> {
|
|
213
|
-
const prisma = getPrisma();
|
|
214
|
-
if (fields.length === 0) {
|
|
215
|
-
const rows = await prisma.$queryRawUnsafe<Array<{ id: unknown; occurred_at: Date }>>(
|
|
216
|
-
`SELECT id::text, occurred_at
|
|
217
|
-
FROM event_log
|
|
218
|
-
WHERE kind = $1::text AND occurred_at <= now()
|
|
219
|
-
ORDER BY occurred_at DESC
|
|
220
|
-
LIMIT 1`,
|
|
221
|
-
kind,
|
|
222
|
-
);
|
|
223
|
-
const top = rows[0];
|
|
224
|
-
return { sample: "", lastEvent: top ? { occurredAt: isoSeconds(top.occurred_at), eventId: String(top.id) } : null };
|
|
225
|
-
}
|
|
226
|
-
for (const field of fields) {
|
|
227
|
-
if (!/^[A-Za-z0-9_]+$/.test(field)) {
|
|
228
|
-
throw new Error(`config/${CONFIG_SLUG}: sample_fields entry is not a bare field name: ${field}`);
|
|
229
|
-
}
|
|
230
|
-
}
|
|
231
|
-
const selection = fields.map((field) => `payload_json->'${field}' AS "${field}"`).join(", ");
|
|
232
|
-
const rows = await prisma.$queryRawUnsafe<Array<Record<string, unknown>>>(
|
|
233
|
-
`SELECT id::text, occurred_at, ${selection}
|
|
234
|
-
FROM event_log
|
|
235
|
-
WHERE kind = $1::text AND occurred_at <= now()
|
|
236
|
-
ORDER BY occurred_at DESC
|
|
237
|
-
LIMIT 1`,
|
|
238
|
-
kind,
|
|
239
|
-
);
|
|
240
|
-
const row = rows[0];
|
|
241
|
-
if (!row) return { sample: "", lastEvent: null };
|
|
242
|
-
const parts: string[] = [];
|
|
243
|
-
for (const field of fields) {
|
|
244
|
-
const value = row[field];
|
|
245
|
-
if (value === null || value === undefined) continue;
|
|
246
|
-
const formatted = formatEscalationSample(value);
|
|
247
|
-
if (formatted) parts.push(`${field}=${formatted}`);
|
|
248
|
-
}
|
|
249
|
-
return {
|
|
250
|
-
sample: parts.join("; "),
|
|
251
|
-
lastEvent: {
|
|
252
|
-
occurredAt: isoSeconds(row.occurred_at as string | Date),
|
|
253
|
-
eventId: String(row.id),
|
|
254
|
-
},
|
|
255
|
-
};
|
|
256
|
-
}
|
|
257
|
-
|
|
258
|
-
/** ISO-8601 to whole seconds — the same precision event_log stores, so the
|
|
259
|
-
* cited string round-trips a `WHERE occurred_at = '<cited>'` query. */
|
|
260
|
-
function isoSeconds(value: Date | string): string {
|
|
261
|
-
return new Date(value).toISOString().replace(/\.\d{3}Z$/, "Z");
|
|
262
|
-
}
|
|
263
|
-
|
|
264
|
-
async function resolveOperatorRelay(): Promise<OperatorRelayBot> {
|
|
265
|
-
const bot = (await (await orpc()).messaging_bots.findFirst({
|
|
266
|
-
where: {
|
|
267
|
-
is_active: true,
|
|
268
|
-
platform: "whatsapp",
|
|
269
|
-
flow_config: { path: ["mode"], equals: "operator-relay" },
|
|
270
|
-
},
|
|
271
|
-
select: { id: true, organisation_id: true, external_id: true, flow_config: true },
|
|
272
|
-
})) as unknown as OperatorRelayBot | null;
|
|
273
|
-
if (!bot) throw new Error("platform-fault-escalation: no active operator-relay WhatsApp bot row");
|
|
274
|
-
const phone = (bot.flow_config as { operator_phone?: string } | null)?.operator_phone;
|
|
275
|
-
if (!phone) throw new Error("platform-fault-escalation: operator-relay bot has no operator_phone");
|
|
276
|
-
return bot as OperatorRelayBot;
|
|
277
|
-
}
|
|
278
|
-
|
|
279
|
-
async function deliverEscalation(
|
|
280
|
-
kind: string,
|
|
281
|
-
aggregate: FaultAggregate,
|
|
282
|
-
config: FaultEscalationConfig,
|
|
283
|
-
): Promise<{ messageId: string; workItemId: string }> {
|
|
284
|
-
const log = getAppLogger();
|
|
285
|
-
const now = new Date();
|
|
286
|
-
|
|
287
|
-
// 1. Claim the episode BEFORE sending, so a send failure leaves a durable
|
|
288
|
-
// claim the next sweep retries instead of a lost escalation.
|
|
289
|
-
await (await orpc()).platform_fault_escalations.upsert({
|
|
290
|
-
where: { kind },
|
|
291
|
-
create: {
|
|
292
|
-
kind,
|
|
293
|
-
first_seen_at: aggregate.firstSeen.toISOString(),
|
|
294
|
-
last_seen_at: aggregate.lastSeen.toISOString(),
|
|
295
|
-
event_count: aggregate.count,
|
|
296
|
-
},
|
|
297
|
-
update: {
|
|
298
|
-
first_seen_at: aggregate.firstSeen.toISOString(),
|
|
299
|
-
last_seen_at: aggregate.lastSeen.toISOString(),
|
|
300
|
-
event_count: aggregate.count,
|
|
301
|
-
resolved_at: null,
|
|
302
|
-
},
|
|
303
|
-
});
|
|
304
|
-
|
|
305
|
-
// 2. Resolve the operator relay and send. The send surface is the
|
|
306
|
-
// `messaging_whatsapp_send` intrinsic (scala-mcp / messaging-mcp-boot),
|
|
307
|
-
// which delegates to the Rust scala-messaging-service — the channel owner
|
|
308
|
-
// that picks the device and records the send in messaging_logs. The retired
|
|
309
|
-
// api_gowa_message tool (deactivated 2026-08-26) is no longer callable;
|
|
310
|
-
// naming it here left every escalation send failing with "Unknown tool".
|
|
311
|
-
const bot = await resolveOperatorRelay();
|
|
312
|
-
const operatorPhone = (bot.flow_config as { operator_phone?: string }).operator_phone!;
|
|
313
|
-
const copy = config.copy[kind] ?? config.default_copy;
|
|
314
|
-
const { sample, lastEvent } = await fetchLatestEvidence(kind, config.sample_fields?.[kind] ?? []);
|
|
315
|
-
const message = composeEscalationMessage({
|
|
316
|
-
kind,
|
|
317
|
-
aggregate,
|
|
318
|
-
sample,
|
|
319
|
-
lastEvent,
|
|
320
|
-
impact: copy.impact,
|
|
321
|
-
recommendation: copy.recommendation,
|
|
322
|
-
});
|
|
323
|
-
|
|
324
|
-
const { userId, organisationId } = await getSystemIdentity();
|
|
325
|
-
const systemCtx: ExecutionContext = { userId, organisationId } as ExecutionContext;
|
|
326
|
-
const { forwardToolToCentralMcp } = await import("@teamscala/chat-completion/central-tool-forwarder");
|
|
327
|
-
const sendResult = await forwardToolToCentralMcp(
|
|
328
|
-
"messaging_whatsapp_send",
|
|
329
|
-
{ phone: operatorPhone, message },
|
|
330
|
-
systemCtx,
|
|
331
|
-
);
|
|
332
|
-
|
|
333
|
-
// 3. VERIFY delivery before recording it: the send must return a WhatsApp
|
|
334
|
-
// message id. No id => throw, the claim stays undelivered, next sweep retries.
|
|
335
|
-
const messageId = extractWhatsAppMessageId(sendResult);
|
|
336
|
-
if (!messageId) {
|
|
337
|
-
throw new Error(
|
|
338
|
-
`platform-fault-escalation: WhatsApp send for '${kind}' returned no message id — not recording delivery. ` +
|
|
339
|
-
`Response: ${JSON.stringify(sendResult).slice(0, 400)}`,
|
|
340
|
-
);
|
|
341
|
-
}
|
|
342
|
-
|
|
343
|
-
// 4. Durable record: a COMMENT on the orchestrator's work item — an
|
|
344
|
-
// escalation is a finding the orchestrator/operator triages, not an
|
|
345
|
-
// assigned task (the fleet-task-placement constraint refuses task rows
|
|
346
|
-
// without session identity by design).
|
|
347
|
-
const comment = await (await orpc()).comments.create({
|
|
348
|
-
data: {
|
|
349
|
-
work_item_id: config.orchestrator_work_item_id,
|
|
350
|
-
content: message,
|
|
351
|
-
source_cli_session_id: null,
|
|
352
|
-
},
|
|
353
|
-
});
|
|
354
|
-
|
|
355
|
-
// 5. Readback-correct the episode: delivered + linked in one update.
|
|
356
|
-
await (await orpc()).platform_fault_escalations.update({
|
|
357
|
-
where: { kind },
|
|
358
|
-
data: {
|
|
359
|
-
escalated_at: now.toISOString(),
|
|
360
|
-
work_item_id: config.orchestrator_work_item_id,
|
|
361
|
-
delivered_comment_id: comment.id,
|
|
362
|
-
delivered_message_id: messageId,
|
|
363
|
-
last_updated_at: now.toISOString(),
|
|
364
|
-
updated_at: now.toISOString(),
|
|
365
|
-
},
|
|
366
|
-
});
|
|
367
|
-
|
|
368
|
-
recordEvent({
|
|
369
|
-
kind: "platform.fault.escalated",
|
|
370
|
-
payload: {
|
|
371
|
-
event_kind: kind,
|
|
372
|
-
count: aggregate.count,
|
|
373
|
-
sample,
|
|
374
|
-
last_event_id: lastEvent?.eventId ?? null,
|
|
375
|
-
last_event_occurred_at: lastEvent?.occurredAt ?? null,
|
|
376
|
-
work_item_id: config.orchestrator_work_item_id,
|
|
377
|
-
delivered_comment_id: comment.id,
|
|
378
|
-
delivered_message_id: messageId,
|
|
379
|
-
},
|
|
380
|
-
});
|
|
381
|
-
return { messageId, workItemId: config.orchestrator_work_item_id };
|
|
382
|
-
}
|
|
383
|
-
|
|
384
|
-
async function resolveEpisode(episode: EscalationEpisode, quietMinutes: number): Promise<void> {
|
|
385
|
-
const now = new Date();
|
|
386
|
-
await (await orpc()).platform_fault_escalations.update({
|
|
387
|
-
where: { kind: episode.kind },
|
|
388
|
-
data: { resolved_at: now.toISOString(), updated_at: now.toISOString() },
|
|
389
|
-
});
|
|
390
|
-
if (episode.workItemId) {
|
|
391
|
-
await (await orpc()).comments.create({
|
|
392
|
-
data: {
|
|
393
|
-
work_item_id: episode.workItemId,
|
|
394
|
-
content: `Fault stopped — no '${episode.kind}' events for ${Math.round(quietMinutes)} min. Episode closed; the fault will re-escalate if it recurs.`,
|
|
395
|
-
sender_role: "system",
|
|
396
|
-
sender_name: "platform-fault-escalation",
|
|
397
|
-
},
|
|
398
|
-
});
|
|
399
|
-
}
|
|
400
|
-
recordEvent({
|
|
401
|
-
kind: "platform.fault.resolved",
|
|
402
|
-
payload: { event_kind: episode.kind, quiet_minutes: Math.round(quietMinutes), work_item_id: episode.workItemId },
|
|
403
|
-
});
|
|
404
|
-
}
|
|
26
|
+
import { loadFaultEscalationConfig } from "./platform-fault-escalation/config.ts";
|
|
27
|
+
import { fetchAggregates } from "./platform-fault-escalation/event-log-queries.ts";
|
|
28
|
+
import { orpc } from "./platform-fault-escalation/orpc-client.ts";
|
|
29
|
+
import {
|
|
30
|
+
deliverEscalation,
|
|
31
|
+
resolveEpisode,
|
|
32
|
+
} from "./platform-fault-escalation/delivery.ts";
|
|
405
33
|
|
|
406
34
|
async function platformFaultEscalationSweep(ctx: { config: ScheduledJobConfig }): Promise<void> {
|
|
407
35
|
const log = getAppLogger();
|
|
408
|
-
const config = await
|
|
36
|
+
const config = await loadFaultEscalationConfig();
|
|
409
37
|
const now = new Date();
|
|
410
38
|
|
|
411
39
|
const aggregates = await fetchAggregates(config.watched_kind_patterns, config.window_minutes);
|