@teamscala/service-boot 0.23.516 → 0.23.518
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +232 -1127
- package/src/agreement-gate/settings.ts +8 -8
- package/src/apply-logger-config.test.ts +46 -0
- package/src/apply-logger-config.ts +31 -0
- package/src/asset-proxy.test.ts +9 -5
- package/src/auth/sign-out.ts +122 -0
- package/src/build-client-assets/hydrate-bundle.ts +14 -4
- package/src/build-client-assets/manifest.ts +5 -18
- package/src/build-client-assets/styles-build.test.ts +11 -8
- package/src/build-client-assets/styles-build.ts +14 -63
- package/src/build-client-assets.test.ts +0 -161
- package/src/build-client-assets.ts +20 -67
- package/src/content-gate/token.ts +9 -2
- package/src/install-logging.ts +10 -26
- package/src/mount-orpc-router.ts +8 -2
- package/src/mutable-app-ref.ts +0 -7
- package/src/page-server-loader-graph.test.ts +31 -0
- package/src/page-server-loader-graph.ts +21 -0
- package/src/pg-boss-mount.ts +3 -16
- package/src/readiness-gate.ts +0 -4
- package/src/register-api-routes.ts +8 -17
- package/src/register-page-server-client.ts +6 -0
- package/src/register-registry-page-route/meta-title.test.ts +50 -0
- package/src/register-registry-page-route/meta-title.ts +1 -1
- package/src/register-registry-page-route/page-data-shared.ts +7 -11
- package/src/register-registry-page-route/resolve-request-graph.test.ts +1 -1
- package/src/register-registry-page-route/resolve-request-graph.ts +1 -2
- package/src/register-registry-page-route.ts +14 -16
- package/src/resolve-brand-favicon.test.ts +63 -0
- package/src/resolve-theme-mode.test.ts +51 -0
- package/src/resolvers/fetch-org-brand.test.ts +48 -0
- package/src/resolvers/website-by-host-map.test.ts +63 -0
- package/src/resolvers/website-by-host-map.ts +15 -3
- package/src/run-boot-hook-list.test.ts +47 -0
- package/src/run-boot-hooks.test.ts +56 -0
- package/src/scala-tools-currency-watchdog.test.ts +43 -0
- package/src/scala-tools-currency-watchdog.ts +0 -5
- package/src/transfers/presigned-upload.ts +165 -0
- package/src/two-phase-minimal-app.test.ts +24 -0
- package/src/two-phase-readiness.test.ts +102 -0
- package/src/types.test.ts +20 -0
- package/src/ui-service.test.ts +24 -0
- package/src/ws-boot-self-test.test.ts +43 -0
- package/src/build-client-assets/input-fingerprint.ts +0 -92
- package/src/build-client-assets/input-hashing.ts +0 -206
- package/src/handlers/bun-stable-watch.ts +0 -86
- package/src/handlers/messaging-send-reconcile-logic.test.ts +0 -79
- package/src/handlers/messaging-send-reconcile-logic.ts +0 -83
- package/src/handlers/messaging-send-recording-logic.test.ts +0 -86
- package/src/handlers/messaging-send-recording-logic.ts +0 -68
- package/src/handlers/messaging-send-recording-watch.test.ts +0 -91
- package/src/handlers/messaging-send-recording-watch.ts +0 -244
- package/src/handlers/messaging-send-sweep-selection.test.ts +0 -47
- package/src/handlers/messaging-send-sweep-selection.ts +0 -35
- package/src/handlers/platform-fault-escalation/config.ts +0 -95
- package/src/handlers/platform-fault-escalation/delivery.ts +0 -327
- package/src/handlers/platform-fault-escalation/event-log-queries.ts +0 -182
- package/src/handlers/platform-fault-escalation/orpc-client.ts +0 -114
- package/src/handlers/platform-fault-escalation/probe.test.ts +0 -63
- package/src/handlers/platform-fault-escalation/probe.ts +0 -19
- package/src/handlers/platform-fault-escalation-logic.ts +0 -382
- package/src/handlers/platform-fault-escalation-refresh.test.ts +0 -72
- package/src/handlers/platform-fault-escalation.test.ts +0 -497
- package/src/handlers/platform-fault-escalation.ts +0 -246
- package/src/handlers/projects-digest.ts +0 -198
- package/src/handlers/quiz-research.ts +0 -178
- package/src/handlers/registry-stale-inactive-sweep.ts +0 -82
- package/src/handlers/scala-tools-currency-watch.test.ts +0 -65
- package/src/handlers/scala-tools-currency-watch.ts +0 -162
- package/src/handlers/singleton-tools-currency-watch.test.ts +0 -51
- package/src/handlers/singleton-tools-currency-watch.ts +0 -380
- package/src/handlers/sweep-event-log.ts +0 -98
- package/src/handlers/upstream-issue-sync.ts +0 -220
- package/src/handlers/whatsapp-liveness-handler.test.ts +0 -56
- package/src/handlers/whatsapp-liveness-logic.ts +0 -53
- package/src/handlers/whatsapp-liveness.test.ts +0 -78
- package/src/handlers/whatsapp-liveness.ts +0 -138
- package/src/interpolate-meta.ts +0 -123
- package/src/load-service-prisma-client.ts +0 -53
- package/src/loader-errors.ts +0 -41
- package/src/read-viewport-from-request.ts +0 -36
- package/src/register-page-server-client.test.ts +0 -185
- package/src/resolve-app-base-url.ts +0 -35
- package/src/resolve-from-loader-config.test.ts +0 -429
- package/src/resolve-from-loader-config.ts +0 -316
- package/src/resolve-service-context.ts +0 -88
|
@@ -1,95 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* @system server-boot
|
|
3
|
-
* @status handwritten
|
|
4
|
-
* @edit edit directly
|
|
5
|
-
*
|
|
6
|
-
* The config row of the platform-fault-escalation pg-boss handler
|
|
7
|
-
* (config/platform-fault-escalation).
|
|
8
|
-
*
|
|
9
|
-
* Split out of handlers/platform-fault-escalation.ts (500 lines) —
|
|
10
|
-
* byte-for-byte move; import paths adjusted only.
|
|
11
|
-
*/
|
|
12
|
-
|
|
13
|
-
import { loadRegistryConfig } from "@teamscala/db/registry/load-config";
|
|
14
|
-
import * as v from "valibot";
|
|
15
|
-
|
|
16
|
-
export const CONFIG_SLUG = "platform-fault-escalation";
|
|
17
|
-
|
|
18
|
-
export const FaultEscalationConfigSchema = v.object({
|
|
19
|
-
/** Postgres regex patterns selecting watched event_log kinds. */
|
|
20
|
-
watched_kind_patterns: v.array(v.string()),
|
|
21
|
-
/** Lookback window for the recurrence/duration aggregate, minutes. */
|
|
22
|
-
window_minutes: v.number(),
|
|
23
|
-
min_count: v.number(),
|
|
24
|
-
min_span_minutes: v.number(),
|
|
25
|
-
/** Per-kind recurrence/duration overrides — low-rate-but-persistent kinds. */
|
|
26
|
-
kind_thresholds: v.optional(
|
|
27
|
-
v.record(
|
|
28
|
-
v.string(),
|
|
29
|
-
v.object({
|
|
30
|
-
min_count: v.optional(v.number()),
|
|
31
|
-
min_span_minutes: v.optional(v.number()),
|
|
32
|
-
}),
|
|
33
|
-
),
|
|
34
|
-
),
|
|
35
|
-
resolve_after_minutes: v.number(),
|
|
36
|
-
/** Kinds a standing agent lane answers, which therefore escalate to the
|
|
37
|
-
* work item WITHOUT paging the operator (operator ruling 2026-08-28: "why
|
|
38
|
-
* is the WhatsApp PA messaging me about stuff that you should be doing").
|
|
39
|
-
* The episode, the comment and the durable record are unchanged — only the
|
|
40
|
-
* phone is spared, so silencing a page never silences the signal. */
|
|
41
|
-
agent_handled_kinds: v.optional(v.array(v.string())),
|
|
42
|
-
/** Per-kind volume sources closing the quiet-vs-dead gap: a fault kind
|
|
43
|
-
* whose producer can DIE maps to the kind whose row VOLUME proves the
|
|
44
|
-
* pipeline alive. Without a source, resolution on quiet is unchanged;
|
|
45
|
-
* with one, resolve additionally requires the volume floor in-window,
|
|
46
|
-
* and quiet-with-zero-volume HOLDS the episode open instead (one hold
|
|
47
|
-
* notice per episode-state — measured 2026-08-27: the dead llm-router
|
|
48
|
-
* emitted zero failures and the sweep closed as healthy). */
|
|
49
|
-
volume_sources: v.optional(
|
|
50
|
-
v.record(
|
|
51
|
-
v.string(),
|
|
52
|
-
v.object({
|
|
53
|
-
volume_kind_pattern: v.string(),
|
|
54
|
-
min_volume: v.optional(v.number()),
|
|
55
|
-
}),
|
|
56
|
-
),
|
|
57
|
-
),
|
|
58
|
-
/** Per-kind CAPABILITY PROBES gating the recovery transition (operator
|
|
59
|
-
* ruling 2026-08-28: a layer emitting a degraded transition owes a
|
|
60
|
-
* recovery transition backed by a capability probe, never a timer over
|
|
61
|
-
* an empty event stream). A kind declared here resolves only when its
|
|
62
|
-
* probe answers HTTP OK; an unreachable probe HOLDS the episode open —
|
|
63
|
-
* once per episode-state, never an alarm. The volume floor above is
|
|
64
|
-
* still event-stream evidence and cannot answer alone: its window is
|
|
65
|
-
* cumulative, so hours of healthy history mask a just-died producer
|
|
66
|
-
* (measured 2026-08-28: llm.provider.call.failed resolved after 31
|
|
67
|
-
* quiet minutes on 12h-old volume while the pipeline was down). */
|
|
68
|
-
probe_sources: v.optional(
|
|
69
|
-
v.record(
|
|
70
|
-
v.string(),
|
|
71
|
-
v.object({
|
|
72
|
-
url: v.pipe(v.string(), v.minLength(1)),
|
|
73
|
-
timeout_seconds: v.optional(v.number()),
|
|
74
|
-
}),
|
|
75
|
-
),
|
|
76
|
-
),
|
|
77
|
-
/** The orchestrator's work item — escalations are COMMENTS on it, never
|
|
78
|
-
* task rows (the fleet-task-placement CHECK constraint refuses a system
|
|
79
|
-
* task without session identity, and an escalation is a finding, not an
|
|
80
|
-
* assigned unit of work). */
|
|
81
|
-
orchestrator_work_item_id: v.string(),
|
|
82
|
-
/** Per-kind copy for the message (impact + recommendation). Falls back to default_copy. */
|
|
83
|
-
copy: v.record(v.string(), v.object({ impact: v.string(), recommendation: v.string() })),
|
|
84
|
-
default_copy: v.object({ impact: v.string(), recommendation: v.string() }),
|
|
85
|
-
/** Per-kind payload-field whitelist sampled into the message's evidence
|
|
86
|
-
* line — named fields ONLY, never the whole payload: some kinds embed
|
|
87
|
-
* request context, and the escalation channel is WhatsApp. */
|
|
88
|
-
sample_fields: v.optional(v.record(v.string(), v.array(v.string()))),
|
|
89
|
-
});
|
|
90
|
-
|
|
91
|
-
export type FaultEscalationConfig = v.InferOutput<typeof FaultEscalationConfigSchema>;
|
|
92
|
-
|
|
93
|
-
export function loadFaultEscalationConfig(): Promise<FaultEscalationConfig> {
|
|
94
|
-
return loadRegistryConfig("config", CONFIG_SLUG, FaultEscalationConfigSchema);
|
|
95
|
-
}
|
|
@@ -1,327 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* @system server-boot
|
|
3
|
-
* @status handwritten
|
|
4
|
-
* @edit edit directly
|
|
5
|
-
*
|
|
6
|
-
* Escalation delivery of the platform-fault-escalation handler: claim the
|
|
7
|
-
* episode, resolve the operator-relay WhatsApp bot, send via
|
|
8
|
-
* messaging_whatsapp_send, verify the message id, record the comment +
|
|
9
|
-
* readback-corrected episode — and close a quiet episode.
|
|
10
|
-
*
|
|
11
|
-
* Split out of handlers/platform-fault-escalation.ts (500 lines) —
|
|
12
|
-
* byte-for-byte move; import paths adjusted only.
|
|
13
|
-
*/
|
|
14
|
-
|
|
15
|
-
import type { ExecutionContext } from "@teamscala/os/contracts/mcp";
|
|
16
|
-
import { idsWhereJsonKey } from "@teamscala/db/json-key-filter";
|
|
17
|
-
import { recordEvent } from "@teamscala/event-log/record-event";
|
|
18
|
-
import {
|
|
19
|
-
composeEscalationMessage,
|
|
20
|
-
composeZeroVolumeMessage,
|
|
21
|
-
extractWhatsAppMessageId,
|
|
22
|
-
pagesOperator,
|
|
23
|
-
type EscalationEpisode,
|
|
24
|
-
type FaultAggregate,
|
|
25
|
-
type ZeroVolumeSource,
|
|
26
|
-
} from "../platform-fault-escalation-logic.ts";
|
|
27
|
-
import type { FaultEscalationConfig } from "./config.ts";
|
|
28
|
-
import { loadFaultEscalationConfig } from "./config.ts";
|
|
29
|
-
import { fetchLatestEvidence } from "./event-log-queries.ts";
|
|
30
|
-
import { getSystemIdentity, orpc } from "./orpc-client.ts";
|
|
31
|
-
|
|
32
|
-
interface OperatorRelayBot {
|
|
33
|
-
id: string;
|
|
34
|
-
organisation_id: string;
|
|
35
|
-
external_id: string | null;
|
|
36
|
-
flow_config: { operator_phone?: string } | null;
|
|
37
|
-
}
|
|
38
|
-
|
|
39
|
-
async function resolveOperatorRelay(): Promise<OperatorRelayBot> {
|
|
40
|
-
// flow_config.mode lives inside json; the v8 builder cannot address it.
|
|
41
|
-
const relayBotIds = await idsWhereJsonKey({
|
|
42
|
-
table: "messaging_bots",
|
|
43
|
-
column: "flow_config",
|
|
44
|
-
key: "mode",
|
|
45
|
-
equals: "operator-relay",
|
|
46
|
-
});
|
|
47
|
-
const bot = (await (await orpc()).messaging_bots.findFirst({
|
|
48
|
-
where: {
|
|
49
|
-
is_active: true,
|
|
50
|
-
platform: "whatsapp",
|
|
51
|
-
id: { in: relayBotIds },
|
|
52
|
-
},
|
|
53
|
-
select: { id: true, organisation_id: true, external_id: true, flow_config: true },
|
|
54
|
-
})) as unknown as OperatorRelayBot | null;
|
|
55
|
-
if (!bot) throw new Error("platform-fault-escalation: no active operator-relay WhatsApp bot row");
|
|
56
|
-
const phone = (bot.flow_config as { operator_phone?: string } | null)?.operator_phone;
|
|
57
|
-
if (!phone) throw new Error("platform-fault-escalation: operator-relay bot has no operator_phone");
|
|
58
|
-
return bot as OperatorRelayBot;
|
|
59
|
-
}
|
|
60
|
-
|
|
61
|
-
export async function deliverEscalation(
|
|
62
|
-
kind: string,
|
|
63
|
-
aggregate: FaultAggregate,
|
|
64
|
-
config: FaultEscalationConfig,
|
|
65
|
-
): Promise<{ messageId: string | null; workItemId: string }> {
|
|
66
|
-
const now = new Date();
|
|
67
|
-
|
|
68
|
-
// 1. Claim the episode BEFORE sending, so a send failure leaves a durable
|
|
69
|
-
// claim the next sweep retries instead of a lost escalation.
|
|
70
|
-
await (await orpc()).platform_fault_escalations.upsert({
|
|
71
|
-
where: { kind },
|
|
72
|
-
create: {
|
|
73
|
-
kind,
|
|
74
|
-
first_seen_at: aggregate.firstSeen.toISOString(),
|
|
75
|
-
last_seen_at: aggregate.lastSeen.toISOString(),
|
|
76
|
-
event_count: aggregate.count,
|
|
77
|
-
},
|
|
78
|
-
update: {
|
|
79
|
-
first_seen_at: aggregate.firstSeen.toISOString(),
|
|
80
|
-
last_seen_at: aggregate.lastSeen.toISOString(),
|
|
81
|
-
event_count: aggregate.count,
|
|
82
|
-
resolved_at: null,
|
|
83
|
-
// CLEARING THIS IS WHAT STOPS AN ESCALATION STORM. A re-armed row
|
|
84
|
-
// keeps resolved_at NULL, so it still loads as open, and
|
|
85
|
-
// `escalate` fires for any episode that is resolved OR re-armed.
|
|
86
|
-
// Leaving rearmed_at set would therefore re-escalate the same kind
|
|
87
|
-
// on EVERY sweep — the firing state must be consumed by the firing.
|
|
88
|
-
rearmed_at: null,
|
|
89
|
-
},
|
|
90
|
-
});
|
|
91
|
-
|
|
92
|
-
// 2. Resolve the operator relay and send. The send surface is the
|
|
93
|
-
// `messaging_whatsapp_send` intrinsic (scala-mcp / messaging-mcp-boot),
|
|
94
|
-
// which delegates to the Rust scala-messaging-service — the channel owner
|
|
95
|
-
// that picks the device and records the send in messaging_logs. The retired
|
|
96
|
-
// api_gowa_message tool (deactivated 2026-08-26) is no longer callable;
|
|
97
|
-
// naming it here left every escalation send failing with "Unknown tool".
|
|
98
|
-
|
|
99
|
-
// PAGING IS PER-KIND (operator ruling 2026-08-28). Only the phone is
|
|
100
|
-
// skipped; episode, comment and record are identical, so a silenced page
|
|
101
|
-
// never becomes a silenced signal.
|
|
102
|
-
const paging = pagesOperator(kind, config.agent_handled_kinds);
|
|
103
|
-
const copy = config.copy[kind] ?? config.default_copy;
|
|
104
|
-
const { sample, lastEvent } = await fetchLatestEvidence(kind, config.sample_fields?.[kind] ?? []);
|
|
105
|
-
const message = composeEscalationMessage({
|
|
106
|
-
kind,
|
|
107
|
-
aggregate,
|
|
108
|
-
sample,
|
|
109
|
-
lastEvent,
|
|
110
|
-
impact: copy.impact,
|
|
111
|
-
recommendation: copy.recommendation,
|
|
112
|
-
});
|
|
113
|
-
|
|
114
|
-
// 3. VERIFY delivery before recording it: the send must return a WhatsApp
|
|
115
|
-
// message id. No id => throw, the claim stays undelivered, next sweep retries.
|
|
116
|
-
// An agent-handled kind sends nothing, so there is no id to verify and none
|
|
117
|
-
// is recorded — absence here means "never paged", never "paged unverified".
|
|
118
|
-
let messageId: string | null = null;
|
|
119
|
-
if (paging) {
|
|
120
|
-
const bot = await resolveOperatorRelay();
|
|
121
|
-
const operatorPhone = (bot.flow_config as { operator_phone?: string }).operator_phone!;
|
|
122
|
-
const { userId, organisationId } = await getSystemIdentity();
|
|
123
|
-
const systemCtx: ExecutionContext = { userId, organisationId } as ExecutionContext;
|
|
124
|
-
const { forwardToolToCentralMcp } = await import(
|
|
125
|
-
"@teamscala/chat-completion/central-tool-forwarder"
|
|
126
|
-
);
|
|
127
|
-
const sendResult = await forwardToolToCentralMcp(
|
|
128
|
-
"messaging_whatsapp_send",
|
|
129
|
-
{ phone: operatorPhone, message },
|
|
130
|
-
systemCtx,
|
|
131
|
-
);
|
|
132
|
-
messageId = extractWhatsAppMessageId(sendResult);
|
|
133
|
-
if (!messageId) {
|
|
134
|
-
throw new Error(
|
|
135
|
-
`platform-fault-escalation: WhatsApp send for '${kind}' returned no message id — not recording delivery. ` +
|
|
136
|
-
`Response: ${JSON.stringify(sendResult).slice(0, 400)}`,
|
|
137
|
-
);
|
|
138
|
-
}
|
|
139
|
-
}
|
|
140
|
-
|
|
141
|
-
// 4. Durable record: a COMMENT on the orchestrator's work item — an
|
|
142
|
-
// escalation is a finding the orchestrator/operator triages, not an
|
|
143
|
-
// assigned task (the fleet-task-placement constraint refuses task rows
|
|
144
|
-
// without session identity by design).
|
|
145
|
-
const comment = await (await orpc()).comments.create({
|
|
146
|
-
data: {
|
|
147
|
-
work_item_id: config.orchestrator_work_item_id,
|
|
148
|
-
content: message,
|
|
149
|
-
source_cli_session_id: null,
|
|
150
|
-
},
|
|
151
|
-
});
|
|
152
|
-
|
|
153
|
-
// 5. Readback-correct the episode: delivered + linked in one update.
|
|
154
|
-
await (await orpc()).platform_fault_escalations.update({
|
|
155
|
-
where: { kind },
|
|
156
|
-
data: {
|
|
157
|
-
escalated_at: now.toISOString(),
|
|
158
|
-
work_item_id: config.orchestrator_work_item_id,
|
|
159
|
-
delivered_comment_id: comment.id,
|
|
160
|
-
delivered_message_id: messageId,
|
|
161
|
-
last_updated_at: now.toISOString(),
|
|
162
|
-
updated_at: now.toISOString(),
|
|
163
|
-
},
|
|
164
|
-
});
|
|
165
|
-
|
|
166
|
-
recordEvent({
|
|
167
|
-
kind: "platform.fault.escalated",
|
|
168
|
-
payload: {
|
|
169
|
-
event_kind: kind,
|
|
170
|
-
count: aggregate.count,
|
|
171
|
-
sample,
|
|
172
|
-
last_event_id: lastEvent?.eventId ?? null,
|
|
173
|
-
last_event_occurred_at: lastEvent?.occurredAt ?? null,
|
|
174
|
-
work_item_id: config.orchestrator_work_item_id,
|
|
175
|
-
delivered_comment_id: comment.id,
|
|
176
|
-
delivered_message_id: messageId,
|
|
177
|
-
},
|
|
178
|
-
});
|
|
179
|
-
return { messageId, workItemId: config.orchestrator_work_item_id };
|
|
180
|
-
}
|
|
181
|
-
|
|
182
|
-
/**
|
|
183
|
-
* Re-arm an episode whose kind went quiet but whose condition NOBODY CHECKED.
|
|
184
|
-
* The alarm becomes audible again and no claim is made that the fault
|
|
185
|
-
* stopped, so there is deliberately NO operator comment: "Fault stopped" on
|
|
186
|
-
* the strength of silence is the sentence this change exists to stop writing.
|
|
187
|
-
* The event is still recorded — re-arming is a state change. See migration 811.
|
|
188
|
-
*/
|
|
189
|
-
export async function rearmEpisode(
|
|
190
|
-
episode: EscalationEpisode,
|
|
191
|
-
quietMinutes: number,
|
|
192
|
-
): Promise<void> {
|
|
193
|
-
const now = new Date();
|
|
194
|
-
await (await orpc()).platform_fault_escalations.update({
|
|
195
|
-
where: { kind: episode.kind },
|
|
196
|
-
data: { rearmed_at: now.toISOString(), updated_at: now.toISOString() },
|
|
197
|
-
});
|
|
198
|
-
recordEvent({
|
|
199
|
-
kind: "platform.fault.rearmed",
|
|
200
|
-
payload: {
|
|
201
|
-
event_kind: episode.kind,
|
|
202
|
-
quiet_minutes: Math.round(quietMinutes),
|
|
203
|
-
work_item_id: episode.workItemId,
|
|
204
|
-
// Named so a reader never mistakes this for a resolution: no probe
|
|
205
|
-
// exists for this kind, so nothing verified the condition ended.
|
|
206
|
-
verified: false,
|
|
207
|
-
reason: "no capability probe declared for this kind",
|
|
208
|
-
},
|
|
209
|
-
});
|
|
210
|
-
}
|
|
211
|
-
|
|
212
|
-
export async function resolveEpisode(
|
|
213
|
-
episode: EscalationEpisode,
|
|
214
|
-
quietMinutes: number,
|
|
215
|
-
): Promise<void> {
|
|
216
|
-
const now = new Date();
|
|
217
|
-
await (await orpc()).platform_fault_escalations.update({
|
|
218
|
-
where: { kind: episode.kind },
|
|
219
|
-
data: { resolved_at: now.toISOString(), updated_at: now.toISOString() },
|
|
220
|
-
});
|
|
221
|
-
if (episode.workItemId) {
|
|
222
|
-
await (await orpc()).comments.create({
|
|
223
|
-
data: {
|
|
224
|
-
work_item_id: episode.workItemId,
|
|
225
|
-
content: `Fault stopped — no '${episode.kind}' events for ${Math.round(quietMinutes)} min. Episode closed; the fault will re-escalate if it recurs.`,
|
|
226
|
-
sender_role: "system",
|
|
227
|
-
sender_name: "platform-fault-escalation",
|
|
228
|
-
},
|
|
229
|
-
});
|
|
230
|
-
}
|
|
231
|
-
recordEvent({
|
|
232
|
-
kind: "platform.fault.resolved",
|
|
233
|
-
payload: { event_kind: episode.kind, quiet_minutes: Math.round(quietMinutes), work_item_id: episode.workItemId },
|
|
234
|
-
});
|
|
235
|
-
}
|
|
236
|
-
|
|
237
|
-
/**
|
|
238
|
-
* Refresh an OPEN episode's window facts from the live aggregate.
|
|
239
|
-
*
|
|
240
|
-
* Escalation wrote the row once and never again, so its facts froze while the
|
|
241
|
-
* fault kept firing (2026-08-28: fifteen hours stale, 1,150 later events).
|
|
242
|
-
* That also corrupts RESOLUTION, since `decideSweep` falls back to
|
|
243
|
-
* `lastSeenAt` once a kind leaves the window. Fields mirror the escalation
|
|
244
|
-
* upsert. `last_updated_at` is untouched: it gates the resolve-hold dedupe.
|
|
245
|
-
*/
|
|
246
|
-
export async function refreshOpenEpisode(
|
|
247
|
-
kind: string,
|
|
248
|
-
aggregate: { firstSeen: Date; lastSeen: Date; count: number },
|
|
249
|
-
): Promise<void> {
|
|
250
|
-
await (await orpc()).platform_fault_escalations.update({
|
|
251
|
-
where: { kind },
|
|
252
|
-
data: {
|
|
253
|
-
first_seen_at: aggregate.firstSeen.toISOString(),
|
|
254
|
-
last_seen_at: aggregate.lastSeen.toISOString(),
|
|
255
|
-
event_count: aggregate.count,
|
|
256
|
-
updated_at: new Date().toISOString(),
|
|
257
|
-
},
|
|
258
|
-
});
|
|
259
|
-
}
|
|
260
|
-
|
|
261
|
-
/** Send one message to the operator through the verified WhatsApp relay and
|
|
262
|
-
* record it as a comment on the orchestrator's work item. Returns the
|
|
263
|
-
* verified ids; throws when the send carries no message id so the caller's
|
|
264
|
-
* durable claim stays undelivered and retries next sweep (same contract as
|
|
265
|
-
* deliverEscalation — a component may not report a delivery it cannot show). */
|
|
266
|
-
async function sendOperatorNotice(message: string): Promise<{ messageId: string; workItemId: string }> {
|
|
267
|
-
const bot = await resolveOperatorRelay();
|
|
268
|
-
const operatorPhone = (bot.flow_config as { operator_phone?: string }).operator_phone!;
|
|
269
|
-
const { userId, organisationId } = await getSystemIdentity();
|
|
270
|
-
const systemCtx: ExecutionContext = { userId, organisationId } as ExecutionContext;
|
|
271
|
-
const { forwardToolToCentralMcp } = await import("@teamscala/chat-completion/central-tool-forwarder");
|
|
272
|
-
const sendResult = await forwardToolToCentralMcp("messaging_whatsapp_send", { phone: operatorPhone, message }, systemCtx);
|
|
273
|
-
const messageId = extractWhatsAppMessageId(sendResult);
|
|
274
|
-
if (!messageId) {
|
|
275
|
-
throw new Error(
|
|
276
|
-
`platform-fault-escalation: zero-volume notice send returned no message id — not recording delivery. ` +
|
|
277
|
-
`Response: ${JSON.stringify(sendResult).slice(0, 400)}`,
|
|
278
|
-
);
|
|
279
|
-
}
|
|
280
|
-
const config = await loadFaultEscalationConfig();
|
|
281
|
-
await (await orpc()).comments.create({
|
|
282
|
-
data: { work_item_id: config.orchestrator_work_item_id, content: message, source_cli_session_id: null },
|
|
283
|
-
});
|
|
284
|
-
return { messageId, workItemId: config.orchestrator_work_item_id };
|
|
285
|
-
}
|
|
286
|
-
|
|
287
|
-
/** Raise the zero-volume hold ONCE for an open episode whose producer went
|
|
288
|
-
* silent alongside its failures. Delivery is VERIFIED before the marker
|
|
289
|
-
* event lands, so a failed send leaves the hold unflagged and the next
|
|
290
|
-
* sweep retries — never a flagged-but-unheard hold. */
|
|
291
|
-
export async function deliverZeroVolumeHold(
|
|
292
|
-
episode: EscalationEpisode,
|
|
293
|
-
quietMinutes: number,
|
|
294
|
-
volumeCount: number,
|
|
295
|
-
source: ZeroVolumeSource,
|
|
296
|
-
): Promise<void> {
|
|
297
|
-
const message = composeZeroVolumeMessage({
|
|
298
|
-
kind: episode.kind,
|
|
299
|
-
volumeKindPattern: source.volume_kind_pattern,
|
|
300
|
-
minVolume: source.min_volume ?? 1,
|
|
301
|
-
quietMinutes,
|
|
302
|
-
volumeCount,
|
|
303
|
-
firstSeenAt: episode.firstSeenAt,
|
|
304
|
-
});
|
|
305
|
-
const { messageId, workItemId } = await sendOperatorNotice(message);
|
|
306
|
-
recordEvent({
|
|
307
|
-
kind: "platform.fault.zero_volume_flagged",
|
|
308
|
-
payload: {
|
|
309
|
-
event_kind: episode.kind,
|
|
310
|
-
volume_kind_pattern: source.volume_kind_pattern,
|
|
311
|
-
volume_count: volumeCount,
|
|
312
|
-
quiet_minutes: Math.round(quietMinutes),
|
|
313
|
-
work_item_id: workItemId,
|
|
314
|
-
delivered_message_id: messageId,
|
|
315
|
-
},
|
|
316
|
-
});
|
|
317
|
-
}
|
|
318
|
-
|
|
319
|
-
/** Clear a standing zero-volume hold: volume is back above floor (or faults
|
|
320
|
-
* recurred), so quiet regains its meaning and the normal resolve path takes
|
|
321
|
-
* over. One clearing emission per flag — transitions, not conditions. */
|
|
322
|
-
export async function clearZeroVolumeHold(episode: EscalationEpisode, volumeCount: number): Promise<void> {
|
|
323
|
-
recordEvent({
|
|
324
|
-
kind: "platform.fault.zero_volume_cleared",
|
|
325
|
-
payload: { event_kind: episode.kind, volume_count: volumeCount, work_item_id: episode.workItemId },
|
|
326
|
-
});
|
|
327
|
-
}
|
|
@@ -1,182 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* @system server-boot
|
|
3
|
-
* @status handwritten
|
|
4
|
-
* @edit edit directly
|
|
5
|
-
*
|
|
6
|
-
* The bounded event_log reads of the platform-fault-escalation handler: the
|
|
7
|
-
* recurrence/duration aggregate and the latest-payload evidence sample, both
|
|
8
|
-
* with the `occurred_at <= now()` database-clock top edge.
|
|
9
|
-
*
|
|
10
|
-
* Split out of handlers/platform-fault-escalation.ts (500 lines) —
|
|
11
|
-
* byte-for-byte move; import paths adjusted only.
|
|
12
|
-
*/
|
|
13
|
-
|
|
14
|
-
import { getPrisma } from "@teamscala/db/prisma-registry";
|
|
15
|
-
import { formatEscalationSample, type FaultAggregate } from "../platform-fault-escalation-logic.ts";
|
|
16
|
-
import { CONFIG_SLUG } from "./config.ts";
|
|
17
|
-
|
|
18
|
-
interface EventLogAggregateRow {
|
|
19
|
-
kind: string;
|
|
20
|
-
n: bigint;
|
|
21
|
-
first_seen: Date;
|
|
22
|
-
last_seen: Date;
|
|
23
|
-
}
|
|
24
|
-
|
|
25
|
-
export async function fetchAggregates(patterns: string[], windowMinutes: number): Promise<FaultAggregate[]> {
|
|
26
|
-
const prisma = getPrisma();
|
|
27
|
-
// Patterns come from the valibot-validated registry row and are regex-only
|
|
28
|
-
// by construction below; embedding them as one alternation keeps the query
|
|
29
|
-
// parameter-free on the pattern side while the window is a real parameter.
|
|
30
|
-
// The window is bounded on BOTH sides by the database's own statement
|
|
31
|
-
// clock: `occurred_at` is producer-claimed, and an unbounded top edge
|
|
32
|
-
// admits rows stamped in the future by a skewed or mocked producer clock —
|
|
33
|
-
// which then fabricate "still recurring" for a quiet fault (measured
|
|
34
|
-
// 2026-08-26). A future row is not evidence of anything happening now.
|
|
35
|
-
const regexClass = /^[A-Za-z0-9_\\^$.*+?()[\]{}|-]+$/;
|
|
36
|
-
for (const pattern of patterns) {
|
|
37
|
-
if (!regexClass.test(pattern)) {
|
|
38
|
-
throw new Error(`config/${CONFIG_SLUG}: watched_kind_pattern is not regex-only: ${pattern}`);
|
|
39
|
-
}
|
|
40
|
-
}
|
|
41
|
-
const combined = `(${patterns.join("|")})`;
|
|
42
|
-
const rows = await prisma.$queryRawUnsafe<EventLogAggregateRow[]>(
|
|
43
|
-
`SELECT kind, count(*)::bigint AS n, min(occurred_at) AS first_seen, max(occurred_at) AS last_seen
|
|
44
|
-
FROM event_log
|
|
45
|
-
WHERE occurred_at > now() - ($1::int * interval '1 minute')
|
|
46
|
-
AND occurred_at <= now()
|
|
47
|
-
AND kind ~ $2::text
|
|
48
|
-
GROUP BY kind`,
|
|
49
|
-
windowMinutes,
|
|
50
|
-
combined,
|
|
51
|
-
);
|
|
52
|
-
return rows.map((row) => ({
|
|
53
|
-
kind: row.kind,
|
|
54
|
-
count: Number(row.n),
|
|
55
|
-
firstSeen: new Date(row.first_seen),
|
|
56
|
-
lastSeen: new Date(row.last_seen),
|
|
57
|
-
}));
|
|
58
|
-
}
|
|
59
|
-
|
|
60
|
-
/**
|
|
61
|
-
* The message's evidence: the whitelisted sample fields AND the row identity
|
|
62
|
-
* (id + occurred_at) of the newest payload for the kind. Field names come
|
|
63
|
-
* from the valibot-validated config row and are constrained to bare
|
|
64
|
-
* identifiers below, so the JSON path cannot break out of the string. The
|
|
65
|
-
* `occurred_at <= now()` bound matches the aggregate's top edge: the cited
|
|
66
|
-
* "last" row must be a real, non-future event whose id rides in the message
|
|
67
|
-
* so the claim resolves against event_log.
|
|
68
|
-
*/
|
|
69
|
-
export async function fetchLatestEvidence(
|
|
70
|
-
kind: string,
|
|
71
|
-
fields: string[],
|
|
72
|
-
): Promise<{ sample: string; lastEvent: { occurredAt: string; eventId: string } | null }> {
|
|
73
|
-
const prisma = getPrisma();
|
|
74
|
-
if (fields.length === 0) {
|
|
75
|
-
const rows = await prisma.$queryRawUnsafe<Array<{ id: unknown; occurred_at: Date }>>(
|
|
76
|
-
`SELECT id::text, occurred_at
|
|
77
|
-
FROM event_log
|
|
78
|
-
WHERE kind = $1::text AND occurred_at <= now()
|
|
79
|
-
ORDER BY occurred_at DESC
|
|
80
|
-
LIMIT 1`,
|
|
81
|
-
kind,
|
|
82
|
-
);
|
|
83
|
-
const top = rows[0];
|
|
84
|
-
return { sample: "", lastEvent: top ? { occurredAt: isoSeconds(top.occurred_at), eventId: String(top.id) } : null };
|
|
85
|
-
}
|
|
86
|
-
for (const field of fields) {
|
|
87
|
-
if (!/^[A-Za-z0-9_]+$/.test(field)) {
|
|
88
|
-
throw new Error(`config/${CONFIG_SLUG}: sample_fields entry is not a bare field name: ${field}`);
|
|
89
|
-
}
|
|
90
|
-
}
|
|
91
|
-
const selection = fields.map((field) => `payload_json->'${field}' AS "${field}"`).join(", ");
|
|
92
|
-
const rows = await prisma.$queryRawUnsafe<Array<Record<string, unknown>>>(
|
|
93
|
-
`SELECT id::text, occurred_at, ${selection}
|
|
94
|
-
FROM event_log
|
|
95
|
-
WHERE kind = $1::text AND occurred_at <= now()
|
|
96
|
-
ORDER BY occurred_at DESC
|
|
97
|
-
LIMIT 1`,
|
|
98
|
-
kind,
|
|
99
|
-
);
|
|
100
|
-
const row = rows[0];
|
|
101
|
-
if (!row) return { sample: "", lastEvent: null };
|
|
102
|
-
const parts: string[] = [];
|
|
103
|
-
for (const field of fields) {
|
|
104
|
-
const value = row[field];
|
|
105
|
-
if (value === null || value === undefined) continue;
|
|
106
|
-
const formatted = formatEscalationSample(value);
|
|
107
|
-
if (formatted) parts.push(`${field}=${formatted}`);
|
|
108
|
-
}
|
|
109
|
-
return {
|
|
110
|
-
sample: parts.join("; "),
|
|
111
|
-
lastEvent: {
|
|
112
|
-
occurredAt: isoSeconds(row.occurred_at as string | Date),
|
|
113
|
-
eventId: String(row.id),
|
|
114
|
-
},
|
|
115
|
-
};
|
|
116
|
-
}
|
|
117
|
-
|
|
118
|
-
/** ISO-8601 to whole seconds — the same precision event_log stores, so the
|
|
119
|
-
* cited string round-trips a `WHERE occurred_at = '<cited>'` query. */
|
|
120
|
-
function isoSeconds(value: Date | string): string {
|
|
121
|
-
return new Date(value).toISOString().replace(/\.\d{3}Z$/, "Z");
|
|
122
|
-
}
|
|
123
|
-
|
|
124
|
-
/** The bounded VOLUME count for one fault kind's volume source: how many
|
|
125
|
-
* rows of `kind ~ pattern` landed inside the window (both edges database-
|
|
126
|
-
* clock-bounded, identical discipline to fetchAggregates). A floor-checked
|
|
127
|
-
* zero here is what separates "failures stopped because things recovered"
|
|
128
|
-
* from "failures stopped because nothing is calling anymore" — 2026-08-27:
|
|
129
|
-
* the dead llm-router produced BOTH zeros and read as healthy. */
|
|
130
|
-
export async function fetchVolumeCount(pattern: string, windowMinutes: number): Promise<number> {
|
|
131
|
-
const prisma = getPrisma();
|
|
132
|
-
const regexClass = /^[A-Za-z0-9_\\^$.*+?()[\]{}|-]+$/;
|
|
133
|
-
if (!regexClass.test(pattern)) {
|
|
134
|
-
throw new Error(`config/${CONFIG_SLUG}: volume_kind_pattern is not regex-only: ${pattern}`);
|
|
135
|
-
}
|
|
136
|
-
const rows = await prisma.$queryRawUnsafe<Array<{ n: bigint }>>(
|
|
137
|
-
`SELECT count(*)::bigint AS n
|
|
138
|
-
FROM event_log
|
|
139
|
-
WHERE occurred_at > now() - ($1::int * interval '1 minute')
|
|
140
|
-
AND occurred_at <= now()
|
|
141
|
-
AND kind ~ $2::text`,
|
|
142
|
-
windowMinutes,
|
|
143
|
-
pattern,
|
|
144
|
-
);
|
|
145
|
-
return Number(rows[0]?.n ?? 0);
|
|
146
|
-
}
|
|
147
|
-
|
|
148
|
-
/** Whether a resolve-HOLD has already been raised for this episode state:
|
|
149
|
-
* the newest `platform.fault.resolve_held` naming the kind is newer than the
|
|
150
|
-
* episode's last transition. Durable, never process memory — a restart must
|
|
151
|
-
* not re-raise the hold, and a NEW episode state (a later last_updated_at)
|
|
152
|
-
* may raise it again. */
|
|
153
|
-
export async function resolveHoldRaisedSince(kind: string, since: Date | null): Promise<boolean> {
|
|
154
|
-
const prisma = getPrisma();
|
|
155
|
-
const rows = await prisma.$queryRawUnsafe<Array<{ occurred_at: Date }>>(
|
|
156
|
-
`SELECT occurred_at FROM event_log
|
|
157
|
-
WHERE kind = 'platform.fault.resolve_held'
|
|
158
|
-
AND payload_json->>'event_kind' = $1::text
|
|
159
|
-
ORDER BY occurred_at DESC LIMIT 1`,
|
|
160
|
-
kind,
|
|
161
|
-
);
|
|
162
|
-
const latest = rows[0]?.occurred_at;
|
|
163
|
-
if (!latest) return false;
|
|
164
|
-
if (!since) return true;
|
|
165
|
-
return new Date(latest) > since;
|
|
166
|
-
}
|
|
167
|
-
|
|
168
|
-
/** Whether a zero-volume hold marker STANDS for a fault kind: derived from
|
|
169
|
-
* the durable stream (latest flagged/cleared wins), never from process
|
|
170
|
-
* memory — a restart must not re-raise or silently drop a standing hold.
|
|
171
|
-
* Returns false when no marker exists (unflagged). */
|
|
172
|
-
export async function zeroVolumeFlagActive(kind: string): Promise<boolean> {
|
|
173
|
-
const prisma = getPrisma();
|
|
174
|
-
const rows = await prisma.$queryRawUnsafe<Array<{ kind: string }>>(
|
|
175
|
-
`SELECT kind FROM event_log
|
|
176
|
-
WHERE kind IN ('platform.fault.zero_volume_flagged', 'platform.fault.zero_volume_cleared')
|
|
177
|
-
AND payload_json->>'event_kind' = $1::text
|
|
178
|
-
ORDER BY occurred_at DESC LIMIT 1`,
|
|
179
|
-
kind,
|
|
180
|
-
);
|
|
181
|
-
return rows[0]?.kind === "platform.fault.zero_volume_flagged";
|
|
182
|
-
}
|