@teamscala/service-boot 0.23.421 → 0.23.425
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +9 -2
- package/src/build-client-assets/input-hashing.ts +1 -1
- package/src/fn-routers/assessment-report-data.test.ts +404 -0
- package/src/fn-routers/assessment-report-data.ts +423 -0
- package/src/fn-routers/assessment-report-email.ts +2 -2
- package/src/fn-routers/chat-session-router-shared.ts +4 -21
- package/src/fn-routers/chat-session-router.ts +1 -1
- package/src/fn-routers/conversation-ownership.ts +23 -0
- package/src/fn-routers/index.ts +1 -1
- package/src/fn-routers/public-agreement-router.ts +1 -1
- package/src/fn-routers/public-assessment-router.ts +2 -2
- package/src/fn-routers/public-gpt-session-router.ts +1 -1
- package/src/handlers/platform-fault-escalation/config.ts +16 -0
- package/src/handlers/platform-fault-escalation/delivery.ts +75 -1
- package/src/handlers/platform-fault-escalation/event-log-queries.ts +40 -0
- package/src/handlers/platform-fault-escalation-logic.ts +83 -4
- package/src/handlers/platform-fault-escalation.test.ts +68 -1
- package/src/handlers/platform-fault-escalation.ts +47 -3
- package/src/page-server-orpc/audit-revert-router.ts +1 -1
- package/src/pg-boss-mount.ts +1 -1
- package/src/resolve-from-loader-config.ts +1 -1
- package/src/two-phase-readiness.ts +2 -1
|
@@ -16,11 +16,14 @@ import type { ExecutionContext } from "@teamscala/os/contracts/mcp";
|
|
|
16
16
|
import { recordEvent } from "@teamscala/event-log/record-event";
|
|
17
17
|
import {
|
|
18
18
|
composeEscalationMessage,
|
|
19
|
+
composeZeroVolumeMessage,
|
|
19
20
|
extractWhatsAppMessageId,
|
|
20
21
|
type EscalationEpisode,
|
|
21
22
|
type FaultAggregate,
|
|
23
|
+
type ZeroVolumeSource,
|
|
22
24
|
} from "../platform-fault-escalation-logic.ts";
|
|
23
25
|
import type { FaultEscalationConfig } from "./config.ts";
|
|
26
|
+
import { loadFaultEscalationConfig } from "./config.ts";
|
|
24
27
|
import { fetchLatestEvidence } from "./event-log-queries.ts";
|
|
25
28
|
import { getSystemIdentity, orpc } from "./orpc-client.ts";
|
|
26
29
|
|
|
@@ -150,7 +153,10 @@ export async function deliverEscalation(
|
|
|
150
153
|
return { messageId, workItemId: config.orchestrator_work_item_id };
|
|
151
154
|
}
|
|
152
155
|
|
|
153
|
-
export async function resolveEpisode(
|
|
156
|
+
export async function resolveEpisode(
|
|
157
|
+
episode: EscalationEpisode,
|
|
158
|
+
quietMinutes: number,
|
|
159
|
+
): Promise<void> {
|
|
154
160
|
const now = new Date();
|
|
155
161
|
await (await orpc()).platform_fault_escalations.update({
|
|
156
162
|
where: { kind: episode.kind },
|
|
@@ -171,3 +177,71 @@ export async function resolveEpisode(episode: EscalationEpisode, quietMinutes: n
|
|
|
171
177
|
payload: { event_kind: episode.kind, quiet_minutes: Math.round(quietMinutes), work_item_id: episode.workItemId },
|
|
172
178
|
});
|
|
173
179
|
}
|
|
180
|
+
|
|
181
|
+
/** Send one message to the operator through the verified WhatsApp relay and
|
|
182
|
+
* record it as a comment on the orchestrator's work item. Returns the
|
|
183
|
+
* verified ids; throws when the send carries no message id so the caller's
|
|
184
|
+
* durable claim stays undelivered and retries next sweep (same contract as
|
|
185
|
+
* deliverEscalation — a component may not report a delivery it cannot show). */
|
|
186
|
+
async function sendOperatorNotice(message: string): Promise<{ messageId: string; workItemId: string }> {
|
|
187
|
+
const bot = await resolveOperatorRelay();
|
|
188
|
+
const operatorPhone = (bot.flow_config as { operator_phone?: string }).operator_phone!;
|
|
189
|
+
const { userId, organisationId } = await getSystemIdentity();
|
|
190
|
+
const systemCtx: ExecutionContext = { userId, organisationId } as ExecutionContext;
|
|
191
|
+
const { forwardToolToCentralMcp } = await import("@teamscala/chat-completion/central-tool-forwarder");
|
|
192
|
+
const sendResult = await forwardToolToCentralMcp("messaging_whatsapp_send", { phone: operatorPhone, message }, systemCtx);
|
|
193
|
+
const messageId = extractWhatsAppMessageId(sendResult);
|
|
194
|
+
if (!messageId) {
|
|
195
|
+
throw new Error(
|
|
196
|
+
`platform-fault-escalation: zero-volume notice send returned no message id — not recording delivery. ` +
|
|
197
|
+
`Response: ${JSON.stringify(sendResult).slice(0, 400)}`,
|
|
198
|
+
);
|
|
199
|
+
}
|
|
200
|
+
const config = await loadFaultEscalationConfig();
|
|
201
|
+
await (await orpc()).comments.create({
|
|
202
|
+
data: { work_item_id: config.orchestrator_work_item_id, content: message, source_cli_session_id: null },
|
|
203
|
+
});
|
|
204
|
+
return { messageId, workItemId: config.orchestrator_work_item_id };
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
/** Raise the zero-volume hold ONCE for an open episode whose producer went
|
|
208
|
+
* silent alongside its failures. Delivery is VERIFIED before the marker
|
|
209
|
+
* event lands, so a failed send leaves the hold unflagged and the next
|
|
210
|
+
* sweep retries — never a flagged-but-unheard hold. */
|
|
211
|
+
export async function deliverZeroVolumeHold(
|
|
212
|
+
episode: EscalationEpisode,
|
|
213
|
+
quietMinutes: number,
|
|
214
|
+
volumeCount: number,
|
|
215
|
+
source: ZeroVolumeSource,
|
|
216
|
+
): Promise<void> {
|
|
217
|
+
const message = composeZeroVolumeMessage({
|
|
218
|
+
kind: episode.kind,
|
|
219
|
+
volumeKindPattern: source.volume_kind_pattern,
|
|
220
|
+
minVolume: source.min_volume ?? 1,
|
|
221
|
+
quietMinutes,
|
|
222
|
+
volumeCount,
|
|
223
|
+
firstSeenAt: episode.firstSeenAt,
|
|
224
|
+
});
|
|
225
|
+
const { messageId, workItemId } = await sendOperatorNotice(message);
|
|
226
|
+
recordEvent({
|
|
227
|
+
kind: "platform.fault.zero_volume_flagged",
|
|
228
|
+
payload: {
|
|
229
|
+
event_kind: episode.kind,
|
|
230
|
+
volume_kind_pattern: source.volume_kind_pattern,
|
|
231
|
+
volume_count: volumeCount,
|
|
232
|
+
quiet_minutes: Math.round(quietMinutes),
|
|
233
|
+
work_item_id: workItemId,
|
|
234
|
+
delivered_message_id: messageId,
|
|
235
|
+
},
|
|
236
|
+
});
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
/** Clear a standing zero-volume hold: volume is back above floor (or faults
|
|
240
|
+
* recurred), so quiet regains its meaning and the normal resolve path takes
|
|
241
|
+
* over. One clearing emission per flag — transitions, not conditions. */
|
|
242
|
+
export async function clearZeroVolumeHold(episode: EscalationEpisode, volumeCount: number): Promise<void> {
|
|
243
|
+
recordEvent({
|
|
244
|
+
kind: "platform.fault.zero_volume_cleared",
|
|
245
|
+
payload: { event_kind: episode.kind, volume_count: volumeCount, work_item_id: episode.workItemId },
|
|
246
|
+
});
|
|
247
|
+
}
|
|
@@ -120,3 +120,43 @@ export async function fetchLatestEvidence(
|
|
|
120
120
|
function isoSeconds(value: Date | string): string {
|
|
121
121
|
return new Date(value).toISOString().replace(/\.\d{3}Z$/, "Z");
|
|
122
122
|
}
|
|
123
|
+
|
|
124
|
+
/** The bounded VOLUME count for one fault kind's volume source: how many
|
|
125
|
+
* rows of `kind ~ pattern` landed inside the window (both edges database-
|
|
126
|
+
* clock-bounded, identical discipline to fetchAggregates). A floor-checked
|
|
127
|
+
* zero here is what separates "failures stopped because things recovered"
|
|
128
|
+
* from "failures stopped because nothing is calling anymore" — 2026-08-27:
|
|
129
|
+
* the dead llm-router produced BOTH zeros and read as healthy. */
|
|
130
|
+
export async function fetchVolumeCount(pattern: string, windowMinutes: number): Promise<number> {
|
|
131
|
+
const prisma = getPrisma();
|
|
132
|
+
const regexClass = /^[A-Za-z0-9_\\^$.*+?()[\]{}|-]+$/;
|
|
133
|
+
if (!regexClass.test(pattern)) {
|
|
134
|
+
throw new Error(`config/${CONFIG_SLUG}: volume_kind_pattern is not regex-only: ${pattern}`);
|
|
135
|
+
}
|
|
136
|
+
const rows = await prisma.$queryRawUnsafe<Array<{ n: bigint }>>(
|
|
137
|
+
`SELECT count(*)::bigint AS n
|
|
138
|
+
FROM event_log
|
|
139
|
+
WHERE occurred_at > now() - ($1::int * interval '1 minute')
|
|
140
|
+
AND occurred_at <= now()
|
|
141
|
+
AND kind ~ $2::text`,
|
|
142
|
+
windowMinutes,
|
|
143
|
+
pattern,
|
|
144
|
+
);
|
|
145
|
+
return Number(rows[0]?.n ?? 0);
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
/** Whether a zero-volume hold marker STANDS for a fault kind: derived from
|
|
149
|
+
* the durable stream (latest flagged/cleared wins), never from process
|
|
150
|
+
* memory — a restart must not re-raise or silently drop a standing hold.
|
|
151
|
+
* Returns false when no marker exists (unflagged). */
|
|
152
|
+
export async function zeroVolumeFlagActive(kind: string): Promise<boolean> {
|
|
153
|
+
const prisma = getPrisma();
|
|
154
|
+
const rows = await prisma.$queryRawUnsafe<Array<{ kind: string }>>(
|
|
155
|
+
`SELECT kind FROM event_log
|
|
156
|
+
WHERE kind IN ('platform.fault.zero_volume_flagged', 'platform.fault.zero_volume_cleared')
|
|
157
|
+
AND payload_json->>'event_kind' = $1::text
|
|
158
|
+
ORDER BY occurred_at DESC LIMIT 1`,
|
|
159
|
+
kind,
|
|
160
|
+
);
|
|
161
|
+
return rows[0]?.kind === "platform.fault.zero_volume_flagged";
|
|
162
|
+
}
|
|
@@ -52,7 +52,36 @@ export interface SweepDecision {
|
|
|
52
52
|
/** Sustained faults needing a FIRST delivery (new episode, or a claim whose send failed). */
|
|
53
53
|
escalate: Array<{ aggregate: FaultAggregate; episode: EscalationEpisode | null }>;
|
|
54
54
|
/** Open episodes whose kind went quiet past the resolve window — close them. */
|
|
55
|
-
resolve: Array<{ episode: EscalationEpisode; quietMinutes: number }>;
|
|
55
|
+
resolve: Array<{ episode: EscalationEpisode; quietMinutes: number; clearZeroVolumeFlag: boolean }>;
|
|
56
|
+
/** Open episodes quiet past the resolve window whose producer ALSO went
|
|
57
|
+
* silent: zero failure rows AND zero call volume that would produce them.
|
|
58
|
+
* Quiet is NOT evidence of health here (measured 2026-08-27: llm-router
|
|
59
|
+
* died and produced zero llm.provider.call.failed events, which the
|
|
60
|
+
* sweep closed as "no failures for 30 min" while crush lanes had no
|
|
61
|
+
* model backend at all). The episode HOLDS OPEN and the hold raises
|
|
62
|
+
* once; it never resolves from silence alone. */
|
|
63
|
+
zeroVolumeHold: Array<{ episode: EscalationEpisode; quietMinutes: number; volumeCount: number }>;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/** Per-kind config naming the kind whose VOLUME proves the producing
|
|
67
|
+
* pipeline is alive — e.g. failures pair with completed calls. A fault kind
|
|
68
|
+
* without a source resolves on quiet exactly as before; with one, resolve
|
|
69
|
+
* additionally requires the volume floor to be met inside the window. */
|
|
70
|
+
export interface ZeroVolumeSource {
|
|
71
|
+
volume_kind_pattern: string;
|
|
72
|
+
min_volume?: number;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/** Everything decideSweep needs to judge quiet-vs-healthy, computed by the
|
|
76
|
+
* caller from the valibot-validated config row + bounded queries. */
|
|
77
|
+
export interface ZeroVolumeInputs {
|
|
78
|
+
/** Configured sources keyed by watched kind. */
|
|
79
|
+
sources: Record<string, ZeroVolumeSource>;
|
|
80
|
+
/** Resolved volume count per watched kind over the window. */
|
|
81
|
+
countByKind: Record<string, number>;
|
|
82
|
+
/** Whether a flagged/cleared marker already stands for the kind (derived
|
|
83
|
+
* from event_log by the caller — never process memory). */
|
|
84
|
+
flagActiveByKind: Record<string, boolean>;
|
|
56
85
|
}
|
|
57
86
|
|
|
58
87
|
function minutesBetween(a: Date, b: Date): number {
|
|
@@ -95,10 +124,11 @@ export function decideSweep(
|
|
|
95
124
|
policy: FaultEscalationPolicy,
|
|
96
125
|
now: Date,
|
|
97
126
|
kindThresholds: Record<string, KindThresholdOverride> = {},
|
|
127
|
+
zeroVolume?: ZeroVolumeInputs,
|
|
98
128
|
): SweepDecision {
|
|
99
129
|
const byKind = new Map(episodes.map((e) => [e.kind, e]));
|
|
100
130
|
const aggregateByKind = new Map(aggregates.map((a) => [a.kind, a]));
|
|
101
|
-
const decision: SweepDecision = { escalate: [], resolve: [] };
|
|
131
|
+
const decision: SweepDecision = { escalate: [], resolve: [], zeroVolumeHold: [] };
|
|
102
132
|
|
|
103
133
|
for (const aggregate of aggregates) {
|
|
104
134
|
const episode = byKind.get(aggregate.kind);
|
|
@@ -132,9 +162,29 @@ export function decideSweep(
|
|
|
132
162
|
const quietReference =
|
|
133
163
|
aggregate && aggregate.lastSeen.getTime() <= now.getTime() ? aggregate.lastSeen : episode.lastSeenAt;
|
|
134
164
|
const quietMinutes = minutesBetween(quietReference, now);
|
|
135
|
-
if (quietMinutes
|
|
136
|
-
|
|
165
|
+
if (quietMinutes < policy.resolveAfterMinutes) continue;
|
|
166
|
+
|
|
167
|
+
// ZERO-VOLUME GATE on resolution. When this fault kind declares a
|
|
168
|
+
// volume source and that volume is below its floor inside the window,
|
|
169
|
+
// "no failure rows" no longer separates healthy from dead-producer:
|
|
170
|
+
// hold the episode open instead of closing it. The hold raises once
|
|
171
|
+
// (when unflagged) through zeroVolumeHold; a flagged hold emits
|
|
172
|
+
// nothing while it persists — same transitions-only rule as an open
|
|
173
|
+
// escalation.
|
|
174
|
+
const source = zeroVolume?.sources[episode.kind];
|
|
175
|
+
const volumeCount = zeroVolume?.countByKind[episode.kind];
|
|
176
|
+
if (source) {
|
|
177
|
+
const floor = source.min_volume ?? 1;
|
|
178
|
+
if ((volumeCount ?? 0) < floor) {
|
|
179
|
+
decision.zeroVolumeHold.push({ episode, quietMinutes, volumeCount: volumeCount ?? 0 });
|
|
180
|
+
continue;
|
|
181
|
+
}
|
|
137
182
|
}
|
|
183
|
+
decision.resolve.push({
|
|
184
|
+
episode,
|
|
185
|
+
quietMinutes,
|
|
186
|
+
clearZeroVolumeFlag: Boolean(source && zeroVolume?.flagActiveByKind[episode.kind]),
|
|
187
|
+
});
|
|
138
188
|
}
|
|
139
189
|
|
|
140
190
|
return decision;
|
|
@@ -216,3 +266,32 @@ export function composeEscalationMessage(input: {
|
|
|
216
266
|
lines.push(`Impact: ${input.impact}`, `Recommendation: ${input.recommendation}`);
|
|
217
267
|
return lines.join("\n");
|
|
218
268
|
}
|
|
269
|
+
|
|
270
|
+
/**
|
|
271
|
+
* The zero-volume hold notice — the same three-field contract, for the shape
|
|
272
|
+
* where the watched failures went quiet AND the call volume that would have
|
|
273
|
+
* produced them went quiet too. That combination is NOT a resolution: it is
|
|
274
|
+
* either "fault ended" or "the producer itself died" (2026-08-27: the dead
|
|
275
|
+
* llm-router emitted zero llm.provider.call.failed rows and the sweep closed
|
|
276
|
+
* the episode as healthy while crush lanes had no model backend). Holds one
|
|
277
|
+
* episode open; raises once per episode-state, never on cadence.
|
|
278
|
+
*/
|
|
279
|
+
export function composeZeroVolumeMessage(input: {
|
|
280
|
+
kind: string;
|
|
281
|
+
volumeKindPattern: string;
|
|
282
|
+
minVolume: number;
|
|
283
|
+
quietMinutes: number;
|
|
284
|
+
volumeCount: number;
|
|
285
|
+
firstSeenAt: Date;
|
|
286
|
+
}): string {
|
|
287
|
+
const observed =
|
|
288
|
+
`Observed: '${input.kind}' open since ${input.firstSeenAt.toISOString()}, ` +
|
|
289
|
+
`quiet ${Math.round(input.quietMinutes)} min — but volume kind ~${input.volumeKindPattern} is ALSO silent ` +
|
|
290
|
+
`(${input.volumeCount} < floor ${input.minVolume} in window). Quiet is indistinguishable from dead.`;
|
|
291
|
+
return [
|
|
292
|
+
`🟡 Platform fault unresolved: ${input.kind}`,
|
|
293
|
+
observed,
|
|
294
|
+
"Impact: failures stopping means EITHER recovery OR its producer died entirely — this exact ambiguity hid llm-router's halt behind an all-clear.",
|
|
295
|
+
`Recommendation: probe/restart the producer of ${input.volumeKindPattern} directly; resolve the episode by hand only once completion is verified.`,
|
|
296
|
+
].join("\n");
|
|
297
|
+
}
|
|
@@ -11,6 +11,7 @@
|
|
|
11
11
|
import { describe, expect, test } from "bun:test";
|
|
12
12
|
import {
|
|
13
13
|
composeEscalationMessage,
|
|
14
|
+
composeZeroVolumeMessage,
|
|
14
15
|
decideSweep,
|
|
15
16
|
extractWhatsAppMessageId,
|
|
16
17
|
formatEscalationSample,
|
|
@@ -114,7 +115,7 @@ describe("decideSweep — dedup (one escalation per fault, no re-fire)", () => {
|
|
|
114
115
|
const decision = decideSweep([sustained], [longOpen], POLICY, NOW);
|
|
115
116
|
expect(decision.escalate).toHaveLength(0);
|
|
116
117
|
expect(decision.resolve).toHaveLength(0);
|
|
117
|
-
expect(Object.keys(decision).sort()).toEqual(["escalate", "resolve"]);
|
|
118
|
+
expect(Object.keys(decision).sort()).toEqual(["escalate", "resolve", "zeroVolumeHold"]);
|
|
118
119
|
});
|
|
119
120
|
|
|
120
121
|
test("a claimed-but-undelivered episode is RETRIED, not counted as escalated", () => {
|
|
@@ -332,3 +333,69 @@ describe("decideSweep — qualification asks 'is this happening', not 'did this
|
|
|
332
333
|
expect(decision.escalate).toHaveLength(1);
|
|
333
334
|
});
|
|
334
335
|
});
|
|
336
|
+
|
|
337
|
+
describe("decideSweep — zero-volume gate (quiet vs dead producer)", () => {
|
|
338
|
+
const SOURCES = {
|
|
339
|
+
"llm.provider.call.failed": { volume_kind_pattern: "^llm\.router\.routing_decision$", min_volume: 1 },
|
|
340
|
+
};
|
|
341
|
+
const VOLUME_BACKED = { sources: SOURCES, countByKind: {}, flagActiveByKind: {} };
|
|
342
|
+
|
|
343
|
+
test("an open episode with NO volume does not resolve — it holds", () => {
|
|
344
|
+
const episodes = [episode({ kind: "llm.provider.call.failed", lastSeenAt: new Date(NOW.getTime() - 45 * 60_000) })];
|
|
345
|
+
const inputs = { ...VOLUME_BACKED, countByKind: { "llm.provider.call.failed": 0 } };
|
|
346
|
+
const decision = decideSweep([], episodes, POLICY, NOW, {}, inputs);
|
|
347
|
+
expect(decision.resolve).toHaveLength(0);
|
|
348
|
+
expect(decision.zeroVolumeHold).toHaveLength(1);
|
|
349
|
+
expect(decision.zeroVolumeHold[0].volumeCount).toBe(0);
|
|
350
|
+
});
|
|
351
|
+
|
|
352
|
+
test("quiet WITH healthy call volume still resolves (the normal close)", () => {
|
|
353
|
+
const episodes = [episode({ kind: "llm.provider.call.failed", lastSeenAt: new Date(NOW.getTime() - 45 * 60_000) })];
|
|
354
|
+
const inputs = { ...VOLUME_BACKED, countByKind: { "llm.provider.call.failed": 120 } };
|
|
355
|
+
const decision = decideSweep([], episodes, POLICY, NOW, {}, inputs);
|
|
356
|
+
expect(decision.zeroVolumeHold).toHaveLength(0);
|
|
357
|
+
expect(decision.resolve).toHaveLength(1);
|
|
358
|
+
expect(decision.resolve[0].clearZeroVolumeFlag).toBe(false);
|
|
359
|
+
});
|
|
360
|
+
|
|
361
|
+
test("a sourceless kind resolves on quiet exactly as before", () => {
|
|
362
|
+
const episodes = [episode({ kind: "guard.fail.open", lastSeenAt: new Date(NOW.getTime() - 45 * 60_000) })];
|
|
363
|
+
const decision = decideSweep([], episodes, POLICY, NOW, {}, { ...VOLUME_BACKED, countByKind: {} });
|
|
364
|
+
expect(decision.resolve).toHaveLength(1);
|
|
365
|
+
expect(decision.zeroVolumeHold).toHaveLength(0);
|
|
366
|
+
});
|
|
367
|
+
|
|
368
|
+
test("recovery from a flagged hold clears the flag on resolve", () => {
|
|
369
|
+
const episodes = [episode({ kind: "llm.provider.call.failed", lastSeenAt: new Date(NOW.getTime() - 45 * 60_000) })];
|
|
370
|
+
const inputs = {
|
|
371
|
+
...VOLUME_BACKED,
|
|
372
|
+
countByKind: { "llm.provider.call.failed": 50 },
|
|
373
|
+
flagActiveByKind: { "llm.provider.call.failed": true },
|
|
374
|
+
};
|
|
375
|
+
const decision = decideSweep([], episodes, POLICY, NOW, {}, inputs);
|
|
376
|
+
expect(decision.resolve[0].clearZeroVolumeFlag).toBe(true);
|
|
377
|
+
});
|
|
378
|
+
|
|
379
|
+
test("without a sixth argument the legacy shape is untouched", () => {
|
|
380
|
+
const episodes = [episode({ kind: "llm.provider.call.failed", lastSeenAt: new Date(NOW.getTime() - 45 * 60_000) })];
|
|
381
|
+
const decision = decideSweep([], episodes, POLICY, NOW);
|
|
382
|
+
expect(decision.resolve).toHaveLength(1);
|
|
383
|
+
});
|
|
384
|
+
});
|
|
385
|
+
|
|
386
|
+
describe("composeZeroVolumeMessage — the quiet-is-not-healthy notice", () => {
|
|
387
|
+
test("names the held kind, both absences and the floors; stays phone-sized", () => {
|
|
388
|
+
const message = composeZeroVolumeMessage({
|
|
389
|
+
kind: "llm.provider.call.failed",
|
|
390
|
+
volumeKindPattern: "^llm\.router\.routing_decision$",
|
|
391
|
+
minVolume: 1,
|
|
392
|
+
quietMinutes: 600,
|
|
393
|
+
volumeCount: 0,
|
|
394
|
+
firstSeenAt: new Date(NOW.getTime() - 700 * 60_000),
|
|
395
|
+
});
|
|
396
|
+
for (const fragment of ["llm.provider.call.failed", "routing_decision", "(0 < floor 1", "Recommendation:"]) {
|
|
397
|
+
expect(message).toContain(fragment);
|
|
398
|
+
}
|
|
399
|
+
expect(message.split("\n").length).toBeLessThanOrEqual(4);
|
|
400
|
+
});
|
|
401
|
+
});
|
|
@@ -22,12 +22,19 @@ import {
|
|
|
22
22
|
decideSweep,
|
|
23
23
|
type EscalationEpisode,
|
|
24
24
|
type KindThresholdOverride,
|
|
25
|
+
type ZeroVolumeInputs,
|
|
25
26
|
} from "./platform-fault-escalation-logic.ts";
|
|
26
27
|
import { loadFaultEscalationConfig } from "./platform-fault-escalation/config.ts";
|
|
27
|
-
import {
|
|
28
|
+
import {
|
|
29
|
+
fetchAggregates,
|
|
30
|
+
fetchVolumeCount,
|
|
31
|
+
zeroVolumeFlagActive,
|
|
32
|
+
} from "./platform-fault-escalation/event-log-queries.ts";
|
|
28
33
|
import { orpc } from "./platform-fault-escalation/orpc-client.ts";
|
|
29
34
|
import {
|
|
35
|
+
clearZeroVolumeHold,
|
|
30
36
|
deliverEscalation,
|
|
37
|
+
deliverZeroVolumeHold,
|
|
31
38
|
resolveEpisode,
|
|
32
39
|
} from "./platform-fault-escalation/delivery.ts";
|
|
33
40
|
|
|
@@ -70,6 +77,22 @@ async function platformFaultEscalationSweep(ctx: { config: ScheduledJobConfig })
|
|
|
70
77
|
minSpanMinutes: override?.min_span_minutes,
|
|
71
78
|
};
|
|
72
79
|
}
|
|
80
|
+
|
|
81
|
+
// ZERO-VOLUME INPUTS: per configured kind, resolve its source's window
|
|
82
|
+
// volume and standing-flag state so decideSweep can tell quiet apart
|
|
83
|
+
// from dead-producer. Computed here, pure decision stays in logic.
|
|
84
|
+
const volumeSources = config.volume_sources ?? {};
|
|
85
|
+
const zeroVolume: ZeroVolumeInputs = { sources: volumeSources, countByKind: {}, flagActiveByKind: {} };
|
|
86
|
+
for (const [kind, source] of Object.entries(volumeSources)) {
|
|
87
|
+
try {
|
|
88
|
+
zeroVolume.countByKind[kind] = await fetchVolumeCount(source.volume_kind_pattern, config.window_minutes);
|
|
89
|
+
} catch (err) {
|
|
90
|
+
log.error(
|
|
91
|
+
`[platform-fault-escalation] volume count for '${kind}' (~${source.volume_kind_pattern}) failed (${err instanceof Error ? err.message : String(err)}) — treating as unmeasured for this sweep`,
|
|
92
|
+
);
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
|
|
73
96
|
const decision = decideSweep(
|
|
74
97
|
aggregates,
|
|
75
98
|
episodes,
|
|
@@ -80,6 +103,7 @@ async function platformFaultEscalationSweep(ctx: { config: ScheduledJobConfig })
|
|
|
80
103
|
},
|
|
81
104
|
now,
|
|
82
105
|
kindThresholds,
|
|
106
|
+
Object.keys(volumeSources).length > 0 ? zeroVolume : undefined,
|
|
83
107
|
);
|
|
84
108
|
|
|
85
109
|
// Per-kind isolation: one fault's delivery failure must never block the
|
|
@@ -97,9 +121,28 @@ async function platformFaultEscalationSweep(ctx: { config: ScheduledJobConfig })
|
|
|
97
121
|
recordEvent({ kind: "platform.fault.escalation_failed", payload: { event_kind: aggregate.kind, reason } });
|
|
98
122
|
}
|
|
99
123
|
}
|
|
100
|
-
for (const { episode, quietMinutes } of decision.
|
|
124
|
+
for (const { episode, quietMinutes } of decision.zeroVolumeHold) {
|
|
125
|
+
const source = volumeSources[episode.kind];
|
|
126
|
+
if (!source) continue;
|
|
127
|
+
try {
|
|
128
|
+
if (await zeroVolumeFlagActive(episode.kind)) continue; // hold already raised once
|
|
129
|
+
await deliverZeroVolumeHold(
|
|
130
|
+
episode,
|
|
131
|
+
quietMinutes,
|
|
132
|
+
zeroVolume.countByKind[episode.kind] ?? 0,
|
|
133
|
+
source,
|
|
134
|
+
);
|
|
135
|
+
log.warn(
|
|
136
|
+
`[platform-fault-escalation] ZERO-VOLUME HOLD raised for '${episode.kind}' — failures quiet AND producer quiet; episode held open`,
|
|
137
|
+
);
|
|
138
|
+
} catch (err) {
|
|
139
|
+
log.error(`[platform-fault-escalation] zero-volume hold for '${episode.kind}' failed (retried next sweep): ${err instanceof Error ? err.message : String(err)}`);
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
for (const { episode, quietMinutes, clearZeroVolumeFlag } of decision.resolve) {
|
|
101
143
|
try {
|
|
102
144
|
await resolveEpisode(episode, quietMinutes);
|
|
145
|
+
if (clearZeroVolumeFlag) await clearZeroVolumeHold(episode, zeroVolume.countByKind[episode.kind] ?? 0);
|
|
103
146
|
} catch (err) {
|
|
104
147
|
log.error(`[platform-fault-escalation] resolve for '${episode.kind}' failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
105
148
|
}
|
|
@@ -113,9 +156,10 @@ async function platformFaultEscalationSweep(ctx: { config: ScheduledJobConfig })
|
|
|
113
156
|
watched_kinds: aggregates.length,
|
|
114
157
|
escalated: decision.escalate.length,
|
|
115
158
|
resolved: decision.resolve.length,
|
|
159
|
+
zero_volume_holds: decision.zeroVolumeHold.length,
|
|
116
160
|
},
|
|
117
161
|
});
|
|
118
|
-
if (decision.escalate.length > 0 || decision.resolve.length > 0) {
|
|
162
|
+
if (decision.escalate.length > 0 || decision.resolve.length > 0 || decision.zeroVolumeHold.length > 0) {
|
|
119
163
|
log.info(
|
|
120
164
|
`[platform-fault-escalation] sweep — ${aggregates.length} kind(s) watched, ${decision.escalate.length} escalated, ${decision.resolve.length} resolved`,
|
|
121
165
|
);
|
|
@@ -47,7 +47,7 @@ export const auditRevertRouter = {
|
|
|
47
47
|
if (!user?.isSuperAdmin) {
|
|
48
48
|
throw new Error("auditRevert: requires super-admin");
|
|
49
49
|
}
|
|
50
|
-
const prisma = getPrisma(context) as unknown as Record<string,
|
|
50
|
+
const prisma = getPrisma(context) as unknown as Record<string, unknown>;
|
|
51
51
|
const log = getAppLogger();
|
|
52
52
|
const { auditTrailId } = input;
|
|
53
53
|
|
package/src/pg-boss-mount.ts
CHANGED
|
@@ -147,7 +147,7 @@ export async function mountPgBoss(
|
|
|
147
147
|
// background degradation (`a-per-consumer-pool-max-is-not-a-bound-on-the-server`:
|
|
148
148
|
// background work degrades to a queue, interactive work degrades to an
|
|
149
149
|
// outage). The sibling total is declared in
|
|
150
|
-
// config/db-pool-sizing.
|
|
150
|
+
// config/db-pool-sizing.perServiceOverhead (pg-boss 1 + LISTEN 1 = 2).
|
|
151
151
|
max: 1,
|
|
152
152
|
// Generous connection-acquisition budget for the cross-internet link so
|
|
153
153
|
// a slow TLS handshake under sustained load doesn't terminate the conn.
|
|
@@ -13,7 +13,7 @@
|
|
|
13
13
|
|
|
14
14
|
import { loadRegistryConfig } from "@teamscala/db/registry/load-config";
|
|
15
15
|
import { ResolverFnConfigSchema } from "@teamscala/db-validation/registry-schemas/resolver-fn";
|
|
16
|
-
import { getServerClient } from "
|
|
16
|
+
import { getServerClient } from "./page-server-orpc/server-client";
|
|
17
17
|
|
|
18
18
|
interface SourceConfig {
|
|
19
19
|
key: string;
|
|
@@ -10,6 +10,7 @@
|
|
|
10
10
|
*/
|
|
11
11
|
|
|
12
12
|
import { buildWakingPageHtml } from "@teamscala/os/brand/waking-page-html";
|
|
13
|
+
import { WAKING_RETRY_SECONDS } from "@teamscala/os/brand/waking-page-budget.generated";
|
|
13
14
|
import type { Context } from "hono";
|
|
14
15
|
import { getServerClientCapabilityFault } from "./page-server-orpc/server-client.ts";
|
|
15
16
|
import { resolveErrorPageLogo } from "./resource-not-found.ts";
|
|
@@ -160,7 +161,7 @@ export function createPhase1PageHandler(serviceSlug: string): CreatePageHandler
|
|
|
160
161
|
headers: {
|
|
161
162
|
"content-type": "text/html; charset=utf-8",
|
|
162
163
|
"cache-control": "no-store, no-transform",
|
|
163
|
-
"retry-after":
|
|
164
|
+
"retry-after": String(WAKING_RETRY_SECONDS),
|
|
164
165
|
},
|
|
165
166
|
});
|
|
166
167
|
}
|