@teamscala/service-boot 0.23.516 → 0.23.518
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +232 -1127
- package/src/agreement-gate/settings.ts +8 -8
- package/src/apply-logger-config.test.ts +46 -0
- package/src/apply-logger-config.ts +31 -0
- package/src/asset-proxy.test.ts +9 -5
- package/src/auth/sign-out.ts +122 -0
- package/src/build-client-assets/hydrate-bundle.ts +14 -4
- package/src/build-client-assets/manifest.ts +5 -18
- package/src/build-client-assets/styles-build.test.ts +11 -8
- package/src/build-client-assets/styles-build.ts +14 -63
- package/src/build-client-assets.test.ts +0 -161
- package/src/build-client-assets.ts +20 -67
- package/src/content-gate/token.ts +9 -2
- package/src/install-logging.ts +10 -26
- package/src/mount-orpc-router.ts +8 -2
- package/src/mutable-app-ref.ts +0 -7
- package/src/page-server-loader-graph.test.ts +31 -0
- package/src/page-server-loader-graph.ts +21 -0
- package/src/pg-boss-mount.ts +3 -16
- package/src/readiness-gate.ts +0 -4
- package/src/register-api-routes.ts +8 -17
- package/src/register-page-server-client.ts +6 -0
- package/src/register-registry-page-route/meta-title.test.ts +50 -0
- package/src/register-registry-page-route/meta-title.ts +1 -1
- package/src/register-registry-page-route/page-data-shared.ts +7 -11
- package/src/register-registry-page-route/resolve-request-graph.test.ts +1 -1
- package/src/register-registry-page-route/resolve-request-graph.ts +1 -2
- package/src/register-registry-page-route.ts +14 -16
- package/src/resolve-brand-favicon.test.ts +63 -0
- package/src/resolve-theme-mode.test.ts +51 -0
- package/src/resolvers/fetch-org-brand.test.ts +48 -0
- package/src/resolvers/website-by-host-map.test.ts +63 -0
- package/src/resolvers/website-by-host-map.ts +15 -3
- package/src/run-boot-hook-list.test.ts +47 -0
- package/src/run-boot-hooks.test.ts +56 -0
- package/src/scala-tools-currency-watchdog.test.ts +43 -0
- package/src/scala-tools-currency-watchdog.ts +0 -5
- package/src/transfers/presigned-upload.ts +165 -0
- package/src/two-phase-minimal-app.test.ts +24 -0
- package/src/two-phase-readiness.test.ts +102 -0
- package/src/types.test.ts +20 -0
- package/src/ui-service.test.ts +24 -0
- package/src/ws-boot-self-test.test.ts +43 -0
- package/src/build-client-assets/input-fingerprint.ts +0 -92
- package/src/build-client-assets/input-hashing.ts +0 -206
- package/src/handlers/bun-stable-watch.ts +0 -86
- package/src/handlers/messaging-send-reconcile-logic.test.ts +0 -79
- package/src/handlers/messaging-send-reconcile-logic.ts +0 -83
- package/src/handlers/messaging-send-recording-logic.test.ts +0 -86
- package/src/handlers/messaging-send-recording-logic.ts +0 -68
- package/src/handlers/messaging-send-recording-watch.test.ts +0 -91
- package/src/handlers/messaging-send-recording-watch.ts +0 -244
- package/src/handlers/messaging-send-sweep-selection.test.ts +0 -47
- package/src/handlers/messaging-send-sweep-selection.ts +0 -35
- package/src/handlers/platform-fault-escalation/config.ts +0 -95
- package/src/handlers/platform-fault-escalation/delivery.ts +0 -327
- package/src/handlers/platform-fault-escalation/event-log-queries.ts +0 -182
- package/src/handlers/platform-fault-escalation/orpc-client.ts +0 -114
- package/src/handlers/platform-fault-escalation/probe.test.ts +0 -63
- package/src/handlers/platform-fault-escalation/probe.ts +0 -19
- package/src/handlers/platform-fault-escalation-logic.ts +0 -382
- package/src/handlers/platform-fault-escalation-refresh.test.ts +0 -72
- package/src/handlers/platform-fault-escalation.test.ts +0 -497
- package/src/handlers/platform-fault-escalation.ts +0 -246
- package/src/handlers/projects-digest.ts +0 -198
- package/src/handlers/quiz-research.ts +0 -178
- package/src/handlers/registry-stale-inactive-sweep.ts +0 -82
- package/src/handlers/scala-tools-currency-watch.test.ts +0 -65
- package/src/handlers/scala-tools-currency-watch.ts +0 -162
- package/src/handlers/singleton-tools-currency-watch.test.ts +0 -51
- package/src/handlers/singleton-tools-currency-watch.ts +0 -380
- package/src/handlers/sweep-event-log.ts +0 -98
- package/src/handlers/upstream-issue-sync.ts +0 -220
- package/src/handlers/whatsapp-liveness-handler.test.ts +0 -56
- package/src/handlers/whatsapp-liveness-logic.ts +0 -53
- package/src/handlers/whatsapp-liveness.test.ts +0 -78
- package/src/handlers/whatsapp-liveness.ts +0 -138
- package/src/interpolate-meta.ts +0 -123
- package/src/load-service-prisma-client.ts +0 -53
- package/src/loader-errors.ts +0 -41
- package/src/read-viewport-from-request.ts +0 -36
- package/src/register-page-server-client.test.ts +0 -185
- package/src/resolve-app-base-url.ts +0 -35
- package/src/resolve-from-loader-config.test.ts +0 -429
- package/src/resolve-from-loader-config.ts +0 -316
- package/src/resolve-service-context.ts +0 -88
|
@@ -1,497 +0,0 @@
|
|
|
1
|
-
// @system codegen
|
|
2
|
-
// @status generated
|
|
3
|
-
// @edit change the suite in the owned-suites band, then re-run codegen. Hand-edits are overwritten.
|
|
4
|
-
//
|
|
5
|
-
// This suite's assertions are OWNED by the codegen band: the band module
|
|
6
|
-
// carries them verbatim, this file is the emission, and hand edits here are
|
|
7
|
-
// overwritten on the next run. The rationale each assertion carries moved
|
|
8
|
-
// with it into the band.
|
|
9
|
-
|
|
10
|
-
import { describe, expect, test } from "bun:test";
|
|
11
|
-
import {
|
|
12
|
-
composeEscalationMessage,
|
|
13
|
-
composeZeroVolumeMessage,
|
|
14
|
-
decideSweep,
|
|
15
|
-
extractWhatsAppMessageId,
|
|
16
|
-
formatEscalationSample,
|
|
17
|
-
type EscalationEpisode,
|
|
18
|
-
type FaultAggregate,
|
|
19
|
-
type FaultEscalationPolicy,
|
|
20
|
-
} from "./platform-fault-escalation-logic.ts";
|
|
21
|
-
|
|
22
|
-
const NOW = new Date("2026-08-15T23:45:00+10:00");
|
|
23
|
-
const POLICY: FaultEscalationPolicy = {
|
|
24
|
-
minCount: 10,
|
|
25
|
-
minSpanMinutes: 25,
|
|
26
|
-
resolveAfterMinutes: 30,
|
|
27
|
-
};
|
|
28
|
-
|
|
29
|
-
function agg(kind: string, count: number, firstSeenMinutesAgo: number, lastSeenMinutesAgo: number): FaultAggregate {
|
|
30
|
-
return {
|
|
31
|
-
kind,
|
|
32
|
-
count,
|
|
33
|
-
firstSeen: new Date(NOW.getTime() - firstSeenMinutesAgo * 60_000),
|
|
34
|
-
lastSeen: new Date(NOW.getTime() - lastSeenMinutesAgo * 60_000),
|
|
35
|
-
};
|
|
36
|
-
}
|
|
37
|
-
|
|
38
|
-
function episode(overrides: Partial<EscalationEpisode> & { kind: string }): EscalationEpisode {
|
|
39
|
-
return {
|
|
40
|
-
firstSeenAt: NOW,
|
|
41
|
-
lastSeenAt: NOW,
|
|
42
|
-
eventCount: 10,
|
|
43
|
-
escalatedAt: new Date(NOW.getTime() - 60 * 60_000),
|
|
44
|
-
workItemId: "wi-1",
|
|
45
|
-
resolvedAt: null,
|
|
46
|
-
rearmedAt: null,
|
|
47
|
-
lastUpdatedAt: null,
|
|
48
|
-
...overrides,
|
|
49
|
-
};
|
|
50
|
-
}
|
|
51
|
-
|
|
52
|
-
describe("decideSweep — thresholds", () => {
|
|
53
|
-
test("a single event is noise — never escalates", () => {
|
|
54
|
-
const decision = decideSweep([agg("push.error", 1, 2, 2)], [], POLICY, NOW);
|
|
55
|
-
expect(decision.escalate).toHaveLength(0);
|
|
56
|
-
});
|
|
57
|
-
|
|
58
|
-
test("recurrence without duration does not escalate (10 events in 5 min)", () => {
|
|
59
|
-
const decision = decideSweep([agg("burst.error", 10, 5, 0)], [], POLICY, NOW);
|
|
60
|
-
expect(decision.escalate).toHaveLength(0);
|
|
61
|
-
});
|
|
62
|
-
|
|
63
|
-
test("duration without recurrence does not escalate (3 events over 40 min)", () => {
|
|
64
|
-
const decision = decideSweep([agg("sparse.error", 3, 40, 0)], [], POLICY, NOW);
|
|
65
|
-
expect(decision.escalate).toHaveLength(0);
|
|
66
|
-
});
|
|
67
|
-
|
|
68
|
-
test("the 490-over-12h shape escalates (duration + recurrence)", () => {
|
|
69
|
-
const sustained = agg("host-command-bus.command.failed", 490, 720, 0);
|
|
70
|
-
const decision = decideSweep([sustained], [], POLICY, NOW);
|
|
71
|
-
expect(decision.escalate).toHaveLength(1);
|
|
72
|
-
expect(decision.escalate[0]!.aggregate.kind).toBe("host-command-bus.command.failed");
|
|
73
|
-
expect(decision.escalate[0]!.episode).toBeNull();
|
|
74
|
-
});
|
|
75
|
-
|
|
76
|
-
test("per-kind threshold override lets a low-rate persistent kind through", () => {
|
|
77
|
-
// guard.fail.open: 8 events over 12h — below the default recurrence floor
|
|
78
|
-
// of 10, but persistent. The override lowers recurrence to 4 over 60 min.
|
|
79
|
-
const drip = agg("guard.fail.open", 8, 720, 0);
|
|
80
|
-
const withoutOverride = decideSweep([drip], [], POLICY, NOW);
|
|
81
|
-
expect(withoutOverride.escalate).toHaveLength(0);
|
|
82
|
-
const withOverride = decideSweep([drip], [], POLICY, NOW, {
|
|
83
|
-
"guard.fail.open": { minCount: 4, minSpanMinutes: 60 },
|
|
84
|
-
});
|
|
85
|
-
expect(withOverride.escalate).toHaveLength(1);
|
|
86
|
-
});
|
|
87
|
-
|
|
88
|
-
test("override raises a floor too — bursty kinds can be tightened", () => {
|
|
89
|
-
const sustained = agg("llm.provider.call.failed", 40, 60, 0); // passes default
|
|
90
|
-
const decision = decideSweep([sustained], [], POLICY, NOW, {
|
|
91
|
-
"llm.provider.call.failed": { minCount: 100 },
|
|
92
|
-
});
|
|
93
|
-
expect(decision.escalate).toHaveLength(0);
|
|
94
|
-
});
|
|
95
|
-
});
|
|
96
|
-
|
|
97
|
-
describe("decideSweep — dedup (one escalation per fault, no re-fire)", () => {
|
|
98
|
-
test("already escalated and unresolved does NOT re-escalate", () => {
|
|
99
|
-
const sustained = agg("host-command-bus.command.failed", 500, 720, 0);
|
|
100
|
-
const open = episode({ kind: "host-command-bus.command.failed", escalatedAt: new Date(NOW.getTime() - 3_600_000) });
|
|
101
|
-
const decision = decideSweep([sustained], [open], POLICY, NOW);
|
|
102
|
-
expect(decision.escalate).toHaveLength(0);
|
|
103
|
-
});
|
|
104
|
-
|
|
105
|
-
test("open delivered episode emits NOTHING while it recurs — transitions, not cadence", () => {
|
|
106
|
-
// 2026-08-26: the hourly "still recurring" comment made the alarm
|
|
107
|
-
// skippable noise and hid a real signal. An open, delivered episode now
|
|
108
|
-
// stays silent until the close transition fires (see the resolve suite).
|
|
109
|
-
const sustained = agg("llm.provider.call.failed", 300, 720, 0);
|
|
110
|
-
const longOpen = episode({
|
|
111
|
-
kind: "llm.provider.call.failed",
|
|
112
|
-
escalatedAt: new Date(NOW.getTime() - 90 * 60_000),
|
|
113
|
-
lastUpdatedAt: new Date(NOW.getTime() - 90 * 60_000),
|
|
114
|
-
});
|
|
115
|
-
const decision = decideSweep([sustained], [longOpen], POLICY, NOW);
|
|
116
|
-
expect(decision.escalate).toHaveLength(0);
|
|
117
|
-
expect(decision.resolve).toHaveLength(0);
|
|
118
|
-
expect(Object.keys(decision).sort()).toEqual([
|
|
119
|
-
"escalate",
|
|
120
|
-
"rearm",
|
|
121
|
-
"resolve",
|
|
122
|
-
"resolveProbeGate",
|
|
123
|
-
"zeroVolumeHold",
|
|
124
|
-
]);
|
|
125
|
-
});
|
|
126
|
-
|
|
127
|
-
test("a claimed-but-undelivered episode is RETRIED, not counted as escalated", () => {
|
|
128
|
-
const sustained = agg("guard.fail.open", 40, 720, 0);
|
|
129
|
-
const claim = episode({ kind: "guard.fail.open", escalatedAt: null, workItemId: null });
|
|
130
|
-
const decision = decideSweep([sustained], [claim], POLICY, NOW);
|
|
131
|
-
expect(decision.escalate).toHaveLength(1);
|
|
132
|
-
expect(decision.escalate[0]!.episode).toBe(claim);
|
|
133
|
-
});
|
|
134
|
-
});
|
|
135
|
-
|
|
136
|
-
describe("decideSweep — resolution + re-arm", () => {
|
|
137
|
-
// QUIET RE-ARMS, IT DOES NOT RESOLVE. These two asserted `resolve` until
|
|
138
|
-
// 2026-08-29, which is the unsound close: only a probe can say a condition
|
|
139
|
-
// ended, and no probe is declared for these kinds.
|
|
140
|
-
test("quiet past the resolve window RE-ARMS the episode, and claims nothing", () => {
|
|
141
|
-
const gone = episode({ kind: "wake.failed", lastSeenAt: new Date(NOW.getTime() - 60 * 60_000) });
|
|
142
|
-
const decision = decideSweep([], [gone], POLICY, NOW);
|
|
143
|
-
expect(decision.rearm).toHaveLength(1);
|
|
144
|
-
expect(decision.rearm[0]!.quietMinutes).toBeGreaterThanOrEqual(30);
|
|
145
|
-
expect(decision.resolve).toHaveLength(0);
|
|
146
|
-
});
|
|
147
|
-
|
|
148
|
-
test("an absent kind re-arms by its episode's last_seen, not the sweep clock", () => {
|
|
149
|
-
const gone = episode({ kind: "listen.connect_failed", lastSeenAt: new Date(NOW.getTime() - 45 * 60_000) });
|
|
150
|
-
const decision = decideSweep([], [gone], POLICY, NOW);
|
|
151
|
-
expect(decision.rearm).toHaveLength(1);
|
|
152
|
-
expect(decision.resolve).toHaveLength(0);
|
|
153
|
-
});
|
|
154
|
-
|
|
155
|
-
// THE INVARIANT THAT MAKES RE-ARMING SAFE. If a re-armed episode did not
|
|
156
|
-
// escalate again, dropping the quiet-close would have left every
|
|
157
|
-
// probe-less alarm permanently silent — worse than the false close.
|
|
158
|
-
test("a RE-ARMED episode escalates again when its fault recurs", () => {
|
|
159
|
-
const recurring = agg("guard.fail.open", 50, 40, 0);
|
|
160
|
-
const rearmed = episode({
|
|
161
|
-
kind: "guard.fail.open",
|
|
162
|
-
rearmedAt: new Date(NOW.getTime() - 35 * 60_000),
|
|
163
|
-
});
|
|
164
|
-
const decision = decideSweep([recurring], [rearmed], POLICY, NOW);
|
|
165
|
-
expect(decision.escalate).toHaveLength(1);
|
|
166
|
-
});
|
|
167
|
-
|
|
168
|
-
// A re-armed episode is already armed; re-arming it every sweep would emit
|
|
169
|
-
// a transition per sweep, which is the repeating-condition shape itself.
|
|
170
|
-
test("an already re-armed episode is not re-armed again while it stays quiet", () => {
|
|
171
|
-
const rearmed = episode({
|
|
172
|
-
kind: "guard.fail.open",
|
|
173
|
-
lastSeenAt: new Date(NOW.getTime() - 60 * 60_000),
|
|
174
|
-
rearmedAt: new Date(NOW.getTime() - 35 * 60_000),
|
|
175
|
-
});
|
|
176
|
-
const decision = decideSweep([], [rearmed], POLICY, NOW);
|
|
177
|
-
expect(decision.rearm).toHaveLength(0);
|
|
178
|
-
expect(decision.resolve).toHaveLength(0);
|
|
179
|
-
});
|
|
180
|
-
|
|
181
|
-
// A kind WITH a probe keeps the stronger path: it is never re-armed on
|
|
182
|
-
// quiet, because something can actually be asked about it.
|
|
183
|
-
test("a kind declaring a capability probe goes to the probe gate, never to re-arm", () => {
|
|
184
|
-
const gone = episode({ kind: "llm.provider.call.failed", lastSeenAt: new Date(NOW.getTime() - 60 * 60_000) });
|
|
185
|
-
const decision = decideSweep([], [gone], POLICY, NOW, {}, undefined, {
|
|
186
|
-
"llm.provider.call.failed": { url: "http://127.0.0.1:9/health" },
|
|
187
|
-
});
|
|
188
|
-
expect(decision.resolveProbeGate).toHaveLength(1);
|
|
189
|
-
expect(decision.rearm).toHaveLength(0);
|
|
190
|
-
expect(decision.resolve).toHaveLength(0);
|
|
191
|
-
});
|
|
192
|
-
|
|
193
|
-
test("after resolution the same fault escalates again (new episode)", () => {
|
|
194
|
-
const recurring = agg("host-command-bus.command.failed", 50, 40, 0);
|
|
195
|
-
const resolved = episode({
|
|
196
|
-
kind: "host-command-bus.command.failed",
|
|
197
|
-
resolvedAt: new Date(NOW.getTime() - 35 * 60_000),
|
|
198
|
-
});
|
|
199
|
-
const decision = decideSweep([recurring], [resolved], POLICY, NOW);
|
|
200
|
-
expect(decision.escalate).toHaveLength(1);
|
|
201
|
-
});
|
|
202
|
-
|
|
203
|
-
test("still-active kind within the resolve window is not resolved even if below threshold", () => {
|
|
204
|
-
const active = agg("slow.error", 3, 20, 0); // below threshold, but recent
|
|
205
|
-
const open = episode({ kind: "slow.error", lastSeenAt: active.lastSeen });
|
|
206
|
-
const decision = decideSweep([active], [open], POLICY, NOW);
|
|
207
|
-
expect(decision.resolve).toHaveLength(0);
|
|
208
|
-
expect(decision.rearm).toHaveLength(0);
|
|
209
|
-
});
|
|
210
|
-
});
|
|
211
|
-
|
|
212
|
-
describe("decideSweep — a future-dated lastSeen is not recurrence evidence", () => {
|
|
213
|
-
// 2026-08-26: an escalation sent 08:20Z cited "last at 09:04:54Z" — 44
|
|
214
|
-
// minutes in its own future. The rows existed (producer envelopes whose
|
|
215
|
-
// clock ran 70 minutes ahead), and `minutesBetween(lastSeen, now)` went
|
|
216
|
-
// negative, passing every recency check while the kind was actually quiet.
|
|
217
|
-
// A timestamp that has not happened yet cannot evidence a fault happening
|
|
218
|
-
// now — no matter how the aggregate was built.
|
|
219
|
-
test("a future-dated aggregate does not escalate", () => {
|
|
220
|
-
const phantom = agg("guard.fail.open", 40, 720, -44);
|
|
221
|
-
const decision = decideSweep([phantom], [], POLICY, NOW);
|
|
222
|
-
expect(decision.escalate).toHaveLength(0);
|
|
223
|
-
});
|
|
224
|
-
|
|
225
|
-
test("a future-dated aggregate does not hold an open episode open past the quiet window", () => {
|
|
226
|
-
// lastSeen 44 min in the future means the kind's real events are quiet:
|
|
227
|
-
// the episode's own last_seen_at (honest) drives the resolve decision,
|
|
228
|
-
// and the phantom aggregate must not veto it.
|
|
229
|
-
const quietEpisode = episode({
|
|
230
|
-
kind: "guard.fail.open",
|
|
231
|
-
lastSeenAt: new Date(NOW.getTime() - 60 * 60_000),
|
|
232
|
-
});
|
|
233
|
-
const phantom = agg("guard.fail.open", 40, 720, -44);
|
|
234
|
-
const decision = decideSweep([phantom], [quietEpisode], POLICY, NOW);
|
|
235
|
-
expect(decision.escalate).toHaveLength(0);
|
|
236
|
-
// Re-armed rather than resolved: no probe is declared for this kind.
|
|
237
|
-
expect(decision.rearm).toHaveLength(1);
|
|
238
|
-
expect(decision.resolve).toHaveLength(0);
|
|
239
|
-
});
|
|
240
|
-
});
|
|
241
|
-
|
|
242
|
-
describe("extractWhatsAppMessageId — delivery verification", () => {
|
|
243
|
-
test("the messaging_whatsapp_send result shape yields message_id", () => {
|
|
244
|
-
expect(extractWhatsAppMessageId({ ok: true, message_id: "3EB089B9D6ADD58153C561" })).toBe(
|
|
245
|
-
"3EB089B9D6ADD58153C561",
|
|
246
|
-
);
|
|
247
|
-
});
|
|
248
|
-
|
|
249
|
-
test("a 'true' ack or an error body is NOT a message id — delivery stays unverified", () => {
|
|
250
|
-
// A boolean-ish ack string must never be recorded as a delivered id:
|
|
251
|
-
// that would mark the episode delivered with an unfollowable handle.
|
|
252
|
-
expect(extractWhatsAppMessageId({ ok: true })).toBeNull();
|
|
253
|
-
expect(extractWhatsAppMessageId({ ok: false, error: "no device" })).toBeNull();
|
|
254
|
-
expect(extractWhatsAppMessageId("true")).toBeNull();
|
|
255
|
-
expect(extractWhatsAppMessageId(null)).toBeNull();
|
|
256
|
-
});
|
|
257
|
-
});
|
|
258
|
-
|
|
259
|
-
describe("composeEscalationMessage", () => {
|
|
260
|
-
test("carries the three escalation-routing fields and stays phone-sized", () => {
|
|
261
|
-
const message = composeEscalationMessage({
|
|
262
|
-
kind: "host-command-bus.command.failed",
|
|
263
|
-
aggregate: agg("host-command-bus.command.failed", 490, 720, 0),
|
|
264
|
-
impact: "scala-tools verbs are failing host-wide",
|
|
265
|
-
recommendation: "check oxmgr logs for the bus consumer",
|
|
266
|
-
sample: "",
|
|
267
|
-
lastEvent: null,
|
|
268
|
-
});
|
|
269
|
-
expect(message).toContain("Platform fault: host-command-bus.command.failed");
|
|
270
|
-
expect(message).toContain("Observed: 490 errors over 12 h");
|
|
271
|
-
expect(message).toContain("Impact: scala-tools verbs are failing host-wide");
|
|
272
|
-
expect(message).toContain("Recommendation: check oxmgr logs for the bus consumer");
|
|
273
|
-
expect(message.split("\n")).toHaveLength(4);
|
|
274
|
-
});
|
|
275
|
-
|
|
276
|
-
// 2026-08-26: an escalation sent 08:20Z cited "last at 09:04:54Z", a
|
|
277
|
-
// timestamp 44 minutes in its own future matching no event the operator
|
|
278
|
-
// could find. The Observed line now cites the newest backing row's own
|
|
279
|
-
// occurred_at and id, so "last at T" resolves to a real event_log row.
|
|
280
|
-
test("cites the newest backing row's timestamp and event id — resolvable evidence", () => {
|
|
281
|
-
const message = composeEscalationMessage({
|
|
282
|
-
kind: "guard.fail.open",
|
|
283
|
-
aggregate: agg("guard.fail.open", 490, 720, 0),
|
|
284
|
-
impact: "guard enforcement is failing open",
|
|
285
|
-
recommendation: "check the guard daemon",
|
|
286
|
-
sample: "",
|
|
287
|
-
lastEvent: { occurredAt: "2026-08-26T08:20:29Z", eventId: "01a03b79-c124-748f-a79f-a9a436729544" },
|
|
288
|
-
});
|
|
289
|
-
expect(message).toContain("last 2026-08-26T08:20:29Z (event 01a03b79)");
|
|
290
|
-
});
|
|
291
|
-
|
|
292
|
-
// 2026-08-26: three escalations reached the orchestrator seat carrying
|
|
293
|
-
// kind, count and duration only — triaging each required a hand event_log
|
|
294
|
-
// query, and the llm.provider.call.failed impact line pointed AWAY from
|
|
295
|
-
// the defect (payloads held our own 'Invalid length: Expected <=1200 but
|
|
296
|
-
// received 1255' contract violations recorded against healthy providers).
|
|
297
|
-
// The sample line carries the evidence in the first read.
|
|
298
|
-
test("carries the sampled payload evidence as its own line", () => {
|
|
299
|
-
const message = composeEscalationMessage({
|
|
300
|
-
kind: "llm.provider.call.failed",
|
|
301
|
-
aggregate: agg("llm.provider.call.failed", 26, 560, 0),
|
|
302
|
-
sample: 'status=429; message=Usage limit reached for 5 hour. Your limit will reset at 2026-08-26 01:19:03',
|
|
303
|
-
impact: "AI chat/completion calls are failing",
|
|
304
|
-
recommendation: "check provider quota",
|
|
305
|
-
lastEvent: null,
|
|
306
|
-
});
|
|
307
|
-
const lines = message.split("\n");
|
|
308
|
-
expect(lines).toHaveLength(5);
|
|
309
|
-
expect(lines[2]).toContain("Sample: status=429");
|
|
310
|
-
expect(lines[2]).toContain("reset at 2026-08-26 01:19:03");
|
|
311
|
-
});
|
|
312
|
-
});
|
|
313
|
-
|
|
314
|
-
describe("formatEscalationSample — evidence truncated hard, secrets never shipped", () => {
|
|
315
|
-
test("non-string values are JSON-encoded so structure survives the phone channel", () => {
|
|
316
|
-
expect(formatEscalationSample(429)).toBe("429");
|
|
317
|
-
expect(formatEscalationSample(true)).toBe("true");
|
|
318
|
-
expect(formatEscalationSample({ expected: 1200, received: 1255 })).toBe('{"expected":1200,"received":1255}');
|
|
319
|
-
});
|
|
320
|
-
|
|
321
|
-
test("truncates hard at the bound with an ellipsis so the reader knows the cut happened", () => {
|
|
322
|
-
const formatted = formatEscalationSample("x".repeat(500));
|
|
323
|
-
expect(formatted.length).toBe(201); // 200 chars + ellipsis
|
|
324
|
-
expect(formatted.endsWith("…")).toBe(true);
|
|
325
|
-
});
|
|
326
|
-
|
|
327
|
-
test("redacts credential shapes before truncation — a cut can never expose a secret tail", () => {
|
|
328
|
-
expect(formatEscalationSample("key sk-abcdefghijklmnopqrstuvwxyz failed")).toBe("key [redacted] failed");
|
|
329
|
-
expect(formatEscalationSample("Authorization: Bearer abcdefghijklmnop")).toBe(
|
|
330
|
-
"Authorization=[redacted]",
|
|
331
|
-
);
|
|
332
|
-
expect(formatEscalationSample("api_key=abcdefgh1234 rejected")).toBe("api_key=[redacted] rejected");
|
|
333
|
-
});
|
|
334
|
-
|
|
335
|
-
test("collapses whitespace to one line — the sample is one message line", () => {
|
|
336
|
-
expect(formatEscalationSample("line one\nline two\t tabbed")).toBe("line one line two tabbed");
|
|
337
|
-
});
|
|
338
|
-
|
|
339
|
-
test("empty input yields an empty sample (no Sample line)", () => {
|
|
340
|
-
expect(formatEscalationSample(undefined)).toBe("");
|
|
341
|
-
expect(formatEscalationSample("")).toBe("");
|
|
342
|
-
});
|
|
343
|
-
});
|
|
344
|
-
|
|
345
|
-
describe("decideSweep — qualification asks 'is this happening', not 'did this happen'", () => {
|
|
346
|
-
// 2026-08-26: llm.provider.call.failed re-escalated every ~10 minutes with
|
|
347
|
-
// lastSeen frozen at 16:54 — escalated 19:40 on a fault three hours dead,
|
|
348
|
-
// resolved 19:45, repeat. The tell was the closure counter climbing
|
|
349
|
-
// (141/151/171 min): quiet measured from one frozen aggregate lastSeen.
|
|
350
|
-
// isSustained checked count and span but never recency, so a dead fault
|
|
351
|
-
// kept qualifying for the rest of the 720-minute lookback while a resolved
|
|
352
|
-
// episode read as "no episode" and re-escalated it.
|
|
353
|
-
const resolved = (kind: string) =>
|
|
354
|
-
episode({ kind, resolvedAt: new Date(NOW.getTime() - 5 * 60_000) });
|
|
355
|
-
|
|
356
|
-
test("a fault quiet past the resolve window does not re-escalate after resolution", () => {
|
|
357
|
-
const decision = decideSweep(
|
|
358
|
-
[agg("stale.error", 30, 400, 170)],
|
|
359
|
-
[resolved("stale.error")],
|
|
360
|
-
POLICY,
|
|
361
|
-
NOW,
|
|
362
|
-
);
|
|
363
|
-
expect(decision.escalate).toHaveLength(0);
|
|
364
|
-
});
|
|
365
|
-
|
|
366
|
-
test("a fault quiet past the resolve window does not escalate with no episode at all", () => {
|
|
367
|
-
const decision = decideSweep([agg("stale.error", 30, 400, 170)], [], POLICY, NOW);
|
|
368
|
-
expect(decision.escalate).toHaveLength(0);
|
|
369
|
-
});
|
|
370
|
-
|
|
371
|
-
// The other direction, and the one that protects live escalations: the fix
|
|
372
|
-
// must not silence a fault that is still emitting.
|
|
373
|
-
test("a fault still emitting escalates", () => {
|
|
374
|
-
const decision = decideSweep([agg("live.error", 30, 400, 2)], [], POLICY, NOW);
|
|
375
|
-
expect(decision.escalate).toHaveLength(1);
|
|
376
|
-
});
|
|
377
|
-
|
|
378
|
-
test("a recurring fault opens a NEW episode after its previous one resolved", () => {
|
|
379
|
-
const decision = decideSweep(
|
|
380
|
-
[agg("live.error", 30, 400, 2)],
|
|
381
|
-
[resolved("live.error")],
|
|
382
|
-
POLICY,
|
|
383
|
-
NOW,
|
|
384
|
-
);
|
|
385
|
-
expect(decision.escalate).toHaveLength(1);
|
|
386
|
-
});
|
|
387
|
-
});
|
|
388
|
-
|
|
389
|
-
describe("decideSweep — zero-volume gate (quiet vs dead producer)", () => {
|
|
390
|
-
const SOURCES = {
|
|
391
|
-
"llm.provider.call.failed": { volume_kind_pattern: "^llm\.router\.routing_decision$", min_volume: 1 },
|
|
392
|
-
};
|
|
393
|
-
const VOLUME_BACKED = { sources: SOURCES, countByKind: {}, flagActiveByKind: {} };
|
|
394
|
-
|
|
395
|
-
test("an open episode with NO volume does not resolve — it holds", () => {
|
|
396
|
-
const episodes = [episode({ kind: "llm.provider.call.failed", lastSeenAt: new Date(NOW.getTime() - 45 * 60_000) })];
|
|
397
|
-
const inputs = { ...VOLUME_BACKED, countByKind: { "llm.provider.call.failed": 0 } };
|
|
398
|
-
const decision = decideSweep([], episodes, POLICY, NOW, {}, inputs);
|
|
399
|
-
expect(decision.resolve).toHaveLength(0);
|
|
400
|
-
expect(decision.zeroVolumeHold).toHaveLength(1);
|
|
401
|
-
expect(decision.zeroVolumeHold[0].volumeCount).toBe(0);
|
|
402
|
-
});
|
|
403
|
-
|
|
404
|
-
test("quiet WITH healthy call volume re-arms (the normal path for a probe-less kind)", () => {
|
|
405
|
-
const episodes = [episode({ kind: "llm.provider.call.failed", lastSeenAt: new Date(NOW.getTime() - 45 * 60_000) })];
|
|
406
|
-
const inputs = { ...VOLUME_BACKED, countByKind: { "llm.provider.call.failed": 120 } };
|
|
407
|
-
const decision = decideSweep([], episodes, POLICY, NOW, {}, inputs);
|
|
408
|
-
expect(decision.zeroVolumeHold).toHaveLength(0);
|
|
409
|
-
expect(decision.rearm).toHaveLength(1);
|
|
410
|
-
expect(decision.resolve).toHaveLength(0);
|
|
411
|
-
expect(decision.rearm[0]!.clearZeroVolumeFlag).toBe(false);
|
|
412
|
-
});
|
|
413
|
-
|
|
414
|
-
test("a sourceless kind re-arms on quiet", () => {
|
|
415
|
-
const episodes = [episode({ kind: "guard.fail.open", lastSeenAt: new Date(NOW.getTime() - 45 * 60_000) })];
|
|
416
|
-
const decision = decideSweep([], episodes, POLICY, NOW, {}, { ...VOLUME_BACKED, countByKind: {} });
|
|
417
|
-
expect(decision.rearm).toHaveLength(1);
|
|
418
|
-
expect(decision.resolve).toHaveLength(0);
|
|
419
|
-
expect(decision.zeroVolumeHold).toHaveLength(0);
|
|
420
|
-
});
|
|
421
|
-
|
|
422
|
-
test("recovery from a flagged hold clears the flag on resolve", () => {
|
|
423
|
-
const episodes = [episode({ kind: "llm.provider.call.failed", lastSeenAt: new Date(NOW.getTime() - 45 * 60_000) })];
|
|
424
|
-
const inputs = {
|
|
425
|
-
...VOLUME_BACKED,
|
|
426
|
-
countByKind: { "llm.provider.call.failed": 50 },
|
|
427
|
-
flagActiveByKind: { "llm.provider.call.failed": true },
|
|
428
|
-
};
|
|
429
|
-
const decision = decideSweep([], episodes, POLICY, NOW, {}, inputs);
|
|
430
|
-
expect(decision.rearm[0]!.clearZeroVolumeFlag).toBe(true);
|
|
431
|
-
});
|
|
432
|
-
|
|
433
|
-
test("without a sixth argument the legacy shape is untouched", () => {
|
|
434
|
-
const episodes = [episode({ kind: "llm.provider.call.failed", lastSeenAt: new Date(NOW.getTime() - 45 * 60_000) })];
|
|
435
|
-
const decision = decideSweep([], episodes, POLICY, NOW);
|
|
436
|
-
expect(decision.rearm).toHaveLength(1);
|
|
437
|
-
expect(decision.resolve).toHaveLength(0);
|
|
438
|
-
});
|
|
439
|
-
});
|
|
440
|
-
|
|
441
|
-
describe("composeZeroVolumeMessage — the quiet-is-not-healthy notice", () => {
|
|
442
|
-
test("names the held kind, both absences and the floors; stays phone-sized", () => {
|
|
443
|
-
const message = composeZeroVolumeMessage({
|
|
444
|
-
kind: "llm.provider.call.failed",
|
|
445
|
-
volumeKindPattern: "^llm\.router\.routing_decision$",
|
|
446
|
-
minVolume: 1,
|
|
447
|
-
quietMinutes: 600,
|
|
448
|
-
volumeCount: 0,
|
|
449
|
-
firstSeenAt: new Date(NOW.getTime() - 700 * 60_000),
|
|
450
|
-
});
|
|
451
|
-
for (const fragment of ["llm.provider.call.failed", "routing_decision", "(0 < floor 1", "Recommendation:"]) {
|
|
452
|
-
expect(message).toContain(fragment);
|
|
453
|
-
}
|
|
454
|
-
expect(message.split("\n").length).toBeLessThanOrEqual(4);
|
|
455
|
-
});
|
|
456
|
-
});
|
|
457
|
-
|
|
458
|
-
describe("decideSweep — probe-gated resolution (operator ruling 2026-08-28)", () => {
|
|
459
|
-
const PROBE_SOURCES = {
|
|
460
|
-
"llm.provider.call.failed": { url: "https://ai.scala.business/health", timeout_seconds: 5 },
|
|
461
|
-
};
|
|
462
|
-
|
|
463
|
-
test("a kind with a declared probe is gated, not resolved on quiet", () => {
|
|
464
|
-
const gone = episode({
|
|
465
|
-
kind: "llm.provider.call.failed",
|
|
466
|
-
lastSeenAt: new Date(NOW.getTime() - 60 * 60_000),
|
|
467
|
-
});
|
|
468
|
-
const decision = decideSweep([agg("llm.provider.call.failed", 10, 700, 60)], [gone], POLICY, NOW, {}, undefined, PROBE_SOURCES);
|
|
469
|
-
expect(decision.resolve).toHaveLength(0);
|
|
470
|
-
expect(decision.resolveProbeGate).toHaveLength(1);
|
|
471
|
-
expect(decision.resolveProbeGate[0]!.probeUrl).toBe("https://ai.scala.business/health");
|
|
472
|
-
expect(decision.resolveProbeGate[0]!.quietMinutes).toBeGreaterThanOrEqual(30);
|
|
473
|
-
});
|
|
474
|
-
|
|
475
|
-
test("a kind without a declared probe RE-ARMS on quiet — it is never closed unverified", () => {
|
|
476
|
-
const gone = episode({ kind: "wake.failed", lastSeenAt: new Date(NOW.getTime() - 60 * 60_000) });
|
|
477
|
-
const decision = decideSweep([agg("wake.failed", 10, 700, 60)], [gone], POLICY, NOW, {}, undefined, PROBE_SOURCES);
|
|
478
|
-
expect(decision.rearm).toHaveLength(1);
|
|
479
|
-
expect(decision.resolve).toHaveLength(0);
|
|
480
|
-
expect(decision.resolveProbeGate).toHaveLength(0);
|
|
481
|
-
});
|
|
482
|
-
|
|
483
|
-
test("the gate passes through the zero-volume flag so a healthy probe clears it", () => {
|
|
484
|
-
const gone = episode({
|
|
485
|
-
kind: "llm.provider.call.failed",
|
|
486
|
-
lastSeenAt: new Date(NOW.getTime() - 60 * 60_000),
|
|
487
|
-
});
|
|
488
|
-
const zeroVolume = {
|
|
489
|
-
sources: { "llm.provider.call.failed": { volume_kind_pattern: "^llm\.router\.routing_decision$", min_volume: 1 } },
|
|
490
|
-
countByKind: { "llm.provider.call.failed": 12 },
|
|
491
|
-
flagActiveByKind: { "llm.provider.call.failed": true },
|
|
492
|
-
};
|
|
493
|
-
const decision = decideSweep([agg("llm.provider.call.failed", 10, 700, 60)], [gone], POLICY, NOW, {}, zeroVolume, PROBE_SOURCES);
|
|
494
|
-
expect(decision.resolveProbeGate).toHaveLength(1);
|
|
495
|
-
expect(decision.resolveProbeGate[0]!.clearZeroVolumeFlag).toBe(true);
|
|
496
|
-
});
|
|
497
|
-
});
|