@teamscala/service-boot 0.23.428 → 0.23.429
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json
CHANGED
|
@@ -178,6 +178,30 @@ export async function resolveEpisode(
|
|
|
178
178
|
});
|
|
179
179
|
}
|
|
180
180
|
|
|
181
|
+
/**
|
|
182
|
+
* Refresh an OPEN episode's window facts from the live aggregate.
|
|
183
|
+
*
|
|
184
|
+
* Escalation wrote the row once and never again, so its facts froze while the
|
|
185
|
+
* fault kept firing (2026-08-28: fifteen hours stale, 1,150 later events).
|
|
186
|
+
* That also corrupts RESOLUTION, since `decideSweep` falls back to
|
|
187
|
+
* `lastSeenAt` once a kind leaves the window. Fields mirror the escalation
|
|
188
|
+
* upsert. `last_updated_at` is untouched: it gates the resolve-hold dedupe.
|
|
189
|
+
*/
|
|
190
|
+
export async function refreshOpenEpisode(
|
|
191
|
+
kind: string,
|
|
192
|
+
aggregate: { firstSeen: Date; lastSeen: Date; count: number },
|
|
193
|
+
): Promise<void> {
|
|
194
|
+
await (await orpc()).platform_fault_escalations.update({
|
|
195
|
+
where: { kind },
|
|
196
|
+
data: {
|
|
197
|
+
first_seen_at: aggregate.firstSeen.toISOString(),
|
|
198
|
+
last_seen_at: aggregate.lastSeen.toISOString(),
|
|
199
|
+
event_count: aggregate.count,
|
|
200
|
+
updated_at: new Date().toISOString(),
|
|
201
|
+
},
|
|
202
|
+
});
|
|
203
|
+
}
|
|
204
|
+
|
|
181
205
|
/** Send one message to the operator through the verified WhatsApp relay and
|
|
182
206
|
* record it as a comment on the orchestrator's work item. Returns the
|
|
183
207
|
* verified ids; throws when the send carries no message id so the caller's
|
|
@@ -229,6 +229,21 @@ export function decideSweep(
|
|
|
229
229
|
}
|
|
230
230
|
|
|
231
231
|
/** Human "N min" span, rounded — for the phone-sized escalation message. */
|
|
232
|
+
/**
|
|
233
|
+
* Do this OPEN episode's stored window facts lag the live aggregate?
|
|
234
|
+
*
|
|
235
|
+
* Both NEGATIVE answers carry weight: a quiet kind keeps its last true
|
|
236
|
+
* sighting (the value resolve falls back to), and an aggregate no newer than
|
|
237
|
+
* the row is never written, so a steady-state sweep writes nothing.
|
|
238
|
+
*/
|
|
239
|
+
export function episodeNeedsRefresh(
|
|
240
|
+
episode: Pick<EscalationEpisode, "lastSeenAt">,
|
|
241
|
+
aggregate: { lastSeen: Date } | undefined,
|
|
242
|
+
): boolean {
|
|
243
|
+
if (!aggregate) return false;
|
|
244
|
+
return aggregate.lastSeen.getTime() > episode.lastSeenAt.getTime();
|
|
245
|
+
}
|
|
246
|
+
|
|
232
247
|
export function formatSpanMinutes(minutes: number): string {
|
|
233
248
|
if (minutes < 60) return `${Math.round(minutes)} min`;
|
|
234
249
|
const hours = minutes / 60;
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @system server-boot
|
|
3
|
+
* @status handwritten
|
|
4
|
+
* @edit edit directly
|
|
5
|
+
*
|
|
6
|
+
* Unit tests for episode FACT CURRENCY — a separate concern from the
|
|
7
|
+
* threshold/dedup semantics locked by `platform-fault-escalation.test.ts`,
|
|
8
|
+
* and kept in its own file because that one already sits over the size
|
|
9
|
+
* budget.
|
|
10
|
+
*
|
|
11
|
+
* WHAT THIS LOCKS. The sweep escalated once and then never wrote the row
|
|
12
|
+
* again, so an open episode's window facts froze at their escalation-time
|
|
13
|
+
* values while the fault kept firing. Measured 2026-08-28: the
|
|
14
|
+
* `host-command-bus.command.failed` row read `last_seen_at` fifteen hours old
|
|
15
|
+
* with 1,150 further events recorded after it, and a reader — this lane
|
|
16
|
+
* included — concluded from the row that the fault had stopped.
|
|
17
|
+
*
|
|
18
|
+
* The stale value is not only cosmetic: `decideSweep` falls back to
|
|
19
|
+
* `episode.lastSeenAt` once a kind drops out of the window, so a frozen
|
|
20
|
+
* timestamp makes a just-gone-quiet fault look quiet for hours and closes the
|
|
21
|
+
* episode immediately instead of after its configured resolve window.
|
|
22
|
+
*/
|
|
23
|
+
import { describe, expect, test } from "bun:test";
|
|
24
|
+
import { episodeNeedsRefresh } from "./platform-fault-escalation-logic.ts";
|
|
25
|
+
|
|
26
|
+
const at = (iso: string) => ({ lastSeenAt: new Date(iso) });
|
|
27
|
+
const seen = (iso: string) => ({ lastSeen: new Date(iso) });
|
|
28
|
+
|
|
29
|
+
describe("episodeNeedsRefresh", () => {
|
|
30
|
+
test("a still-firing fault whose row has fallen behind is refreshed", () => {
|
|
31
|
+
// The measured incident, in miniature: row frozen at escalation time,
|
|
32
|
+
// events still arriving fifteen hours later.
|
|
33
|
+
expect(episodeNeedsRefresh(at("2026-08-27T12:25:03Z"), seen("2026-08-28T03:00:41Z"))).toBe(
|
|
34
|
+
true,
|
|
35
|
+
);
|
|
36
|
+
});
|
|
37
|
+
|
|
38
|
+
test("a kind that has gone quiet keeps its last true sighting", () => {
|
|
39
|
+
// No aggregate means nothing landed in the window. The stored value must
|
|
40
|
+
// STAND rather than advance, because it is the reference the resolve
|
|
41
|
+
// decision falls back to — advancing it here would hold every quiet
|
|
42
|
+
// episode open forever.
|
|
43
|
+
expect(episodeNeedsRefresh(at("2026-08-27T12:25:03Z"), undefined)).toBe(false);
|
|
44
|
+
});
|
|
45
|
+
|
|
46
|
+
test("an unchanged fault is not written back", () => {
|
|
47
|
+
// Equal timestamps must not write: a sweep over steady state performs
|
|
48
|
+
// zero updates, so the refresh cannot become a per-sweep write storm.
|
|
49
|
+
const same = "2026-08-28T03:00:41Z";
|
|
50
|
+
expect(episodeNeedsRefresh(at(same), seen(same))).toBe(false);
|
|
51
|
+
});
|
|
52
|
+
|
|
53
|
+
test("an aggregate OLDER than the row never rewinds it", () => {
|
|
54
|
+
// A shrinking window can report an older newest-event than the row
|
|
55
|
+
// already holds. Rewinding would fabricate quiet time and resolve a
|
|
56
|
+
// live fault early.
|
|
57
|
+
expect(episodeNeedsRefresh(at("2026-08-28T03:00:41Z"), seen("2026-08-28T01:00:00Z"))).toBe(
|
|
58
|
+
false,
|
|
59
|
+
);
|
|
60
|
+
});
|
|
61
|
+
});
|
|
@@ -20,6 +20,7 @@ import { getAppLogger } from "@teamscala/logger/app-loggers";
|
|
|
20
20
|
import { pgBossHandlerRegistry } from "../pg-boss-handler-registry.ts";
|
|
21
21
|
import {
|
|
22
22
|
decideSweep,
|
|
23
|
+
episodeNeedsRefresh,
|
|
23
24
|
type EscalationEpisode,
|
|
24
25
|
type KindThresholdOverride,
|
|
25
26
|
type ZeroVolumeInputs,
|
|
@@ -37,6 +38,7 @@ import {
|
|
|
37
38
|
clearZeroVolumeHold,
|
|
38
39
|
deliverEscalation,
|
|
39
40
|
deliverZeroVolumeHold,
|
|
41
|
+
refreshOpenEpisode,
|
|
40
42
|
resolveEpisode,
|
|
41
43
|
} from "./platform-fault-escalation/delivery.ts";
|
|
42
44
|
|
|
@@ -125,6 +127,24 @@ async function platformFaultEscalationSweep(ctx: { config: ScheduledJobConfig })
|
|
|
125
127
|
recordEvent({ kind: "platform.fault.escalation_failed", payload: { event_kind: aggregate.kind, reason } });
|
|
126
128
|
}
|
|
127
129
|
}
|
|
130
|
+
// AN OPEN EPISODE KEEPS ITS FACTS CURRENT. Escalation is a TRANSITION and
|
|
131
|
+
// fires once (`emit-transitions-not-conditions`) — but the ROW is the
|
|
132
|
+
// durable record of the condition, and a record that stops updating starts
|
|
133
|
+
// lying. Refreshing is not emitting: nothing is delivered here, no comment
|
|
134
|
+
// is written, no message is sent. Only the stored window facts move.
|
|
135
|
+
const aggregateByKind = new Map(aggregates.map((aggregate) => [aggregate.kind, aggregate]));
|
|
136
|
+
for (const episode of episodes) {
|
|
137
|
+
const aggregate = aggregateByKind.get(episode.kind);
|
|
138
|
+
if (!episodeNeedsRefresh(episode, aggregate) || !aggregate) continue;
|
|
139
|
+
try {
|
|
140
|
+
await refreshOpenEpisode(episode.kind, aggregate);
|
|
141
|
+
} catch (err) {
|
|
142
|
+
log.error(
|
|
143
|
+
`[platform-fault-escalation] refresh for '${episode.kind}' failed (facts stay stale for this sweep, retried next): ${err instanceof Error ? err.message : String(err)}`,
|
|
144
|
+
);
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
|
|
128
148
|
for (const { episode, quietMinutes } of decision.zeroVolumeHold) {
|
|
129
149
|
const source = volumeSources[episode.kind];
|
|
130
150
|
if (!source) continue;
|