@checkstack/dependency-backend 0.2.15 â 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +26 -0
- package/package.json +4 -3
- package/src/index.ts +4 -0
- package/src/notifications.ts +243 -42
- package/tests/notifications.test.ts +186 -0
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,31 @@
|
|
|
1
1
|
# @checkstack/dependency-backend
|
|
2
2
|
|
|
3
|
+
## 0.3.0
|
|
4
|
+
|
|
5
|
+
### Minor Changes
|
|
6
|
+
|
|
7
|
+
- 298bf42: ### Notification System Optimizations
|
|
8
|
+
|
|
9
|
+
**System context in notifications**: All notification senders (healthcheck, incident, maintenance, dependency) now include the affected system name in the notification title and body. Users can immediately identify which system is affected without clicking through to the detail page.
|
|
10
|
+
|
|
11
|
+
**Upstream notification deduplication**: When an upstream dependency goes down affecting multiple downstream systems, the dependency notification sidecar now sends **one personalized notification per user** instead of one notification per affected system. Each user's notification lists only the systems they are subscribed to, with a link to the upstream root cause system. This prevents notification floods for users subscribed to groups containing many dependent systems.
|
|
12
|
+
|
|
13
|
+
**New catalog endpoint**: Added `getSystemGroupIds` S2S RPC endpoint on the catalog to resolve which catalog groups contain a given system, used by the dependency plugin for efficient subscriber resolution during batched notification dispatch.
|
|
14
|
+
|
|
15
|
+
### Patch Changes
|
|
16
|
+
|
|
17
|
+
- Updated dependencies [298bf42]
|
|
18
|
+
- @checkstack/healthcheck-backend@0.17.0
|
|
19
|
+
- @checkstack/catalog-common@1.5.0
|
|
20
|
+
- @checkstack/catalog-backend@0.6.0
|
|
21
|
+
|
|
22
|
+
## 0.2.16
|
|
23
|
+
|
|
24
|
+
### Patch Changes
|
|
25
|
+
|
|
26
|
+
- Updated dependencies [9a320fe]
|
|
27
|
+
- @checkstack/healthcheck-backend@0.16.5
|
|
28
|
+
|
|
3
29
|
## 0.2.15
|
|
4
30
|
|
|
5
31
|
### Patch Changes
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@checkstack/dependency-backend",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.3.0",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"main": "src/index.ts",
|
|
6
6
|
"checkstack": {
|
|
@@ -16,11 +16,12 @@
|
|
|
16
16
|
"@checkstack/backend-api": "0.12.0",
|
|
17
17
|
"@checkstack/dependency-common": "0.2.1",
|
|
18
18
|
"@checkstack/catalog-common": "1.4.1",
|
|
19
|
-
"@checkstack/catalog-backend": "0.5.
|
|
19
|
+
"@checkstack/catalog-backend": "0.5.4",
|
|
20
20
|
"@checkstack/healthcheck-common": "0.11.0",
|
|
21
|
-
"@checkstack/healthcheck-backend": "0.16.
|
|
21
|
+
"@checkstack/healthcheck-backend": "0.16.5",
|
|
22
22
|
"@checkstack/maintenance-common": "0.4.9",
|
|
23
23
|
"@checkstack/incident-common": "0.4.7",
|
|
24
|
+
"@checkstack/notification-common": "0.2.8",
|
|
24
25
|
"@checkstack/signal-common": "0.1.9",
|
|
25
26
|
"@checkstack/common": "0.6.5",
|
|
26
27
|
"drizzle-orm": "^0.45.0",
|
package/src/index.ts
CHANGED
|
@@ -14,6 +14,7 @@ import { CatalogApi } from "@checkstack/catalog-common";
|
|
|
14
14
|
import { HealthCheckApi } from "@checkstack/healthcheck-common";
|
|
15
15
|
import { MaintenanceApi } from "@checkstack/maintenance-common";
|
|
16
16
|
import { IncidentApi } from "@checkstack/incident-common";
|
|
17
|
+
import { NotificationApi } from "@checkstack/notification-common";
|
|
17
18
|
import { catalogHooks } from "@checkstack/catalog-backend";
|
|
18
19
|
import { healthCheckHooks } from "@checkstack/healthcheck-backend";
|
|
19
20
|
import { evaluateAndNotifyDownstream } from "./notifications";
|
|
@@ -73,6 +74,7 @@ export default createBackendPlugin({
|
|
|
73
74
|
const healthCheckClient = rpcClient.forPlugin(HealthCheckApi);
|
|
74
75
|
const maintenanceClient = rpcClient.forPlugin(MaintenanceApi);
|
|
75
76
|
const incidentClient = rpcClient.forPlugin(IncidentApi);
|
|
77
|
+
const notificationClient = rpcClient.forPlugin(NotificationApi);
|
|
76
78
|
|
|
77
79
|
/**
|
|
78
80
|
* Build system statuses for warning evaluation.
|
|
@@ -161,6 +163,7 @@ export default createBackendPlugin({
|
|
|
161
163
|
warningService,
|
|
162
164
|
fetchSystemStatuses,
|
|
163
165
|
catalogClient,
|
|
166
|
+
notificationClient,
|
|
164
167
|
maintenanceClient,
|
|
165
168
|
incidentClient,
|
|
166
169
|
signalService,
|
|
@@ -186,6 +189,7 @@ export default createBackendPlugin({
|
|
|
186
189
|
warningService,
|
|
187
190
|
fetchSystemStatuses,
|
|
188
191
|
catalogClient,
|
|
192
|
+
notificationClient,
|
|
189
193
|
maintenanceClient,
|
|
190
194
|
incidentClient,
|
|
191
195
|
signalService,
|
package/src/notifications.ts
CHANGED
|
@@ -5,6 +5,7 @@ import type { CatalogApi } from "@checkstack/catalog-common";
|
|
|
5
5
|
import { catalogRoutes } from "@checkstack/catalog-common";
|
|
6
6
|
import type { MaintenanceApi } from "@checkstack/maintenance-common";
|
|
7
7
|
import type { IncidentApi } from "@checkstack/incident-common";
|
|
8
|
+
import type { NotificationApi } from "@checkstack/notification-common";
|
|
8
9
|
import type { DerivedState } from "@checkstack/dependency-common";
|
|
9
10
|
import { DEPENDENCY_WARNINGS_CHANGED } from "@checkstack/dependency-common";
|
|
10
11
|
import type { DependencyService } from "./services/dependency-service";
|
|
@@ -45,26 +46,30 @@ function derivedStateToImportance(
|
|
|
45
46
|
export function buildNotificationTitle({
|
|
46
47
|
derivedState,
|
|
47
48
|
isRecovery,
|
|
49
|
+
systemName,
|
|
48
50
|
}: {
|
|
49
51
|
derivedState?: DerivedState;
|
|
50
52
|
isRecovery: boolean;
|
|
53
|
+
systemName?: string;
|
|
51
54
|
}): string {
|
|
55
|
+
const prefix = systemName ? `${systemName}: ` : "";
|
|
56
|
+
|
|
52
57
|
if (isRecovery) {
|
|
53
|
-
return
|
|
58
|
+
return `${prefix}Dependency impact resolved`;
|
|
54
59
|
}
|
|
55
60
|
|
|
56
61
|
switch (derivedState) {
|
|
57
62
|
case "info": {
|
|
58
|
-
return
|
|
63
|
+
return `${prefix}Upstream dependency issue (informational)`;
|
|
59
64
|
}
|
|
60
65
|
case "degraded": {
|
|
61
|
-
return
|
|
66
|
+
return `${prefix}Availability impacted by upstream dependency`;
|
|
62
67
|
}
|
|
63
68
|
case "down": {
|
|
64
|
-
return
|
|
69
|
+
return `${prefix}Availability critically impacted by upstream dependency`;
|
|
65
70
|
}
|
|
66
71
|
default: {
|
|
67
|
-
return
|
|
72
|
+
return `${prefix}Dependency impact changed`;
|
|
68
73
|
}
|
|
69
74
|
}
|
|
70
75
|
}
|
|
@@ -76,26 +81,29 @@ export function buildNotificationBody({
|
|
|
76
81
|
upstreamNames,
|
|
77
82
|
derivedState,
|
|
78
83
|
isRecovery,
|
|
84
|
+
systemName,
|
|
79
85
|
}: {
|
|
80
86
|
upstreamNames: string[];
|
|
81
87
|
derivedState?: DerivedState;
|
|
82
88
|
isRecovery: boolean;
|
|
89
|
+
systemName?: string;
|
|
83
90
|
}): string {
|
|
84
91
|
const upstreamList = upstreamNames.join(", ");
|
|
92
|
+
const systemRef = systemName ? `**${systemName}**` : "This system";
|
|
85
93
|
|
|
86
94
|
if (isRecovery) {
|
|
87
|
-
return
|
|
95
|
+
return `All upstream dependencies have recovered. ${systemRef} is no longer affected by dependency failures.`;
|
|
88
96
|
}
|
|
89
97
|
|
|
90
98
|
switch (derivedState) {
|
|
91
99
|
case "info": {
|
|
92
|
-
return `An upstream dependency (${upstreamList}) is experiencing issues.
|
|
100
|
+
return `An upstream dependency (${upstreamList}) is experiencing issues. ${systemRef} â this is informational, no direct impact expected.`;
|
|
93
101
|
}
|
|
94
102
|
case "degraded": {
|
|
95
|
-
return `An upstream dependency (${upstreamList}) is experiencing issues.
|
|
103
|
+
return `An upstream dependency (${upstreamList}) is experiencing issues. ${systemRef}'s availability may be degraded.`;
|
|
96
104
|
}
|
|
97
105
|
case "down": {
|
|
98
|
-
return `A critical upstream dependency (${upstreamList}) is down.
|
|
106
|
+
return `A critical upstream dependency (${upstreamList}) is down. ${systemRef} is expected to be unavailable.`;
|
|
99
107
|
}
|
|
100
108
|
default: {
|
|
101
109
|
return `Upstream dependency status has changed (${upstreamList}).`;
|
|
@@ -103,12 +111,68 @@ export function buildNotificationBody({
|
|
|
103
111
|
}
|
|
104
112
|
}
|
|
105
113
|
|
|
114
|
+
/**
|
|
115
|
+
* Represents a downstream system that needs notification due to a state change.
|
|
116
|
+
*/
|
|
117
|
+
export interface SystemNotificationEntry {
|
|
118
|
+
systemId: string;
|
|
119
|
+
systemName: string;
|
|
120
|
+
derivedState?: DerivedState;
|
|
121
|
+
isRecovery: boolean;
|
|
122
|
+
importance: "info" | "warning" | "critical";
|
|
123
|
+
upstreamNames: string[];
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
/**
|
|
127
|
+
* Resolve the worst importance level from a list of notification entries.
|
|
128
|
+
*/
|
|
129
|
+
function resolveWorstImportance(
|
|
130
|
+
entries: SystemNotificationEntry[],
|
|
131
|
+
): "info" | "warning" | "critical" {
|
|
132
|
+
let worst: "info" | "warning" | "critical" = "info";
|
|
133
|
+
for (const entry of entries) {
|
|
134
|
+
if (entry.importance === "critical") return "critical";
|
|
135
|
+
if (entry.importance === "warning") worst = "warning";
|
|
136
|
+
}
|
|
137
|
+
return worst;
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
/**
|
|
141
|
+
* Format a per-system impact line with criticality indicator for multi-system
|
|
142
|
+
* notification bodies.
|
|
143
|
+
*/
|
|
144
|
+
export function formatSystemImpactLine(entry: SystemNotificationEntry): string {
|
|
145
|
+
if (entry.isRecovery) {
|
|
146
|
+
return `- â
**${entry.systemName}** â recovered`;
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
switch (entry.derivedState) {
|
|
150
|
+
case "down": {
|
|
151
|
+
return `- đ´ **${entry.systemName}** â critically impacted`;
|
|
152
|
+
}
|
|
153
|
+
case "degraded": {
|
|
154
|
+
return `- đĄ **${entry.systemName}** â degraded`;
|
|
155
|
+
}
|
|
156
|
+
case "info": {
|
|
157
|
+
return `- âšī¸ **${entry.systemName}** â informational`;
|
|
158
|
+
}
|
|
159
|
+
default: {
|
|
160
|
+
return `- **${entry.systemName}** â impact changed`;
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
|
|
106
165
|
/**
|
|
107
166
|
* Evaluate downstream systems for dependency-driven state changes
|
|
108
167
|
* and notify subscribers when the derived state transitions.
|
|
109
168
|
*
|
|
110
169
|
* This is the Sidecar Notification Orchestration function.
|
|
111
170
|
* It runs when an upstream system's health status changes.
|
|
171
|
+
*
|
|
172
|
+
* Notification deduplication: Instead of sending one notification per
|
|
173
|
+
* downstream system (which floods users subscribed to groups), we resolve
|
|
174
|
+
* all affected subscribers and send one personalized notification per user
|
|
175
|
+
* listing only the systems they are subscribed to.
|
|
112
176
|
*/
|
|
113
177
|
export async function evaluateAndNotifyDownstream({
|
|
114
178
|
changedSystemId,
|
|
@@ -117,6 +181,7 @@ export async function evaluateAndNotifyDownstream({
|
|
|
117
181
|
warningService,
|
|
118
182
|
fetchSystemStatuses,
|
|
119
183
|
catalogClient,
|
|
184
|
+
notificationClient,
|
|
120
185
|
maintenanceClient,
|
|
121
186
|
incidentClient,
|
|
122
187
|
signalService,
|
|
@@ -130,6 +195,7 @@ export async function evaluateAndNotifyDownstream({
|
|
|
130
195
|
systemIds: string[],
|
|
131
196
|
) => Promise<Map<string, SystemStatus>>;
|
|
132
197
|
catalogClient: InferClient<typeof CatalogApi>;
|
|
198
|
+
notificationClient: InferClient<typeof NotificationApi>;
|
|
133
199
|
maintenanceClient: InferClient<typeof MaintenanceApi>;
|
|
134
200
|
incidentClient: InferClient<typeof IncidentApi>;
|
|
135
201
|
signalService: SignalService;
|
|
@@ -236,8 +302,10 @@ export async function evaluateAndNotifyDownstream({
|
|
|
236
302
|
}
|
|
237
303
|
}
|
|
238
304
|
|
|
239
|
-
// 6.
|
|
305
|
+
// 6. Evaluate state changes and collect systems that need notification
|
|
240
306
|
const changedSystemIds: string[] = [];
|
|
307
|
+
const systemsToNotify: SystemNotificationEntry[] = [];
|
|
308
|
+
|
|
241
309
|
for (const systemId of downstreamIds) {
|
|
242
310
|
const currentWarning = warningMap.get(systemId);
|
|
243
311
|
const currentState = currentWarning?.derivedState;
|
|
@@ -278,52 +346,40 @@ export async function evaluateAndNotifyDownstream({
|
|
|
278
346
|
continue;
|
|
279
347
|
}
|
|
280
348
|
|
|
281
|
-
//
|
|
349
|
+
// Collect notification entry instead of sending immediately
|
|
282
350
|
const isRecovery = !currentState && !!previousState;
|
|
283
351
|
const upstreamNames =
|
|
284
352
|
currentWarning?.affectedUpstreams.map(
|
|
285
353
|
(u) => u.systemName ?? u.systemId,
|
|
286
354
|
) ?? [];
|
|
355
|
+
const systemName =
|
|
356
|
+
statuses.get(systemId)?.systemName ?? systemId;
|
|
287
357
|
|
|
288
|
-
|
|
358
|
+
systemsToNotify.push({
|
|
359
|
+
systemId,
|
|
360
|
+
systemName,
|
|
289
361
|
derivedState: currentState,
|
|
290
362
|
isRecovery,
|
|
291
|
-
|
|
292
|
-
|
|
363
|
+
importance: isRecovery
|
|
364
|
+
? "info"
|
|
365
|
+
: derivedStateToImportance(currentState!),
|
|
293
366
|
upstreamNames,
|
|
294
|
-
derivedState: currentState,
|
|
295
|
-
isRecovery,
|
|
296
367
|
});
|
|
297
|
-
|
|
298
|
-
? ("info" as const)
|
|
299
|
-
: derivedStateToImportance(currentState!);
|
|
368
|
+
}
|
|
300
369
|
|
|
301
|
-
|
|
302
|
-
|
|
370
|
+
// 7. Send batched per-user notifications (deduplication)
|
|
371
|
+
if (systemsToNotify.length > 0) {
|
|
372
|
+
await sendBatchedNotifications({
|
|
373
|
+
systemsToNotify,
|
|
374
|
+
changedSystemId,
|
|
375
|
+
statuses,
|
|
376
|
+
catalogClient,
|
|
377
|
+
notificationClient,
|
|
378
|
+
logger,
|
|
303
379
|
});
|
|
304
|
-
|
|
305
|
-
try {
|
|
306
|
-
await catalogClient.notifySystemSubscribers({
|
|
307
|
-
systemId,
|
|
308
|
-
title,
|
|
309
|
-
body,
|
|
310
|
-
importance,
|
|
311
|
-
action: { label: "View System", url: systemDetailPath },
|
|
312
|
-
includeGroupSubscribers: true,
|
|
313
|
-
});
|
|
314
|
-
logger.debug(
|
|
315
|
-
`Dependency notification sent: ${systemId} ${previousState ?? "none"} â ${currentState ?? "none"}`,
|
|
316
|
-
);
|
|
317
|
-
} catch (error) {
|
|
318
|
-
// Notifications are best-effort
|
|
319
|
-
logger.warn(
|
|
320
|
-
`Failed to send dependency notification for ${systemId}:`,
|
|
321
|
-
error,
|
|
322
|
-
);
|
|
323
|
-
}
|
|
324
380
|
}
|
|
325
381
|
|
|
326
|
-
//
|
|
382
|
+
// 8. Broadcast signal so frontends can react
|
|
327
383
|
if (changedSystemIds.length > 0) {
|
|
328
384
|
await signalService.broadcast(DEPENDENCY_WARNINGS_CHANGED, {
|
|
329
385
|
affectedSystemIds: changedSystemIds,
|
|
@@ -337,3 +393,148 @@ export async function evaluateAndNotifyDownstream({
|
|
|
337
393
|
);
|
|
338
394
|
}
|
|
339
395
|
}
|
|
396
|
+
|
|
397
|
+
/**
|
|
398
|
+
* Send batched, per-user personalized notifications.
|
|
399
|
+
*
|
|
400
|
+
* Instead of sending one notification per affected system (which causes
|
|
401
|
+
* floods for group subscribers), this function:
|
|
402
|
+
* 1. Resolves all notification group IDs for each affected system
|
|
403
|
+
* 2. Resolves subscribers per group and builds a user â systems reverse map
|
|
404
|
+
* 3. Sends one personalized notification per user with only their relevant systems
|
|
405
|
+
*/
|
|
406
|
+
async function sendBatchedNotifications({
|
|
407
|
+
systemsToNotify,
|
|
408
|
+
changedSystemId,
|
|
409
|
+
statuses,
|
|
410
|
+
catalogClient,
|
|
411
|
+
notificationClient,
|
|
412
|
+
logger,
|
|
413
|
+
}: {
|
|
414
|
+
systemsToNotify: SystemNotificationEntry[];
|
|
415
|
+
changedSystemId: string;
|
|
416
|
+
statuses: Map<string, SystemStatus>;
|
|
417
|
+
catalogClient: InferClient<typeof CatalogApi>;
|
|
418
|
+
notificationClient: InferClient<typeof NotificationApi>;
|
|
419
|
+
logger: Logger;
|
|
420
|
+
}): Promise<void> {
|
|
421
|
+
// Phase 1: Resolve all notification group IDs for affected systems
|
|
422
|
+
const systemToGroupIds = new Map<string, string[]>();
|
|
423
|
+
for (const system of systemsToNotify) {
|
|
424
|
+
const groupIds = [`catalog.system.${system.systemId}`];
|
|
425
|
+
|
|
426
|
+
try {
|
|
427
|
+
const { groupIds: catalogGroupIds } =
|
|
428
|
+
await catalogClient.getSystemGroupIds({
|
|
429
|
+
systemId: system.systemId,
|
|
430
|
+
});
|
|
431
|
+
groupIds.push(
|
|
432
|
+
...catalogGroupIds.map((gid) => `catalog.group.${gid}`),
|
|
433
|
+
);
|
|
434
|
+
} catch (error) {
|
|
435
|
+
logger.warn(
|
|
436
|
+
`Failed to resolve group IDs for system ${system.systemId}:`,
|
|
437
|
+
error,
|
|
438
|
+
);
|
|
439
|
+
}
|
|
440
|
+
|
|
441
|
+
systemToGroupIds.set(system.systemId, groupIds);
|
|
442
|
+
}
|
|
443
|
+
|
|
444
|
+
// Phase 2: Resolve subscribers per group â build user â systems reverse map
|
|
445
|
+
const userSystems = new Map<string, Set<string>>();
|
|
446
|
+
for (const system of systemsToNotify) {
|
|
447
|
+
const groupIds = systemToGroupIds.get(system.systemId) ?? [];
|
|
448
|
+
for (const groupId of groupIds) {
|
|
449
|
+
try {
|
|
450
|
+
const { userIds } =
|
|
451
|
+
await notificationClient.getGroupSubscribers({ groupId });
|
|
452
|
+
for (const userId of userIds) {
|
|
453
|
+
if (!userSystems.has(userId)) {
|
|
454
|
+
userSystems.set(userId, new Set());
|
|
455
|
+
}
|
|
456
|
+
userSystems.get(userId)!.add(system.systemId);
|
|
457
|
+
}
|
|
458
|
+
} catch (error) {
|
|
459
|
+
logger.warn(
|
|
460
|
+
`Failed to resolve subscribers for group ${groupId}:`,
|
|
461
|
+
error,
|
|
462
|
+
);
|
|
463
|
+
}
|
|
464
|
+
}
|
|
465
|
+
}
|
|
466
|
+
|
|
467
|
+
if (userSystems.size === 0) {
|
|
468
|
+
return;
|
|
469
|
+
}
|
|
470
|
+
|
|
471
|
+
// Phase 3: Send one personalized notification per user
|
|
472
|
+
const upstreamName =
|
|
473
|
+
statuses.get(changedSystemId)?.systemName ?? changedSystemId;
|
|
474
|
+
const upstreamSystemDetailPath = resolveRoute(
|
|
475
|
+
catalogRoutes.routes.systemDetail,
|
|
476
|
+
{ systemId: changedSystemId },
|
|
477
|
+
);
|
|
478
|
+
|
|
479
|
+
for (const [userId, subscribedSystemIds] of userSystems) {
|
|
480
|
+
const relevantSystems = systemsToNotify.filter((s) =>
|
|
481
|
+
subscribedSystemIds.has(s.systemId),
|
|
482
|
+
);
|
|
483
|
+
if (relevantSystems.length === 0) continue;
|
|
484
|
+
|
|
485
|
+
const allRecovery = relevantSystems.every((s) => s.isRecovery);
|
|
486
|
+
const systemNames = relevantSystems.map((s) => s.systemName);
|
|
487
|
+
|
|
488
|
+
let title: string;
|
|
489
|
+
let body: string;
|
|
490
|
+
|
|
491
|
+
if (relevantSystems.length === 1) {
|
|
492
|
+
// Single system â use detailed title/body
|
|
493
|
+
const entry = relevantSystems[0];
|
|
494
|
+
title = buildNotificationTitle({
|
|
495
|
+
derivedState: entry.derivedState,
|
|
496
|
+
isRecovery: entry.isRecovery,
|
|
497
|
+
systemName: entry.systemName,
|
|
498
|
+
});
|
|
499
|
+
body = buildNotificationBody({
|
|
500
|
+
upstreamNames: entry.upstreamNames,
|
|
501
|
+
derivedState: entry.derivedState,
|
|
502
|
+
isRecovery: entry.isRecovery,
|
|
503
|
+
systemName: entry.systemName,
|
|
504
|
+
});
|
|
505
|
+
} else if (allRecovery) {
|
|
506
|
+
title = `Dependency impact resolved: ${upstreamName}`;
|
|
507
|
+
body = `All upstream dependencies of **${systemNames.join("**, **")}** have recovered.`;
|
|
508
|
+
} else {
|
|
509
|
+
title = `Upstream dependency issue: ${upstreamName} â ${relevantSystems.length} systems affected`;
|
|
510
|
+
const systemLines = relevantSystems
|
|
511
|
+
.map((entry) => formatSystemImpactLine(entry))
|
|
512
|
+
.join("\n");
|
|
513
|
+
body = `An upstream dependency (**${upstreamName}**) is affecting:\n\n${systemLines}`;
|
|
514
|
+
}
|
|
515
|
+
|
|
516
|
+
const importance = allRecovery
|
|
517
|
+
? ("info" as const)
|
|
518
|
+
: resolveWorstImportance(relevantSystems);
|
|
519
|
+
|
|
520
|
+
try {
|
|
521
|
+
await notificationClient.notifyUsers({
|
|
522
|
+
userIds: [userId],
|
|
523
|
+
title,
|
|
524
|
+
body,
|
|
525
|
+
importance,
|
|
526
|
+
action: { label: "View Root Cause", url: upstreamSystemDetailPath },
|
|
527
|
+
});
|
|
528
|
+
logger.debug(
|
|
529
|
+
`Dependency notification sent to user ${userId}: ${relevantSystems.length} system(s)`,
|
|
530
|
+
);
|
|
531
|
+
} catch (error) {
|
|
532
|
+
// Notifications are best-effort
|
|
533
|
+
logger.warn(
|
|
534
|
+
`Failed to send dependency notification to user ${userId}:`,
|
|
535
|
+
error,
|
|
536
|
+
);
|
|
537
|
+
}
|
|
538
|
+
}
|
|
539
|
+
}
|
|
540
|
+
|
|
@@ -2,7 +2,9 @@ import { describe, test, expect } from "bun:test";
|
|
|
2
2
|
import {
|
|
3
3
|
buildNotificationTitle,
|
|
4
4
|
buildNotificationBody,
|
|
5
|
+
formatSystemImpactLine,
|
|
5
6
|
} from "../src/notifications";
|
|
7
|
+
import type { SystemNotificationEntry } from "../src/notifications";
|
|
6
8
|
import type { DerivedState } from "@checkstack/dependency-common";
|
|
7
9
|
|
|
8
10
|
describe("Dependency Notification Sidecar", () => {
|
|
@@ -15,6 +17,15 @@ describe("Dependency Notification Sidecar", () => {
|
|
|
15
17
|
expect(title).toBe("Dependency impact resolved");
|
|
16
18
|
});
|
|
17
19
|
|
|
20
|
+
test("returns recovery title with system name prefix", () => {
|
|
21
|
+
const title = buildNotificationTitle({
|
|
22
|
+
derivedState: undefined,
|
|
23
|
+
isRecovery: true,
|
|
24
|
+
systemName: "API Gateway",
|
|
25
|
+
});
|
|
26
|
+
expect(title).toBe("API Gateway: Dependency impact resolved");
|
|
27
|
+
});
|
|
28
|
+
|
|
18
29
|
test("returns info title for info derived state", () => {
|
|
19
30
|
const title = buildNotificationTitle({
|
|
20
31
|
derivedState: "info",
|
|
@@ -23,6 +34,16 @@ describe("Dependency Notification Sidecar", () => {
|
|
|
23
34
|
expect(title).toContain("informational");
|
|
24
35
|
});
|
|
25
36
|
|
|
37
|
+
test("returns info title with system name prefix", () => {
|
|
38
|
+
const title = buildNotificationTitle({
|
|
39
|
+
derivedState: "info",
|
|
40
|
+
isRecovery: false,
|
|
41
|
+
systemName: "Web Frontend",
|
|
42
|
+
});
|
|
43
|
+
expect(title).toStartWith("Web Frontend: ");
|
|
44
|
+
expect(title).toContain("informational");
|
|
45
|
+
});
|
|
46
|
+
|
|
26
47
|
test("returns degraded title for degraded derived state", () => {
|
|
27
48
|
const title = buildNotificationTitle({
|
|
28
49
|
derivedState: "degraded",
|
|
@@ -31,6 +52,16 @@ describe("Dependency Notification Sidecar", () => {
|
|
|
31
52
|
expect(title).toContain("impacted");
|
|
32
53
|
});
|
|
33
54
|
|
|
55
|
+
test("returns degraded title with system name prefix", () => {
|
|
56
|
+
const title = buildNotificationTitle({
|
|
57
|
+
derivedState: "degraded",
|
|
58
|
+
isRecovery: false,
|
|
59
|
+
systemName: "Payment Service",
|
|
60
|
+
});
|
|
61
|
+
expect(title).toStartWith("Payment Service: ");
|
|
62
|
+
expect(title).toContain("impacted");
|
|
63
|
+
});
|
|
64
|
+
|
|
34
65
|
test("returns critical title for down derived state", () => {
|
|
35
66
|
const title = buildNotificationTitle({
|
|
36
67
|
derivedState: "down",
|
|
@@ -39,6 +70,16 @@ describe("Dependency Notification Sidecar", () => {
|
|
|
39
70
|
expect(title).toContain("critically impacted");
|
|
40
71
|
});
|
|
41
72
|
|
|
73
|
+
test("returns critical title with system name prefix", () => {
|
|
74
|
+
const title = buildNotificationTitle({
|
|
75
|
+
derivedState: "down",
|
|
76
|
+
isRecovery: false,
|
|
77
|
+
systemName: "Database",
|
|
78
|
+
});
|
|
79
|
+
expect(title).toStartWith("Database: ");
|
|
80
|
+
expect(title).toContain("critically impacted");
|
|
81
|
+
});
|
|
82
|
+
|
|
42
83
|
test("returns fallback title for undefined state", () => {
|
|
43
84
|
const title = buildNotificationTitle({
|
|
44
85
|
derivedState: undefined,
|
|
@@ -46,6 +87,15 @@ describe("Dependency Notification Sidecar", () => {
|
|
|
46
87
|
});
|
|
47
88
|
expect(title).toBe("Dependency impact changed");
|
|
48
89
|
});
|
|
90
|
+
|
|
91
|
+
test("returns fallback title with system name prefix", () => {
|
|
92
|
+
const title = buildNotificationTitle({
|
|
93
|
+
derivedState: undefined,
|
|
94
|
+
isRecovery: false,
|
|
95
|
+
systemName: "Monitoring",
|
|
96
|
+
});
|
|
97
|
+
expect(title).toBe("Monitoring: Dependency impact changed");
|
|
98
|
+
});
|
|
49
99
|
});
|
|
50
100
|
|
|
51
101
|
describe("buildNotificationBody", () => {
|
|
@@ -59,6 +109,17 @@ describe("Dependency Notification Sidecar", () => {
|
|
|
59
109
|
expect(body).toContain("no longer affected");
|
|
60
110
|
});
|
|
61
111
|
|
|
112
|
+
test("returns recovery body with system name reference", () => {
|
|
113
|
+
const body = buildNotificationBody({
|
|
114
|
+
upstreamNames: ["Database"],
|
|
115
|
+
derivedState: undefined,
|
|
116
|
+
isRecovery: true,
|
|
117
|
+
systemName: "API Gateway",
|
|
118
|
+
});
|
|
119
|
+
expect(body).toContain("recovered");
|
|
120
|
+
expect(body).toContain("**API Gateway**");
|
|
121
|
+
});
|
|
122
|
+
|
|
62
123
|
test("includes upstream name in info body", () => {
|
|
63
124
|
const body = buildNotificationBody({
|
|
64
125
|
upstreamNames: ["Redis"],
|
|
@@ -69,6 +130,17 @@ describe("Dependency Notification Sidecar", () => {
|
|
|
69
130
|
expect(body).toContain("informational");
|
|
70
131
|
});
|
|
71
132
|
|
|
133
|
+
test("includes system name in info body", () => {
|
|
134
|
+
const body = buildNotificationBody({
|
|
135
|
+
upstreamNames: ["Redis"],
|
|
136
|
+
derivedState: "info",
|
|
137
|
+
isRecovery: false,
|
|
138
|
+
systemName: "Cache Layer",
|
|
139
|
+
});
|
|
140
|
+
expect(body).toContain("Redis");
|
|
141
|
+
expect(body).toContain("**Cache Layer**");
|
|
142
|
+
});
|
|
143
|
+
|
|
72
144
|
test("includes upstream name in degraded body", () => {
|
|
73
145
|
const body = buildNotificationBody({
|
|
74
146
|
upstreamNames: ["API Gateway"],
|
|
@@ -97,6 +169,26 @@ describe("Dependency Notification Sidecar", () => {
|
|
|
97
169
|
});
|
|
98
170
|
expect(body).toContain("Service A, Service B");
|
|
99
171
|
});
|
|
172
|
+
|
|
173
|
+
test("uses 'This system' when no systemName is provided", () => {
|
|
174
|
+
const body = buildNotificationBody({
|
|
175
|
+
upstreamNames: ["Redis"],
|
|
176
|
+
derivedState: "degraded",
|
|
177
|
+
isRecovery: false,
|
|
178
|
+
});
|
|
179
|
+
expect(body).toContain("This system");
|
|
180
|
+
});
|
|
181
|
+
|
|
182
|
+
test("uses bold system name when provided", () => {
|
|
183
|
+
const body = buildNotificationBody({
|
|
184
|
+
upstreamNames: ["Redis"],
|
|
185
|
+
derivedState: "degraded",
|
|
186
|
+
isRecovery: false,
|
|
187
|
+
systemName: "My App",
|
|
188
|
+
});
|
|
189
|
+
expect(body).toContain("**My App**");
|
|
190
|
+
expect(body).not.toContain("This system");
|
|
191
|
+
});
|
|
100
192
|
});
|
|
101
193
|
|
|
102
194
|
describe("importance mapping", () => {
|
|
@@ -151,5 +243,99 @@ describe("Dependency Notification Sidecar", () => {
|
|
|
151
243
|
expect(body).toContain("Test System");
|
|
152
244
|
});
|
|
153
245
|
}
|
|
246
|
+
|
|
247
|
+
for (const state of ["info", "degraded", "down"] as DerivedState[]) {
|
|
248
|
+
test(`non-recovery '${state}' with system name produces meaningful title and body`, () => {
|
|
249
|
+
const title = buildNotificationTitle({
|
|
250
|
+
derivedState: state,
|
|
251
|
+
isRecovery: false,
|
|
252
|
+
systemName: "My Service",
|
|
253
|
+
});
|
|
254
|
+
const body = buildNotificationBody({
|
|
255
|
+
upstreamNames: ["Upstream DB"],
|
|
256
|
+
derivedState: state,
|
|
257
|
+
isRecovery: false,
|
|
258
|
+
systemName: "My Service",
|
|
259
|
+
});
|
|
260
|
+
|
|
261
|
+
expect(title).toStartWith("My Service: ");
|
|
262
|
+
expect(body).toContain("**My Service**");
|
|
263
|
+
expect(body).toContain("Upstream DB");
|
|
264
|
+
});
|
|
265
|
+
}
|
|
266
|
+
});
|
|
267
|
+
|
|
268
|
+
describe("formatSystemImpactLine", () => {
|
|
269
|
+
const baseEntry: SystemNotificationEntry = {
|
|
270
|
+
systemId: "sys-1",
|
|
271
|
+
systemName: "API Gateway",
|
|
272
|
+
isRecovery: false,
|
|
273
|
+
importance: "info",
|
|
274
|
+
upstreamNames: ["Database"],
|
|
275
|
+
};
|
|
276
|
+
|
|
277
|
+
test("formats critically impacted system with red indicator", () => {
|
|
278
|
+
const line = formatSystemImpactLine({
|
|
279
|
+
...baseEntry,
|
|
280
|
+
derivedState: "down",
|
|
281
|
+
importance: "critical",
|
|
282
|
+
});
|
|
283
|
+
expect(line).toContain("đ´");
|
|
284
|
+
expect(line).toContain("**API Gateway**");
|
|
285
|
+
expect(line).toContain("critically impacted");
|
|
286
|
+
});
|
|
287
|
+
|
|
288
|
+
test("formats degraded system with yellow indicator", () => {
|
|
289
|
+
const line = formatSystemImpactLine({
|
|
290
|
+
...baseEntry,
|
|
291
|
+
derivedState: "degraded",
|
|
292
|
+
importance: "warning",
|
|
293
|
+
});
|
|
294
|
+
expect(line).toContain("đĄ");
|
|
295
|
+
expect(line).toContain("**API Gateway**");
|
|
296
|
+
expect(line).toContain("degraded");
|
|
297
|
+
});
|
|
298
|
+
|
|
299
|
+
test("formats informational system with info indicator", () => {
|
|
300
|
+
const line = formatSystemImpactLine({
|
|
301
|
+
...baseEntry,
|
|
302
|
+
derivedState: "info",
|
|
303
|
+
importance: "info",
|
|
304
|
+
});
|
|
305
|
+
expect(line).toContain("âšī¸");
|
|
306
|
+
expect(line).toContain("**API Gateway**");
|
|
307
|
+
expect(line).toContain("informational");
|
|
308
|
+
});
|
|
309
|
+
|
|
310
|
+
test("formats recovered system with checkmark indicator", () => {
|
|
311
|
+
const line = formatSystemImpactLine({
|
|
312
|
+
...baseEntry,
|
|
313
|
+
isRecovery: true,
|
|
314
|
+
importance: "info",
|
|
315
|
+
});
|
|
316
|
+
expect(line).toContain("â
");
|
|
317
|
+
expect(line).toContain("**API Gateway**");
|
|
318
|
+
expect(line).toContain("recovered");
|
|
319
|
+
});
|
|
320
|
+
|
|
321
|
+
test("formats system with undefined state", () => {
|
|
322
|
+
const line = formatSystemImpactLine({
|
|
323
|
+
...baseEntry,
|
|
324
|
+
derivedState: undefined,
|
|
325
|
+
importance: "info",
|
|
326
|
+
});
|
|
327
|
+
expect(line).toContain("**API Gateway**");
|
|
328
|
+
expect(line).toContain("impact changed");
|
|
329
|
+
});
|
|
330
|
+
|
|
331
|
+
test("all lines start with markdown list prefix", () => {
|
|
332
|
+
for (const state of ["info", "degraded", "down"] as DerivedState[]) {
|
|
333
|
+
const line = formatSystemImpactLine({
|
|
334
|
+
...baseEntry,
|
|
335
|
+
derivedState: state,
|
|
336
|
+
});
|
|
337
|
+
expect(line).toStartWith("- ");
|
|
338
|
+
}
|
|
339
|
+
});
|
|
154
340
|
});
|
|
155
341
|
});
|