@checkstack/incident-backend 1.9.4 → 1.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +80 -0
- package/drizzle/0004_flaky_malcolm_colcord.sql +2 -0
- package/drizzle/meta/0004_snapshot.json +315 -0
- package/drizzle/meta/_journal.json +7 -0
- package/package.json +22 -22
- package/src/ai/incident-create.test.ts +1 -0
- package/src/ai/incident-resolve.test.ts +1 -0
- package/src/ai/incident-update.test.ts +1 -0
- package/src/cache.test.ts +2 -0
- package/src/router.ts +12 -0
- package/src/schema.ts +12 -0
- package/src/service.test.ts +89 -0
- package/src/service.ts +51 -1
- package/src/system-signals.test.ts +1 -0
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,85 @@
|
|
|
1
1
|
# @checkstack/incident-backend
|
|
2
2
|
|
|
3
|
+
## 1.10.0
|
|
4
|
+
|
|
5
|
+
### Minor Changes
|
|
6
|
+
|
|
7
|
+
- 9d30324: Incidents can now optionally override the health status of their affected
|
|
8
|
+
systems. When creating or editing an incident you can pick "Override system
|
|
9
|
+
health" (Degraded or Unhealthy); while the incident is active (not resolved)
|
|
10
|
+
that status is folded into every affected system's derived health via
|
|
11
|
+
worst-wins, so it shows on every health surface (status pages, dashboards,
|
|
12
|
+
dependency map, catalog badges). A health check reporting a worse status still
|
|
13
|
+
wins, and the override lifts automatically when the incident resolves. This
|
|
14
|
+
covers components that no automated check can monitor (e.g. a running app whose
|
|
15
|
+
licenses were revoked so it won't open).
|
|
16
|
+
|
|
17
|
+
The override is a deliberate operator choice, independent of the incident's
|
|
18
|
+
severity. A new service-typed incident RPC `getActiveHealthOverrides` exposes
|
|
19
|
+
active overrides per system, which `@checkstack/healthcheck-backend` reads and
|
|
20
|
+
folds into `getSystemHealthStatus`. The system-health response gains an optional
|
|
21
|
+
`override` field naming the contributing incident so UIs can explain why a
|
|
22
|
+
system reads unhealthy when its checks look fine. The system health badge uses
|
|
23
|
+
it to show, on hover, when a status was forced by an incident.
|
|
24
|
+
|
|
25
|
+
The dashboard "problem system" signal attributes an override-forced status to
|
|
26
|
+
the incident ("Forced by incident: <title>") instead of misreporting
|
|
27
|
+
"0 of N checks failing", while a genuinely worse health check still drives the
|
|
28
|
+
signal and its detail. Public status pages reflect the forced status but never
|
|
29
|
+
carry the incident title (the widget DTOs project only the status), so an
|
|
30
|
+
override cannot leak the name of a hidden incident.
|
|
31
|
+
|
|
32
|
+
Behavior change: a system's derived health now reflects active incident
|
|
33
|
+
overrides in addition to its health checks. Adds a forward-only migration for
|
|
34
|
+
the new nullable `incidents.health_override` column.
|
|
35
|
+
|
|
36
|
+
Thanks to [@stuajnht](https://github.com/stuajnht) for the valuable feedback
|
|
37
|
+
that shaped this release.
|
|
38
|
+
|
|
39
|
+
### Patch Changes
|
|
40
|
+
|
|
41
|
+
- Updated dependencies [390d9cf]
|
|
42
|
+
- Updated dependencies [390d9cf]
|
|
43
|
+
- Updated dependencies [fc64fad]
|
|
44
|
+
- Updated dependencies [9d30324]
|
|
45
|
+
- Updated dependencies [9d30324]
|
|
46
|
+
- Updated dependencies [b218e3e]
|
|
47
|
+
- @checkstack/ai-backend@0.10.8
|
|
48
|
+
- @checkstack/backend-api@0.30.0
|
|
49
|
+
- @checkstack/incident-common@1.8.0
|
|
50
|
+
- @checkstack/automation-backend@0.10.10
|
|
51
|
+
- @checkstack/catalog-backend@1.6.8
|
|
52
|
+
- @checkstack/command-backend@0.2.20
|
|
53
|
+
- @checkstack/integration-backend@0.7.2
|
|
54
|
+
- @checkstack/status-page-backend@0.4.7
|
|
55
|
+
|
|
56
|
+
## 1.9.5
|
|
57
|
+
|
|
58
|
+
### Patch Changes
|
|
59
|
+
|
|
60
|
+
- Updated dependencies [c55d7c6]
|
|
61
|
+
- Updated dependencies [a83bcc2]
|
|
62
|
+
- Updated dependencies [c55d7c6]
|
|
63
|
+
- @checkstack/ai-backend@0.10.7
|
|
64
|
+
- @checkstack/common@0.21.0
|
|
65
|
+
- @checkstack/automation-backend@0.10.9
|
|
66
|
+
- @checkstack/catalog-backend@1.6.7
|
|
67
|
+
- @checkstack/backend-api@0.29.1
|
|
68
|
+
- @checkstack/ai-common@0.6.5
|
|
69
|
+
- @checkstack/auth-common@0.12.2
|
|
70
|
+
- @checkstack/automation-common@0.9.2
|
|
71
|
+
- @checkstack/cache-api@0.3.18
|
|
72
|
+
- @checkstack/catalog-common@2.6.2
|
|
73
|
+
- @checkstack/command-backend@0.2.19
|
|
74
|
+
- @checkstack/incident-common@1.7.2
|
|
75
|
+
- @checkstack/integration-backend@0.7.1
|
|
76
|
+
- @checkstack/integration-common@0.9.7
|
|
77
|
+
- @checkstack/notification-common@1.5.2
|
|
78
|
+
- @checkstack/signal-common@0.2.16
|
|
79
|
+
- @checkstack/status-page-backend@0.4.6
|
|
80
|
+
- @checkstack/status-page-common@0.5.2
|
|
81
|
+
- @checkstack/cache-utils@0.2.23
|
|
82
|
+
|
|
3
83
|
## 1.9.4
|
|
4
84
|
|
|
5
85
|
### Patch Changes
|
|
@@ -0,0 +1,315 @@
|
|
|
1
|
+
{
|
|
2
|
+
"id": "c158fa99-0e1b-452f-85d3-6343af3349e1",
|
|
3
|
+
"prevId": "1d8635bb-6fc2-4dfb-bc27-794bfb92b6fc",
|
|
4
|
+
"version": "7",
|
|
5
|
+
"dialect": "postgresql",
|
|
6
|
+
"tables": {
|
|
7
|
+
"public.incident_links": {
|
|
8
|
+
"name": "incident_links",
|
|
9
|
+
"schema": "",
|
|
10
|
+
"columns": {
|
|
11
|
+
"id": {
|
|
12
|
+
"name": "id",
|
|
13
|
+
"type": "text",
|
|
14
|
+
"primaryKey": true,
|
|
15
|
+
"notNull": true
|
|
16
|
+
},
|
|
17
|
+
"incident_id": {
|
|
18
|
+
"name": "incident_id",
|
|
19
|
+
"type": "text",
|
|
20
|
+
"primaryKey": false,
|
|
21
|
+
"notNull": true
|
|
22
|
+
},
|
|
23
|
+
"label": {
|
|
24
|
+
"name": "label",
|
|
25
|
+
"type": "text",
|
|
26
|
+
"primaryKey": false,
|
|
27
|
+
"notNull": false
|
|
28
|
+
},
|
|
29
|
+
"url": {
|
|
30
|
+
"name": "url",
|
|
31
|
+
"type": "text",
|
|
32
|
+
"primaryKey": false,
|
|
33
|
+
"notNull": true
|
|
34
|
+
},
|
|
35
|
+
"created_at": {
|
|
36
|
+
"name": "created_at",
|
|
37
|
+
"type": "timestamp",
|
|
38
|
+
"primaryKey": false,
|
|
39
|
+
"notNull": true,
|
|
40
|
+
"default": "now()"
|
|
41
|
+
}
|
|
42
|
+
},
|
|
43
|
+
"indexes": {
|
|
44
|
+
"incident_links_incident_url_unique": {
|
|
45
|
+
"name": "incident_links_incident_url_unique",
|
|
46
|
+
"columns": [
|
|
47
|
+
{
|
|
48
|
+
"expression": "incident_id",
|
|
49
|
+
"isExpression": false,
|
|
50
|
+
"asc": true,
|
|
51
|
+
"nulls": "last"
|
|
52
|
+
},
|
|
53
|
+
{
|
|
54
|
+
"expression": "url",
|
|
55
|
+
"isExpression": false,
|
|
56
|
+
"asc": true,
|
|
57
|
+
"nulls": "last"
|
|
58
|
+
}
|
|
59
|
+
],
|
|
60
|
+
"isUnique": true,
|
|
61
|
+
"concurrently": false,
|
|
62
|
+
"method": "btree",
|
|
63
|
+
"with": {}
|
|
64
|
+
}
|
|
65
|
+
},
|
|
66
|
+
"foreignKeys": {
|
|
67
|
+
"incident_links_incident_id_incidents_id_fk": {
|
|
68
|
+
"name": "incident_links_incident_id_incidents_id_fk",
|
|
69
|
+
"tableFrom": "incident_links",
|
|
70
|
+
"tableTo": "incidents",
|
|
71
|
+
"columnsFrom": [
|
|
72
|
+
"incident_id"
|
|
73
|
+
],
|
|
74
|
+
"columnsTo": [
|
|
75
|
+
"id"
|
|
76
|
+
],
|
|
77
|
+
"onDelete": "cascade",
|
|
78
|
+
"onUpdate": "no action"
|
|
79
|
+
}
|
|
80
|
+
},
|
|
81
|
+
"compositePrimaryKeys": {},
|
|
82
|
+
"uniqueConstraints": {},
|
|
83
|
+
"policies": {},
|
|
84
|
+
"checkConstraints": {},
|
|
85
|
+
"isRLSEnabled": false
|
|
86
|
+
},
|
|
87
|
+
"public.incident_systems": {
|
|
88
|
+
"name": "incident_systems",
|
|
89
|
+
"schema": "",
|
|
90
|
+
"columns": {
|
|
91
|
+
"incident_id": {
|
|
92
|
+
"name": "incident_id",
|
|
93
|
+
"type": "text",
|
|
94
|
+
"primaryKey": false,
|
|
95
|
+
"notNull": true
|
|
96
|
+
},
|
|
97
|
+
"system_id": {
|
|
98
|
+
"name": "system_id",
|
|
99
|
+
"type": "text",
|
|
100
|
+
"primaryKey": false,
|
|
101
|
+
"notNull": true
|
|
102
|
+
}
|
|
103
|
+
},
|
|
104
|
+
"indexes": {},
|
|
105
|
+
"foreignKeys": {
|
|
106
|
+
"incident_systems_incident_id_incidents_id_fk": {
|
|
107
|
+
"name": "incident_systems_incident_id_incidents_id_fk",
|
|
108
|
+
"tableFrom": "incident_systems",
|
|
109
|
+
"tableTo": "incidents",
|
|
110
|
+
"columnsFrom": [
|
|
111
|
+
"incident_id"
|
|
112
|
+
],
|
|
113
|
+
"columnsTo": [
|
|
114
|
+
"id"
|
|
115
|
+
],
|
|
116
|
+
"onDelete": "cascade",
|
|
117
|
+
"onUpdate": "no action"
|
|
118
|
+
}
|
|
119
|
+
},
|
|
120
|
+
"compositePrimaryKeys": {
|
|
121
|
+
"incident_systems_incident_id_system_id_pk": {
|
|
122
|
+
"name": "incident_systems_incident_id_system_id_pk",
|
|
123
|
+
"columns": [
|
|
124
|
+
"incident_id",
|
|
125
|
+
"system_id"
|
|
126
|
+
]
|
|
127
|
+
}
|
|
128
|
+
},
|
|
129
|
+
"uniqueConstraints": {},
|
|
130
|
+
"policies": {},
|
|
131
|
+
"checkConstraints": {},
|
|
132
|
+
"isRLSEnabled": false
|
|
133
|
+
},
|
|
134
|
+
"public.incident_updates": {
|
|
135
|
+
"name": "incident_updates",
|
|
136
|
+
"schema": "",
|
|
137
|
+
"columns": {
|
|
138
|
+
"id": {
|
|
139
|
+
"name": "id",
|
|
140
|
+
"type": "text",
|
|
141
|
+
"primaryKey": true,
|
|
142
|
+
"notNull": true
|
|
143
|
+
},
|
|
144
|
+
"incident_id": {
|
|
145
|
+
"name": "incident_id",
|
|
146
|
+
"type": "text",
|
|
147
|
+
"primaryKey": false,
|
|
148
|
+
"notNull": true
|
|
149
|
+
},
|
|
150
|
+
"message": {
|
|
151
|
+
"name": "message",
|
|
152
|
+
"type": "text",
|
|
153
|
+
"primaryKey": false,
|
|
154
|
+
"notNull": true
|
|
155
|
+
},
|
|
156
|
+
"status_change": {
|
|
157
|
+
"name": "status_change",
|
|
158
|
+
"type": "incident_status",
|
|
159
|
+
"typeSchema": "public",
|
|
160
|
+
"primaryKey": false,
|
|
161
|
+
"notNull": false
|
|
162
|
+
},
|
|
163
|
+
"created_at": {
|
|
164
|
+
"name": "created_at",
|
|
165
|
+
"type": "timestamp",
|
|
166
|
+
"primaryKey": false,
|
|
167
|
+
"notNull": true,
|
|
168
|
+
"default": "now()"
|
|
169
|
+
},
|
|
170
|
+
"created_by": {
|
|
171
|
+
"name": "created_by",
|
|
172
|
+
"type": "text",
|
|
173
|
+
"primaryKey": false,
|
|
174
|
+
"notNull": false
|
|
175
|
+
}
|
|
176
|
+
},
|
|
177
|
+
"indexes": {},
|
|
178
|
+
"foreignKeys": {
|
|
179
|
+
"incident_updates_incident_id_incidents_id_fk": {
|
|
180
|
+
"name": "incident_updates_incident_id_incidents_id_fk",
|
|
181
|
+
"tableFrom": "incident_updates",
|
|
182
|
+
"tableTo": "incidents",
|
|
183
|
+
"columnsFrom": [
|
|
184
|
+
"incident_id"
|
|
185
|
+
],
|
|
186
|
+
"columnsTo": [
|
|
187
|
+
"id"
|
|
188
|
+
],
|
|
189
|
+
"onDelete": "cascade",
|
|
190
|
+
"onUpdate": "no action"
|
|
191
|
+
}
|
|
192
|
+
},
|
|
193
|
+
"compositePrimaryKeys": {},
|
|
194
|
+
"uniqueConstraints": {},
|
|
195
|
+
"policies": {},
|
|
196
|
+
"checkConstraints": {},
|
|
197
|
+
"isRLSEnabled": false
|
|
198
|
+
},
|
|
199
|
+
"public.incidents": {
|
|
200
|
+
"name": "incidents",
|
|
201
|
+
"schema": "",
|
|
202
|
+
"columns": {
|
|
203
|
+
"id": {
|
|
204
|
+
"name": "id",
|
|
205
|
+
"type": "text",
|
|
206
|
+
"primaryKey": true,
|
|
207
|
+
"notNull": true
|
|
208
|
+
},
|
|
209
|
+
"title": {
|
|
210
|
+
"name": "title",
|
|
211
|
+
"type": "text",
|
|
212
|
+
"primaryKey": false,
|
|
213
|
+
"notNull": true
|
|
214
|
+
},
|
|
215
|
+
"description": {
|
|
216
|
+
"name": "description",
|
|
217
|
+
"type": "text",
|
|
218
|
+
"primaryKey": false,
|
|
219
|
+
"notNull": false
|
|
220
|
+
},
|
|
221
|
+
"status": {
|
|
222
|
+
"name": "status",
|
|
223
|
+
"type": "incident_status",
|
|
224
|
+
"typeSchema": "public",
|
|
225
|
+
"primaryKey": false,
|
|
226
|
+
"notNull": true,
|
|
227
|
+
"default": "'investigating'"
|
|
228
|
+
},
|
|
229
|
+
"severity": {
|
|
230
|
+
"name": "severity",
|
|
231
|
+
"type": "incident_severity",
|
|
232
|
+
"typeSchema": "public",
|
|
233
|
+
"primaryKey": false,
|
|
234
|
+
"notNull": true,
|
|
235
|
+
"default": "'major'"
|
|
236
|
+
},
|
|
237
|
+
"suppress_notifications": {
|
|
238
|
+
"name": "suppress_notifications",
|
|
239
|
+
"type": "boolean",
|
|
240
|
+
"primaryKey": false,
|
|
241
|
+
"notNull": true,
|
|
242
|
+
"default": false
|
|
243
|
+
},
|
|
244
|
+
"health_override": {
|
|
245
|
+
"name": "health_override",
|
|
246
|
+
"type": "incident_health_override",
|
|
247
|
+
"typeSchema": "public",
|
|
248
|
+
"primaryKey": false,
|
|
249
|
+
"notNull": false
|
|
250
|
+
},
|
|
251
|
+
"created_at": {
|
|
252
|
+
"name": "created_at",
|
|
253
|
+
"type": "timestamp",
|
|
254
|
+
"primaryKey": false,
|
|
255
|
+
"notNull": true,
|
|
256
|
+
"default": "now()"
|
|
257
|
+
},
|
|
258
|
+
"updated_at": {
|
|
259
|
+
"name": "updated_at",
|
|
260
|
+
"type": "timestamp",
|
|
261
|
+
"primaryKey": false,
|
|
262
|
+
"notNull": true,
|
|
263
|
+
"default": "now()"
|
|
264
|
+
}
|
|
265
|
+
},
|
|
266
|
+
"indexes": {},
|
|
267
|
+
"foreignKeys": {},
|
|
268
|
+
"compositePrimaryKeys": {},
|
|
269
|
+
"uniqueConstraints": {},
|
|
270
|
+
"policies": {},
|
|
271
|
+
"checkConstraints": {},
|
|
272
|
+
"isRLSEnabled": false
|
|
273
|
+
}
|
|
274
|
+
},
|
|
275
|
+
"enums": {
|
|
276
|
+
"public.incident_health_override": {
|
|
277
|
+
"name": "incident_health_override",
|
|
278
|
+
"schema": "public",
|
|
279
|
+
"values": [
|
|
280
|
+
"degraded",
|
|
281
|
+
"unhealthy"
|
|
282
|
+
]
|
|
283
|
+
},
|
|
284
|
+
"public.incident_severity": {
|
|
285
|
+
"name": "incident_severity",
|
|
286
|
+
"schema": "public",
|
|
287
|
+
"values": [
|
|
288
|
+
"minor",
|
|
289
|
+
"major",
|
|
290
|
+
"critical"
|
|
291
|
+
]
|
|
292
|
+
},
|
|
293
|
+
"public.incident_status": {
|
|
294
|
+
"name": "incident_status",
|
|
295
|
+
"schema": "public",
|
|
296
|
+
"values": [
|
|
297
|
+
"investigating",
|
|
298
|
+
"identified",
|
|
299
|
+
"fixing",
|
|
300
|
+
"monitoring",
|
|
301
|
+
"resolved"
|
|
302
|
+
]
|
|
303
|
+
}
|
|
304
|
+
},
|
|
305
|
+
"schemas": {},
|
|
306
|
+
"sequences": {},
|
|
307
|
+
"roles": {},
|
|
308
|
+
"policies": {},
|
|
309
|
+
"views": {},
|
|
310
|
+
"_meta": {
|
|
311
|
+
"columns": {},
|
|
312
|
+
"schemas": {},
|
|
313
|
+
"tables": {}
|
|
314
|
+
}
|
|
315
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@checkstack/incident-backend",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.10.0",
|
|
4
4
|
"license": "Elastic-2.0",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "src/index.ts",
|
|
@@ -14,25 +14,25 @@
|
|
|
14
14
|
"lint:code": "eslint . --max-warnings 0"
|
|
15
15
|
},
|
|
16
16
|
"dependencies": {
|
|
17
|
-
"@checkstack/ai-backend": "0.10.
|
|
18
|
-
"@checkstack/ai-common": "0.6.
|
|
19
|
-
"@checkstack/backend-api": "0.
|
|
20
|
-
"@checkstack/cache-api": "0.3.
|
|
21
|
-
"@checkstack/cache-utils": "0.2.
|
|
22
|
-
"@checkstack/incident-common": "1.
|
|
23
|
-
"@checkstack/catalog-common": "2.6.
|
|
24
|
-
"@checkstack/catalog-backend": "1.6.
|
|
25
|
-
"@checkstack/notification-common": "1.5.
|
|
26
|
-
"@checkstack/auth-common": "0.12.
|
|
27
|
-
"@checkstack/command-backend": "0.2.
|
|
28
|
-
"@checkstack/signal-common": "0.2.
|
|
29
|
-
"@checkstack/status-page-backend": "0.4.
|
|
30
|
-
"@checkstack/status-page-common": "0.5.
|
|
31
|
-
"@checkstack/integration-backend": "0.7.
|
|
32
|
-
"@checkstack/integration-common": "0.9.
|
|
33
|
-
"@checkstack/automation-backend": "0.10.
|
|
34
|
-
"@checkstack/automation-common": "0.9.
|
|
35
|
-
"@checkstack/common": "0.
|
|
17
|
+
"@checkstack/ai-backend": "0.10.8",
|
|
18
|
+
"@checkstack/ai-common": "0.6.5",
|
|
19
|
+
"@checkstack/backend-api": "0.30.0",
|
|
20
|
+
"@checkstack/cache-api": "0.3.18",
|
|
21
|
+
"@checkstack/cache-utils": "0.2.23",
|
|
22
|
+
"@checkstack/incident-common": "1.8.0",
|
|
23
|
+
"@checkstack/catalog-common": "2.6.2",
|
|
24
|
+
"@checkstack/catalog-backend": "1.6.8",
|
|
25
|
+
"@checkstack/notification-common": "1.5.2",
|
|
26
|
+
"@checkstack/auth-common": "0.12.2",
|
|
27
|
+
"@checkstack/command-backend": "0.2.20",
|
|
28
|
+
"@checkstack/signal-common": "0.2.16",
|
|
29
|
+
"@checkstack/status-page-backend": "0.4.7",
|
|
30
|
+
"@checkstack/status-page-common": "0.5.2",
|
|
31
|
+
"@checkstack/integration-backend": "0.7.2",
|
|
32
|
+
"@checkstack/integration-common": "0.9.7",
|
|
33
|
+
"@checkstack/automation-backend": "0.10.10",
|
|
34
|
+
"@checkstack/automation-common": "0.9.2",
|
|
35
|
+
"@checkstack/common": "0.21.0",
|
|
36
36
|
"drizzle-orm": "^0.45.0",
|
|
37
37
|
"zod": "^4.2.1",
|
|
38
38
|
"@orpc/contract": "^1.14.4",
|
|
@@ -40,8 +40,8 @@
|
|
|
40
40
|
},
|
|
41
41
|
"devDependencies": {
|
|
42
42
|
"@checkstack/drizzle-helper": "0.0.6",
|
|
43
|
-
"@checkstack/scripts": "0.7.
|
|
44
|
-
"@checkstack/test-utils-backend": "0.1.
|
|
43
|
+
"@checkstack/scripts": "0.7.2",
|
|
44
|
+
"@checkstack/test-utils-backend": "0.1.54",
|
|
45
45
|
"@checkstack/tsconfig": "0.0.7",
|
|
46
46
|
"@types/bun": "^1.0.0",
|
|
47
47
|
"drizzle-kit": "^0.31.10",
|
package/src/cache.test.ts
CHANGED
|
@@ -22,6 +22,7 @@ const sampleIncident: IncidentDetail = {
|
|
|
22
22
|
status: "investigating",
|
|
23
23
|
severity: "major",
|
|
24
24
|
suppressNotifications: false,
|
|
25
|
+
healthOverride: null,
|
|
25
26
|
createdAt: new Date("2026-06-01T00:00:00Z"),
|
|
26
27
|
updatedAt: new Date("2026-06-01T00:00:00Z"),
|
|
27
28
|
systemIds: ["s1"],
|
|
@@ -35,6 +36,7 @@ const sampleWithSystems: IncidentWithSystems = {
|
|
|
35
36
|
status: sampleIncident.status,
|
|
36
37
|
severity: sampleIncident.severity,
|
|
37
38
|
suppressNotifications: sampleIncident.suppressNotifications,
|
|
39
|
+
healthOverride: sampleIncident.healthOverride,
|
|
38
40
|
createdAt: sampleIncident.createdAt,
|
|
39
41
|
updatedAt: sampleIncident.updatedAt,
|
|
40
42
|
systemIds: sampleIncident.systemIds,
|
package/src/router.ts
CHANGED
|
@@ -260,6 +260,18 @@ export function createRouter({
|
|
|
260
260
|
},
|
|
261
261
|
),
|
|
262
262
|
|
|
263
|
+
getActiveHealthOverrides: os.getActiveHealthOverrides.handler(
|
|
264
|
+
async ({ input }) => {
|
|
265
|
+
// Deliberately un-cached: this feeds live system-health derivation, so
|
|
266
|
+
// an override must lift/apply the instant an incident is edited or
|
|
267
|
+
// resolved. It is a single indexed join (see service), not an N+1.
|
|
268
|
+
const overrides = await service.getActiveHealthOverrides(
|
|
269
|
+
input.systemIds,
|
|
270
|
+
);
|
|
271
|
+
return { overrides };
|
|
272
|
+
},
|
|
273
|
+
),
|
|
274
|
+
|
|
263
275
|
createIncident: os.createIncident.handler(async ({ input, context }) => {
|
|
264
276
|
const userId =
|
|
265
277
|
context.user && "id" in context.user ? context.user.id : undefined;
|
package/src/schema.ts
CHANGED
|
@@ -28,6 +28,17 @@ export const incidentSeverityEnum = pgEnum("incident_severity", [
|
|
|
28
28
|
"critical",
|
|
29
29
|
]);
|
|
30
30
|
|
|
31
|
+
/**
|
|
32
|
+
* Optional health status an active incident forces onto its affected systems.
|
|
33
|
+
* A subset of the health-check status vocabulary (no "healthy" - forcing a
|
|
34
|
+
* system healthy via an incident makes no sense). Nullable column: `null` means
|
|
35
|
+
* the incident does not touch derived system health.
|
|
36
|
+
*/
|
|
37
|
+
export const incidentHealthOverrideEnum = pgEnum("incident_health_override", [
|
|
38
|
+
"degraded",
|
|
39
|
+
"unhealthy",
|
|
40
|
+
]);
|
|
41
|
+
|
|
31
42
|
/**
|
|
32
43
|
* Main incidents table
|
|
33
44
|
*/
|
|
@@ -40,6 +51,7 @@ export const incidents = pgTable("incidents", {
|
|
|
40
51
|
suppressNotifications: boolean("suppress_notifications")
|
|
41
52
|
.default(false)
|
|
42
53
|
.notNull(),
|
|
54
|
+
healthOverride: incidentHealthOverrideEnum("health_override"),
|
|
43
55
|
createdAt: timestamp("created_at").defaultNow().notNull(),
|
|
44
56
|
updatedAt: timestamp("updated_at").defaultNow().notNull(),
|
|
45
57
|
});
|
package/src/service.test.ts
CHANGED
|
@@ -457,3 +457,92 @@ describe("IncidentService.createIncidentDedupedForSystem (M3)", () => {
|
|
|
457
457
|
expect([a.reused, b.reused].filter(Boolean)).toHaveLength(1);
|
|
458
458
|
});
|
|
459
459
|
});
|
|
460
|
+
|
|
461
|
+
/**
|
|
462
|
+
* Mock DB for the join query in `getActiveHealthOverrides`
|
|
463
|
+
* (`select().from().innerJoin().where()` resolving to a flat joined-row list).
|
|
464
|
+
* The status/override filtering is enforced by the real query builder, so the
|
|
465
|
+
* test supplies rows as the DB would AFTER filtering and asserts the mapping +
|
|
466
|
+
* per-system grouping into the RPC shape.
|
|
467
|
+
*/
|
|
468
|
+
function createJoinDb(rows: unknown[]) {
|
|
469
|
+
const select = mock(() => {
|
|
470
|
+
const whereResult = Promise.resolve(rows);
|
|
471
|
+
const innerJoin = mock(() => ({ where: mock(() => whereResult) }));
|
|
472
|
+
const from = mock(() => ({ innerJoin }));
|
|
473
|
+
return { from };
|
|
474
|
+
});
|
|
475
|
+
return { db: { select } as unknown };
|
|
476
|
+
}
|
|
477
|
+
|
|
478
|
+
describe("IncidentService.getActiveHealthOverrides", () => {
|
|
479
|
+
it("returns an empty record without querying when given no systems", async () => {
|
|
480
|
+
const { db } = createJoinDb([]);
|
|
481
|
+
const service = new IncidentService(db as never, makeFakeAdvisoryLock());
|
|
482
|
+
expect(await service.getActiveHealthOverrides([])).toEqual({});
|
|
483
|
+
});
|
|
484
|
+
|
|
485
|
+
it("groups active overrides by system and shapes each into { status, incidentId, incidentTitle }", async () => {
|
|
486
|
+
const { db } = createJoinDb([
|
|
487
|
+
{
|
|
488
|
+
systemId: "sys-1",
|
|
489
|
+
incidentId: "inc-1",
|
|
490
|
+
incidentTitle: "License server revoked",
|
|
491
|
+
healthOverride: "unhealthy",
|
|
492
|
+
},
|
|
493
|
+
{
|
|
494
|
+
systemId: "sys-1",
|
|
495
|
+
incidentId: "inc-2",
|
|
496
|
+
incidentTitle: "Slow logins",
|
|
497
|
+
healthOverride: "degraded",
|
|
498
|
+
},
|
|
499
|
+
{
|
|
500
|
+
systemId: "sys-2",
|
|
501
|
+
incidentId: "inc-3",
|
|
502
|
+
incidentTitle: "Vendor maintenance",
|
|
503
|
+
healthOverride: "degraded",
|
|
504
|
+
},
|
|
505
|
+
]);
|
|
506
|
+
const service = new IncidentService(db as never, makeFakeAdvisoryLock());
|
|
507
|
+
|
|
508
|
+
const result = await service.getActiveHealthOverrides(["sys-1", "sys-2"]);
|
|
509
|
+
|
|
510
|
+
expect(result).toEqual({
|
|
511
|
+
"sys-1": [
|
|
512
|
+
{
|
|
513
|
+
status: "unhealthy",
|
|
514
|
+
incidentId: "inc-1",
|
|
515
|
+
incidentTitle: "License server revoked",
|
|
516
|
+
},
|
|
517
|
+
{
|
|
518
|
+
status: "degraded",
|
|
519
|
+
incidentId: "inc-2",
|
|
520
|
+
incidentTitle: "Slow logins",
|
|
521
|
+
},
|
|
522
|
+
],
|
|
523
|
+
"sys-2": [
|
|
524
|
+
{
|
|
525
|
+
status: "degraded",
|
|
526
|
+
incidentId: "inc-3",
|
|
527
|
+
incidentTitle: "Vendor maintenance",
|
|
528
|
+
},
|
|
529
|
+
],
|
|
530
|
+
});
|
|
531
|
+
});
|
|
532
|
+
|
|
533
|
+
it("omits systems with no active override", async () => {
|
|
534
|
+
const { db } = createJoinDb([
|
|
535
|
+
{
|
|
536
|
+
systemId: "sys-1",
|
|
537
|
+
incidentId: "inc-1",
|
|
538
|
+
incidentTitle: "Down",
|
|
539
|
+
healthOverride: "unhealthy",
|
|
540
|
+
},
|
|
541
|
+
]);
|
|
542
|
+
const service = new IncidentService(db as never, makeFakeAdvisoryLock());
|
|
543
|
+
|
|
544
|
+
const result = await service.getActiveHealthOverrides(["sys-1", "sys-2"]);
|
|
545
|
+
|
|
546
|
+
expect(Object.keys(result)).toEqual(["sys-1"]);
|
|
547
|
+
});
|
|
548
|
+
});
|
package/src/service.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { eq, and, inArray, ne } from "drizzle-orm";
|
|
1
|
+
import { eq, and, inArray, ne, isNotNull } from "drizzle-orm";
|
|
2
2
|
import type { AdvisoryLockService, SafeDatabase } from "@checkstack/backend-api";
|
|
3
3
|
import * as schema from "./schema";
|
|
4
4
|
import {
|
|
@@ -18,6 +18,7 @@ import type {
|
|
|
18
18
|
AddIncidentUpdateInput,
|
|
19
19
|
IncidentStatus,
|
|
20
20
|
IncidentSeverity,
|
|
21
|
+
SystemHealthOverride,
|
|
21
22
|
} from "@checkstack/incident-common";
|
|
22
23
|
|
|
23
24
|
type Db = SafeDatabase<typeof schema>;
|
|
@@ -277,6 +278,50 @@ export class IncidentService {
|
|
|
277
278
|
return result;
|
|
278
279
|
}
|
|
279
280
|
|
|
281
|
+
/**
|
|
282
|
+
* For each requested system, the health overrides contributed by its ACTIVE
|
|
283
|
+
* incidents (status != resolved AND `healthOverride` set). Powers the
|
|
284
|
+
* healthcheck plugin's system-health derivation (worst-wins fold). Reads the
|
|
285
|
+
* authoritative `incidents` + `incident_systems` tables, so the answer is
|
|
286
|
+
* identical on every pod (state-and-scale rule). Systems with no active
|
|
287
|
+
* override are omitted; resolved incidents never appear, so an override
|
|
288
|
+
* auto-lifts the moment its incident resolves.
|
|
289
|
+
*/
|
|
290
|
+
async getActiveHealthOverrides(
|
|
291
|
+
systemIds: string[],
|
|
292
|
+
): Promise<Record<string, SystemHealthOverride[]>> {
|
|
293
|
+
if (systemIds.length === 0) return {};
|
|
294
|
+
|
|
295
|
+
const rows = await this.db
|
|
296
|
+
.select({
|
|
297
|
+
systemId: incidentSystems.systemId,
|
|
298
|
+
incidentId: incidents.id,
|
|
299
|
+
incidentTitle: incidents.title,
|
|
300
|
+
healthOverride: incidents.healthOverride,
|
|
301
|
+
})
|
|
302
|
+
.from(incidentSystems)
|
|
303
|
+
.innerJoin(incidents, eq(incidentSystems.incidentId, incidents.id))
|
|
304
|
+
.where(
|
|
305
|
+
and(
|
|
306
|
+
inArray(incidentSystems.systemId, systemIds),
|
|
307
|
+
ne(incidents.status, "resolved"),
|
|
308
|
+
isNotNull(incidents.healthOverride),
|
|
309
|
+
),
|
|
310
|
+
);
|
|
311
|
+
|
|
312
|
+
const result: Record<string, SystemHealthOverride[]> = {};
|
|
313
|
+
for (const row of rows) {
|
|
314
|
+
// `healthOverride` is guaranteed non-null by the `isNotNull` predicate.
|
|
315
|
+
if (row.healthOverride === null) continue;
|
|
316
|
+
(result[row.systemId] ??= []).push({
|
|
317
|
+
status: row.healthOverride,
|
|
318
|
+
incidentId: row.incidentId,
|
|
319
|
+
incidentTitle: row.incidentTitle,
|
|
320
|
+
});
|
|
321
|
+
}
|
|
322
|
+
return result;
|
|
323
|
+
}
|
|
324
|
+
|
|
280
325
|
/**
|
|
281
326
|
* Create a new incident.
|
|
282
327
|
*
|
|
@@ -302,6 +347,7 @@ export class IncidentService {
|
|
|
302
347
|
status: "investigating",
|
|
303
348
|
severity: input.severity,
|
|
304
349
|
suppressNotifications: input.suppressNotifications ?? false,
|
|
350
|
+
healthOverride: input.healthOverride ?? null,
|
|
305
351
|
});
|
|
306
352
|
|
|
307
353
|
// Insert system associations
|
|
@@ -350,6 +396,10 @@ export class IncidentService {
|
|
|
350
396
|
if (input.severity !== undefined) updateData.severity = input.severity;
|
|
351
397
|
if (input.suppressNotifications !== undefined)
|
|
352
398
|
updateData.suppressNotifications = input.suppressNotifications;
|
|
399
|
+
// `null` clears a previously-set override; `undefined` leaves it unchanged
|
|
400
|
+
// (same nullable-optional semantics as `description`).
|
|
401
|
+
if (input.healthOverride !== undefined)
|
|
402
|
+
updateData.healthOverride = input.healthOverride;
|
|
353
403
|
|
|
354
404
|
// Atomic: the field update and the delete-then-reinsert of system links must
|
|
355
405
|
// commit together. Without the transaction a failure after the delete left
|
|
@@ -12,6 +12,7 @@ const openIncident: IncidentWithSystems = {
|
|
|
12
12
|
status: "investigating",
|
|
13
13
|
severity: "critical",
|
|
14
14
|
suppressNotifications: false,
|
|
15
|
+
healthOverride: null,
|
|
15
16
|
createdAt: new Date("2026-06-01T10:00:00.000Z"),
|
|
16
17
|
updatedAt: new Date("2026-06-01T10:00:00.000Z"),
|
|
17
18
|
systemIds: ["sys-1"],
|