@checkstack/incident-backend 1.10.0 → 1.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +416 -0
- package/drizzle/0005_real_leper_queen.sql +4 -0
- package/drizzle/0006_last_princess_powerful.sql +1 -0
- package/drizzle/meta/0005_snapshot.json +346 -0
- package/drizzle/meta/0006_snapshot.json +353 -0
- package/drizzle/meta/_journal.json +14 -0
- package/package.json +24 -22
- package/src/ai/incident-add-link.test.ts +1 -0
- package/src/ai/incident-add-update.test.ts +6 -1
- package/src/ai/incident-delete-update.test.ts +59 -0
- package/src/ai/incident-delete-update.ts +73 -0
- package/src/ai/incident-remove-link.test.ts +5 -5
- package/src/ai/incident-remove-link.ts +9 -5
- package/src/ai/register-ai-tools.ts +2 -0
- package/src/automations.test.ts +70 -0
- package/src/automations.ts +77 -4
- package/src/hooks.ts +54 -4
- package/src/index.ts +4 -0
- package/src/notifications.test.ts +181 -0
- package/src/notifications.ts +12 -1
- package/src/read-visibility.test.ts +158 -0
- package/src/read-visibility.ts +99 -0
- package/src/router.test.ts +69 -1
- package/src/router.ts +196 -16
- package/src/schema.ts +27 -0
- package/src/service-reads.it.test.ts +353 -0
- package/src/service-updates.it.test.ts +340 -0
- package/src/service.it.test.ts +187 -0
- package/src/service.test.ts +235 -1
- package/src/service.ts +501 -180
- package/src/status-page-widget.test.ts +149 -0
- package/src/status-page-widget.ts +133 -37
|
@@ -36,6 +36,20 @@
|
|
|
36
36
|
"when": 1783287429703,
|
|
37
37
|
"tag": "0004_flaky_malcolm_colcord",
|
|
38
38
|
"breakpoints": true
|
|
39
|
+
},
|
|
40
|
+
{
|
|
41
|
+
"idx": 5,
|
|
42
|
+
"version": "7",
|
|
43
|
+
"when": 1783498751737,
|
|
44
|
+
"tag": "0005_real_leper_queen",
|
|
45
|
+
"breakpoints": true
|
|
46
|
+
},
|
|
47
|
+
{
|
|
48
|
+
"idx": 6,
|
|
49
|
+
"version": "7",
|
|
50
|
+
"when": 1783535507970,
|
|
51
|
+
"tag": "0006_last_princess_powerful",
|
|
52
|
+
"breakpoints": true
|
|
39
53
|
}
|
|
40
54
|
]
|
|
41
55
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@checkstack/incident-backend",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.12.0",
|
|
4
4
|
"license": "Elastic-2.0",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "src/index.ts",
|
|
@@ -14,25 +14,25 @@
|
|
|
14
14
|
"lint:code": "eslint . --max-warnings 0"
|
|
15
15
|
},
|
|
16
16
|
"dependencies": {
|
|
17
|
-
"@checkstack/ai-backend": "0.10.
|
|
18
|
-
"@checkstack/ai-common": "0.6.
|
|
19
|
-
"@checkstack/backend-api": "0.
|
|
20
|
-
"@checkstack/cache-api": "0.3.
|
|
21
|
-
"@checkstack/cache-utils": "0.2.
|
|
22
|
-
"@checkstack/incident-common": "1.
|
|
23
|
-
"@checkstack/catalog-common": "2.
|
|
24
|
-
"@checkstack/catalog-backend": "1.
|
|
25
|
-
"@checkstack/notification-common": "1.
|
|
26
|
-
"@checkstack/auth-common": "0.
|
|
27
|
-
"@checkstack/command-backend": "0.2.
|
|
28
|
-
"@checkstack/signal-common": "0.2.
|
|
29
|
-
"@checkstack/status-page-backend": "0.
|
|
30
|
-
"@checkstack/status-page-common": "0.
|
|
31
|
-
"@checkstack/integration-backend": "0.7.
|
|
32
|
-
"@checkstack/integration-common": "0.9.
|
|
33
|
-
"@checkstack/automation-backend": "0.
|
|
34
|
-
"@checkstack/automation-common": "0.
|
|
35
|
-
"@checkstack/common": "0.
|
|
17
|
+
"@checkstack/ai-backend": "0.10.10",
|
|
18
|
+
"@checkstack/ai-common": "0.6.6",
|
|
19
|
+
"@checkstack/backend-api": "0.31.1",
|
|
20
|
+
"@checkstack/cache-api": "0.3.19",
|
|
21
|
+
"@checkstack/cache-utils": "0.2.24",
|
|
22
|
+
"@checkstack/incident-common": "1.10.0",
|
|
23
|
+
"@checkstack/catalog-common": "2.7.0",
|
|
24
|
+
"@checkstack/catalog-backend": "1.7.0",
|
|
25
|
+
"@checkstack/notification-common": "1.6.0",
|
|
26
|
+
"@checkstack/auth-common": "0.13.0",
|
|
27
|
+
"@checkstack/command-backend": "0.2.22",
|
|
28
|
+
"@checkstack/signal-common": "0.2.17",
|
|
29
|
+
"@checkstack/status-page-backend": "0.5.0",
|
|
30
|
+
"@checkstack/status-page-common": "0.6.0",
|
|
31
|
+
"@checkstack/integration-backend": "0.7.4",
|
|
32
|
+
"@checkstack/integration-common": "0.9.8",
|
|
33
|
+
"@checkstack/automation-backend": "0.11.1",
|
|
34
|
+
"@checkstack/automation-common": "0.10.0",
|
|
35
|
+
"@checkstack/common": "0.22.0",
|
|
36
36
|
"drizzle-orm": "^0.45.0",
|
|
37
37
|
"zod": "^4.2.1",
|
|
38
38
|
"@orpc/contract": "^1.14.4",
|
|
@@ -40,11 +40,13 @@
|
|
|
40
40
|
},
|
|
41
41
|
"devDependencies": {
|
|
42
42
|
"@checkstack/drizzle-helper": "0.0.6",
|
|
43
|
-
"@checkstack/scripts": "0.7.
|
|
44
|
-
"@checkstack/test-utils-backend": "0.1.
|
|
43
|
+
"@checkstack/scripts": "0.7.3",
|
|
44
|
+
"@checkstack/test-utils-backend": "0.1.56",
|
|
45
45
|
"@checkstack/tsconfig": "0.0.7",
|
|
46
46
|
"@types/bun": "^1.0.0",
|
|
47
|
+
"@types/pg": "^8.20.0",
|
|
47
48
|
"drizzle-kit": "^0.31.10",
|
|
49
|
+
"pg": "^8.21.0",
|
|
48
50
|
"typescript": "^5.0.0"
|
|
49
51
|
}
|
|
50
52
|
}
|
|
@@ -45,7 +45,12 @@ describe("incident.addUpdate tool", () => {
|
|
|
45
45
|
});
|
|
46
46
|
|
|
47
47
|
test("execute (apply) posts via addUpdate", async () => {
|
|
48
|
-
const created = {
|
|
48
|
+
const created = {
|
|
49
|
+
id: "upd1",
|
|
50
|
+
...input,
|
|
51
|
+
visibility: "public" as const,
|
|
52
|
+
createdAt: new Date(),
|
|
53
|
+
};
|
|
49
54
|
const addUpdate = mock(() => Promise.resolve(created));
|
|
50
55
|
const rpcClient = fakeRpcClient({ addUpdate });
|
|
51
56
|
const tool = createIncidentAddUpdateTool();
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
import { describe, expect, test, mock } from "bun:test";
|
|
2
|
+
import type { AuthUser, RpcClient } from "@checkstack/backend-api";
|
|
3
|
+
import { createIncidentDeleteUpdateTool } from "./incident-delete-update";
|
|
4
|
+
|
|
5
|
+
const principal: AuthUser = {
|
|
6
|
+
type: "user",
|
|
7
|
+
id: "u1",
|
|
8
|
+
accessRules: ["incident.incident.manage"],
|
|
9
|
+
};
|
|
10
|
+
|
|
11
|
+
function fakeRpcClient({
|
|
12
|
+
deleteUpdate,
|
|
13
|
+
}: {
|
|
14
|
+
deleteUpdate: ReturnType<typeof mock>;
|
|
15
|
+
}): RpcClient {
|
|
16
|
+
return {
|
|
17
|
+
forPlugin: () => ({ deleteUpdate }),
|
|
18
|
+
} as unknown as RpcClient;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
describe("incident.deleteUpdate tool", () => {
|
|
22
|
+
test("declares destructive effect + the manage rule", () => {
|
|
23
|
+
const tool = createIncidentDeleteUpdateTool();
|
|
24
|
+
expect(tool.name).toBe("incident.deleteUpdate");
|
|
25
|
+
expect(tool.effect).toBe("destructive");
|
|
26
|
+
expect(tool.requiredAccessRules).toEqual(["incident.incident.manage"]);
|
|
27
|
+
expect(typeof tool.dryRun).toBe("function");
|
|
28
|
+
});
|
|
29
|
+
|
|
30
|
+
test("dryRun returns a payload and NEVER deletes the update", async () => {
|
|
31
|
+
const deleteUpdate = mock(() => Promise.resolve({ success: true }));
|
|
32
|
+
const rpcClient = fakeRpcClient({ deleteUpdate });
|
|
33
|
+
const tool = createIncidentDeleteUpdateTool();
|
|
34
|
+
const preview = await tool.dryRun!({
|
|
35
|
+
input: { id: "upd1", incidentId: "inc1" },
|
|
36
|
+
principal,
|
|
37
|
+
rpcClient,
|
|
38
|
+
});
|
|
39
|
+
expect(deleteUpdate).not.toHaveBeenCalled();
|
|
40
|
+
expect(preview.summary).toContain("permanent");
|
|
41
|
+
expect(preview.payload).toEqual({ id: "upd1", incidentId: "inc1" });
|
|
42
|
+
});
|
|
43
|
+
|
|
44
|
+
test("execute (apply) deletes via deleteUpdate with {id, incidentId}", async () => {
|
|
45
|
+
const deleteUpdate = mock(() => Promise.resolve({ success: true }));
|
|
46
|
+
const rpcClient = fakeRpcClient({ deleteUpdate });
|
|
47
|
+
const tool = createIncidentDeleteUpdateTool();
|
|
48
|
+
const result = await tool.execute({
|
|
49
|
+
input: { id: "upd1", incidentId: "inc1" },
|
|
50
|
+
principal,
|
|
51
|
+
rpcClient,
|
|
52
|
+
});
|
|
53
|
+
expect(deleteUpdate).toHaveBeenCalledWith({
|
|
54
|
+
id: "upd1",
|
|
55
|
+
incidentId: "inc1",
|
|
56
|
+
});
|
|
57
|
+
expect(result).toEqual({ id: "upd1", removed: true });
|
|
58
|
+
});
|
|
59
|
+
});
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
import { qualifyAccessRuleId } from "@checkstack/common";
|
|
3
|
+
import type { RpcClient, AuthUser } from "@checkstack/backend-api";
|
|
4
|
+
import {
|
|
5
|
+
IncidentApi,
|
|
6
|
+
incidentAccess,
|
|
7
|
+
pluginMetadata,
|
|
8
|
+
} from "@checkstack/incident-common";
|
|
9
|
+
import type { AiProposalPreview } from "@checkstack/ai-common";
|
|
10
|
+
import type { RegisteredAiTool } from "@checkstack/ai-backend";
|
|
11
|
+
|
|
12
|
+
/** Input for `incident.deleteUpdate`: the update id + its owning incident. */
|
|
13
|
+
export const IncidentDeleteUpdateInputSchema = z.object({
|
|
14
|
+
id: z.string(),
|
|
15
|
+
incidentId: z.string(),
|
|
16
|
+
});
|
|
17
|
+
export type IncidentDeleteUpdateInput = z.infer<
|
|
18
|
+
typeof IncidentDeleteUpdateInputSchema
|
|
19
|
+
>;
|
|
20
|
+
|
|
21
|
+
/** Output returned once a human applies the update removal. */
|
|
22
|
+
export interface IncidentDeleteUpdateApplyResult {
|
|
23
|
+
id: string;
|
|
24
|
+
removed: true;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* `incident.deleteUpdate` - delete a published status update by id.
|
|
29
|
+
*
|
|
30
|
+
* `effect: "destructive"` - deleting an update is irreversible, so it ALWAYS
|
|
31
|
+
* routes through the propose/apply confirm card in BOTH permission modes (it can
|
|
32
|
+
* never auto-apply). `dryRun` returns the captured payload for human review
|
|
33
|
+
* WITHOUT mutating; `execute` (reached only via `apply`) deletes the update. The
|
|
34
|
+
* underlying RPC uses the USER-SCOPED client passed at call time, so
|
|
35
|
+
* handler-side authorization is enforced exactly as a direct UI/RPC call.
|
|
36
|
+
*/
|
|
37
|
+
export function createIncidentDeleteUpdateTool(): RegisteredAiTool<
|
|
38
|
+
IncidentDeleteUpdateInput,
|
|
39
|
+
IncidentDeleteUpdateApplyResult
|
|
40
|
+
> {
|
|
41
|
+
const dryRun = async ({
|
|
42
|
+
input,
|
|
43
|
+
}: {
|
|
44
|
+
input: IncidentDeleteUpdateInput;
|
|
45
|
+
principal: AuthUser;
|
|
46
|
+
rpcClient: RpcClient;
|
|
47
|
+
}): Promise<AiProposalPreview<IncidentDeleteUpdateInput>> => {
|
|
48
|
+
return {
|
|
49
|
+
summary: `Delete update ${input.id} from incident ${input.incidentId}. This is permanent.`,
|
|
50
|
+
payload: { id: input.id, incidentId: input.incidentId },
|
|
51
|
+
};
|
|
52
|
+
};
|
|
53
|
+
|
|
54
|
+
return {
|
|
55
|
+
name: "incident.deleteUpdate",
|
|
56
|
+
description:
|
|
57
|
+
"Delete a published status update from an incident by update id and its owning incidentId. DESTRUCTIVE and irreversible. Never deletes directly; a person must approve the confirmation. Find the update id and incidentId by reading the incident's details first.",
|
|
58
|
+
effect: "destructive",
|
|
59
|
+
input: IncidentDeleteUpdateInputSchema,
|
|
60
|
+
requiredAccessRules: [
|
|
61
|
+
qualifyAccessRuleId(pluginMetadata, incidentAccess.incident.manage),
|
|
62
|
+
],
|
|
63
|
+
dryRun,
|
|
64
|
+
async execute({ input, rpcClient }) {
|
|
65
|
+
const incidentClient = rpcClient.forPlugin(IncidentApi);
|
|
66
|
+
await incidentClient.deleteUpdate({
|
|
67
|
+
id: input.id,
|
|
68
|
+
incidentId: input.incidentId,
|
|
69
|
+
});
|
|
70
|
+
return { id: input.id, removed: true };
|
|
71
|
+
},
|
|
72
|
+
};
|
|
73
|
+
}
|
|
@@ -32,25 +32,25 @@ describe("incident.removeLink tool", () => {
|
|
|
32
32
|
const rpcClient = fakeRpcClient({ removeLink });
|
|
33
33
|
const tool = createIncidentRemoveLinkTool();
|
|
34
34
|
const preview = await tool.dryRun!({
|
|
35
|
-
input: { id: "lnk1" },
|
|
35
|
+
input: { id: "lnk1", incidentId: "inc1" },
|
|
36
36
|
principal,
|
|
37
37
|
rpcClient,
|
|
38
38
|
});
|
|
39
39
|
expect(removeLink).not.toHaveBeenCalled();
|
|
40
40
|
expect(preview.summary).toContain("permanent");
|
|
41
|
-
expect(preview.payload).toEqual({ id: "lnk1" });
|
|
41
|
+
expect(preview.payload).toEqual({ id: "lnk1", incidentId: "inc1" });
|
|
42
42
|
});
|
|
43
43
|
|
|
44
|
-
test("execute (apply) removes via removeLink with {id}", async () => {
|
|
44
|
+
test("execute (apply) removes via removeLink with {id, incidentId}", async () => {
|
|
45
45
|
const removeLink = mock(() => Promise.resolve({ success: true }));
|
|
46
46
|
const rpcClient = fakeRpcClient({ removeLink });
|
|
47
47
|
const tool = createIncidentRemoveLinkTool();
|
|
48
48
|
const result = await tool.execute({
|
|
49
|
-
input: { id: "lnk1" },
|
|
49
|
+
input: { id: "lnk1", incidentId: "inc1" },
|
|
50
50
|
principal,
|
|
51
51
|
rpcClient,
|
|
52
52
|
});
|
|
53
|
-
expect(removeLink).toHaveBeenCalledWith({ id: "lnk1" });
|
|
53
|
+
expect(removeLink).toHaveBeenCalledWith({ id: "lnk1", incidentId: "inc1" });
|
|
54
54
|
expect(result).toEqual({ id: "lnk1", removed: true });
|
|
55
55
|
});
|
|
56
56
|
});
|
|
@@ -9,9 +9,10 @@ import {
|
|
|
9
9
|
import type { AiProposalPreview } from "@checkstack/ai-common";
|
|
10
10
|
import type { RegisteredAiTool } from "@checkstack/ai-backend";
|
|
11
11
|
|
|
12
|
-
/** Input for `incident.removeLink`: the link id to remove. */
|
|
12
|
+
/** Input for `incident.removeLink`: the link id to remove + its owning incident. */
|
|
13
13
|
export const IncidentRemoveLinkInputSchema = z.object({
|
|
14
14
|
id: z.string(),
|
|
15
|
+
incidentId: z.string(),
|
|
15
16
|
});
|
|
16
17
|
export type IncidentRemoveLinkInput = z.infer<
|
|
17
18
|
typeof IncidentRemoveLinkInputSchema
|
|
@@ -45,15 +46,15 @@ export function createIncidentRemoveLinkTool(): RegisteredAiTool<
|
|
|
45
46
|
rpcClient: RpcClient;
|
|
46
47
|
}): Promise<AiProposalPreview<IncidentRemoveLinkInput>> => {
|
|
47
48
|
return {
|
|
48
|
-
summary: `Remove link ${input.id} from
|
|
49
|
-
payload: { id: input.id },
|
|
49
|
+
summary: `Remove link ${input.id} from incident ${input.incidentId}. This is permanent.`,
|
|
50
|
+
payload: { id: input.id, incidentId: input.incidentId },
|
|
50
51
|
};
|
|
51
52
|
};
|
|
52
53
|
|
|
53
54
|
return {
|
|
54
55
|
name: "incident.removeLink",
|
|
55
56
|
description:
|
|
56
|
-
"Remove a hotlink from an incident by link id. DESTRUCTIVE and irreversible. Never removes directly; a person must approve the confirmation. Find the link id by reading the incident's details first.",
|
|
57
|
+
"Remove a hotlink from an incident by link id and its owning incidentId. DESTRUCTIVE and irreversible. Never removes directly; a person must approve the confirmation. Find the link id and incidentId by reading the incident's details first.",
|
|
57
58
|
effect: "destructive",
|
|
58
59
|
input: IncidentRemoveLinkInputSchema,
|
|
59
60
|
requiredAccessRules: [
|
|
@@ -62,7 +63,10 @@ export function createIncidentRemoveLinkTool(): RegisteredAiTool<
|
|
|
62
63
|
dryRun,
|
|
63
64
|
async execute({ input, rpcClient }) {
|
|
64
65
|
const incidentClient = rpcClient.forPlugin(IncidentApi);
|
|
65
|
-
await incidentClient.removeLink({
|
|
66
|
+
await incidentClient.removeLink({
|
|
67
|
+
id: input.id,
|
|
68
|
+
incidentId: input.incidentId,
|
|
69
|
+
});
|
|
66
70
|
return { id: input.id, removed: true };
|
|
67
71
|
},
|
|
68
72
|
};
|
|
@@ -6,6 +6,7 @@ import { createIncidentAddUpdateTool } from "./incident-add-update";
|
|
|
6
6
|
import { createIncidentResolveTool } from "./incident-resolve";
|
|
7
7
|
import { createIncidentAddLinkTool } from "./incident-add-link";
|
|
8
8
|
import { createIncidentRemoveLinkTool } from "./incident-remove-link";
|
|
9
|
+
import { createIncidentDeleteUpdateTool } from "./incident-delete-update";
|
|
9
10
|
|
|
10
11
|
/**
|
|
11
12
|
* The incident plugin's AI tools, registered into the AI registry via
|
|
@@ -29,5 +30,6 @@ export function buildIncidentAiTools(): RegisteredAiTool[] {
|
|
|
29
30
|
createIncidentResolveTool(),
|
|
30
31
|
createIncidentAddLinkTool(),
|
|
31
32
|
createIncidentRemoveLinkTool(),
|
|
33
|
+
createIncidentDeleteUpdateTool(),
|
|
32
34
|
];
|
|
33
35
|
}
|
package/src/automations.test.ts
CHANGED
|
@@ -324,6 +324,76 @@ describe("incident automation actions", () => {
|
|
|
324
324
|
expect(service.resolveIncident).toHaveBeenCalledWith("INC-1", "Fixed");
|
|
325
325
|
});
|
|
326
326
|
|
|
327
|
+
// SLO-1 regression: an automation-driven resolve mutates the incident WITHOUT
|
|
328
|
+
// going through the RPC router, so it must emit `incident.lifecycle.changed`
|
|
329
|
+
// itself — otherwise an override-bearing incident resolved by an automation
|
|
330
|
+
// never triggers SLO downtime reconciliation and its incident-sourced event
|
|
331
|
+
// leaks open.
|
|
332
|
+
it("emits incident.lifecycle.changed on an automation resolve (SLO downtime reconcile)", async () => {
|
|
333
|
+
const resolved = {
|
|
334
|
+
id: "INC-1",
|
|
335
|
+
status: "resolved",
|
|
336
|
+
severity: "critical",
|
|
337
|
+
systemIds: ["sys-1", "sys-2"],
|
|
338
|
+
};
|
|
339
|
+
const service = makeServiceStub({
|
|
340
|
+
resolveIncident: mock(
|
|
341
|
+
async () => resolved,
|
|
342
|
+
) as unknown as IncidentService["resolveIncident"],
|
|
343
|
+
});
|
|
344
|
+
const emit = mock(async () => {});
|
|
345
|
+
const eventBus = { emit } as unknown as Parameters<
|
|
346
|
+
typeof createIncidentActions
|
|
347
|
+
>[0]["eventBus"];
|
|
348
|
+
const resolveAction = createIncidentActions({ service, eventBus })[1];
|
|
349
|
+
|
|
350
|
+
await resolveAction.execute({
|
|
351
|
+
...actionContext,
|
|
352
|
+
config: { incidentId: "INC-1", message: "Fixed" } as never,
|
|
353
|
+
});
|
|
354
|
+
|
|
355
|
+
expect(emit).toHaveBeenCalledTimes(1);
|
|
356
|
+
const emitArgs = emit.mock.calls[0] as unknown[] | undefined;
|
|
357
|
+
const hook = emitArgs?.[0] as { id?: string } | undefined;
|
|
358
|
+
const payload = emitArgs?.[1] as
|
|
359
|
+
| { systemIds?: string[]; action?: string }
|
|
360
|
+
| undefined;
|
|
361
|
+
expect(hook?.id).toBe("incident.lifecycle.changed");
|
|
362
|
+
expect(payload?.action).toBe("resolved");
|
|
363
|
+
expect(payload?.systemIds).toEqual(["sys-1", "sys-2"]);
|
|
364
|
+
});
|
|
365
|
+
|
|
366
|
+
it("emits incident.lifecycle.changed on an automation update_status → resolved", async () => {
|
|
367
|
+
const update = {
|
|
368
|
+
id: "upd-3",
|
|
369
|
+
incidentId: "INC-1",
|
|
370
|
+
message: "Status changed to resolved",
|
|
371
|
+
createdAt: new Date(),
|
|
372
|
+
};
|
|
373
|
+
const service = makeServiceStub({
|
|
374
|
+
addUpdate: mock(
|
|
375
|
+
async () => update,
|
|
376
|
+
) as unknown as IncidentService["addUpdate"],
|
|
377
|
+
});
|
|
378
|
+
const emit = mock(async () => {});
|
|
379
|
+
const eventBus = { emit } as unknown as Parameters<
|
|
380
|
+
typeof createIncidentActions
|
|
381
|
+
>[0]["eventBus"];
|
|
382
|
+
const updateStatusAction = createIncidentActions({ service, eventBus })[3];
|
|
383
|
+
|
|
384
|
+
await updateStatusAction.execute({
|
|
385
|
+
...actionContext,
|
|
386
|
+
config: { incidentId: "INC-1", status: "resolved" } as never,
|
|
387
|
+
});
|
|
388
|
+
|
|
389
|
+
expect(emit).toHaveBeenCalledTimes(1);
|
|
390
|
+
const payload = (emit.mock.calls[0] as unknown[] | undefined)?.[1] as
|
|
391
|
+
| { systemIds?: string[]; action?: string }
|
|
392
|
+
| undefined;
|
|
393
|
+
expect(payload?.action).toBe("resolved");
|
|
394
|
+
expect(payload?.systemIds).toEqual(["sys-1"]);
|
|
395
|
+
});
|
|
396
|
+
|
|
327
397
|
// 6(b) regression: an action-driven resolve must route through the reactive
|
|
328
398
|
// entity (like the RPC router) so it appends an `entity_transitions` row,
|
|
329
399
|
// emits `ENTITY_CHANGED` (waking `wait_until`), and fires the
|
package/src/automations.ts
CHANGED
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
* and to match `wait_for_trigger` waits against the same incident.
|
|
12
12
|
*/
|
|
13
13
|
import { z } from "zod";
|
|
14
|
-
import { Versioned } from "@checkstack/backend-api";
|
|
14
|
+
import { Versioned, type EventBus } from "@checkstack/backend-api";
|
|
15
15
|
import type {
|
|
16
16
|
ActionDefinition,
|
|
17
17
|
ArtifactTypeDefinition,
|
|
@@ -22,9 +22,11 @@ import { makeEntityDrivenTriggerSetup } from "@checkstack/automation-backend";
|
|
|
22
22
|
import {
|
|
23
23
|
IncidentSeverityEnum,
|
|
24
24
|
IncidentStatusEnum,
|
|
25
|
+
type IncidentLifecycleAction,
|
|
25
26
|
} from "@checkstack/incident-common";
|
|
26
27
|
|
|
27
28
|
import type { IncidentService } from "./service";
|
|
29
|
+
import { emitIncidentLifecycleChanged } from "./hooks";
|
|
28
30
|
import {
|
|
29
31
|
toIncidentEntityState,
|
|
30
32
|
writeIncidentEntity,
|
|
@@ -259,12 +261,20 @@ export interface IncidentActionDeps {
|
|
|
259
261
|
* non-reactively.
|
|
260
262
|
*/
|
|
261
263
|
getIncidentEntity?: () => EntityHandle<IncidentEntityState> | undefined;
|
|
264
|
+
/**
|
|
265
|
+
* Distributed event bus used to emit `incident.lifecycle.changed` after an
|
|
266
|
+
* automation-driven lifecycle mutation. Automations mutate incidents WITHOUT
|
|
267
|
+
* going through the RPC router, so without this an automation resolving an
|
|
268
|
+
* override-bearing incident would never notify SLO downtime reconciliation and
|
|
269
|
+
* the incident-sourced downtime event would leak open. Undefined in tests.
|
|
270
|
+
*/
|
|
271
|
+
eventBus?: EventBus;
|
|
262
272
|
}
|
|
263
273
|
|
|
264
274
|
export function createIncidentActions(
|
|
265
275
|
deps: IncidentActionDeps,
|
|
266
276
|
): ActionDefinition<unknown, unknown>[] {
|
|
267
|
-
const { service, getIncidentEntity } = deps;
|
|
277
|
+
const { service, getIncidentEntity, eventBus } = deps;
|
|
268
278
|
|
|
269
279
|
const createAction: ActionDefinition<
|
|
270
280
|
z.infer<typeof incidentCreateConfigSchema>,
|
|
@@ -339,6 +349,17 @@ export function createIncidentActions(
|
|
|
339
349
|
);
|
|
340
350
|
} else {
|
|
341
351
|
logger.info(`Automation created incident ${incident.id}`);
|
|
352
|
+
// Only a real create is a lifecycle mutation; a REUSE changed nothing,
|
|
353
|
+
// so SLO already reconciled it at its original creation.
|
|
354
|
+
await emitIncidentLifecycleChanged({
|
|
355
|
+
eventBus,
|
|
356
|
+
logger,
|
|
357
|
+
payload: {
|
|
358
|
+
incidentId: incident.id,
|
|
359
|
+
systemIds: incident.systemIds,
|
|
360
|
+
action: "created",
|
|
361
|
+
},
|
|
362
|
+
});
|
|
342
363
|
}
|
|
343
364
|
return {
|
|
344
365
|
success: true,
|
|
@@ -364,6 +385,15 @@ export function createIncidentActions(
|
|
|
364
385
|
},
|
|
365
386
|
});
|
|
366
387
|
logger.info(`Automation created incident ${incident.id}`);
|
|
388
|
+
await emitIncidentLifecycleChanged({
|
|
389
|
+
eventBus,
|
|
390
|
+
logger,
|
|
391
|
+
payload: {
|
|
392
|
+
incidentId: incident.id,
|
|
393
|
+
systemIds: incident.systemIds,
|
|
394
|
+
action: "created",
|
|
395
|
+
},
|
|
396
|
+
});
|
|
367
397
|
return {
|
|
368
398
|
success: true,
|
|
369
399
|
externalId: incident.id,
|
|
@@ -442,6 +472,17 @@ export function createIncidentActions(
|
|
|
442
472
|
};
|
|
443
473
|
}
|
|
444
474
|
logger.info(`Automation resolved incident ${incident.id}`);
|
|
475
|
+
// A resolve clears any active health override, so SLO must close the
|
|
476
|
+
// incident-forced downtime — the defect this notifier fixes.
|
|
477
|
+
await emitIncidentLifecycleChanged({
|
|
478
|
+
eventBus,
|
|
479
|
+
logger,
|
|
480
|
+
payload: {
|
|
481
|
+
incidentId: incident.id,
|
|
482
|
+
systemIds: incident.systemIds,
|
|
483
|
+
action: "resolved",
|
|
484
|
+
},
|
|
485
|
+
});
|
|
445
486
|
return {
|
|
446
487
|
success: true,
|
|
447
488
|
externalId: incident.id,
|
|
@@ -485,7 +526,8 @@ export function createIncidentActions(
|
|
|
485
526
|
// is a no-op diff (no event). `opts.runId` masks run-resolved secrets.
|
|
486
527
|
const captured: {
|
|
487
528
|
update: Awaited<ReturnType<typeof service.addUpdate>> | null;
|
|
488
|
-
|
|
529
|
+
systemIds: string[];
|
|
530
|
+
} = { update: null, systemIds: [] };
|
|
489
531
|
await writeIncidentEntity({
|
|
490
532
|
handle: getIncidentEntity?.(),
|
|
491
533
|
incidentId,
|
|
@@ -500,6 +542,7 @@ export function createIncidentActions(
|
|
|
500
542
|
if (!incident) {
|
|
501
543
|
throw new Error(`Incident ${incidentId} not found`);
|
|
502
544
|
}
|
|
545
|
+
captured.systemIds = incident.systemIds;
|
|
503
546
|
return toIncidentEntityState(incident);
|
|
504
547
|
},
|
|
505
548
|
});
|
|
@@ -513,6 +556,21 @@ export function createIncidentActions(
|
|
|
513
556
|
logger.info(
|
|
514
557
|
`Automation added update ${update.id} to incident ${incidentId}`,
|
|
515
558
|
);
|
|
559
|
+
// Only a status-changing update is a lifecycle mutation SLO cares about; a
|
|
560
|
+
// comment-only update changes no health-relevant state. A change to
|
|
561
|
+
// resolved clears any override, so SLO must reconcile.
|
|
562
|
+
if (config.statusChange) {
|
|
563
|
+
await emitIncidentLifecycleChanged({
|
|
564
|
+
eventBus,
|
|
565
|
+
logger,
|
|
566
|
+
payload: {
|
|
567
|
+
incidentId,
|
|
568
|
+
systemIds: captured.systemIds,
|
|
569
|
+
action:
|
|
570
|
+
config.statusChange === "resolved" ? "resolved" : "updated",
|
|
571
|
+
},
|
|
572
|
+
});
|
|
573
|
+
}
|
|
516
574
|
return {
|
|
517
575
|
success: true,
|
|
518
576
|
externalId: update.id,
|
|
@@ -553,7 +611,8 @@ export function createIncidentActions(
|
|
|
553
611
|
// re-reads post-write state. `opts.runId` masks run-resolved secrets.
|
|
554
612
|
const captured: {
|
|
555
613
|
update: Awaited<ReturnType<typeof service.addUpdate>> | null;
|
|
556
|
-
|
|
614
|
+
systemIds: string[];
|
|
615
|
+
} = { update: null, systemIds: [] };
|
|
557
616
|
await writeIncidentEntity({
|
|
558
617
|
handle: getIncidentEntity?.(),
|
|
559
618
|
incidentId,
|
|
@@ -568,6 +627,7 @@ export function createIncidentActions(
|
|
|
568
627
|
if (!incident) {
|
|
569
628
|
throw new Error(`Incident ${incidentId} not found`);
|
|
570
629
|
}
|
|
630
|
+
captured.systemIds = incident.systemIds;
|
|
571
631
|
return toIncidentEntityState(incident);
|
|
572
632
|
},
|
|
573
633
|
});
|
|
@@ -581,6 +641,19 @@ export function createIncidentActions(
|
|
|
581
641
|
logger.info(
|
|
582
642
|
`Automation set incident ${incidentId} status → ${config.status}`,
|
|
583
643
|
);
|
|
644
|
+
// A status flip is a lifecycle mutation; a flip to resolved clears any
|
|
645
|
+
// override, so SLO must reconcile the affected systems' downtime.
|
|
646
|
+
const action: IncidentLifecycleAction =
|
|
647
|
+
config.status === "resolved" ? "resolved" : "updated";
|
|
648
|
+
await emitIncidentLifecycleChanged({
|
|
649
|
+
eventBus,
|
|
650
|
+
logger,
|
|
651
|
+
payload: {
|
|
652
|
+
incidentId,
|
|
653
|
+
systemIds: captured.systemIds,
|
|
654
|
+
action,
|
|
655
|
+
},
|
|
656
|
+
});
|
|
584
657
|
return {
|
|
585
658
|
success: true,
|
|
586
659
|
externalId: update.id,
|
package/src/hooks.ts
CHANGED
|
@@ -1,11 +1,61 @@
|
|
|
1
|
+
import { createHook, type EventBus, type Logger } from "@checkstack/backend-api";
|
|
2
|
+
import {
|
|
3
|
+
INCIDENT_LIFECYCLE_CHANGED_HOOK_ID,
|
|
4
|
+
type IncidentLifecycleChangedPayload,
|
|
5
|
+
} from "@checkstack/incident-common";
|
|
6
|
+
|
|
1
7
|
/**
|
|
2
8
|
* Incident cross-plugin hooks.
|
|
3
9
|
*
|
|
4
10
|
* The `incident.created` / `.updated` / `.resolved` hooks were removed in
|
|
5
11
|
* Phase 4 (§10.1): incidents are now the reactive `incident` entity, whose
|
|
6
12
|
* change deriver fires the matching `incident.created` / `.updated` /
|
|
7
|
-
* `.resolved` trigger events through Stage-1 routing.
|
|
8
|
-
*
|
|
9
|
-
* `
|
|
13
|
+
* `.resolved` trigger events through Stage-1 routing.
|
|
14
|
+
*
|
|
15
|
+
* `lifecycleChanged` is a distinct, lower-level cross-plugin hook: it fires on
|
|
16
|
+
* EVERY incident lifecycle mutation (create, update — including a health
|
|
17
|
+
* override added / changed / cleared —, resolve, delete, and the auto-incident
|
|
18
|
+
* paths), carrying `{ incidentId, systemIds, action }`. Unlike the reactive
|
|
19
|
+
* `incident` entity change (state `{ status, severity, systemIds }`, so an
|
|
20
|
+
* override-only edit emits nothing), this catches override changes, which
|
|
21
|
+
* `@checkstack/slo-backend` needs to open/close incident-forced SLO downtime.
|
|
22
|
+
* The id + payload contract live in `@checkstack/incident-common` so consumers
|
|
23
|
+
* subscribe without depending on this backend.
|
|
24
|
+
*/
|
|
25
|
+
export const incidentLifecycleChangedHook =
|
|
26
|
+
createHook<IncidentLifecycleChangedPayload>(
|
|
27
|
+
INCIDENT_LIFECYCLE_CHANGED_HOOK_ID,
|
|
28
|
+
);
|
|
29
|
+
|
|
30
|
+
export const incidentHooks = {
|
|
31
|
+
lifecycleChanged: incidentLifecycleChangedHook,
|
|
32
|
+
} as const;
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* Emit `incident.lifecycle.changed` on the distributed event bus. THE single
|
|
36
|
+
* place the hook is fired, so every incident lifecycle path - the RPC router,
|
|
37
|
+
* the bulk paths, AND the automation actions - shares identical guard/failure
|
|
38
|
+
* semantics and can never drift. Best-effort by contract: the incident write is
|
|
39
|
+
* already committed by the time this runs, so a delivery failure must never turn
|
|
40
|
+
* a successful mutation into a client/run error - consumers reconcile on the
|
|
41
|
+
* next lifecycle event. A no-op when no event bus is wired (tests).
|
|
10
42
|
*/
|
|
11
|
-
export
|
|
43
|
+
export async function emitIncidentLifecycleChanged({
|
|
44
|
+
eventBus,
|
|
45
|
+
logger,
|
|
46
|
+
payload,
|
|
47
|
+
}: {
|
|
48
|
+
eventBus: EventBus | undefined;
|
|
49
|
+
logger: Logger;
|
|
50
|
+
payload: IncidentLifecycleChangedPayload;
|
|
51
|
+
}): Promise<void> {
|
|
52
|
+
if (!eventBus) return;
|
|
53
|
+
try {
|
|
54
|
+
await eventBus.emit(incidentLifecycleChangedHook, payload);
|
|
55
|
+
} catch (error) {
|
|
56
|
+
logger.warn(
|
|
57
|
+
`Failed to emit incident.lifecycle.changed hook for incident ${payload.incidentId}; consumers will reconcile on the next lifecycle event.`,
|
|
58
|
+
{ error },
|
|
59
|
+
);
|
|
60
|
+
}
|
|
61
|
+
}
|