@checkstack/incident-backend 1.11.0 → 1.13.0

package/src/automations.test.ts

@@ -324,6 +324,76 @@ describe("incident automation actions", () => {
       expect(service.resolveIncident).toHaveBeenCalledWith("INC-1", "Fixed");
     });
 
+    // SLO-1 regression: an automation-driven resolve mutates the incident WITHOUT
+    // going through the RPC router, so it must emit `incident.lifecycle.changed`
+    // itself — otherwise an override-bearing incident resolved by an automation
+    // never triggers SLO downtime reconciliation and its incident-sourced event
+    // leaks open.
+    it("emits incident.lifecycle.changed on an automation resolve (SLO downtime reconcile)", async () => {
+      const resolved = {
+        id: "INC-1",
+        status: "resolved",
+        severity: "critical",
+        systemIds: ["sys-1", "sys-2"],
+      };
+      const service = makeServiceStub({
+        resolveIncident: mock(
+          async () => resolved,
+        ) as unknown as IncidentService["resolveIncident"],
+      });
+      const emit = mock(async () => {});
+      const eventBus = { emit } as unknown as Parameters<
+        typeof createIncidentActions
+      >[0]["eventBus"];
+      const resolveAction = createIncidentActions({ service, eventBus })[1];
+
+      await resolveAction.execute({
+        ...actionContext,
+        config: { incidentId: "INC-1", message: "Fixed" } as never,
+      });
+
+      expect(emit).toHaveBeenCalledTimes(1);
+      const emitArgs = emit.mock.calls[0] as unknown[] | undefined;
+      const hook = emitArgs?.[0] as { id?: string } | undefined;
+      const payload = emitArgs?.[1] as
+        | { systemIds?: string[]; action?: string }
+        | undefined;
+      expect(hook?.id).toBe("incident.lifecycle.changed");
+      expect(payload?.action).toBe("resolved");
+      expect(payload?.systemIds).toEqual(["sys-1", "sys-2"]);
+    });
+
+    it("emits incident.lifecycle.changed on an automation update_status → resolved", async () => {
+      const update = {
+        id: "upd-3",
+        incidentId: "INC-1",
+        message: "Status changed to resolved",
+        createdAt: new Date(),
+      };
+      const service = makeServiceStub({
+        addUpdate: mock(
+          async () => update,
+        ) as unknown as IncidentService["addUpdate"],
+      });
+      const emit = mock(async () => {});
+      const eventBus = { emit } as unknown as Parameters<
+        typeof createIncidentActions
+      >[0]["eventBus"];
+      const updateStatusAction = createIncidentActions({ service, eventBus })[3];
+
+      await updateStatusAction.execute({
+        ...actionContext,
+        config: { incidentId: "INC-1", status: "resolved" } as never,
+      });
+
+      expect(emit).toHaveBeenCalledTimes(1);
+      const payload = (emit.mock.calls[0] as unknown[] | undefined)?.[1] as
+        | { systemIds?: string[]; action?: string }
+        | undefined;
+      expect(payload?.action).toBe("resolved");
+      expect(payload?.systemIds).toEqual(["sys-1"]);
+    });
+
     // 6(b) regression: an action-driven resolve must route through the reactive
     // entity (like the RPC router) so it appends an `entity_transitions` row,
     // emits `ENTITY_CHANGED` (waking `wait_until`), and fires the

package/src/automations.ts

@@ -11,7 +11,7 @@
  * and to match `wait_for_trigger` waits against the same incident.
  */
 import { z } from "zod";
-import { Versioned } from "@checkstack/backend-api";
+import { Versioned, type EventBus } from "@checkstack/backend-api";
 import type {
   ActionDefinition,
   ArtifactTypeDefinition,
@@ -22,9 +22,11 @@ import { makeEntityDrivenTriggerSetup } from "@checkstack/automation-backend";
 import {
   IncidentSeverityEnum,
   IncidentStatusEnum,
+  type IncidentLifecycleAction,
 } from "@checkstack/incident-common";
 
 import type { IncidentService } from "./service";
+import { emitIncidentLifecycleChanged } from "./hooks";
 import {
   toIncidentEntityState,
   writeIncidentEntity,
@@ -259,12 +261,20 @@ export interface IncidentActionDeps {
    * non-reactively.
    */
   getIncidentEntity?: () => EntityHandle<IncidentEntityState> | undefined;
+  /**
+   * Distributed event bus used to emit `incident.lifecycle.changed` after an
+   * automation-driven lifecycle mutation. Automations mutate incidents WITHOUT
+   * going through the RPC router, so without this an automation resolving an
+   * override-bearing incident would never notify SLO downtime reconciliation and
+   * the incident-sourced downtime event would leak open. Undefined in tests.
+   */
+  eventBus?: EventBus;
 }
 
 export function createIncidentActions(
   deps: IncidentActionDeps,
 ): ActionDefinition<unknown, unknown>[] {
-  const { service, getIncidentEntity } = deps;
+  const { service, getIncidentEntity, eventBus } = deps;
 
   const createAction: ActionDefinition<
     z.infer<typeof incidentCreateConfigSchema>,
@@ -339,6 +349,17 @@ export function createIncidentActions(
           );
         } else {
           logger.info(`Automation created incident ${incident.id}`);
+          // Only a real create is a lifecycle mutation; a REUSE changed nothing,
+          // so SLO already reconciled it at its original creation.
+          await emitIncidentLifecycleChanged({
+            eventBus,
+            logger,
+            payload: {
+              incidentId: incident.id,
+              systemIds: incident.systemIds,
+              action: "created",
+            },
+          });
         }
         return {
           success: true,
@@ -364,6 +385,15 @@ export function createIncidentActions(
         },
       });
       logger.info(`Automation created incident ${incident.id}`);
+      await emitIncidentLifecycleChanged({
+        eventBus,
+        logger,
+        payload: {
+          incidentId: incident.id,
+          systemIds: incident.systemIds,
+          action: "created",
+        },
+      });
       return {
         success: true,
         externalId: incident.id,
@@ -442,6 +472,17 @@ export function createIncidentActions(
         };
       }
       logger.info(`Automation resolved incident ${incident.id}`);
+      // A resolve clears any active health override, so SLO must close the
+      // incident-forced downtime — the defect this notifier fixes.
+      await emitIncidentLifecycleChanged({
+        eventBus,
+        logger,
+        payload: {
+          incidentId: incident.id,
+          systemIds: incident.systemIds,
+          action: "resolved",
+        },
+      });
       return {
         success: true,
         externalId: incident.id,
@@ -485,7 +526,8 @@ export function createIncidentActions(
       // is a no-op diff (no event). `opts.runId` masks run-resolved secrets.
       const captured: {
         update: Awaited<ReturnType<typeof service.addUpdate>> | null;
-      } = { update: null };
+        systemIds: string[];
+      } = { update: null, systemIds: [] };
       await writeIncidentEntity({
         handle: getIncidentEntity?.(),
         incidentId,
@@ -500,6 +542,7 @@ export function createIncidentActions(
           if (!incident) {
             throw new Error(`Incident ${incidentId} not found`);
           }
+          captured.systemIds = incident.systemIds;
           return toIncidentEntityState(incident);
         },
       });
@@ -513,6 +556,21 @@ export function createIncidentActions(
       logger.info(
         `Automation added update ${update.id} to incident ${incidentId}`,
       );
+      // Only a status-changing update is a lifecycle mutation SLO cares about; a
+      // comment-only update changes no health-relevant state. A change to
+      // resolved clears any override, so SLO must reconcile.
+      if (config.statusChange) {
+        await emitIncidentLifecycleChanged({
+          eventBus,
+          logger,
+          payload: {
+            incidentId,
+            systemIds: captured.systemIds,
+            action:
+              config.statusChange === "resolved" ? "resolved" : "updated",
+          },
+        });
+      }
       return {
         success: true,
         externalId: update.id,
@@ -553,7 +611,8 @@ export function createIncidentActions(
       // re-reads post-write state. `opts.runId` masks run-resolved secrets.
       const captured: {
         update: Awaited<ReturnType<typeof service.addUpdate>> | null;
-      } = { update: null };
+        systemIds: string[];
+      } = { update: null, systemIds: [] };
       await writeIncidentEntity({
         handle: getIncidentEntity?.(),
         incidentId,
@@ -568,6 +627,7 @@ export function createIncidentActions(
           if (!incident) {
             throw new Error(`Incident ${incidentId} not found`);
           }
+          captured.systemIds = incident.systemIds;
           return toIncidentEntityState(incident);
         },
       });
@@ -581,6 +641,19 @@ export function createIncidentActions(
       logger.info(
         `Automation set incident ${incidentId} status → ${config.status}`,
       );
+      // A status flip is a lifecycle mutation; a flip to resolved clears any
+      // override, so SLO must reconcile the affected systems' downtime.
+      const action: IncidentLifecycleAction =
+        config.status === "resolved" ? "resolved" : "updated";
+      await emitIncidentLifecycleChanged({
+        eventBus,
+        logger,
+        payload: {
+          incidentId,
+          systemIds: captured.systemIds,
+          action,
+        },
+      });
       return {
         success: true,
         externalId: update.id,

package/src/hooks.ts

@@ -1,11 +1,61 @@
+import { createHook, type EventBus, type Logger } from "@checkstack/backend-api";
+import {
+  INCIDENT_LIFECYCLE_CHANGED_HOOK_ID,
+  type IncidentLifecycleChangedPayload,
+} from "@checkstack/incident-common";
+
 /**
  * Incident cross-plugin hooks.
  *
  * The `incident.created` / `.updated` / `.resolved` hooks were removed in
  * Phase 4 (§10.1): incidents are now the reactive `incident` entity, whose
  * change deriver fires the matching `incident.created` / `.updated` /
- * `.resolved` trigger events through Stage-1 routing. No cross-plugin hook
- * remains, so this object is intentionally empty (kept for the stable
- * `export { incidentHooks }` surface).
+ * `.resolved` trigger events through Stage-1 routing.
+ *
+ * `lifecycleChanged` is a distinct, lower-level cross-plugin hook: it fires on
+ * EVERY incident lifecycle mutation (create, update — including a health
+ * override added / changed / cleared —, resolve, delete, and the auto-incident
+ * paths), carrying `{ incidentId, systemIds, action }`. Unlike the reactive
+ * `incident` entity change (state `{ status, severity, systemIds }`, so an
+ * override-only edit emits nothing), this catches override changes, which
+ * `@checkstack/slo-backend` needs to open/close incident-forced SLO downtime.
+ * The id + payload contract live in `@checkstack/incident-common` so consumers
+ * subscribe without depending on this backend.
+ */
+export const incidentLifecycleChangedHook =
+  createHook<IncidentLifecycleChangedPayload>(
+    INCIDENT_LIFECYCLE_CHANGED_HOOK_ID,
+  );
+
+export const incidentHooks = {
+  lifecycleChanged: incidentLifecycleChangedHook,
+} as const;
+
+/**
+ * Emit `incident.lifecycle.changed` on the distributed event bus. THE single
+ * place the hook is fired, so every incident lifecycle path - the RPC router,
+ * the bulk paths, AND the automation actions - shares identical guard/failure
+ * semantics and can never drift. Best-effort by contract: the incident write is
+ * already committed by the time this runs, so a delivery failure must never turn
+ * a successful mutation into a client/run error - consumers reconcile on the
+ * next lifecycle event. A no-op when no event bus is wired (tests).
  */
-export const incidentHooks = {} as const;
+export async function emitIncidentLifecycleChanged({
+  eventBus,
+  logger,
+  payload,
+}: {
+  eventBus: EventBus | undefined;
+  logger: Logger;
+  payload: IncidentLifecycleChangedPayload;
+}): Promise<void> {
+  if (!eventBus) return;
+  try {
+    await eventBus.emit(incidentLifecycleChangedHook, payload);
+  } catch (error) {
+    logger.warn(
+      `Failed to emit incident.lifecycle.changed hook for incident ${payload.incidentId}; consumers will reconcile on the next lifecycle event.`,
+      { error },
+    );
+  }
+}

package/src/index.ts

@@ -145,6 +145,7 @@ export default createBackendPlugin({
         rpc: coreServices.rpc,
         rpcClient: coreServices.rpcClient,
         signalService: coreServices.signalService,
+        eventBus: coreServices.eventBus,
         cacheManager: coreServices.cacheManager,
         advisoryLock: coreServices.advisoryLock,
         resourceResolverRegistry: coreServices.resourceResolverRegistry,
@@ -155,6 +156,7 @@ export default createBackendPlugin({
         rpc,
         rpcClient,
         signalService,
+        eventBus,
         cacheManager,
         advisoryLock,
         resourceResolverRegistry,
@@ -198,6 +200,7 @@ export default createBackendPlugin({
         const router = createRouter({
           service,
           signalService,
+          eventBus,
           catalogClient,
           notificationClient,
           authClient,
@@ -218,6 +221,7 @@ export default createBackendPlugin({
         for (const action of createIncidentActions({
           service,
           getIncidentEntity: () => incidentEntity,
+          eventBus,
         })) {
           automationActions.registerAction(action, pluginMetadata);
         }

package/src/notifications.test.ts

@@ -317,6 +317,187 @@ describe("notifyAffectedSystems", () => {
     });
   });
 
+  describe("update message in body", () => {
+    it("appends the escaped update message as a blockquote", async () => {
+      await notifyAffectedSystems({
+        catalogClient: mockCatalogClient as never,
+        notificationClient: mockNotificationClient as never,
+        logger: mockLogger as never,
+        incidentId: "inc-1",
+        incidentTitle: "API Outage",
+        systemIds: ["sys-1"],
+        action: "updated",
+        severity: "minor",
+        updateMessage: "Rolled back the bad deploy, monitoring recovery.",
+      });
+
+      const call = (
+        mockNotificationClient.notifyForSubscription.mock
+          .calls[0] as unknown as [{ body?: string }]
+      )[0];
+      expect(call?.body).toContain(
+        "\n\n> Rolled back the bad deploy, monitoring recovery",
+      );
+    });
+
+    it("escapes markdown control characters in the message", async () => {
+      await notifyAffectedSystems({
+        catalogClient: mockCatalogClient as never,
+        notificationClient: mockNotificationClient as never,
+        logger: mockLogger as never,
+        incidentId: "inc-1",
+        incidentTitle: "API Outage",
+        systemIds: ["sys-1"],
+        action: "updated",
+        severity: "minor",
+        updateMessage: "See [here](http://evil) **now** `code`",
+      });
+
+      const call = (
+        mockNotificationClient.notifyForSubscription.mock
+          .calls[0] as unknown as [{ body?: string }]
+      )[0];
+      // No unescaped link/bold/code syntax survives into the body.
+      expect(call?.body).not.toContain("[here](http://evil)");
+      expect(call?.body).not.toContain("**now**");
+      expect(call?.body).toContain("\\[here\\]");
+      expect(call?.body).toContain("\\*\\*now\\*\\*");
+    });
+
+    it("strips non-whitespace control characters (ESC/NUL/BEL/DEL)", async () => {
+      await notifyAffectedSystems({
+        catalogClient: mockCatalogClient as never,
+        notificationClient: mockNotificationClient as never,
+        logger: mockLogger as never,
+        incidentId: "inc-1",
+        incidentTitle: "API Outage",
+        systemIds: ["sys-1"],
+        action: "updated",
+        severity: "minor",
+        updateMessage: "before\u001B\u0000\u0007\u007Fafter",
+      });
+
+      const call = (
+        mockNotificationClient.notifyForSubscription.mock
+          .calls[0] as unknown as [{ body?: string }]
+      )[0];
+      const blockquoteLine = (call?.body ?? "").split("\n\n> ")[1] ?? "";
+      expect(blockquoteLine).toBe("beforeafter");
+      // No control characters survive into the excerpt.
+      expect(/[\u0000-\u001F\u007F-\u009F]/u.test(blockquoteLine)).toBe(
+        false,
+      );
+    });
+
+    it("escapes HTML-significant < and & so markup cannot be injected", async () => {
+      await notifyAffectedSystems({
+        catalogClient: mockCatalogClient as never,
+        notificationClient: mockNotificationClient as never,
+        logger: mockLogger as never,
+        incidentId: "inc-1",
+        incidentTitle: "API Outage",
+        systemIds: ["sys-1"],
+        action: "updated",
+        severity: "minor",
+        updateMessage: "watch <img onerror=x> & <script>",
+      });
+
+      const call = (
+        mockNotificationClient.notifyForSubscription.mock
+          .calls[0] as unknown as [{ body?: string }]
+      )[0];
+      const blockquoteLine = (call?.body ?? "").split("\n\n> ")[1] ?? "";
+      expect(blockquoteLine).not.toContain("<");
+      expect(blockquoteLine).toContain("&lt;img");
+      expect(blockquoteLine).toContain("&amp;");
+    });
+
+    it("collapses newlines so the message cannot break out of the blockquote", async () => {
+      await notifyAffectedSystems({
+        catalogClient: mockCatalogClient as never,
+        notificationClient: mockNotificationClient as never,
+        logger: mockLogger as never,
+        incidentId: "inc-1",
+        incidentTitle: "API Outage",
+        systemIds: ["sys-1"],
+        action: "updated",
+        severity: "minor",
+        updateMessage: "line one\n\nline two\ninjected",
+      });
+
+      const call = (
+        mockNotificationClient.notifyForSubscription.mock
+          .calls[0] as unknown as [{ body?: string }]
+      )[0];
+      const blockquoteLine = (call?.body ?? "").split("\n\n> ")[1] ?? "";
+      expect(blockquoteLine).not.toContain("\n");
+      expect(blockquoteLine).toBe("line one line two injected");
+    });
+
+    it("truncates an over-long message to a bounded length", async () => {
+      const longMessage = "a".repeat(1000);
+      await notifyAffectedSystems({
+        catalogClient: mockCatalogClient as never,
+        notificationClient: mockNotificationClient as never,
+        logger: mockLogger as never,
+        incidentId: "inc-1",
+        incidentTitle: "API Outage",
+        systemIds: ["sys-1"],
+        action: "updated",
+        severity: "minor",
+        updateMessage: longMessage,
+      });
+
+      const call = (
+        mockNotificationClient.notifyForSubscription.mock
+          .calls[0] as unknown as [{ body?: string }]
+      )[0];
+      const blockquoteLine = (call?.body ?? "").split("\n\n> ")[1] ?? "";
+      expect(blockquoteLine.endsWith("...")).toBe(true);
+      // 500 chars + the "..." indicator.
+      expect(blockquoteLine.length).toBeLessThanOrEqual(503);
+    });
+
+    it("omits the blockquote entirely for a blank/whitespace message", async () => {
+      await notifyAffectedSystems({
+        catalogClient: mockCatalogClient as never,
+        notificationClient: mockNotificationClient as never,
+        logger: mockLogger as never,
+        incidentId: "inc-1",
+        incidentTitle: "API Outage",
+        systemIds: ["sys-1"],
+        action: "updated",
+        severity: "minor",
+        updateMessage: "   \n  ",
+      });
+
+      const call = (
+        mockNotificationClient.notifyForSubscription.mock
+          .calls[0] as unknown as [{ body?: string }]
+      )[0];
+      expect(call?.body).not.toContain("\n\n>");
+    });
+
+    it("omits the blockquote when no message is provided", async () => {
+      await notifyAffectedSystems({
+        catalogClient: mockCatalogClient as never,
+        notificationClient: mockNotificationClient as never,
+        logger: mockLogger as never,
+        incidentId: "inc-1",
+        incidentTitle: "API Outage",
+        systemIds: ["sys-1"],
+        action: "created",
+        severity: "minor",
+      });
+
+      const call = (
+        mockNotificationClient.notifyForSubscription.mock
+          .calls[0] as unknown as [{ body?: string }]
+      )[0];
+      expect(call?.body).not.toContain("\n\n>");
+    });
+  });
+
   describe("error handling", () => {
     it("logs a warning but does not throw when the notify call fails", async () => {
       mockNotificationClient.notifyForSubscription.mockRejectedValue(

package/src/notifications.ts

@@ -7,6 +7,7 @@ import type { Logger } from "@checkstack/backend-api";
 import type { InferClient } from "@checkstack/common";
 import { resolveRoute } from "@checkstack/common";
 import type { NotificationApi } from "@checkstack/notification-common";
+import { buildUpdateMessageSuffix } from "@checkstack/notification-common";
 import {
   incidentRoutes,
   incidentCollapseKey,
@@ -29,6 +30,13 @@ export async function notifyAffectedSystems(props: {
   systemNames?: Map<string, string>;
   action: "created" | "updated" | "resolved" | "reopened";
   severity: string;
+  /**
+   * The latest incident update's free-text message. When present it is
+   * escaped, single-lined, truncated, and appended to the notification body as
+   * a blockquote so subscribers see WHAT changed, not just that something did.
+   * User-supplied, so it is always sanitized before it reaches a markdown body.
+   */
+  updateMessage?: string;
 }): Promise<void> {
   const {
     notificationClient,
@@ -39,6 +47,7 @@ export async function notifyAffectedSystems(props: {
     systemNames,
     action,
     severity,
+    updateMessage,
   } = props;
   void props.catalogClient;
 
@@ -64,12 +73,14 @@ export async function notifyAffectedSystems(props: {
     }),
   );
 
+  const messageSuffix = buildUpdateMessageSuffix({ message: updateMessage });
+
   try {
     await notificationClient.notifyForSubscription({
       specId: incidentSystemSubscription.specId,
       resourceKeys: uniqueSystemIds,
       title: `Incident ${actionText}: ${incidentTitle}`,
-      body: `Incident **"${incidentTitle}"** has been ${actionText}.`,
+      body: `Incident **"${incidentTitle}"** has been ${actionText}.${messageSuffix}`,
       importance,
       action: { label: "View Incident", url: incidentDetailPath },
       collapseKey: incidentCollapseKey(incidentId),