@checkstack/incident-backend 1.4.0 → 1.6.0

package/src/service.ts

@@ -1,5 +1,5 @@
 import { eq, and, inArray, ne } from "drizzle-orm";
-import { withXactLock, type SafeDatabase } from "@checkstack/backend-api";
+import type { AdvisoryLockService, SafeDatabase } from "@checkstack/backend-api";
 import * as schema from "./schema";
 import {
   incidents,
@@ -27,7 +27,10 @@ function generateId(): string {
 }
 
 export class IncidentService {
-  constructor(private db: Db) {}
+  constructor(
+    private db: Db,
+    private advisoryLock: AdvisoryLockService,
+  ) {}
 
   /**
    * List incidents with optional filters
@@ -235,33 +238,38 @@ export class IncidentService {
     id: string = generateId(),
   ): Promise<IncidentWithSystems> {
 
-    await this.db.insert(incidents).values({
-      id,
-      title: input.title,
-      description: input.description,
-      status: "investigating",
-      severity: input.severity,
-      suppressNotifications: input.suppressNotifications ?? false,
-    });
-
-    // Insert system associations
-    for (const systemId of input.systemIds) {
-      await this.db.insert(incidentSystems).values({
-        incidentId: id,
-        systemId,
+    // Atomic: the incident row, its system associations, and any initial update
+    // must all commit together. Without the transaction a failure mid-loop left
+    // a committed incident with only some (or none) of its system links.
+    await this.db.transaction(async (tx) => {
+      await tx.insert(incidents).values({
+        id,
+        title: input.title,
+        description: input.description,
+        status: "investigating",
+        severity: input.severity,
+        suppressNotifications: input.suppressNotifications ?? false,
       });
-    }
 
-    // Add initial update if provided
-    if (input.initialMessage) {
-      await this.db.insert(incidentUpdates).values({
-        id: generateId(),
-        incidentId: id,
-        message: input.initialMessage,
-        statusChange: "investigating",
-        createdBy: userId,
-      });
-    }
+      // Insert system associations
+      for (const systemId of input.systemIds) {
+        await tx.insert(incidentSystems).values({
+          incidentId: id,
+          systemId,
+        });
+      }
+
+      // Add initial update if provided
+      if (input.initialMessage) {
+        await tx.insert(incidentUpdates).values({
+          id: generateId(),
+          incidentId: id,
+          message: input.initialMessage,
+          statusChange: "investigating",
+          createdBy: userId,
+        });
+      }
+    });
 
     return (await this.getIncident(id))!;
   }
@@ -290,24 +298,29 @@ export class IncidentService {
     if (input.suppressNotifications !== undefined)
       updateData.suppressNotifications = input.suppressNotifications;
 
-    await this.db
-      .update(incidents)
-      .set(updateData)
-      .where(eq(incidents.id, input.id));
-
-    // Update system associations if provided
-    if (input.systemIds !== undefined) {
-      await this.db
-        .delete(incidentSystems)
-        .where(eq(incidentSystems.incidentId, input.id));
-
-      for (const systemId of input.systemIds) {
-        await this.db.insert(incidentSystems).values({
-          incidentId: input.id,
-          systemId,
-        });
+    // Atomic: the field update and the delete-then-reinsert of system links must
+    // commit together. Without the transaction a failure after the delete left
+    // the incident with ALL system associations wiped.
+    await this.db.transaction(async (tx) => {
+      await tx
+        .update(incidents)
+        .set(updateData)
+        .where(eq(incidents.id, input.id));
+
+      // Update system associations if provided
+      if (input.systemIds !== undefined) {
+        await tx
+          .delete(incidentSystems)
+          .where(eq(incidentSystems.incidentId, input.id));
+
+        for (const systemId of input.systemIds) {
+          await tx.insert(incidentSystems).values({
+            incidentId: input.id,
+            systemId,
+          });
+        }
       }
-    }
+    });
 
     return (await this.getIncident(input.id))!;
   }
@@ -321,20 +334,25 @@ export class IncidentService {
   ): Promise<IncidentUpdate> {
     const id = generateId();
 
-    // If status change is provided, update the incident status
-    if (input.statusChange) {
-      await this.db
-        .update(incidents)
-        .set({ status: input.statusChange, updatedAt: new Date() })
-        .where(eq(incidents.id, input.incidentId));
-    }
+    // Atomic: the status flip and the timeline entry that records it must commit
+    // together. Without the transaction a failed insert left the incident in a
+    // new status with no update row explaining it (status/timeline divergence).
+    await this.db.transaction(async (tx) => {
+      // If status change is provided, update the incident status
+      if (input.statusChange) {
+        await tx
+          .update(incidents)
+          .set({ status: input.statusChange, updatedAt: new Date() })
+          .where(eq(incidents.id, input.incidentId));
+      }
 
-    await this.db.insert(incidentUpdates).values({
-      id,
-      incidentId: input.incidentId,
-      message: input.message,
-      statusChange: input.statusChange,
-      createdBy: userId,
+      await tx.insert(incidentUpdates).values({
+        id,
+        incidentId: input.incidentId,
+        message: input.message,
+        statusChange: input.statusChange,
+        createdBy: userId,
+      });
     });
 
     const [update] = await this.db
@@ -364,18 +382,21 @@ export class IncidentService {
 
     if (!existing) return undefined;
 
-    await this.db
-      .update(incidents)
-      .set({ status: "resolved", updatedAt: new Date() })
-      .where(eq(incidents.id, id));
+    // Atomic: mark resolved + write the resolution timeline entry together.
+    await this.db.transaction(async (tx) => {
+      await tx
+        .update(incidents)
+        .set({ status: "resolved", updatedAt: new Date() })
+        .where(eq(incidents.id, id));
 
-    // Add resolution update entry
-    await this.db.insert(incidentUpdates).values({
-      id: generateId(),
-      incidentId: id,
-      message: message ?? "Incident resolved",
-      statusChange: "resolved",
-      createdBy: userId,
+      // Add resolution update entry
+      await tx.insert(incidentUpdates).values({
+        id: generateId(),
+        incidentId: id,
+        message: message ?? "Incident resolved",
+        statusChange: "resolved",
+        createdBy: userId,
+      });
     });
 
     return (await this.getIncident(id))!;
@@ -528,15 +549,16 @@ export class IncidentService {
       create: () => Promise<IncidentWithSystems>,
     ) => Promise<IncidentWithSystems> = (create) => create(),
   ): Promise<{ incident: IncidentWithSystems; reused: boolean }> {
-    return withXactLock({
-      db: this.db,
+    return this.advisoryLock.withXactLock({
       key: `incident.dedupe-open-for-system:${dedupeSystemId}`,
-      // The find + create run on `this.db` (the pool), NOT on `tx`. That is
-      // safe here because `pg_advisory_xact_lock` BLOCKS every other holder
-      // of this key until this transaction commits: a racing caller waits
-      // at lock-acquire, so its find can't observe "no open incident" until
-      // ours has already committed the insert. The critical section is thus
-      // serialized by the lock window even though it doesn't ride `tx`.
+      // The find + create deliberately run on `this.db` (the admin pool), NOT
+      // on the lock connection. That is safe because `pg_advisory_xact_lock`
+      // BLOCKS every other holder of this key until this lock transaction
+      // commits: a racing caller waits at lock-acquire, so its find can't
+      // observe "no open incident" until ours has already committed the
+      // insert. Crucially, the lock transaction lives on the DEDICATED lock
+      // pool (see `createAdvisoryLockService(lockPool)`), so holding it open
+      // while the work runs on the admin pool cannot starve the admin pool.
       fn: async () => {
         const existing = await this.findActiveIncidentForSystem(dedupeSystemId);
         if (existing) {

package/tsconfig.json

@@ -4,6 +4,12 @@
     "src"
   ],
   "references": [
+    {
+      "path": "../ai-backend"
+    },
+    {
+      "path": "../ai-common"
+    },
     {
       "path": "../auth-common"
     },