@checkstack/healthcheck-backend 1.16.0 → 1.18.0

package/src/schedule-reconciler.test.ts

@@ -0,0 +1,418 @@
+import { describe, it, expect, mock } from "bun:test";
+import {
+  planReconcile,
+  reconcileHealthCheckJobs,
+  type DesiredJob,
+} from "./schedule-reconciler";
+import {
+  encodeHealthCheckJobId,
+  type HealthCheckJobPayload,
+} from "./queue-executor";
+
+function desiredJob(props: {
+  configId: string;
+  systemId: string;
+  environmentId: string | null;
+  intervalSeconds: number;
+}): DesiredJob {
+  const jobId = encodeHealthCheckJobId({
+    configId: props.configId,
+    systemId: props.systemId,
+    environmentId: props.environmentId,
+  });
+  return {
+    jobId,
+    payload: {
+      configId: props.configId,
+      systemId: props.systemId,
+      environmentId: props.environmentId,
+    },
+    intervalSeconds: props.intervalSeconds,
+    jitterKey: jobId,
+  };
+}
+
+describe("planReconcile (pure diff)", () => {
+  it("schedules desired jobs that do not exist yet", () => {
+    const a = desiredJob({
+      configId: "c1",
+      systemId: "s1",
+      environmentId: "prod",
+      intervalSeconds: 30,
+    });
+    const plan = planReconcile({
+      desired: [a],
+      actualJobIds: new Set(),
+      actualIntervalsById: new Map(),
+      cancelOrphans: true,
+    });
+    expect(plan.toSchedule).toEqual([a]);
+    expect(plan.toReschedule).toEqual([]);
+    expect(plan.toCancel).toEqual([]);
+  });
+
+  it("leaves an unchanged existing job alone (no reschedule)", () => {
+    const a = desiredJob({
+      configId: "c1",
+      systemId: "s1",
+      environmentId: "prod",
+      intervalSeconds: 30,
+    });
+    const plan = planReconcile({
+      desired: [a],
+      actualJobIds: new Set([a.jobId]),
+      actualIntervalsById: new Map([[a.jobId, 30]]),
+      cancelOrphans: true,
+    });
+    expect(plan.toSchedule).toEqual([]);
+    expect(plan.toReschedule).toEqual([]);
+    expect(plan.toCancel).toEqual([]);
+  });
+
+  it("reschedules an existing job only when its interval changed", () => {
+    const a = desiredJob({
+      configId: "c1",
+      systemId: "s1",
+      environmentId: "prod",
+      intervalSeconds: 60,
+    });
+    const plan = planReconcile({
+      desired: [a],
+      actualJobIds: new Set([a.jobId]),
+      actualIntervalsById: new Map([[a.jobId, 30]]),
+      cancelOrphans: true,
+    });
+    expect(plan.toSchedule).toEqual([]);
+    expect(plan.toReschedule).toEqual([a]);
+    expect(plan.toCancel).toEqual([]);
+  });
+
+  it("cancels orphaned health-check jobs on a full reconcile", () => {
+    const a = desiredJob({
+      configId: "c1",
+      systemId: "s1",
+      environmentId: "prod",
+      intervalSeconds: 30,
+    });
+    const orphan = encodeHealthCheckJobId({
+      configId: "c9",
+      systemId: "s9",
+      environmentId: "staging",
+    });
+    const plan = planReconcile({
+      desired: [a],
+      actualJobIds: new Set([a.jobId, orphan]),
+      actualIntervalsById: new Map([[a.jobId, 30]]),
+      cancelOrphans: true,
+    });
+    expect(plan.toSchedule).toEqual([]);
+    expect(plan.toCancel).toEqual([orphan]);
+  });
+
+  it("never cancels a non-health-check job even if it is not desired", () => {
+    const a = desiredJob({
+      configId: "c1",
+      systemId: "s1",
+      environmentId: null,
+      intervalSeconds: 30,
+    });
+    const foreign = "some-other-plugin:job:42";
+    const plan = planReconcile({
+      desired: [a],
+      actualJobIds: new Set([a.jobId, foreign]),
+      actualIntervalsById: new Map([[a.jobId, 30]]),
+      cancelOrphans: true,
+    });
+    expect(plan.toCancel).toEqual([]);
+  });
+
+  it("does NOT cancel orphans on a system-scoped reconcile (add/update only)", () => {
+    const a = desiredJob({
+      configId: "c1",
+      systemId: "s1",
+      environmentId: "prod",
+      intervalSeconds: 30,
+    });
+    const orphan = encodeHealthCheckJobId({
+      configId: "c9",
+      systemId: "s9",
+      environmentId: "staging",
+    });
+    const plan = planReconcile({
+      desired: [a],
+      actualJobIds: new Set([orphan]),
+      actualIntervalsById: new Map(),
+      cancelOrphans: false,
+    });
+    expect(plan.toSchedule).toEqual([a]);
+    expect(plan.toCancel).toEqual([]);
+  });
+});
+
+// ─── Integration: reconcileHealthCheckJobs against in-memory mocks ──────────
+
+interface FakeQueue {
+  scheduled: Array<{ payload: HealthCheckJobPayload; intervalSeconds: number }>;
+  cancelled: string[];
+  recurring: Map<string, number>;
+}
+
+function makeQueueManager(existing: Map<string, number> = new Map()) {
+  const state: FakeQueue = {
+    scheduled: [],
+    cancelled: [],
+    recurring: existing,
+  };
+  const queue = {
+    scheduleRecurring: mock(
+      async (
+        payload: HealthCheckJobPayload,
+        opts: { jobId: string; intervalSeconds: number },
+      ) => {
+        state.scheduled.push({
+          payload,
+          intervalSeconds: opts.intervalSeconds,
+        });
+        state.recurring.set(opts.jobId, opts.intervalSeconds);
+        return opts.jobId;
+      },
+    ),
+    cancelRecurring: mock(async (jobId: string) => {
+      state.cancelled.push(jobId);
+      state.recurring.delete(jobId);
+    }),
+    listRecurringJobs: mock(async () => [...state.recurring.keys()]),
+    getRecurringJobDetails: mock(async (jobId: string) => {
+      const intervalSeconds = state.recurring.get(jobId);
+      return intervalSeconds === undefined
+        ? undefined
+        : { intervalSeconds };
+    }),
+  };
+  const queueManager = {
+    getQueue: () => queue,
+  } as unknown as Parameters<typeof reconcileHealthCheckJobs>[0]["queueManager"];
+  return { queueManager, state };
+}
+
+/**
+ * Mock db whose two select() calls return, in order: (1) the enabled checks
+ * join rows, (2) the last-run-per-slice rows. Mirrors the query order in
+ * `reconcileHealthCheckJobs` (buildDesiredJobs first, then the lastRun query).
+ */
+function makeDb(props: {
+  checks: Array<{
+    systemId: string;
+    configId: string;
+    interval: number;
+    environmentIds: string[] | null;
+  }>;
+  lastRuns?: Array<{
+    systemId: string;
+    configurationId: string;
+    environmentId: string | null;
+    maxTimestamp: Date | null;
+  }>;
+}) {
+  let call = 0;
+  const db = {
+    select: mock(() => {
+      call++;
+      if (call === 1) {
+        return {
+          from: () => ({
+            innerJoin: () => ({
+              where: () => Promise.resolve(props.checks),
+            }),
+          }),
+        };
+      }
+      return {
+        from: () => ({
+          groupBy: () => Promise.resolve(props.lastRuns ?? []),
+        }),
+      };
+    }),
+  };
+  return db as unknown as Parameters<typeof reconcileHealthCheckJobs>[0]["db"];
+}
+
+function makeCatalogClient(
+  membershipBySystem: Record<
+    string,
+    Array<{ id: string; name: string; metadata: Record<string, unknown> | null }>
+  >,
+) {
+  return {
+    resolveSystemEnvironments: mock(
+      async ({ systemId }: { systemId: string }) =>
+        (membershipBySystem[systemId] ?? []).map((m) => ({
+          ...m,
+          description: null,
+          systemIds: [],
+          createdAt: new Date(),
+          updatedAt: new Date(),
+        })),
+    ),
+  } as unknown as Parameters<
+    typeof reconcileHealthCheckJobs
+  >[0]["catalogClient"];
+}
+
+const silentLogger = {
+  debug: () => {},
+  info: () => {},
+  warn: () => {},
+  error: () => {},
+} as unknown as Parameters<typeof reconcileHealthCheckJobs>[0]["logger"];
+
+describe("reconcileHealthCheckJobs (integration)", () => {
+  it("schedules one job per effective environment for a full reconcile", async () => {
+    const { queueManager, state } = makeQueueManager();
+    const db = makeDb({
+      checks: [
+        {
+          systemId: "s1",
+          configId: "c1",
+          interval: 30,
+          environmentIds: null,
+        },
+      ],
+    });
+    const catalogClient = makeCatalogClient({
+      s1: [
+        { id: "prod", name: "Production", metadata: {} },
+        { id: "staging", name: "Staging", metadata: {} },
+      ],
+    });
+
+    await reconcileHealthCheckJobs({
+      db,
+      queueManager,
+      catalogClient,
+      logger: silentLogger,
+      now: 1_000_000,
+    });
+
+    expect(state.scheduled).toHaveLength(2);
+    const envs = state.scheduled.map((s) => s.payload.environmentId).sort();
+    expect(envs).toEqual(["prod", "staging"]);
+    expect(state.cancelled).toEqual([]);
+  });
+
+  it("schedules a single env-less job when the system has no environments", async () => {
+    const { queueManager, state } = makeQueueManager();
+    const db = makeDb({
+      checks: [
+        { systemId: "s1", configId: "c1", interval: 30, environmentIds: null },
+      ],
+    });
+    const catalogClient = makeCatalogClient({ s1: [] });
+
+    await reconcileHealthCheckJobs({
+      db,
+      queueManager,
+      catalogClient,
+      logger: silentLogger,
+      now: 1_000_000,
+    });
+
+    expect(state.scheduled).toHaveLength(1);
+    expect(state.scheduled[0]?.payload.environmentId).toBeNull();
+  });
+
+  it("cancels an orphaned health-check job that is no longer desired", async () => {
+    const orphan = encodeHealthCheckJobId({
+      configId: "old",
+      systemId: "gone",
+      environmentId: "prod",
+    });
+    const { queueManager, state } = makeQueueManager(
+      new Map([[orphan, 30]]),
+    );
+    const db = makeDb({
+      checks: [
+        { systemId: "s1", configId: "c1", interval: 30, environmentIds: [] },
+      ],
+    });
+    const catalogClient = makeCatalogClient({ s1: [] });
+
+    await reconcileHealthCheckJobs({
+      db,
+      queueManager,
+      catalogClient,
+      logger: silentLogger,
+      now: 1_000_000,
+    });
+
+    expect(state.cancelled).toEqual([orphan]);
+    // The env-less desired job was scheduled.
+    expect(state.scheduled).toHaveLength(1);
+  });
+
+  it("takes the health:reconcile advisory lock for a full reconcile", async () => {
+    const { queueManager } = makeQueueManager();
+    const db = makeDb({ checks: [] });
+    const catalogClient = makeCatalogClient({});
+    const withXactLock = mock(
+      async ({ fn }: { key: string; fn: () => Promise<void> }) => {
+        await fn();
+      },
+    );
+    const advisoryLock = {
+      withXactLock,
+    } as unknown as Parameters<
+      typeof reconcileHealthCheckJobs
+    >[0]["advisoryLock"];
+
+    await reconcileHealthCheckJobs({
+      db,
+      queueManager,
+      catalogClient,
+      logger: silentLogger,
+      advisoryLock,
+      now: 1_000_000,
+    });
+
+    expect(withXactLock).toHaveBeenCalledTimes(1);
+    expect(withXactLock.mock.calls[0]![0]!.key).toBe("health:reconcile");
+  });
+
+  it("does not cancel orphans on a system-scoped reconcile", async () => {
+    const orphan = encodeHealthCheckJobId({
+      configId: "old",
+      systemId: "gone",
+      environmentId: "prod",
+    });
+    const { queueManager, state } = makeQueueManager(
+      new Map([[orphan, 30]]),
+    );
+    const db = makeDb({
+      checks: [
+        {
+          systemId: "s1",
+          configId: "c1",
+          interval: 30,
+          environmentIds: null,
+        },
+      ],
+    });
+    const catalogClient = makeCatalogClient({
+      s1: [{ id: "prod", name: "Production", metadata: {} }],
+    });
+
+    await reconcileHealthCheckJobs({
+      db,
+      queueManager,
+      catalogClient,
+      logger: silentLogger,
+      systemId: "s1",
+      now: 1_000_000,
+    });
+
+    // The system's prod job is scheduled; the foreign orphan is left intact.
+    expect(state.scheduled).toHaveLength(1);
+    expect(state.scheduled[0]?.payload.environmentId).toBe("prod");
+    expect(state.cancelled).toEqual([]);
+  });
+});

package/src/schedule-reconciler.ts

@@ -0,0 +1,304 @@
+/**
+ * Convergence reconciler for per-environment health-check recurring jobs.
+ *
+ * Every health check is scheduled as ONE recurring job per
+ * `(configId, systemId, environmentId)` slice (env-less systems use the bare
+ * `(configId, systemId)` form). The set of environments is DYNAMIC - it comes
+ * from catalog membership, which is not pushed to this backend - so we cannot
+ * rely on catching every add/remove event. Instead this reconciler computes the
+ * DESIRED job set from the durable tables + current membership and converges the
+ * queue's ACTUAL recurring jobs toward it (schedule missing, cancel orphans,
+ * reschedule interval changes). It is idempotent and self-healing: run it at
+ * boot, periodically, and (scoped to a system) right after a mutation for low
+ * latency.
+ *
+ * Scale-correctness (state-and-scale): desired state derives entirely from
+ * Postgres (`system_health_checks`, `health_check_configurations`,
+ * `health_check_runs`) + the catalog membership RPC, so every pod computes the
+ * same plan. A full reconcile takes the `health:reconcile` advisory lock so only
+ * one pod mutates the schedule at a time.
+ */
+import { eq, and, max } from "drizzle-orm";
+import type { Logger, AdvisoryLockService } from "@checkstack/backend-api";
+import type { QueueManager } from "@checkstack/queue-api";
+import type { InferClient } from "@checkstack/common";
+import type { CatalogApi } from "@checkstack/catalog-common";
+import type { SafeDatabase } from "@checkstack/backend-api";
+import { healthCheckConfigurations, systemHealthChecks, healthCheckRuns } from "./schema";
+import * as schema from "./schema";
+import { resolveEffectiveEnvironments } from "./effective-environments";
+import { computeScheduleJitterSeconds } from "./schedule-jitter";
+import {
+  scheduleHealthCheck,
+  encodeHealthCheckJobId,
+  HEALTH_CHECK_QUEUE,
+  HEALTH_CHECK_JOB_PREFIX,
+  type HealthCheckJobPayload,
+} from "./queue-executor";
+
+type Db = SafeDatabase<typeof schema>;
+type CatalogClient = InferClient<typeof CatalogApi>;
+
+/** A recurring job the reconciler wants to exist. */
+export interface DesiredJob {
+  jobId: string;
+  payload: HealthCheckJobPayload;
+  intervalSeconds: number;
+  /** Key for the deterministic first-fire jitter (per env slice). */
+  jitterKey: string;
+}
+
+export interface ReconcilePlan {
+  /** Desired jobs that do not exist yet - schedule them (with jitter). */
+  toSchedule: DesiredJob[];
+  /** Actual jobIds not desired anymore - cancel them. */
+  toCancel: string[];
+  /** Desired jobs that exist but whose interval changed - reschedule them. */
+  toReschedule: DesiredJob[];
+}
+
+/**
+ * Pure diff: given the desired jobs and the actual recurring jobs (with their
+ * current intervals), decide what to schedule / cancel / reschedule.
+ *
+ * `cancelOrphans: false` (a system-scoped reconcile) skips cancellation - a
+ * scoped run cannot see the whole actual set, so it only ADDS/updates; the
+ * periodic FULL reconcile owns orphan cleanup.
+ */
+export function planReconcile(props: {
+  desired: DesiredJob[];
+  actualJobIds: Set<string>;
+  actualIntervalsById: Map<string, number>;
+  cancelOrphans: boolean;
+}): ReconcilePlan {
+  const { desired, actualJobIds, actualIntervalsById, cancelOrphans } = props;
+  const desiredById = new Map(desired.map((d) => [d.jobId, d]));
+
+  const toSchedule: DesiredJob[] = [];
+  const toReschedule: DesiredJob[] = [];
+  for (const job of desired) {
+    if (!actualJobIds.has(job.jobId)) {
+      toSchedule.push(job);
+    } else if (actualIntervalsById.get(job.jobId) !== job.intervalSeconds) {
+      // Interval changed - reschedule (scheduleRecurring updates in place). Only
+      // when it actually differs, so we don't reset a job's fire phase every run.
+      toReschedule.push(job);
+    }
+  }
+
+  const toCancel = cancelOrphans
+    ? [...actualJobIds].filter(
+        (jobId) =>
+          jobId.startsWith(HEALTH_CHECK_JOB_PREFIX) && !desiredById.has(jobId),
+      )
+    : [];
+
+  return { toSchedule, toCancel, toReschedule };
+}
+
+/**
+ * Build the desired per-env recurring jobs for one or all systems from the
+ * durable tables + current catalog membership.
+ */
+async function buildDesiredJobs(props: {
+  db: Db;
+  catalogClient: CatalogClient;
+  logger: Logger;
+  systemId?: string;
+}): Promise<DesiredJob[]> {
+  const { db, catalogClient, logger, systemId } = props;
+
+  const where = systemId
+    ? and(
+        eq(systemHealthChecks.enabled, true),
+        eq(systemHealthChecks.systemId, systemId),
+      )
+    : eq(systemHealthChecks.enabled, true);
+
+  const checks = await db
+    .select({
+      systemId: systemHealthChecks.systemId,
+      configId: healthCheckConfigurations.id,
+      interval: healthCheckConfigurations.intervalSeconds,
+      environmentIds: systemHealthChecks.environmentIds,
+    })
+    .from(systemHealthChecks)
+    .innerJoin(
+      healthCheckConfigurations,
+      eq(systemHealthChecks.configurationId, healthCheckConfigurations.id),
+    )
+    .where(where);
+
+  // Resolve each distinct system's membership once.
+  const membershipBySystem = new Map<
+    string,
+    Awaited<ReturnType<CatalogClient["resolveSystemEnvironments"]>>
+  >();
+  for (const systemIdToResolve of new Set(checks.map((c) => c.systemId))) {
+    try {
+      membershipBySystem.set(
+        systemIdToResolve,
+        await catalogClient.resolveSystemEnvironments({ systemId: systemIdToResolve }),
+      );
+    } catch (error) {
+      // Fail-open: a system we cannot resolve keeps its env-less job (no fan-out)
+      // rather than being dropped from the schedule entirely.
+      logger.warn(
+        `Reconcile: could not resolve environments for system ${systemIdToResolve}`,
+        error,
+      );
+      membershipBySystem.set(systemIdToResolve, []);
+    }
+  }
+
+  const desired: DesiredJob[] = [];
+  for (const check of checks) {
+    const effectiveEnvs = resolveEffectiveEnvironments({
+      environmentIds: check.environmentIds,
+      membership: membershipBySystem.get(check.systemId) ?? [],
+    });
+    const environmentIds: (string | null)[] =
+      effectiveEnvs.length > 0 ? effectiveEnvs.map((e) => e.id) : [null];
+
+    for (const environmentId of environmentIds) {
+      const jobId = encodeHealthCheckJobId({
+        configId: check.configId,
+        systemId: check.systemId,
+        environmentId,
+      });
+      desired.push({
+        jobId,
+        payload: {
+          configId: check.configId,
+          systemId: check.systemId,
+          environmentId,
+        },
+        intervalSeconds: check.interval,
+        jitterKey: jobId,
+      });
+    }
+  }
+  return desired;
+}
+
+/**
+ * Compute the startDelay for a newly-scheduled job: keep an in-flight cadence by
+ * waiting out the remainder of the interval since the slice's last run, then add
+ * the deterministic de-clustering jitter. Overdue slices (no recent run) fire
+ * after just the jitter.
+ */
+function computeStartDelay(props: {
+  intervalSeconds: number;
+  lastRun: Date | undefined;
+  jitterKey: string;
+  now: number;
+}): number {
+  const { intervalSeconds, lastRun, jitterKey, now } = props;
+  let startDelay = 0;
+  if (lastRun) {
+    const elapsed = Math.floor((now - lastRun.getTime()) / 1000);
+    if (elapsed < intervalSeconds) startDelay = intervalSeconds - elapsed;
+  }
+  return startDelay + computeScheduleJitterSeconds({ key: jitterKey, intervalSeconds });
+}
+
+/**
+ * Converge the queue's recurring health-check jobs toward the desired per-env
+ * set. Pass `systemId` to reconcile just one system (add/update only, no orphan
+ * cleanup); omit it for a full reconcile (also cancels orphaned jobs).
+ */
+export async function reconcileHealthCheckJobs(props: {
+  db: Db;
+  queueManager: QueueManager;
+  catalogClient: CatalogClient;
+  logger: Logger;
+  advisoryLock?: AdvisoryLockService;
+  systemId?: string;
+  now?: number;
+}): Promise<void> {
+  const { db, queueManager, catalogClient, logger, advisoryLock, systemId, now } =
+    props;
+  const isFullReconcile = systemId === undefined;
+
+  const run = async (): Promise<void> => {
+    const desired = await buildDesiredJobs({ db, catalogClient, logger, systemId });
+
+    // Last run per (system, config, environment) slice for startDelay.
+    const lastRunRows = await db
+      .select({
+        systemId: healthCheckRuns.systemId,
+        configurationId: healthCheckRuns.configurationId,
+        environmentId: healthCheckRuns.environmentId,
+        maxTimestamp: max(healthCheckRuns.timestamp),
+      })
+      .from(healthCheckRuns)
+      .groupBy(
+        healthCheckRuns.systemId,
+        healthCheckRuns.configurationId,
+        healthCheckRuns.environmentId,
+      );
+    const lastRunByJobId = new Map<string, Date>();
+    for (const row of lastRunRows) {
+      if (!row.maxTimestamp) continue;
+      lastRunByJobId.set(
+        encodeHealthCheckJobId({
+          configId: row.configurationId,
+          systemId: row.systemId,
+          environmentId: row.environmentId,
+        }),
+        row.maxTimestamp,
+      );
+    }
+
+    const queue =
+      queueManager.getQueue<HealthCheckJobPayload>(HEALTH_CHECK_QUEUE);
+    const actualJobIds = new Set(await queue.listRecurringJobs());
+    const actualIntervalsById = new Map<string, number>();
+    for (const job of desired) {
+      if (!actualJobIds.has(job.jobId)) continue;
+      const details = await queue.getRecurringJobDetails(job.jobId);
+      const interval = details?.intervalSeconds;
+      if (typeof interval === "number") actualIntervalsById.set(job.jobId, interval);
+    }
+
+    const plan = planReconcile({
+      desired,
+      actualJobIds,
+      actualIntervalsById,
+      cancelOrphans: isFullReconcile,
+    });
+
+    const stampNow = now ?? Date.now();
+    for (const job of [...plan.toSchedule, ...plan.toReschedule]) {
+      await scheduleHealthCheck({
+        queueManager,
+        payload: job.payload,
+        intervalSeconds: job.intervalSeconds,
+        startDelay: computeStartDelay({
+          intervalSeconds: job.intervalSeconds,
+          lastRun: lastRunByJobId.get(job.jobId),
+          jitterKey: job.jitterKey,
+          now: stampNow,
+        }),
+        logger,
+      });
+    }
+    for (const jobId of plan.toCancel) {
+      await queue.cancelRecurring(jobId);
+      logger.debug(`Reconcile: cancelled orphaned job ${jobId}`);
+    }
+
+    logger.debug(
+      `Reconcile${systemId ? ` [system ${systemId}]` : ""}: ` +
+        `${plan.toSchedule.length} scheduled, ${plan.toReschedule.length} rescheduled, ` +
+        `${plan.toCancel.length} cancelled (${desired.length} desired)`,
+    );
+  };
+
+  // A full reconcile mutates the whole schedule - serialize across pods. A
+  // system-scoped reconcile only adds/updates that system's jobs (jobId-keyed,
+  // idempotent), so it needs no cluster lock.
+  await (isFullReconcile && advisoryLock
+    ? advisoryLock.withXactLock({ key: "health:reconcile", fn: run })
+    : run());
+}