@celilo/cli 0.13.3 → 0.14.1

package/src/services/alerting/quiet-hours.test.ts

@@ -0,0 +1,140 @@
+import { describe, expect, test } from 'bun:test';
+import {
+  type QuietHoursWindow,
+  isWithinQuietHours,
+  localMinutesOfDay,
+  parseClockTime,
+  quietHoursEndAfter,
+} from './quiet-hours';
+
+const LA = 'America/Los_Angeles';
+
+/** 22:00–07:00 local — the common overnight window, which wraps midnight. */
+const OVERNIGHT: QuietHoursWindow = { start: '22:00', end: '07:00', timezone: LA };
+/** 09:00–17:00 — a same-day window, which does not wrap. */
+const WORKDAY: QuietHoursWindow = { start: '09:00', end: '17:00', timezone: LA };
+
+/**
+ * A UTC instant for a given LA wall-clock hour (PDT = UTC-7 in July).
+ *
+ * No modulo on the hour: `Date.UTC` rolls the date forward on overflow, which
+ * is what makes 17:00 PDT land on the following UTC day rather than silently
+ * on the same one.
+ */
+const laTime = (hour: number, minute = 0) => new Date(Date.UTC(2026, 6, 28, hour + 7, minute));
+
+describe('parseClockTime', () => {
+  test.each([
+    ['00:00', 0],
+    ['07:00', 420],
+    ['22:30', 1350],
+    ['23:59', 1439],
+  ])('parses %p', (value, expected) => {
+    expect(parseClockTime(value)).toBe(expected);
+  });
+
+  test.each(['24:00', '7:00', '22:60', '', 'noon', '22'])('rejects %p', (value) => {
+    expect(parseClockTime(value)).toBeNull();
+  });
+});
+
+describe('localMinutesOfDay', () => {
+  test('converts a UTC instant to local minutes', () => {
+    expect(localMinutesOfDay(laTime(23, 30), LA)).toBe(23 * 60 + 30);
+  });
+
+  test('midnight is zero, not 1440', () => {
+    expect(localMinutesOfDay(laTime(0), LA)).toBe(0);
+  });
+});
+
+describe('isWithinQuietHours — overnight window (wraps midnight)', () => {
+  test.each([
+    [23, true, 'after the start, before midnight'],
+    [2, true, 'after midnight, before the end'],
+    [6, true, 'just before the end'],
+    [7, false, 'at the end — exclusive'],
+    [12, false, 'midday'],
+    [21, false, 'just before the start'],
+    [22, true, 'at the start — inclusive'],
+  ])('%p:00 → %p (%s)', (hour, expected) => {
+    expect(isWithinQuietHours(OVERNIGHT, laTime(hour))).toBe(expected);
+  });
+});
+
+describe('isWithinQuietHours — same-day window', () => {
+  test.each([
+    [8, false],
+    [9, true],
+    [12, true],
+    [16, true],
+    [17, false],
+    [23, false],
+  ])('%p:00 → %p', (hour, expected) => {
+    expect(isWithinQuietHours(WORKDAY, laTime(hour))).toBe(expected);
+  });
+});
+
+describe('isWithinQuietHours — degenerate configurations', () => {
+  test('no window configured means always reachable', () => {
+    const none: QuietHoursWindow = { start: null, end: null, timezone: LA };
+    expect(isWithinQuietHours(none, laTime(3))).toBe(false);
+  });
+
+  test('a half-configured window means always reachable', () => {
+    const half: QuietHoursWindow = { start: '22:00', end: null, timezone: LA };
+    expect(isWithinQuietHours(half, laTime(3))).toBe(false);
+  });
+
+  // A typo must not silence someone permanently.
+  test('start equal to end is treated as no window, not as always quiet', () => {
+    const degenerate: QuietHoursWindow = { start: '09:00', end: '09:00', timezone: LA };
+    expect(isWithinQuietHours(degenerate, laTime(3))).toBe(false);
+    expect(isWithinQuietHours(degenerate, laTime(9))).toBe(false);
+  });
+
+  test('a malformed window means always reachable', () => {
+    const bad: QuietHoursWindow = { start: 'evening', end: 'morning', timezone: LA };
+    expect(isWithinQuietHours(bad, laTime(3))).toBe(false);
+  });
+});
+
+describe('quietHoursEndAfter', () => {
+  test('a page at 02:00 waits until 07:00', () => {
+    expect(quietHoursEndAfter(OVERNIGHT, laTime(2))).toEqual(laTime(7));
+  });
+
+  // The window started last night; the end is tomorrow morning, not today's.
+  test('a page at 23:00 waits until 07:00 the next morning', () => {
+    const deliverAt = quietHoursEndAfter(OVERNIGHT, laTime(23));
+    expect(deliverAt).toEqual(new Date(laTime(23).getTime() + 8 * 60 * 60_000));
+  });
+
+  test('outside the window there is nothing to defer', () => {
+    expect(quietHoursEndAfter(OVERNIGHT, laTime(12))).toBeNull();
+  });
+
+  test('a same-day window defers to its end on the same day', () => {
+    expect(quietHoursEndAfter(WORKDAY, laTime(10))).toEqual(laTime(17));
+  });
+});
+
+describe('timezone is honoured, not assumed', () => {
+  // The same instant is inside one person's window and outside another's.
+  test('two people in different zones disagree about the same moment', () => {
+    const instant = laTime(23);
+    const pacific: QuietHoursWindow = { start: '22:00', end: '07:00', timezone: LA };
+    const newYork: QuietHoursWindow = {
+      start: '22:00',
+      end: '07:00',
+      timezone: 'America/New_York',
+    };
+    expect(isWithinQuietHours(pacific, instant)).toBe(true);
+    // 23:00 Pacific is 02:00 Eastern — also inside, but for a different reason.
+    expect(isWithinQuietHours(newYork, instant)).toBe(true);
+
+    const midday = laTime(12); // 15:00 Eastern
+    expect(isWithinQuietHours(pacific, midday)).toBe(false);
+    expect(isWithinQuietHours(newYork, midday)).toBe(false);
+  });
+});

package/src/services/alerting/quiet-hours.ts

@@ -0,0 +1,99 @@
+/**
+ * Quiet hours — when a person may be reached.
+ *
+ * Quiet hours defer delivery of EVERY severity, criticals included. This is a
+ * home lab, not a pager rotation: very little here is worth waking someone for,
+ * and an alerting system that wakes you for things you would not have acted on
+ * until morning trains you to ignore it.
+ *
+ * Escalation steps continue to ADVANCE during a quiet window — only delivery
+ * defers — so nothing is skipped; it arrives when the window opens. And a
+ * deferred alert that resolves before the window ends is delivered as neither a
+ * page nor an all-clear, because delivery re-evaluates current state rather
+ * than replaying history.
+ *
+ * `escalation_policies.bypass_quiet_hours` is the escape hatch for the few
+ * conditions that genuinely cannot wait. Reserved, unused in MVP.
+ *
+ * See openspec/changes/add-alerting/design.md D13.
+ */
+
+export interface QuietHoursWindow {
+  /** Local "HH:MM", inclusive. Null (either field) means always reachable. */
+  start: string | null;
+  end: string | null;
+  /** IANA timezone the window is expressed in, e.g. "America/Los_Angeles". */
+  timezone: string;
+}
+
+const HHMM = /^([01]\d|2[0-3]):([0-5]\d)$/;
+
+/** Parse "HH:MM" to minutes since local midnight, or null if malformed. */
+export function parseClockTime(value: string): number | null {
+  const match = HHMM.exec(value);
+  if (!match) return null;
+  return Number.parseInt(match[1], 10) * 60 + Number.parseInt(match[2], 10);
+}
+
+/**
+ * Minutes since local midnight for `instant` in `timezone`.
+ *
+ * Uses Intl rather than manual offset arithmetic so DST is handled by the
+ * platform's tz database. A window is expressed in wall-clock terms — "22:00
+ * to 07:00" means those local hours on whichever side of a DST boundary the
+ * instant falls.
+ */
+export function localMinutesOfDay(instant: Date, timezone: string): number {
+  const parts = new Intl.DateTimeFormat('en-US', {
+    timeZone: timezone,
+    hour: '2-digit',
+    minute: '2-digit',
+    hour12: false,
+  }).formatToParts(instant);
+  const hour = Number.parseInt(parts.find((p) => p.type === 'hour')?.value ?? '0', 10);
+  const minute = Number.parseInt(parts.find((p) => p.type === 'minute')?.value ?? '0', 10);
+  // Intl renders midnight as 24 in some locales/options combinations.
+  return (hour % 24) * 60 + minute;
+}
+
+/**
+ * Whether `instant` falls inside the window.
+ *
+ * Windows normally wrap midnight (22:00–07:00), so a naive `start <= t <= end`
+ * comparison would be wrong for the common case rather than an edge case.
+ */
+export function isWithinQuietHours(window: QuietHoursWindow, instant: Date): boolean {
+  if (!window.start || !window.end) return false;
+
+  const start = parseClockTime(window.start);
+  const end = parseClockTime(window.end);
+  if (start === null || end === null) return false;
+
+  // Degenerate: start === end is treated as "no window", not "always quiet".
+  // The alternative silences someone permanently through a typo.
+  if (start === end) return false;
+
+  const now = localMinutesOfDay(instant, window.timezone);
+  return start < end
+    ? now >= start && now < end
+    : // Wraps midnight: inside if after the start OR before the end.
+      now >= start || now < end;
+}
+
+/**
+ * The instant a deferred notification becomes deliverable — the next time the
+ * window's end is reached, at or after `instant`.
+ *
+ * Returns null when the instant is not inside a window (nothing to defer).
+ */
+export function quietHoursEndAfter(window: QuietHoursWindow, instant: Date): Date | null {
+  if (!isWithinQuietHours(window, instant)) return null;
+
+  const end = parseClockTime(window.end ?? '');
+  if (end === null) return null;
+
+  const nowLocal = localMinutesOfDay(instant, window.timezone);
+  // Minutes until the window's end, wrapping to tomorrow when already past it.
+  const minutesUntilEnd = end > nowLocal ? end - nowLocal : 24 * 60 - nowLocal + end;
+  return new Date(instant.getTime() + minutesUntilEnd * 60_000);
+}

package/src/services/alerting/reconcile.test.ts

@@ -0,0 +1,190 @@
+import { describe, expect, test } from 'bun:test';
+import { moduleAlertKey, moduleCheckAlertKey } from './keys';
+import { type ReconcileAction, type ReconcileInput, reconcile } from './reconcile';
+
+const NOW = new Date('2026-07-28T03:00:00Z');
+const GRACE_MS = 60_000;
+
+const MODULE = 'caddy';
+const MODULE_KEY = moduleAlertKey(MODULE);
+const DISK = moduleCheckAlertKey(MODULE, 'disk-space');
+const CERT = moduleCheckAlertKey(MODULE, 'cert-validity');
+
+function input(over: Partial<ReconcileInput> = {}): ReconcileInput {
+  return {
+    liveAlerts: [],
+    outcome: 'success',
+    failingKeys: [],
+    monitorLevelKey: MODULE_KEY,
+    monitorSeverity: 'critical',
+    now: NOW,
+    graceMs: GRACE_MS,
+    ...over,
+  };
+}
+
+const failing = (key: string, message = 'bad') => ({ key, severity: 'critical', message }) as const;
+
+const keysOf = (actions: ReconcileAction[], type: ReconcileAction['type']) =>
+  actions.filter((a) => a.type === type).map((a) => a.key);
+
+describe('successful run', () => {
+  test('a newly failing key is created with a grace window', () => {
+    const actions = reconcile(input({ failingKeys: [failing(DISK)] }));
+    expect(actions).toHaveLength(1);
+    const [action] = actions;
+    expect(action.type).toBe('create');
+    if (action.type !== 'create') throw new Error('expected create');
+    expect(action.key).toBe(DISK);
+    expect(action.graceUntil).toEqual(new Date(NOW.getTime() + GRACE_MS));
+  });
+
+  test('a still-failing key refreshes rather than duplicating', () => {
+    const actions = reconcile(
+      input({
+        liveAlerts: [{ id: 'a1', key: DISK }],
+        failingKeys: [failing(DISK, '/var 96% used')],
+      }),
+    );
+    expect(actions).toEqual([
+      {
+        type: 'refresh',
+        alertId: 'a1',
+        key: DISK,
+        severity: 'critical',
+        message: '/var 96% used',
+        details: undefined,
+      },
+    ]);
+  });
+
+  test('a key that stopped being reported resolves', () => {
+    const actions = reconcile(input({ liveAlerts: [{ id: 'a1', key: DISK }], failingKeys: [] }));
+    expect(actions).toEqual([{ type: 'resolve', alertId: 'a1', key: DISK }]);
+  });
+
+  test('resolves only what disappeared, keeping what persists', () => {
+    const actions = reconcile(
+      input({
+        liveAlerts: [
+          { id: 'a1', key: DISK },
+          { id: 'a2', key: CERT },
+        ],
+        failingKeys: [failing(DISK)],
+      }),
+    );
+    expect(keysOf(actions, 'refresh')).toEqual([DISK]);
+    expect(keysOf(actions, 'resolve')).toEqual([CERT]);
+  });
+
+  // A module upgrade renaming a check is indistinguishable from one problem
+  // clearing and another appearing — which is correct, and beats a zombie
+  // alert firing forever for a key nobody reports any more.
+  test('a renamed check resolves the old key and fires the new one', () => {
+    const renamed = moduleCheckAlertKey(MODULE, 'disk-usage');
+    const actions = reconcile(
+      input({ liveAlerts: [{ id: 'a1', key: DISK }], failingKeys: [failing(renamed)] }),
+    );
+    expect(keysOf(actions, 'resolve')).toEqual([DISK]);
+    expect(keysOf(actions, 'create')).toEqual([renamed]);
+  });
+
+  // The hook ran, so "could not run" is no longer true. It is never in the
+  // failing set, so set difference retires it without a special case.
+  test('a successful run resolves a standing monitor-level alert', () => {
+    const actions = reconcile(
+      input({ liveAlerts: [{ id: 'm1', key: MODULE_KEY }], failingKeys: [failing(DISK)] }),
+    );
+    expect(keysOf(actions, 'resolve')).toEqual([MODULE_KEY]);
+    expect(keysOf(actions, 'create')).toEqual([DISK]);
+  });
+
+  test('a clean run against no live alerts does nothing', () => {
+    expect(reconcile(input())).toEqual([]);
+  });
+});
+
+describe('errored run — the false-all-clear guard', () => {
+  const errored = (liveAlerts: ReconcileInput['liveAlerts']) =>
+    reconcile(
+      input({
+        outcome: 'error',
+        liveAlerts,
+        // A caller that leaks an errored run's empty item list into failingKeys
+        // must still not cause a resolve. The branch ignores this field.
+        failingKeys: [],
+        errorMessage: 'ssh: connect to host caddy.dmz port 22: timed out',
+      }),
+    );
+
+  // THE case. Disk has been failing since yesterday; now the box drops off the
+  // network. A naive set-difference sends "✅ RESOLVED — disk-space" at 3am for
+  // a problem that just got much worse, and fires nothing to replace it.
+  test('an unreachable system does NOT resolve a firing alert', () => {
+    const actions = errored([{ id: 'a1', key: DISK }]);
+    expect(keysOf(actions, 'resolve')).toEqual([]);
+    expect(actions.some((a) => a.type === 'resolve')).toBe(false);
+  });
+
+  test('it fires the monitor-level alert carrying the real error', () => {
+    const actions = errored([{ id: 'a1', key: DISK }]);
+    const created = actions.filter((a) => a.type === 'create');
+    expect(created).toHaveLength(1);
+    expect(created[0].key).toBe(MODULE_KEY);
+    expect(created[0].type === 'create' && created[0].message).toContain('timed out');
+  });
+
+  test('existing item alerts are frozen — not refreshed either', () => {
+    const actions = errored([
+      { id: 'a1', key: DISK },
+      { id: 'a2', key: CERT },
+    ]);
+    const touched = actions.filter((a) => a.key === DISK || a.key === CERT);
+    expect(touched).toEqual([]);
+  });
+
+  test('a repeated failure refreshes the monitor-level alert instead of duplicating', () => {
+    const actions = errored([
+      { id: 'm1', key: MODULE_KEY },
+      { id: 'a1', key: DISK },
+    ]);
+    expect(actions).toHaveLength(1);
+    expect(actions[0].type).toBe('refresh');
+    expect(actions[0].key).toBe(MODULE_KEY);
+  });
+
+  test('falls back to a message when the caller supplies no error text', () => {
+    const actions = reconcile(input({ outcome: 'error', errorMessage: undefined }));
+    expect(actions[0].type === 'create' && actions[0].message).toBe('Monitor run failed');
+  });
+});
+
+describe('recovery after an outage', () => {
+  // The frozen alerts are only ever cleaned up by a run that actually executed.
+  test('a later successful run resolves what is genuinely gone', () => {
+    const afterOutage = reconcile(
+      input({
+        liveAlerts: [
+          { id: 'm1', key: MODULE_KEY },
+          { id: 'a1', key: DISK },
+        ],
+        failingKeys: [],
+      }),
+    );
+    expect(keysOf(afterOutage, 'resolve').sort()).toEqual([MODULE_KEY, DISK].sort());
+  });
+
+  test('a later successful run keeps what is still broken', () => {
+    const afterOutage = reconcile(
+      input({
+        liveAlerts: [
+          { id: 'm1', key: MODULE_KEY },
+          { id: 'a1', key: DISK },
+        ],
+        failingKeys: [failing(DISK)],
+      }),
+    );
+    expect(keysOf(afterOutage, 'resolve')).toEqual([MODULE_KEY]);
+    expect(keysOf(afterOutage, 'refresh')).toEqual([DISK]);
+  });
+});

package/src/services/alerting/reconcile.ts

@@ -0,0 +1,166 @@
+/**
+ * Alert reconciliation — turning one monitor run into alert state changes.
+ *
+ * This is the most consequential logic in the alerting system, and it is pure
+ * so that it can be tested exhaustively without a database, a bus, or an SSH
+ * connection. The caller applies the returned actions.
+ *
+ * The whole design rests on one asymmetry:
+ *
+ *   a run that SUCCEEDED tells you about every key it did not report
+ *     → absent means resolved (set difference)
+ *
+ *   a run that FAILED TO EXECUTE tells you nothing about any key
+ *     → absent means unknown, and nothing may be resolved
+ *
+ * Both produce an empty failing set. Treating them alike is not a subtle bug:
+ * it emits "✅ RESOLVED" for every outstanding problem at the exact moment a
+ * machine goes offline, and then goes quiet — the worst possible behaviour for
+ * an alerting system, which is confident false reassurance.
+ *
+ * See openspec/changes/add-alerting/design.md D5.
+ */
+
+import type { AlertSeverity, MonitorRunOutcome } from '../../db/schema';
+import type { FailingKey } from './keys';
+
+/** An alert that has not yet resolved, as the reconciler needs to see it. */
+export interface LiveAlert {
+  id: string;
+  key: string;
+}
+
+export interface ReconcileInput {
+  /** Every non-resolved alert currently owned by this monitor. */
+  liveAlerts: LiveAlert[];
+  outcome: MonitorRunOutcome;
+  /**
+   * The complete set of currently-failing keys. Meaningful ONLY when
+   * `outcome` is `success`; ignored otherwise, because a run that could not
+   * execute has no opinion about what is failing.
+   */
+  failingKeys: FailingKey[];
+  /**
+   * The key meaning "this monitor could not run" — `module:<id>` for a module
+   * hook, `builtin:<check>` for a built-in check.
+   */
+  monitorLevelKey: string;
+  /** Populated when `outcome` is `error`; becomes the monitor-level message. */
+  errorMessage?: string;
+  /** Severity carried by the monitor-level alert. */
+  monitorSeverity: AlertSeverity;
+  now: Date;
+  /** Grace window before a newly-fired alert may notify. */
+  graceMs: number;
+}
+
+export type ReconcileAction =
+  | {
+      type: 'create';
+      key: string;
+      severity: AlertSeverity;
+      message: string;
+      details?: string;
+      graceUntil: Date;
+    }
+  | {
+      type: 'refresh';
+      alertId: string;
+      key: string;
+      severity: AlertSeverity;
+      message: string;
+      details?: string;
+    }
+  | { type: 'resolve'; alertId: string; key: string };
+
+/**
+ * Reconcile one monitor run against the alerts that monitor currently owns.
+ *
+ * Returns the actions to apply, in no particular order. Applying them is the
+ * caller's job — keeping that separate is what lets every branch below be
+ * asserted directly.
+ */
+export function reconcile(input: ReconcileInput): ReconcileAction[] {
+  return input.outcome === 'error' ? reconcileErroredRun(input) : reconcileSuccessfulRun(input);
+}
+
+/**
+ * A run that executed. The failing set is authoritative: what is present is
+ * firing, what is absent has recovered.
+ */
+function reconcileSuccessfulRun(input: ReconcileInput): ReconcileAction[] {
+  const actions: ReconcileAction[] = [];
+  const byKey = new Map(input.liveAlerts.map((alert) => [alert.key, alert]));
+  const reported = new Set<string>();
+
+  for (const failing of input.failingKeys) {
+    reported.add(failing.key);
+    const existing = byKey.get(failing.key);
+    if (existing) {
+      actions.push({
+        type: 'refresh',
+        alertId: existing.id,
+        key: failing.key,
+        severity: failing.severity,
+        message: failing.message,
+        details: failing.details,
+      });
+    } else {
+      actions.push({
+        type: 'create',
+        key: failing.key,
+        severity: failing.severity,
+        message: failing.message,
+        details: failing.details,
+        graceUntil: new Date(input.now.getTime() + input.graceMs),
+      });
+    }
+  }
+
+  // Set difference: anything live that this run did NOT report has recovered.
+  // The monitor-level alert falls out of this naturally — the run executed, so
+  // "could not run" is no longer true and it is never in `failingKeys`.
+  for (const alert of input.liveAlerts) {
+    if (!reported.has(alert.key)) {
+      actions.push({ type: 'resolve', alertId: alert.id, key: alert.key });
+    }
+  }
+
+  return actions;
+}
+
+/**
+ * A run that could not execute. It reports exactly one thing — that it could
+ * not run — and says nothing whatsoever about the individual checks.
+ *
+ * Every existing alert is therefore left FROZEN: not refreshed, not resolved,
+ * not touched. They are neither confirmed nor cleared, which is the honest
+ * position. When the system comes back and a run succeeds, ordinary set
+ * difference cleans them up.
+ */
+function reconcileErroredRun(input: ReconcileInput): ReconcileAction[] {
+  const message = input.errorMessage ?? 'Monitor run failed';
+  const existing = input.liveAlerts.find((alert) => alert.key === input.monitorLevelKey);
+
+  if (existing) {
+    return [
+      {
+        type: 'refresh',
+        alertId: existing.id,
+        key: input.monitorLevelKey,
+        severity: input.monitorSeverity,
+        message,
+      },
+    ];
+  }
+
+  return [
+    {
+      type: 'create',
+      key: input.monitorLevelKey,
+      severity: input.monitorSeverity,
+      message,
+      graceUntil: new Date(input.now.getTime() + input.graceMs),
+    },
+  ];
+}