@cronwatch/sdk 0.1.0 → 0.3.1

package/dist/index.js

@@ -1,4 +1,5 @@
 import { Cron } from 'croner';
+import { randomBytes, createHash } from 'crypto';
 
 // src/duration.ts
 var UNIT_MS = {
@@ -53,6 +54,8 @@ function formatRelative(at, now) {
   return diff < 0 ? `${text} ago` : `in ${text}`;
 }
 var cache = /* @__PURE__ */ new Map();
+var crons = /* @__PURE__ */ new WeakMap();
+var EARLY_SLACK_MS = 6e4;
 function parseSchedule(schedule, timezone) {
   const key = `${timezone ?? ""}|${schedule}`;
   const hit = cache.get(key);
@@ -71,58 +74,94 @@ function parseSchedule(schedule, timezone) {
     } catch (error) {
       throw new Error(`schedule "${schedule}" is not a cron expression or "every <duration>": ${error.message}`);
     }
-    parsed = { kind: "cron", source: text, cron };
+    parsed = { kind: "cron", source: text, ...timezone ? { timezone } : {} };
+    crons.set(parsed, cron);
   }
   cache.set(key, parsed);
   return parsed;
 }
+function fireAfter(parsed, from) {
+  const cron = crons.get(parsed);
+  if (!cron) throw new Error(`schedule "${parsed.source}" was not made by parseSchedule`);
+  let probe = from;
+  for (let attempt = 0; attempt < 4; attempt++) {
+    const runs = cron.nextRuns(8, new Date(probe));
+    if (runs.length === 0) return null;
+    const found = runs.find((d) => d.getTime() > from);
+    if (found) return found.getTime();
+    probe += 36e5;
+  }
+  return null;
+}
 function nextFire(parsed, from, lastRunAt) {
   if (parsed.kind === "interval") {
     const base = lastRunAt ?? from;
     return base + parsed.everyMs;
   }
-  const next = parsed.cron.nextRun(new Date(from));
-  return next ? next.getTime() : null;
-}
-function previousFire(parsed, now) {
-  if (parsed.kind !== "cron") throw new Error("previousFire is for cron schedules");
-  const cron = parsed.cron;
-  const fireAtOrBefore = (from) => {
-    const next = cron.nextRun(new Date(from - 1));
-    if (!next) return null;
-    const t = next.getTime();
-    return t <= now ? t : null;
-  };
-  const YEAR = 366 * 864e5;
-  let step = 6e4;
-  let lo = null;
-  while (step <= YEAR) {
-    if (fireAtOrBefore(now - step) !== null) {
-      lo = now - step;
-      break;
-    }
-    step *= 2;
-  }
-  if (lo === null) return null;
-  let hi = now;
-  while (hi - lo > 1) {
-    const mid = Math.floor((lo + hi) / 2);
-    if (fireAtOrBefore(mid) !== null) lo = mid;
-    else hi = mid;
-  }
-  return fireAtOrBefore(lo);
+  return fireAfter(parsed, from);
 }
-function expectation(parsed, now, lastRunAt, registeredAt, graceMs2) {
+function expectation(parsed, lastRunAt, registeredAt, graceMs2) {
+  let dueAt;
   if (parsed.kind === "interval") {
-    const dueAt = (lastRunAt ?? registeredAt) + parsed.everyMs;
-    return { dueAt, deadline: dueAt + graceMs2 };
+    dueAt = (lastRunAt ?? registeredAt) + parsed.everyMs;
+  } else if (lastRunAt === null) {
+    dueAt = fireAfter(parsed, registeredAt - 1);
+  } else {
+    dueAt = dueAfterRun(parsed, lastRunAt);
   }
-  const prev = previousFire(parsed, now);
-  if (prev === null || prev < registeredAt) return null;
-  return { dueAt: prev, deadline: prev + graceMs2 };
+  return dueAt === null ? null : { dueAt, deadline: dueAt + graceMs2 };
+}
+function dueAfterRun(parsed, startedAt) {
+  const next = fireAfter(parsed, startedAt);
+  if (next === null) return null;
+  const following = fireAfter(parsed, next);
+  const covers = runCovers(startedAt, next, following) || inSpringForwardGap(parsed, startedAt, next);
+  return covers ? following : next;
+}
+function runCovers(startedAt, dueAt, followingAt = null) {
+  const slack = followingAt === null ? EARLY_SLACK_MS : Math.min(EARLY_SLACK_MS, Math.floor((followingAt - dueAt) / 2));
+  return startedAt >= dueAt - slack;
 }
-function runCovers(startedAt, dueAt) {
-  return startedAt >= dueAt - 6e4;
+function inSpringForwardGap(parsed, startedAt, fireAt) {
+  const LOOKBACK = 3 * 36e5;
+  const after = utcOffset(fireAt, parsed.timezone);
+  const before = utcOffset(fireAt - LOOKBACK, parsed.timezone);
+  const gap = after - before;
+  if (gap <= 0) return false;
+  let lo = fireAt - LOOKBACK;
+  let hi = fireAt;
+  while (hi - lo > 6e4) {
+    const mid = lo + Math.floor((hi - lo) / 2);
+    if (utcOffset(mid, parsed.timezone) === after) hi = mid;
+    else lo = mid;
+  }
+  const jumpAt = Math.floor(hi / 6e4) * 6e4;
+  if (fireAt - jumpAt >= gap || startedAt < jumpAt - EARLY_SLACK_MS || startedAt >= fireAt) return false;
+  return fireAfter(parsed, jumpAt - 1) === fireAt;
+}
+var formatters = /* @__PURE__ */ new Map();
+function utcOffset(at, timezone) {
+  const key = timezone ?? "";
+  let format = formatters.get(key);
+  if (!format) {
+    format = new Intl.DateTimeFormat("en-US", {
+      ...timezone ? { timeZone: timezone } : {},
+      hourCycle: "h23",
+      year: "numeric",
+      month: "numeric",
+      day: "numeric",
+      hour: "numeric",
+      minute: "numeric",
+      second: "numeric"
+    });
+    formatters.set(key, format);
+  }
+  const parts = {};
+  for (const part of format.formatToParts(new Date(at))) {
+    if (part.type !== "literal") parts[part.type] = Number(part.value);
+  }
+  const wall = Date.UTC(parts.year, parts.month - 1, parts.day, parts.hour, parts.minute, parts.second);
+  return wall - Math.floor(at / 1e3) * 1e3;
 }
 
 // src/stats.ts
@@ -146,10 +185,20 @@ var SLOW_FLOOR_MS = 1e4;
 var BASELINE_MIN_RUNS = 5;
 var BASELINE_WINDOW = 20;
 function emptyState(job) {
-  return { job, open: {}, consecutiveFailures: 0, silencedUntil: null, lastAlertAt: null };
+  return { job, open: {}, consecutiveFailures: 0, silencedUntil: null, lastAlertAt: null, pendingRecovery: [], undelivered: [] };
+}
+function normalizeState(state, job) {
+  if (!state) return emptyState(job);
+  return {
+    ...emptyState(job),
+    ...state,
+    open: { ...state.open },
+    pendingRecovery: [...state.pendingRecovery ?? []],
+    undelivered: [...state.undelivered ?? []]
+  };
 }
 function cloneState(state) {
-  return { ...state, open: { ...state.open } };
+  return normalizeState(state, state.job);
 }
 function openCondition(state, condition, now) {
   if (state.open[condition] !== void 0) return false;
@@ -159,6 +208,8 @@ function openCondition(state, condition, now) {
 function closeCondition(state, condition) {
   if (state.open[condition] === void 0) return false;
   delete state.open[condition];
+  const pending = state.pendingRecovery ??= [];
+  if (!pending.includes(condition)) pending.push(condition);
   return true;
 }
 function openConditions(state) {
@@ -196,25 +247,27 @@ function budgetBreaches(def, run, history) {
   }
   return breaches;
 }
+function hasFullBaseline(history) {
+  return history.filter((r) => r.status === "ok").length >= BASELINE_WINDOW;
+}
 function formatNumber(n) {
   if (Number.isInteger(n)) return n.toLocaleString("en-US");
   return n.toLocaleString("en-US", { maximumFractionDigits: 4 });
 }
 function onRunStart(state) {
   const next = cloneState(state);
-  const openBefore = openConditions(next);
   closeCondition(next, "missed");
   closeCondition(next, "stuck");
-  return { state: next, openBefore };
+  return next;
 }
-function onRunFinish(def, run, state, history, openBefore, now) {
+function onRunFinish(def, run, state, history, now) {
   const next = cloneState(state);
   const alerts = [];
   if (run.status === "ok") {
     next.consecutiveFailures = 0;
-    closeCondition(next, "failed");
     closeCondition(next, "missed");
     closeCondition(next, "stuck");
+    closeCondition(next, "failed");
     const slow = slowThreshold(def, history);
     if (slow && run.durationMs !== null && run.durationMs > slow.thresholdMs) {
       if (openCondition(next, "slow", now)) {
@@ -231,9 +284,10 @@ function onRunFinish(def, run, state, history, openBefore, now) {
     } else {
       closeCondition(next, "over_budget");
     }
-    const wasOpen = /* @__PURE__ */ new Set([...openBefore, ...openConditions(state)]);
-    if (wasOpen.size > 0 && openConditions(next).length === 0) {
-      alerts.push({ type: "recovered", run, details: { after: [...wasOpen] } });
+    const pending = next.pendingRecovery ?? [];
+    if (pending.length > 0 && openConditions(next).length === 0) {
+      alerts.push({ type: "recovered", run, details: { after: [...pending] } });
+      next.pendingRecovery = [];
     }
     return { state: next, alerts };
   }
@@ -254,26 +308,19 @@ function onCheck(def, stored, lastRun, state, now) {
   if (!def.schedule) return { state: next, alerts, nextExpectedAt: null, dueAt: null };
   const parsed = parseSchedule(def.schedule, def.timezone);
   const grace = graceMs(def);
-  const exp = expectation(parsed, now, lastRun?.startedAt ?? null, stored.createdAt, grace);
-  let nextExpectedAt = null;
-  if (parsed.kind === "interval") {
-    nextExpectedAt = (lastRun?.startedAt ?? stored.createdAt) + parsed.everyMs;
-  } else {
-    const upcoming = parsed.cron.nextRun(new Date(now));
-    nextExpectedAt = upcoming ? upcoming.getTime() : null;
-  }
+  const lastRunAt = lastRun?.startedAt ?? null;
+  const exp = expectation(parsed, lastRunAt, stored.createdAt, grace);
+  const nextExpectedAt = parsed.kind === "interval" ? nextFire(parsed, stored.createdAt, lastRunAt) : nextFire(parsed, now, null);
   if (!exp) return { state: next, alerts, nextExpectedAt, dueAt: null };
-  const covered = parsed.kind === "cron" && lastRun !== null && runCovers(lastRun.startedAt, exp.dueAt);
-  if (covered) {
-    closeCondition(next, "missed");
-  } else if (now > exp.deadline) {
+  if (parsed.kind === "interval" && lastRun?.status === "running") {
+    return { state: next, alerts, nextExpectedAt, dueAt: exp.dueAt };
+  }
+  if (now > exp.deadline) {
     if (openCondition(next, "missed", now)) {
-      alerts.push({
-        type: "missed",
-        run: lastRun,
-        details: { dueAt: exp.dueAt, deadline: exp.deadline, graceMs: grace, lastRunAt: lastRun?.startedAt ?? null }
-      });
+      alerts.push({ type: "missed", run: lastRun, details: { dueAt: exp.dueAt, deadline: exp.deadline, graceMs: grace, lastRunAt } });
     }
+  } else {
+    closeCondition(next, "missed");
   }
   return { state: next, alerts, nextExpectedAt, dueAt: exp.dueAt };
 }
@@ -290,6 +337,51 @@ function muteOpens(previous, next) {
 function isSilenced(state, now) {
   return state.silencedUntil !== null && state.silencedUntil > now;
 }
+function applySilence(previous, evaluation, now) {
+  if (!isSilenced(previous, now)) return evaluation;
+  return { state: muteOpens(previous, evaluation.state), alerts: [] };
+}
+function staleAlert(alert, state) {
+  if (alert.type === "recovered") return alert.details.after.some((condition) => state.open[condition] !== void 0);
+  return state.open[alert.type] !== alert.at;
+}
+function jobHealth(def, lastRun, state, now) {
+  const open = openConditions(state);
+  if (isSilenced(state, now)) return "silenced";
+  if (open.includes("stuck") || lastRun && isStuck(def, lastRun, now)) return "stuck";
+  if (open.includes("failed") || lastRun?.status === "failed" || lastRun?.status === "timeout") return "failing";
+  if (open.includes("missed")) return "late";
+  if (!lastRun) return "never_ran";
+  return "healthy";
+}
+function summarize(stored, recent, state, nextExpectedAt, now) {
+  return summary(stored, recent, state, nextExpectedAt, (lastRun) => jobHealth(stored.definition, lastRun, state, now));
+}
+function unevaluableSummary(stored, recent, state, now) {
+  return summary(stored, recent, state, null, () => isSilenced(state, now) ? "silenced" : "failing");
+}
+function summary(stored, recent, state, nextExpectedAt, health) {
+  const window = recent.slice(0, BASELINE_WINDOW);
+  const lastRun = window[0] ?? null;
+  const finished = window.filter((r) => r.status !== "running");
+  const okDurations = window.filter((r) => r.status === "ok" && r.durationMs !== null).map((r) => r.durationMs);
+  return {
+    name: stored.name,
+    definition: stored.definition,
+    health: health(lastRun),
+    open: openConditions(state),
+    lastRun,
+    nextExpectedAt,
+    consecutiveFailures: state.consecutiveFailures,
+    silencedUntil: state.silencedUntil,
+    stats: {
+      runs: finished.length,
+      okRate: finished.length === 0 ? 1 : finished.filter((r) => r.status === "ok").length / finished.length,
+      p50Ms: percentile(okDurations, 50),
+      p95Ms: percentile(okDurations, 95)
+    }
+  };
+}
 
 // src/format.ts
 function when(at, now) {
@@ -305,14 +397,18 @@ function tail(text, n) {
   const lines = text.trimEnd().split("\n");
   return lines.slice(Math.max(0, lines.length - n)).join("\n");
 }
+function errorLine(error) {
+  const text = firstLines(error, 4);
+  return /^[A-Za-z_$][\w$]*: /.test(text) ? text : `Error: ${text}`;
+}
 function composeAlert(draft, def, now) {
   const name = def.name;
   const run = draft.run;
-  const d = draft.details;
   let title;
   const lines = [];
   switch (draft.type) {
     case "missed": {
+      const d = draft.details;
       title = `${name} missed its scheduled run`;
       lines.push(`Due ${when(d.dueAt, now)}, and no run had started by ${when(d.deadline, now)} (grace ${formatDuration(d.graceMs)}).`);
       lines.push(`Schedule: ${def.schedule}${def.timezone ? ` (${def.timezone})` : ""}.`);
@@ -321,11 +417,11 @@ function composeAlert(draft, def, now) {
     }
     case "failed": {
       title = `${name} failed`;
-      const n = d.consecutiveFailures;
+      const n = draft.details.consecutiveFailures;
       if (n > 1) lines.push(`${n} consecutive failures.`);
       if (run) {
         lines.push(`Started ${when(run.startedAt, now)}${run.durationMs !== null ? `, ran ${formatDuration(run.durationMs)}` : ""}.`);
-        if (run.error) lines.push(`Error: ${firstLines(run.error, 4)}`);
+        if (run.error) lines.push(errorLine(run.error));
         const out = tail(run.output, 8);
         if (out) lines.push(`Output (tail):
 ${out}`);
@@ -344,6 +440,7 @@ ${out}`);
       break;
     }
     case "slow": {
+      const d = draft.details;
       title = `${name} was slow`;
       lines.push(`Took ${formatDuration(d.durationMs)}; the limit is ${formatDuration(d.thresholdMs)} (${d.basis}).`);
       if (run) lines.push(`Started ${when(run.startedAt, now)}.`);
@@ -351,7 +448,7 @@ ${out}`);
     }
     case "over_budget": {
       title = `${name} went over budget`;
-      for (const b of d.breaches) {
+      for (const b of draft.details.breaches) {
         lines.push(`${b.metric}: ${formatNumber(b.value)}, limit ${formatNumber(b.limit)} (${b.basis}).`);
       }
       if (run) lines.push(`Started ${when(run.startedAt, now)}.`);
@@ -359,22 +456,13 @@ ${out}`);
     }
     case "recovered": {
       title = `${name} recovered`;
-      const after = d.after.map((c) => c.replace("_", " ")).join(", ");
+      const after = draft.details.after.map((c) => c.replace("_", " ")).join(", ");
       lines.push(`A run ${run ? when(run.startedAt, now) : "just now"} succeeded${after ? ` after: ${after}` : ""}.`);
       if (run?.durationMs !== null && run?.durationMs !== void 0) lines.push(`Ran ${formatDuration(run.durationMs)}.`);
       break;
     }
   }
-  return {
-    type: draft.type,
-    job: name,
-    definition: def,
-    run,
-    title,
-    message: lines.join("\n"),
-    details: d,
-    at: now
-  };
+  return { ...draft, job: name, definition: def, title, message: lines.join("\n"), at: now };
 }
 
 // src/http.ts
@@ -384,26 +472,85 @@ function json(body, status = 200, headers = {}) {
     headers: { "content-type": "application/json; charset=utf-8", "cache-control": "no-store", ...headers }
   });
 }
+function constantTimeEqual(a, b) {
+  if (a.length !== b.length) return false;
+  let diff = 0;
+  for (let i = 0; i < a.length; i++) diff |= a.charCodeAt(i) ^ b.charCodeAt(i);
+  return diff === 0;
+}
 
 // src/output.ts
 var OUTPUT_CAP = 16 * 1024;
 function capOutput(text) {
-  if (text.length <= OUTPUT_CAP) return text;
-  return "[earlier output trimmed]\n" + text.slice(text.length - OUTPUT_CAP);
+  const clean = stripNul(text);
+  if (clean.length <= OUTPUT_CAP) return clean;
+  return "[earlier output trimmed]\n" + clean.slice(clean.length - OUTPUT_CAP);
+}
+function stripNul(text) {
+  return text.includes("\0") ? text.replace(/\u0000/g, "") : text;
 }
 function errorMessage(error) {
+  return capOutput(describeError(error));
+}
+function describeError(error) {
   if (error instanceof Error) {
-    const stack = error.stack ? `
-${error.stack.split("\n").slice(1, 6).join("\n")}` : "";
-    return `${error.name}: ${error.message}${stack}`;
+    const frames = (error.stack ?? "").split("\n").filter((line) => /^\s+at /.test(line)).slice(0, 5);
+    return `${error.name}: ${error.message}${frames.length ? `
+${frames.join("\n")}` : ""}`;
   }
   if (typeof error === "string") return error;
   try {
-    return JSON.stringify(error);
+    return JSON.stringify(error) ?? String(error);
   } catch {
     return String(error);
   }
 }
+var REDACTED = "[redacted]";
+var SECRET_PATTERNS = [
+  // A PEM private key, header to footer. Without a footer (the output was
+  // trimmed) it runs to the end of the base64 body. A "-" that starts five
+  // dashes ends the body, so the footer is never swallowed into it.
+  [
+    /-----BEGIN (?:[A-Z0-9]{1,20} ){0,3}PRIVATE KEY-----(?:[A-Za-z0-9+/=\s,:]|-(?!----)){0,16384}(?:-----END (?:[A-Z0-9]{1,20} ){0,3}PRIVATE KEY-----)?/g,
+    REDACTED
+  ],
+  // password=..., API_KEY: ..., "client_secret": "...", TOKEN='...', token=...,
+  // :password=>"..." (but not max_tokens: 800). A quoted value is blanked to
+  // its closing quote, spaces and all, and keeps its quotes.
+  [
+    /\b([A-Za-z0-9_-]{0,40}(?:secret|token|passw(?:or)?d|pwd|api[_-]?key|access[_-]?key|private[_-]?key|credential)[A-Za-z0-9_-]{0,40}(?<![Tt][Oo][Kk][Ee][Nn][Ss])"?\s{0,3}(?:=>|[=:])\s{0,3})(?:(")[^"\n]{1,4096}"|(')[^'\n]{1,4096}'|["']?[^\s"',;&]{1,4096})/gi,
+    (_match, name, double, single) => {
+      const quote = double ?? single ?? "";
+      return `${name}${quote}${REDACTED}${quote}`;
+    }
+  ],
+  // Authorization: Basic <base64> and Authorization: Token <token>, also as a JSON or hash entry.
+  [/\b((?:proxy-)?authorization["']?\s{0,3}(?:=>|[=:])\s{0,3}["']?\s{0,3}(?:basic|token)\s{1,3})[A-Za-z0-9._~+/=:-]{1,4096}/gi, `$1${REDACTED}`],
+  // Credentials inside a URL: postgres://user:password@host. The password
+  // runs to the last "@" before a "/" or a space, so one that contains "@"
+  // is blanked whole.
+  [/(\b[a-z][a-z0-9+.-]{0,30}:\/\/[^\s/:@]{0,256}:)[^\s/]{1,256}@/gi, `$1${REDACTED}@`],
+  // Authorization: Bearer <token>
+  [/\b(Bearer\s{1,3})[A-Za-z0-9._~+/=-]{8,4096}/g, `$1${REDACTED}`],
+  // A bare JWT: three base64url segments, the first starting eyJ.
+  [/\beyJ[A-Za-z0-9_-]{4,4096}\.[A-Za-z0-9_-]{4,4096}\.[A-Za-z0-9_-]{0,4096}/g, REDACTED],
+  // Incoming webhook URLs carry their secret in the path.
+  [/(\bhooks\.slack\.com\/(?:services|workflows|triggers)\/)[A-Za-z0-9/_-]{1,255}/gi, `$1${REDACTED}`],
+  [/(\bdiscord(?:app)?\.com\/api\/(?:v\d{1,2}\/)?webhooks\/)[A-Za-z0-9/_-]{1,255}/gi, `$1${REDACTED}`],
+  // Well-known token shapes: AWS, GitHub, Slack, Stripe, Anthropic, OpenAI and Google style keys.
+  [/\b(?:AKIA|ASIA)[0-9A-Z]{16}\b/g, REDACTED],
+  [/\b(?:gh[pousr]_[A-Za-z0-9]{30,255}|github_pat_[A-Za-z0-9_]{20,255})\b/g, REDACTED],
+  [/\bxox[abposr]-[A-Za-z0-9-]{10,255}/g, REDACTED],
+  [/\b[rsp]k_(?:live|test)_[A-Za-z0-9]{10,255}\b/g, REDACTED],
+  [/\bwhsec_[A-Za-z0-9+/=]{16,255}/g, REDACTED],
+  [/\bsk-[A-Za-z0-9_-]{20,255}/g, REDACTED],
+  [/\bAIza[0-9A-Za-z_-]{35}(?![0-9A-Za-z_-])/g, REDACTED]
+];
+function redactSecrets(text) {
+  let out = text;
+  for (const [pattern, replacement] of SECRET_PATTERNS) out = out.replace(pattern, replacement);
+  return out;
+}
 
 // src/job.ts
 function stringify(part) {
@@ -418,6 +565,9 @@ function stringify(part) {
 function createRecorder(run) {
   const lines = [];
   let size = 0;
+  const head = [];
+  let headSize = 0;
+  let dropped = false;
   const metrics = {};
   const controller = new AbortController();
   const context = {
@@ -427,10 +577,15 @@ function createRecorder(run) {
     signal: controller.signal,
     log(...parts) {
       const line = parts.map(stringify).join(" ");
+      if (headSize < OUTPUT_CAP) {
+        head.push(line);
+        headSize += line.length + 1;
+      }
       lines.push(line);
       size += line.length + 1;
       while (size > 64 * 1024 && lines.length > 1) {
         size -= lines.shift().length + 1;
+        dropped = true;
       }
     },
     metric(name, value) {
@@ -446,6 +601,12 @@ function createRecorder(run) {
   return {
     context,
     output: () => lines.length === 0 ? null : capOutput(lines.join("\n")),
+    expectText: () => {
+      if (lines.length === 0) return null;
+      const all = lines.join("\n");
+      if (!dropped && all.length <= 2 * OUTPUT_CAP) return all;
+      return head.join("\n").slice(0, OUTPUT_CAP) + "\n" + all.slice(-OUTPUT_CAP);
+    },
     metrics: () => ({ ...metrics }),
     abort: () => controller.abort()
   };
@@ -467,6 +628,7 @@ function checkExpectation(expect, output) {
     return text.includes(expect) ? null : `Output did not contain ${JSON.stringify(expect)}`;
   }
   if (expect instanceof RegExp) {
+    expect.lastIndex = 0;
     return expect.test(text) ? null : `Output did not match ${expect.toString()}`;
   }
   let ok = false;
@@ -511,6 +673,8 @@ button:hover{border-color:var(--muted)}
 .stat{padding:12px 14px}.stat .k{font-size:12px;color:var(--muted);text-transform:uppercase;letter-spacing:.04em}.stat .v{font-size:20px;font-weight:600;margin-top:2px}
 pre{margin:0;padding:10px 12px;background:var(--pill);border-radius:8px;font:12.5px/1.45 var(--mono);white-space:pre-wrap;word-break:break-word;max-height:320px;overflow:auto}
 details summary{cursor:pointer;color:var(--muted);font-size:13px}details{margin-top:6px}
+details.confirm{margin:0}details.confirm summary{list-style:none;display:inline-block;font-size:13px;padding:5px 10px;border:1px solid var(--line);border-radius:7px;background:var(--card);color:var(--fg)}
+details.confirm summary::-webkit-details-marker{display:none}details.confirm[open] summary{border-color:var(--muted)}details.confirm form{margin-left:8px;font-size:13px}
 .empty{padding:40px 16px;text-align:center;color:var(--muted)}
 dl{display:grid;grid-template-columns:max-content 1fr;gap:6px 16px;margin:0;padding:14px 16px;font-size:14px}dt{color:var(--muted)}dd{margin:0}
 footer{margin-top:28px;color:var(--muted);font-size:12px}
@@ -614,13 +778,13 @@ function jobPage(job, runs, now, base) {
   <div class="actions">
     ${healthPill(job)}
     ${silenced ? `<form class="inline" method="post" action="${h(base)}/jobs/${encodeURIComponent(job.name)}/unsilence"><button type="submit">Unsilence (until ${h(formatRelative(job.silencedUntil, now))})</button></form>` : `<form class="inline" method="post" action="${h(base)}/jobs/${encodeURIComponent(job.name)}/silence"><select name="for"><option value="1h">1 hour</option><option value="4h">4 hours</option><option value="1d">1 day</option><option value="7d">1 week</option></select> <button type="submit">Silence</button></form>`}
-    <form class="inline" method="post" action="${h(base)}/jobs/${encodeURIComponent(job.name)}/forget" onsubmit="return confirm('Remove this job and its runs from the store?')"><button type="submit">Forget</button></form>
+    <details class="confirm"><summary>Forget</summary><form class="inline" method="post" action="${h(base)}/jobs/${encodeURIComponent(job.name)}/forget"><span class="muted">Remove this job and its runs from the store?</span> <button type="submit">Forget</button></form></details>
   </div>
 </header>
 <div class="grid">
   <div class="card stat"><div class="k">Last run</div><div class="v">${job.lastRun ? h(formatRelative(job.lastRun.startedAt, now)) : "never"}</div></div>
   <div class="card stat"><div class="k">Next due</div><div class="v">${job.nextExpectedAt ? h(formatRelative(job.nextExpectedAt, now)) : "no schedule"}</div></div>
-  <div class="card stat"><div class="k">Success, last ${job.stats.runs}</div><div class="v">${okRate}</div></div>
+  <div class="card stat"><div class="k">Success, last ${h(job.stats.runs)}</div><div class="v">${h(okRate)}</div></div>
   <div class="card stat"><div class="k">p50 / p95</div><div class="v">${job.stats.p50Ms !== null ? h(formatDuration(job.stats.p50Ms)) : "?"} <span class="muted">/</span> ${job.stats.p95Ms !== null ? h(formatDuration(job.stats.p95Ms)) : "?"}</div></div>
 </div>
 <div class="card" style="margin-bottom:20px">
@@ -631,11 +795,11 @@ function jobPage(job, runs, now, base) {
   ${d.maxDuration ? `<dt>Max duration</dt><dd class="mono">${h(d.maxDuration)}</dd>` : ""}
   ${d.budget ? `<dt>Budget</dt><dd class="mono">${h(Object.entries(d.budget).map(([k, v]) => `${k} \u2264 ${v}`).join(", "))}</dd>` : ""}
   ${d.expect ? `<dt>Expect</dt><dd class="mono">${h(d.expect)}</dd>` : ""}
-  ${d.failuresBeforeAlert && d.failuresBeforeAlert > 1 ? `<dt>Alert after</dt><dd>${d.failuresBeforeAlert} consecutive failures</dd>` : ""}
+  ${d.failuresBeforeAlert && d.failuresBeforeAlert > 1 ? `<dt>Alert after</dt><dd>${h(d.failuresBeforeAlert)} consecutive failures</dd>` : ""}
   ${d.description ? `<dt>Description</dt><dd>${h(d.description)}</dd>` : ""}
   ${d.tags?.length ? `<dt>Tags</dt><dd>${d.tags.map((t) => `<span class="pill mutedpill">${h(t)}</span>`).join(" ")}</dd>` : ""}
   ${job.open.length ? `<dt>Open</dt><dd>${job.open.map((c) => `<span class="pill warn">${h(c.replace("_", " "))}</span>`).join(" ")}</dd>` : ""}
-  ${job.consecutiveFailures > 0 ? `<dt>Consecutive failures</dt><dd>${job.consecutiveFailures}</dd>` : ""}
+  ${job.consecutiveFailures > 0 ? `<dt>Consecutive failures</dt><dd>${h(job.consecutiveFailures)}</dd>` : ""}
 </dl>
 </div>
 <div class="card">
@@ -652,18 +816,32 @@ function messagePage(title, message, base) {
 
 // src/routes/index.ts
 var COOKIE = "cronwatch_token";
-function constantTimeEqual(a, b) {
-  if (a.length !== b.length) return false;
-  let diff = 0;
-  for (let i = 0; i < a.length; i++) diff |= a.charCodeAt(i) ^ b.charCodeAt(i);
-  return diff === 0;
+function cookieValue(token) {
+  return createHash("sha256").update(`cronwatch-cookie:${token}`).digest("hex");
+}
+var DEFAULT_RUNS = 20;
+var MAX_RUNS = 500;
+var CSP = "default-src 'none'; style-src 'unsafe-inline'; img-src 'self' data:; form-action 'self'; frame-ancestors 'none'; base-uri 'none'";
+var SECURITY_HEADERS = { "x-content-type-options": "nosniff", "referrer-policy": "same-origin", "x-robots-tag": "noindex" };
+function developmentToken() {
+  return randomBytes(32).toString("base64url");
+}
+function developmentSignInLine(origin, base, token) {
+  return `[cronwatch] CRONWATCH_TOKEN is not set, so this development server made a token for the dashboard. Sign in: ${origin}${base}/?token=${token}`;
+}
+function safeDecode(value) {
+  try {
+    return decodeURIComponent(value);
+  } catch {
+    return null;
+  }
 }
 function readCookie(request, name) {
   const header = request.headers.get("cookie");
   if (!header) return null;
   for (const part of header.split(";")) {
     const [k, ...rest] = part.trim().split("=");
-    if (k === name) return decodeURIComponent(rest.join("="));
+    if (k === name) return safeDecode(rest.join("="));
   }
   return null;
 }
@@ -691,56 +869,81 @@ async function readBody(request) {
   }
   return {};
 }
+function crossSite(request, url) {
+  const origin = request.headers.get("origin");
+  if (origin !== null && origin !== url.origin) return true;
+  const site = request.headers.get("sec-fetch-site");
+  return site !== null && site !== "same-origin" && site !== "none";
+}
+function silenceDuration(value) {
+  if (value === void 0 || value === null) return "1h";
+  const text = value.trim();
+  const duration = /^\d+(\.\d+)?$/.test(text) ? Number(text) : text;
+  parseDuration(duration, "silence duration");
+  return duration;
+}
+function runsLimit(value) {
+  const n = value === null || value.trim() === "" ? NaN : Math.trunc(Number(value));
+  return Number.isFinite(n) ? Math.min(MAX_RUNS, Math.max(1, n)) : DEFAULT_RUNS;
+}
 function createRoutes(cw, options = {}) {
-  const token = options.token === void 0 ? process.env.CRONWATCH_TOKEN ?? null : options.token;
+  const optedOut = options.token === null;
+  const configured = optedOut ? null : options.token || process.env.CRONWATCH_TOKEN || null;
   const base = (options.basePath ?? "/cronwatch").replace(/\/+$/, "");
-  const production = process.env.NODE_ENV === "production";
-  const handler = async (request) => {
-    const url = new URL(request.url);
-    const path = stripBase(url.pathname, base);
-    const wantsHtml = !path.startsWith("/api");
+  const developing = process.env.NODE_ENV === "development" || process.env.NODE_ENV === "test";
+  const generated = !configured && !optedOut && developing;
+  const token = generated ? developmentToken() : configured;
+  let announced = false;
+  const serve = async (request, url, path, wantsHtml) => {
     const method = request.method.toUpperCase();
-    if (!token && production) {
-      return wantsHtml ? html(messagePage("CronWatch routes are locked", "Set CRONWATCH_TOKEN (or pass token to cw.routes()) to use them in production.", base), 503) : json({ ok: false, error: "CRONWATCH_TOKEN is not set" }, 503);
+    if (generated && !announced) {
+      announced = true;
+      console.info(developmentSignInLine(url.origin, base, token));
     }
+    if (!token && !optedOut) {
+      return wantsHtml ? html(messagePage("CronWatch routes are locked", "Set CRONWATCH_TOKEN (or pass token to cw.routes()), or pass token: null to serve them open behind your own auth.", base), 503) : api({ ok: false, error: "CRONWATCH_TOKEN is not set" }, 503);
+    }
+    if (method !== "GET" && method !== "HEAD" && crossSite(request, url)) {
+      return wantsHtml ? html(messagePage("Cross-site request refused", "Changes can only be made from the dashboard itself.", base), 403) : api({ ok: false, error: "Cross-site request refused" }, 403);
+    }
+    const bearer = request.headers.get("authorization")?.replace(/^Bearer\s+/i, "") ?? null;
     if (token) {
-      const bearer = request.headers.get("authorization")?.replace(/^Bearer\s+/i, "") ?? null;
-      const query = url.searchParams.get("token");
+      const query = wantsHtml && method === "GET" ? url.searchParams.get("token") : null;
       const cookie = readCookie(request, COOKIE);
-      const presented = bearer ?? query ?? cookie;
       const isCheck = path === "/api/check";
       const cronSecretOk = isCheck && bearer !== null && cw.cronSecret !== null && constantTimeEqual(bearer, cw.cronSecret);
-      if (!cronSecretOk && (presented === null || !constantTimeEqual(presented, token))) {
-        return wantsHtml ? html(messagePage("Sign in", `Open this page with ?token=<your CRONWATCH_TOKEN> once and it will stay signed in.`, base), 401) : json({ ok: false, error: "Unauthorized" }, 401);
+      const tokenOk = bearer !== null ? constantTimeEqual(bearer, token) : query !== null ? constantTimeEqual(query, token) : cookie !== null && constantTimeEqual(cookie, cookieValue(token));
+      if (!cronSecretOk && !tokenOk) {
+        if (generated) {
+          return wantsHtml ? html(messagePage("Sign in", "CRONWATCH_TOKEN is not set, so this development server made a token. The sign-in link is in the server log: open it once and this browser stays signed in.", base), 401) : api({ ok: false, error: "Unauthorized: CRONWATCH_TOKEN is not set, so this development server made a token; it is in the server log" }, 401);
+        }
+        return wantsHtml ? html(messagePage("Sign in", `Open this page with ?token=<your CRONWATCH_TOKEN> once and it will stay signed in.`, base), 401) : api({ ok: false, error: "Unauthorized" }, 401);
       }
-      if (query !== null && wantsHtml && method === "GET") {
+      if (query !== null) {
         url.searchParams.delete("token");
         const secure = url.protocol === "https:" ? "; Secure" : "";
-        return new Response(null, {
-          status: 303,
-          headers: {
-            location: url.pathname + (url.search || ""),
-            "set-cookie": `${COOKIE}=${encodeURIComponent(token)}; Path=${base || "/"}; HttpOnly; SameSite=Lax; Max-Age=${60 * 60 * 24 * 30}${secure}`
-          }
+        return redirect(url.pathname + (url.search || ""), {
+          "set-cookie": `${COOKIE}=${cookieValue(token)}; Path=${base || "/"}; HttpOnly; SameSite=Lax; Max-Age=${60 * 60 * 24 * 30}${secure}`
         });
       }
     }
     const redirectBack = () => {
       const referer = request.headers.get("referer") ?? "";
-      const location = referer.startsWith(url.origin + "/") ? referer : `${base}/`;
-      return new Response(null, { status: 303, headers: { location } });
+      return redirect(referer.startsWith(url.origin + "/") ? referer : `${base}/`);
     };
-    const jobName = (segment) => segment ? decodeURIComponent(segment) : "";
-    const parts = path.split("/").filter(Boolean);
+    const decoded = path.split("/").filter(Boolean).map(safeDecode);
+    if (decoded.some((part) => part === null)) {
+      return wantsHtml ? html(messagePage("Bad request", "The path is not valid.", base), 400) : api({ ok: false, error: "Bad path" }, 400);
+    }
+    const parts = decoded;
     if (method === "GET" && path === "/") {
-      const jobs = await cw.jobs();
-      const runsByJob = /* @__PURE__ */ new Map();
-      for (const job of jobs) runsByJob.set(job.name, await cw.runs(job.name, 20));
-      return html(dashboardPage(jobs, runsByJob, cw.now(), base));
+      const entries = await cw.jobsWithRuns(20);
+      const runsByJob = new Map(entries.map((entry) => [entry.job.name, entry.runs]));
+      return html(dashboardPage(entries.map((entry) => entry.job), runsByJob, cw.now(), base));
     }
     if (method === "GET" && parts[0] === "jobs" && parts.length === 2) {
-      const job = await cw.jobSummary(jobName(parts[1]));
-      if (!job) return html(messagePage("No such job", `${jobName(parts[1])} is not in the store.`, base), 404);
+      const job = await cw.jobSummary(parts[1]);
+      if (!job) return html(messagePage("No such job", `${parts[1]} is not in the store.`, base), 404);
       return html(jobPage(job, await cw.runs(job.name, 50), cw.now(), base));
     }
     if (method === "POST" && path === "/check") {
@@ -748,68 +951,107 @@ function createRoutes(cw, options = {}) {
       return redirectBack();
     }
     if (method === "POST" && parts[0] === "jobs" && parts.length === 3) {
-      const name = jobName(parts[1]);
-      const body = await readBody(request);
-      if (parts[2] === "silence") await cw.silence(name, safeDuration(body.for, "1h"));
-      else if (parts[2] === "unsilence") await cw.unsilence(name);
-      else if (parts[2] === "forget") {
+      const name = parts[1];
+      if (parts[2] === "forget") {
         await cw.forget(name);
-        return new Response(null, { status: 303, headers: { location: `${base}/` } });
-      } else return html(messagePage("Not found", path, base), 404);
+        return redirect(`${base}/`);
+      }
+      if (parts[2] !== "silence" && parts[2] !== "unsilence") return html(messagePage("Not found", path, base), 404);
+      if (!await cw.jobSummary(name)) return html(messagePage("No such job", `${name} is not in the store.`, base), 404);
+      if (parts[2] === "silence") {
+        let duration;
+        try {
+          duration = silenceDuration((await readBody(request)).for);
+        } catch (e) {
+          return html(messagePage("Not silenced", e.message, base), 400);
+        }
+        await cw.silence(name, duration);
+      } else await cw.unsilence(name);
       return redirectBack();
     }
     if (parts[0] === "api") {
       const rest = parts.slice(1);
       if (method === "GET" && rest[0] === "jobs" && rest.length === 1) {
-        return json({ ok: true, jobs: await cw.jobs() });
+        return api({ ok: true, jobs: await cw.jobs() });
       }
       if (rest[0] === "jobs" && rest.length === 2) {
-        const name = jobName(rest[1]);
+        const name = rest[1];
         if (method === "GET") {
           const job = await cw.jobSummary(name);
-          if (!job) return json({ ok: false, error: "No such job" }, 404);
-          const limit = Number(url.searchParams.get("runs") ?? 20);
-          return json({ ok: true, job, runs: await cw.runs(name, Number.isFinite(limit) ? limit : 20) });
+          if (!job) return api({ ok: false, error: "No such job" }, 404);
+          return api({ ok: true, job, runs: await cw.runs(name, runsLimit(url.searchParams.get("runs"))) });
         }
         if (method === "DELETE") {
+          if (!await cw.jobSummary(name)) return api({ ok: false, error: "No such job" }, 404);
           await cw.forget(name);
-          return json({ ok: true });
+          return api({ ok: true });
         }
       }
       if (method === "POST" && rest[0] === "jobs" && rest.length === 3) {
-        const name = jobName(rest[1]);
-        if (!await cw.jobSummary(name)) return json({ ok: false, error: "No such job" }, 404);
-        const body = await readBody(request);
-        if (rest[2] === "silence") return json({ ok: true, state: await cw.silence(name, safeDuration(body.for ?? url.searchParams.get("for"), "1h")) });
-        if (rest[2] === "unsilence") return json({ ok: true, state: await cw.unsilence(name) });
+        const name = rest[1];
+        if (!await cw.jobSummary(name)) return api({ ok: false, error: "No such job" }, 404);
+        if (rest[2] === "silence") {
+          const body = await readBody(request);
+          let duration;
+          try {
+            duration = silenceDuration(body.for ?? url.searchParams.get("for"));
+          } catch (e) {
+            return api({ ok: false, error: e.message }, 400);
+          }
+          return api({ ok: true, state: await cw.silence(name, duration) });
+        }
+        if (rest[2] === "unsilence") return api({ ok: true, state: await cw.unsilence(name) });
       }
-      if ((method === "GET" || method === "POST") && rest[0] === "check" && rest.length === 1) {
-        const result = await cw.check();
-        return json({ ok: true, ...result });
+      if (rest[0] === "check" && rest.length === 1) {
+        if (method === "GET" && bearer === null) {
+          return api({ ok: false, error: "Use POST, or GET with an Authorization bearer" }, 405, { allow: "POST" });
+        }
+        if (method === "GET" || method === "POST") {
+          const result = await cw.check();
+          return api({ ok: true, ...result });
+        }
       }
       if (method === "GET" && rest[0] === "runs" && rest.length === 2) {
         const run = await cw.getRun(rest[1]);
-        return run ? json({ ok: true, run }) : json({ ok: false, error: "No such run" }, 404);
+        return run ? api({ ok: true, run }) : api({ ok: false, error: "No such run" }, 404);
       }
-      return json({ ok: false, error: "Not found" }, 404);
+      return api({ ok: false, error: "Not found" }, 404);
     }
     return html(messagePage("Not found", path, base), 404);
   };
+  const handler = async (request) => {
+    let wantsHtml = true;
+    try {
+      const url = new URL(request.url);
+      const path = stripBase(url.pathname, base);
+      wantsHtml = !path.startsWith("/api");
+      return await serve(request, url, path, wantsHtml);
+    } catch (e) {
+      try {
+        cw.onError(e, "routes");
+      } catch {
+      }
+      return wantsHtml ? html(messagePage("Something went wrong", "The request failed and the error was reported.", base), 500) : api({ ok: false, error: "Internal error" }, 500);
+    }
+  };
   return { handler, GET: handler, POST: handler, DELETE: handler };
 }
-function safeDuration(value, fallback) {
-  if (!value) return fallback;
-  try {
-    parseDuration(value);
-    return value;
-  } catch {
-    return fallback;
-  }
+function api(body, status = 200, headers = {}) {
+  return json(body, status, { ...SECURITY_HEADERS, ...headers });
+}
+function redirect(location, headers = {}) {
+  return new Response(null, { status: 303, headers: { location, "cache-control": "no-store", ...SECURITY_HEADERS, ...headers } });
 }
 function html(body, status = 200) {
   return new Response(body, {
     status,
-    headers: { "content-type": "text/html; charset=utf-8", "cache-control": "no-store", "x-robots-tag": "noindex" }
+    headers: {
+      "content-type": "text/html; charset=utf-8",
+      "cache-control": "no-store",
+      "content-security-policy": CSP,
+      "x-frame-options": "DENY",
+      ...SECURITY_HEADERS
+    }
   });
 }
 
@@ -821,6 +1063,7 @@ function memory() {
   const states = /* @__PURE__ */ new Map();
   let seq = 0;
   const clone = (v) => JSON.parse(JSON.stringify(v));
+  const byName = (a, b) => a.name < b.name ? -1 : a.name > b.name ? 1 : 0;
   return {
     async upsertJob(definition, now) {
       const existing = jobs.get(definition.name);
@@ -836,7 +1079,7 @@ function memory() {
       return j ? clone(j) : null;
     },
     async listJobs() {
-      return [...jobs.values()].map(clone).sort((a, b) => a.name.localeCompare(b.name));
+      return [...jobs.values()].map(clone).sort(byName);
     },
     async deleteJob(name) {
       jobs.delete(name);
@@ -851,7 +1094,10 @@ function memory() {
       order.set(run.id, ++seq);
     },
     async updateRun(run) {
-      runs.set(run.id, clone(run));
+      const existing = runs.get(run.id);
+      if (!existing) return;
+      const { status, finishedAt, durationMs, error, output, metrics } = clone(run);
+      runs.set(run.id, { ...existing, status, finishedAt, durationMs, error, output, metrics });
     },
     async getRun(id) {
       const r = runs.get(id);
@@ -865,7 +1111,7 @@ function memory() {
       return list[0] ?? null;
     },
     async runningRuns() {
-      return [...runs.values()].filter((r) => r.status === "running").map(clone);
+      return [...runs.values()].filter((r) => r.status === "running").sort((a, b) => a.startedAt - b.startedAt || order.get(a.id) - order.get(b.id)).map(clone);
     },
     async getState(job) {
       const s = states.get(job);
@@ -874,10 +1120,17 @@ function memory() {
     async setState(state) {
       states.set(state.job, clone(state));
     },
+    async compareAndSetState(state, expectedVersion) {
+      if ((states.get(state.job)?.version ?? 0) !== expectedVersion) return false;
+      states.set(state.job, clone(state));
+      return true;
+    },
     async prune(before) {
+      const newest = /* @__PURE__ */ new Map();
+      for (const run of runs.values()) newest.set(run.job, Math.max(newest.get(run.job) ?? -Infinity, run.startedAt));
       let n = 0;
       for (const [id, run] of runs) {
-        if (run.status !== "running" && run.startedAt < before) {
+        if (run.status !== "running" && run.startedAt < before && run.startedAt < newest.get(run.job)) {
           runs.delete(id);
           order.delete(id);
           n++;
@@ -891,37 +1144,104 @@ function memory() {
 // src/client.ts
 var NAME_RE = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,119}$/;
 var TRIAGE_TIMEOUT_MS = 25e3;
+var CHANNEL_TIMEOUT_MS = 15e3;
 var PRUNE_INTERVAL_MS = 60 * 6e4;
-function constantTimeEqual2(a, b) {
-  if (a.length !== b.length) return false;
-  let diff = 0;
-  for (let i = 0; i < a.length; i++) diff |= a.charCodeAt(i) ^ b.charCodeAt(i);
-  return diff === 0;
+var MAX_UNDELIVERED = 20;
+var RETRY_BUDGET_MS = 2e4;
+var STATE_ATTEMPTS = 10;
+var HISTORY_PAGE = BASELINE_WINDOW + 5;
+var HISTORY_MAX = 200;
+function validateDefinition(def) {
+  const name = def.name;
+  if (def.schedule !== void 0) {
+    if (typeof def.schedule !== "string" || def.schedule.trim() === "") throw new Error(`job "${name}": schedule must be a non-empty string`);
+    parseSchedule(def.schedule, def.timezone);
+  }
+  if (def.timezone !== void 0) {
+    try {
+      new Intl.DateTimeFormat("en-US", { timeZone: def.timezone });
+    } catch {
+      throw new Error(`job "${name}": timezone "${def.timezone}" is not an IANA timezone`);
+    }
+  }
+  if (def.grace !== void 0) parseDuration(def.grace, "grace");
+  if (def.timeout !== void 0 && parseDuration(def.timeout, "timeout") <= 0) throw new Error(`job "${name}": timeout must be longer than zero`);
+  if (def.maxDuration !== void 0 && parseDuration(def.maxDuration, "maxDuration") <= 0) {
+    throw new Error(`job "${name}": maxDuration must be longer than zero`);
+  }
+  if (def.failuresBeforeAlert !== void 0 && !(Number.isInteger(def.failuresBeforeAlert) && def.failuresBeforeAlert >= 1)) {
+    throw new Error(`job "${name}": failuresBeforeAlert must be a whole number, 1 or more (got ${String(def.failuresBeforeAlert)})`);
+  }
+  if (def.budget !== void 0) {
+    if (typeof def.budget !== "object" || def.budget === null) throw new Error(`job "${name}": budget must be an object of { metric: ceiling }`);
+    for (const [metric, ceiling] of Object.entries(def.budget)) {
+      if (typeof ceiling !== "number" || !Number.isFinite(ceiling) || ceiling < 0) {
+        throw new Error(`job "${name}": budget.${metric} must be a finite number, 0 or more (got ${String(ceiling)})`);
+      }
+    }
+  }
+  if (def.expect !== void 0 && typeof def.expect !== "string" && !(def.expect instanceof RegExp) && typeof def.expect !== "function") {
+    throw new Error(`job "${name}": expect must be a string, a RegExp or a function`);
+  }
+}
+function sameState(a, b) {
+  return JSON.stringify(a) === JSON.stringify(b);
+}
+function alertKey(alert) {
+  return `${alert.type}|${alert.at}|${alert.run?.id ?? ""}`;
 }
 var CronWatch = class {
   store;
   alerts;
   triage;
+  /** The secret handler() requests must carry, or null when none is set. */
   cronSecret;
   retentionMs;
   now;
   onError;
+  /** cronSecret was passed as null: handlers may run without a secret. */
+  secretOptOut;
+  redact;
+  /** "check": queue alerts for another process's check instead of sending them. See CronWatchOptions.deliver. */
+  deferDelivery;
   defaults;
   definitions = /* @__PURE__ */ new Map();
   synced = /* @__PURE__ */ new Set();
+  /** The tail of each job's queue of state updates. See serial(). */
+  queues = /* @__PURE__ */ new Map();
   ready = null;
   checking = null;
   lastPruneAt = 0;
   timer = null;
+  firstTick = null;
   usingDefaultStore = false;
+  warnedNoSecret = false;
+  warnedDeferredStart = false;
   constructor(options = {}) {
     this.store = options.store ?? (this.usingDefaultStore = true, memory());
     this.alerts = options.alerts ?? [consoleChannel()];
     this.triage = options.triage;
-    this.cronSecret = options.cronSecret === void 0 ? process.env.CRON_SECRET ?? null : options.cronSecret;
+    const secret = options.cronSecret === void 0 ? process.env.CRON_SECRET : options.cronSecret;
+    this.cronSecret = secret ? secret : null;
+    this.secretOptOut = options.cronSecret === null;
     this.retentionMs = parseDuration(options.retention ?? "30d", "retention");
     this.defaults = options.defaults ?? {};
     this.now = options.now ?? (() => Date.now());
+    const redact = options.redact;
+    this.redact = redact === false ? (text) => text : typeof redact === "function" ? (text) => {
+      try {
+        const out = redact(text);
+        if (typeof out !== "string") throw new TypeError(`redact must return a string, not ${out === null ? "null" : typeof out}`);
+        return out;
+      } catch (e) {
+        this.report(e, "redact");
+        return redactSecrets(text);
+      }
+    } : redactSecrets;
+    if (options.deliver !== void 0 && options.deliver !== "now" && options.deliver !== "check") {
+      throw new Error(`deliver must be "now" or "check", not ${JSON.stringify(options.deliver)}`);
+    }
+    this.deferDelivery = options.deliver === "check";
     this.onError = options.onError ?? ((error, where) => console.error(`[cronwatch] ${where}:`, error));
   }
   /** Declare a job. Call it once, at module level, and keep the handle. */
@@ -930,10 +1250,7 @@ var CronWatch = class {
       throw new Error(`job name "${name}" must be 1 to 120 characters of letters, digits, ".", "_", ":" or "-"`);
     }
     const definition = { ...this.defaults, ...options, name };
-    if (definition.schedule) parseSchedule(definition.schedule, definition.timezone);
-    if (definition.grace !== void 0) parseDuration(definition.grace, "grace");
-    if (definition.timeout !== void 0) parseDuration(definition.timeout, "timeout");
-    if (definition.maxDuration !== void 0) parseDuration(definition.maxDuration, "maxDuration");
+    validateDefinition(definition);
     this.definitions.set(name, definition);
     this.synced.delete(name);
     return this.handle(definition);
@@ -959,11 +1276,17 @@ var CronWatch = class {
         return outcome.result;
       },
       handler(fn, options) {
-        const secret = options?.secret === void 0 ? self.cronSecret : options.secret;
+        const own = options?.secret;
+        const secret = own === null ? null : own ? own : self.cronSecret;
+        const optedOut = own === null || !own && self.secretOptOut;
         return async (request) => {
+          if (!secret && !optedOut && !isDevelopment()) {
+            self.warnNoSecret();
+            return json({ ok: false, error: "CRON_SECRET is not set, so this job will not run for an unauthenticated request. Set it, or pass secret: null to handler() to allow anyone." }, 503);
+          }
           if (secret) {
             const header = request.headers.get("authorization") ?? "";
-            if (!constantTimeEqual2(header, `Bearer ${secret}`)) {
+            if (!constantTimeEqual(header, `Bearer ${secret}`)) {
               return json({ ok: false, error: "Unauthorized" }, 401);
             }
           }
@@ -975,13 +1298,26 @@ var CronWatch = class {
             run: outcome.run.id,
             status: outcome.run.status,
             durationMs: outcome.run.durationMs,
-            ...outcome.run.error ? { error: outcome.run.error.split("\n")[0] } : {}
+            // Error text only goes to a caller who proved they hold the secret.
+            ...secret && outcome.run.error ? { error: outcome.run.error.split("\n")[0] } : {}
           };
           return json(body, outcome.run.status === "ok" ? 200 : 500);
         };
       }
     };
   }
+  warnNoSecret() {
+    if (this.warnedNoSecret) return;
+    this.warnedNoSecret = true;
+    this.onError(new Error("handler() refused a request because no CRON_SECRET is set; pass secret: null to allow unauthenticated requests"), "handler");
+  }
+  /** onError, for places that must carry on even when onError itself throws. */
+  report(error, where) {
+    try {
+      this.onError(error, where);
+    } catch {
+    }
+  }
   async ensureReady() {
     if (!this.ready) {
       this.ready = (async () => {
@@ -989,7 +1325,10 @@ var CronWatch = class {
         if (this.usingDefaultStore && process.env.NODE_ENV === "production") {
           console.warn("[cronwatch] using the in-memory store: runs and state are lost on restart. Pass a store from @cronwatch/sdk/sqlite or @cronwatch/sdk/postgres.");
         }
-      })();
+      })().catch((error) => {
+        this.ready = null;
+        throw error;
+      });
     }
     await this.ready;
   }
@@ -999,13 +1338,69 @@ var CronWatch = class {
     await this.store.upsertJob(toStored(definition), this.now());
     this.synced.add(definition.name);
   }
-  /** Runs a function as a recorded run. Never throws for the job's own error; see `threw`. */
+  /**
+   * Runs `fn` after every earlier state update for the same job has settled,
+   * so two runs (or a run and a check) in this process never read and write
+   * the job's state over each other. Other processes are coordinated by
+   * updateState() instead.
+   */
+  serial(job, fn) {
+    const result = (this.queues.get(job) ?? Promise.resolve()).then(fn);
+    const tail2 = result.then(() => {
+    }, () => {
+    });
+    this.queues.set(job, tail2);
+    void tail2.then(() => {
+      if (this.queues.get(job) === tail2) this.queues.delete(job);
+    });
+    return result;
+  }
+  async readState(job) {
+    return normalizeState(await this.store.getState(job), job);
+  }
+  /**
+   * Every read-modify-write of a job's state goes through here. In turn with
+   * this process's other updates to the job (serial()), it reads the state,
+   * asks `change` for the next one, and writes it with the version one
+   * higher, only if the stored version is still the one read. When another
+   * process wrote in between, the write is refused and it starts again from
+   * a fresh read, up to STATE_ATTEMPTS times. So `change` may run more than
+   * once and must only compute: whatever it returns from the attempt that
+   * was written is the result. Nothing is written when the state is
+   * unchanged. Returns the state as stored.
+   */
+  updateState(job, change) {
+    return this.serial(job, async () => {
+      for (let attempt = 1; ; attempt++) {
+        const current = await this.readState(job);
+        const { state, result } = await change(current);
+        if (sameState(state, current)) return { state: current, result };
+        const version = current.version ?? 0;
+        const next = { ...state, version: version + 1 };
+        if (await this.writeState(next, version)) return { state: next, result };
+        if (attempt >= STATE_ATTEMPTS) {
+          throw new Error(`the state of ${job} changed under ${STATE_ATTEMPTS} attempts in a row to update it; gave up`);
+        }
+      }
+    });
+  }
+  /** A conditional write, or for a store without compareAndSetState, a plain one that always succeeds. */
+  async writeState(state, expectedVersion) {
+    if (typeof this.store.compareAndSetState === "function") return this.store.compareAndSetState(state, expectedVersion);
+    await this.store.setState(state);
+    return true;
+  }
+  /**
+   * Runs a function as a recorded run. The function always runs, whatever
+   * the store is doing: store errors go to onError, and the result is the
+   * function's own outcome. Never throws for the job's own error; see `threw`.
+   */
   async execute(definition, fn, trigger) {
-    await this.sync(definition);
+    const name = definition.name;
     const startedAt = this.now();
     const run = {
       id: crypto.randomUUID(),
-      job: definition.name,
+      job: name,
       status: "running",
       startedAt,
       finishedAt: null,
@@ -1015,12 +1410,16 @@ var CronWatch = class {
       metrics: {},
       trigger
     };
-    await this.store.insertRun(run);
-    const before = await this.store.getState(definition.name) ?? emptyState(definition.name);
-    const started = onRunStart(before);
-    if (openConditions(started.state).length !== openConditions(before).length) {
-      await this.store.setState(started.state);
+    let recorded = false;
+    try {
+      await this.sync(definition);
+      await this.store.insertRun({ ...run });
+      recorded = true;
+    } catch (e) {
+      this.onError(e, `recording ${name}`);
     }
+    const started = recorded ? this.updateState(name, (before) => ({ state: onRunStart(before), result: void 0 })).then(() => {
+    }, (e) => this.onError(e, `starting ${name}`)) : Promise.resolve();
     const recorder = createRecorder(run);
     const timer = setTimeout(() => recorder.abort(), timeoutMs(definition));
     if (typeof timer === "object" && "unref" in timer) timer.unref();
@@ -1039,7 +1438,7 @@ var CronWatch = class {
     run.finishedAt = finishedAt;
     run.durationMs = Math.max(0, finishedAt - startedAt);
     run.metrics = recorder.metrics();
-    run.output = recorder.output() ?? (typeof result === "string" ? result : null);
+    run.output = recorder.output() ?? (typeof result === "string" ? capOutput(result) : null);
     if (threw) {
       run.status = "failed";
       run.error = errorMessage(error);
@@ -1047,7 +1446,8 @@ var CronWatch = class {
       run.status = "failed";
       run.error = `HTTP ${result.status}${result.statusText ? ` ${result.statusText}` : ""}`;
     } else {
-      const unmet = checkExpectation(definition.expect, run.output);
+      const expectText = recorder.expectText() ?? (typeof result === "string" ? result : null);
+      const unmet = checkExpectation(definition.expect, expectText);
       if (unmet) {
         run.status = "failed";
         run.error = unmet;
@@ -1055,21 +1455,75 @@ var CronWatch = class {
         run.status = "ok";
       }
     }
-    await this.store.updateRun(run);
+    if (run.output !== null) run.output = stripNul(this.redact(run.output));
+    if (run.error !== null) run.error = stripNul(this.redact(run.error));
+    await started;
+    if (!recorded) {
+      try {
+        await this.sync(definition);
+        await this.store.insertRun(run);
+        recorded = true;
+      } catch (e) {
+        this.onError(e, `recording ${name}`);
+      }
+      if (recorded) await this.finishRun(toStored(definition), run, finishedAt, false);
+    } else if (await this.markedTimedOut(run)) {
+      try {
+        await this.store.updateRun(run);
+      } catch (e) {
+        this.onError(e, `recording ${name}`);
+      }
+    } else {
+      await this.finishRun(toStored(definition), run, finishedAt, true);
+    }
+    return { run, result, error, threw };
+  }
+  /** Whether a check already marked this run as timed out, for a failure that finished late. */
+  async markedTimedOut(run) {
+    if (run.status === "ok") return false;
     try {
-      const history = (await this.store.listRuns(definition.name, BASELINE_WINDOW + 5)).filter((r) => r.id !== run.id);
-      const state = await this.store.getState(definition.name) ?? emptyState(definition.name);
-      const evaluation = onRunFinish(definition, run, state, history, started.openBefore, finishedAt);
-      await this.settle(state, evaluation, toStored(definition), finishedAt);
+      return (await this.store.getRun(run.id))?.status === "timeout";
+    } catch {
+      return false;
+    }
+  }
+  /**
+   * Record a finished run (ok, failed, or timed out by a check), evaluate it
+   * against the job's state and send what that produces. Never throws.
+   */
+  async finishRun(definition, run, now, write) {
+    let drafts;
+    try {
+      if (write) await this.store.updateRun(run);
+      let history = null;
+      ({ result: drafts } = await this.updateState(run.job, async (previous) => {
+        history ??= await this.history(run);
+        const settled = applySilence(previous, onRunFinish(definition, run, previous, history, now), now);
+        return { state: settled.state, result: settled.alerts };
+      }));
     } catch (e) {
-      this.onError(e, `evaluating ${definition.name}`);
+      this.onError(e, `evaluating ${run.job}`);
+      return [];
     }
-    return { run, result, error, threw };
+    return this.dispatch(drafts, definition, now);
   }
   /**
-   * Look for missed and stuck runs across every job, send alerts, and prune
-   * old runs. Call it from an interval (start()), a cron hitting the mounted
-   * routes, or by hand. Concurrent calls share one check.
+   * The runs before `run`, newest first, with up to BASELINE_WINDOW
+   * successful ones when the store has them. One small read normally; a
+   * larger one only when failures crowd the successes out of it.
+   */
+  async history(run) {
+    let runs = await this.store.listRuns(run.job, HISTORY_PAGE);
+    if (runs.length === HISTORY_PAGE && !hasFullBaseline(runs.filter((r) => r.id !== run.id))) {
+      runs = await this.store.listRuns(run.job, HISTORY_MAX);
+    }
+    return runs.filter((r) => r.id !== run.id);
+  }
+  /**
+   * Look for missed and stuck runs across every job, send alerts, retry
+   * alerts no channel accepted, and prune old runs. Call it from an interval
+   * (start()), a cron hitting the mounted routes, or by hand. Concurrent
+   * calls share one check.
    */
   check() {
     if (!this.checking) {
@@ -1085,28 +1539,39 @@ var CronWatch = class {
     const now = this.now();
     const alerts = [];
     for (const run of await this.store.runningRuns()) {
-      const stored = await this.store.getJob(run.job);
-      const declared = this.definitions.get(run.job);
-      const definition = declared ? toStored(declared) : stored?.definition;
-      if (!definition || !isStuck(definition, run, now)) continue;
-      run.status = "timeout";
-      run.finishedAt = now;
-      run.durationMs = now - run.startedAt;
-      run.error = `Still running after ${Math.round(timeoutMs(definition) / 6e4)} minutes; marked as timed out`;
-      await this.store.updateRun(run);
-      const history = (await this.store.listRuns(run.job, BASELINE_WINDOW + 5)).filter((r) => r.id !== run.id);
-      const state = await this.store.getState(run.job) ?? emptyState(run.job);
-      const evaluation = onRunFinish(definition, run, state, history, [], now);
-      alerts.push(...await this.settle(state, evaluation, definition, now));
+      try {
+        const declared = this.definitions.get(run.job);
+        const definition = declared ? toStored(declared) : (await this.store.getJob(run.job))?.definition;
+        if (!definition || !isStuck(definition, run, now)) continue;
+        const timeout = timeoutMs(definition);
+        run.status = "timeout";
+        run.finishedAt = now;
+        run.durationMs = now - run.startedAt;
+        run.error = `Still running after ${formatDuration(timeout)}; marked as timed out`;
+        alerts.push(...await this.finishRun(definition, run, now, true));
+      } catch (e) {
+        this.onError(e, `checking ${run.job}`);
+      }
     }
     const jobs = [];
+    const retries = { spentMs: 0 };
     for (const stored of await this.store.listJobs()) {
-      const definition = stored.definition;
-      const lastRun = await this.store.lastRun(stored.name);
-      const state = await this.store.getState(stored.name) ?? emptyState(stored.name);
-      const evaluation = onCheck(definition, stored, lastRun, state, now);
-      alerts.push(...await this.settle(state, evaluation, definition, now));
-      jobs.push(await this.summarize(stored, lastRun, evaluation.state, evaluation.nextExpectedAt, now));
+      try {
+        const recent = await this.store.listRuns(stored.name, BASELINE_WINDOW);
+        let nextExpectedAt = null;
+        const { state, result: drafts } = await this.updateState(stored.name, (previous) => {
+          const evaluation = onCheck(stored.definition, stored, recent[0] ?? null, previous, now);
+          nextExpectedAt = evaluation.nextExpectedAt;
+          const settled = applySilence(previous, evaluation, now);
+          return { state: settled.state, result: settled.alerts };
+        });
+        alerts.push(...await this.retryUndelivered(stored.name, state, now, retries));
+        alerts.push(...await this.dispatch(drafts, stored.definition, now));
+        jobs.push(summarize(stored, recent, state, nextExpectedAt, now));
+      } catch (e) {
+        this.onError(e, `checking ${stored.name}`);
+        jobs.push(await this.unevaluable(stored, now));
+      }
     }
     let pruned = 0;
     if (now - this.lastPruneAt > PRUNE_INTERVAL_MS) {
@@ -1119,18 +1584,36 @@ var CronWatch = class {
     }
     return { checkedAt: now, jobs, alerts, pruned };
   }
+  /** A job's summary and its newest runs, without alerting. A job that cannot be evaluated is reported and shown as failing. */
+  async snapshot(stored, now, runs) {
+    let recent = [];
+    try {
+      recent = await this.store.listRuns(stored.name, Math.max(runs, BASELINE_WINDOW));
+      const state = await this.readState(stored.name);
+      const { nextExpectedAt } = onCheck(stored.definition, stored, recent[0] ?? null, state, now);
+      return { job: summarize(stored, recent, state, nextExpectedAt, now), runs: recent.slice(0, runs) };
+    } catch (e) {
+      this.onError(e, `reading ${stored.name}`);
+      return { job: await this.unevaluable(stored, now), runs: recent.slice(0, runs) };
+    }
+  }
+  /** The summary of a job whose evaluation failed, from whatever can still be read. */
+  async unevaluable(stored, now) {
+    const recent = await this.store.listRuns(stored.name, BASELINE_WINDOW).catch(() => []);
+    const state = await this.readState(stored.name).catch(() => emptyState(stored.name));
+    return unevaluableSummary(stored, recent, state, now);
+  }
   /** Every job the store knows about, with its health. Does not send alerts. */
   async jobs() {
+    return (await this.jobsWithRuns(0)).map((entry) => entry.job);
+  }
+  /** Every job's summary with its newest `limit` runs, read together. What the dashboard shows. */
+  async jobsWithRuns(limit = 20) {
     await this.ensureReady();
     for (const definition of this.definitions.values()) await this.sync(definition);
     const now = this.now();
     const out = [];
-    for (const stored of await this.store.listJobs()) {
-      const lastRun = await this.store.lastRun(stored.name);
-      const state = await this.store.getState(stored.name) ?? emptyState(stored.name);
-      const { nextExpectedAt } = onCheck(stored.definition, stored, lastRun, state, now);
-      out.push(await this.summarize(stored, lastRun, state, nextExpectedAt, now));
-    }
+    for (const stored of await this.store.listJobs()) out.push(await this.snapshot(stored, now, clampLimit(limit, 20, 0)));
     return out;
   }
   async jobSummary(name) {
@@ -1139,15 +1622,12 @@ var CronWatch = class {
     if (definition) await this.sync(definition);
     const stored = await this.store.getJob(name);
     if (!stored) return null;
-    const now = this.now();
-    const lastRun = await this.store.lastRun(name);
-    const state = await this.store.getState(name) ?? emptyState(name);
-    const { nextExpectedAt } = onCheck(stored.definition, stored, lastRun, state, now);
-    return this.summarize(stored, lastRun, state, nextExpectedAt, now);
+    return (await this.snapshot(stored, this.now(), 0)).job;
   }
+  /** A job's runs, newest first. `limit` is a whole number from 1 to 500. */
   async runs(name, limit = 50) {
     await this.ensureReady();
-    return this.store.listRuns(name, Math.min(500, Math.max(1, limit)));
+    return this.store.listRuns(name, clampLimit(limit, 50, 1));
   }
   async getRun(id) {
     await this.ensureReady();
@@ -1155,17 +1635,24 @@ var CronWatch = class {
   }
   /** Stop alerts for a job for a while. State keeps updating underneath. */
   async silence(name, duration) {
-    await this.ensureReady();
-    const state = await this.store.getState(name) ?? emptyState(name);
-    state.silencedUntil = this.now() + parseDuration(duration, "silence duration");
-    await this.store.setState(state);
-    return state;
+    const ms = parseDuration(duration, "silence duration");
+    return this.patchState(name, (state) => {
+      state.silencedUntil = this.now() + ms;
+    });
   }
   async unsilence(name) {
+    return this.patchState(name, (state) => {
+      state.silencedUntil = null;
+    });
+  }
+  /** Read, change and write one job's state, in turn with every other update to it. */
+  async patchState(name, change) {
     await this.ensureReady();
-    const state = await this.store.getState(name) ?? emptyState(name);
-    state.silencedUntil = null;
-    await this.store.setState(state);
+    const { state } = await this.updateState(name, (current) => {
+      const next = normalizeState(current, name);
+      change(next);
+      return { state: next, result: void 0 };
+    });
     return state;
   }
   /** Remove a job and its runs from the store. A job still declared in code comes back on its next run. */
@@ -1183,88 +1670,146 @@ var CronWatch = class {
   start(every = "1m") {
     if (this.timer) return;
     const ms = Math.max(5e3, parseDuration(every, "check interval"));
+    if (this.deferDelivery && !this.warnedDeferredStart) {
+      this.warnedDeferredStart = true;
+      console.warn('[cronwatch] start() was called with deliver: "check", so these checks send no alerts. Another process must run checks with deliver: "now" (the default) to send them.');
+    }
     const tick = () => this.check().catch((e) => this.onError(e, "check"));
     this.timer = setInterval(tick, ms);
     if (typeof this.timer === "object" && "unref" in this.timer) this.timer.unref();
-    setTimeout(tick, 1e3).unref?.();
+    this.firstTick = setTimeout(() => {
+      this.firstTick = null;
+      void tick();
+    }, 1e3);
+    if (typeof this.firstTick === "object" && "unref" in this.firstTick) this.firstTick.unref();
   }
   stop() {
     if (this.timer) clearInterval(this.timer);
+    if (this.firstTick) clearTimeout(this.firstTick);
     this.timer = null;
+    this.firstTick = null;
   }
   async close() {
     this.stop();
     if (this.store.close) await this.store.close();
   }
-  async summarize(stored, lastRun, state, nextExpectedAt, now) {
-    const recent = await this.store.listRuns(stored.name, BASELINE_WINDOW);
-    const finished = recent.filter((r) => r.status !== "running");
-    const okDurations = recent.filter((r) => r.status === "ok" && r.durationMs !== null).map((r) => r.durationMs);
-    const open = openConditions(state);
-    let health;
-    if (state.silencedUntil !== null && state.silencedUntil > now) health = "silenced";
-    else if (open.includes("stuck") || lastRun && isStuck(stored.definition, lastRun, now)) health = "stuck";
-    else if (open.includes("failed") || lastRun?.status === "failed" || lastRun?.status === "timeout") health = "failing";
-    else if (open.includes("missed")) health = "late";
-    else if (!lastRun) health = "never_ran";
-    else health = "healthy";
-    return {
-      name: stored.name,
-      definition: stored.definition,
-      health,
-      open,
-      lastRun,
-      nextExpectedAt,
-      consecutiveFailures: state.consecutiveFailures,
-      silencedUntil: state.silencedUntil,
-      stats: {
-        runs: finished.length,
-        okRate: finished.length === 0 ? 1 : finished.filter((r) => r.status === "ok").length / finished.length,
-        p50Ms: percentile(okDurations, 50),
-        p95Ms: percentile(okDurations, 95)
+  /**
+   * Compose, triage and send each draft. The state was saved before this
+   * (updateState), so a slow channel holds up nothing else; afterwards only
+   * the delivery fields are written back, onto a fresh read of the state.
+   */
+  async dispatch(drafts, definition, now) {
+    if (drafts.length === 0) return [];
+    const composed = [];
+    const delivered = [];
+    const failed = [];
+    for (const draft of drafts) {
+      const alert = composeAlert(draft, definition, now);
+      if (this.deferDelivery) {
+        failed.push(alert);
+      } else {
+        if (this.triage && alert.type !== "recovered") await this.addTriage(alert, TRIAGE_TIMEOUT_MS);
+        (await this.deliver(alert) ? delivered : failed).push(alert);
       }
-    };
+      composed.push(alert);
+    }
+    await this.recordDelivery(definition.name, delivered, failed, [], now);
+    return composed;
   }
-  /** Save an evaluation's state, honouring silence, and send its alerts. */
-  async settle(previous, evaluation, definition, now) {
-    if (isSilenced(previous, now)) {
-      evaluation.state = muteOpens(previous, evaluation.state);
-      evaluation.alerts = [];
+  /**
+   * Send the alerts that no channel accepted last time, once each, oldest
+   * first. `state` is the job's state as this check left it: an alert that
+   * no longer describes it (staleAlert) is dropped instead. Retries across a
+   * check share RETRY_BUDGET_MS of wall-clock time; once it is spent the rest
+   * stay queued for the next check.
+   */
+  async retryUndelivered(name, state, now, budget) {
+    const pending = state.undelivered ?? [];
+    if (pending.length === 0 || isSilenced(state, now) || this.deferDelivery) return [];
+    const delivered = [];
+    const failed = [];
+    const dropped = pending.filter((alert) => staleAlert(alert, state));
+    for (const alert of pending) {
+      if (dropped.includes(alert)) continue;
+      const left = RETRY_BUDGET_MS - budget.spentMs;
+      if (left <= 0) break;
+      const started = Date.now();
+      if (this.triage && alert.type !== "recovered" && alert.triage === void 0) await this.addTriage(alert, Math.min(TRIAGE_TIMEOUT_MS, left));
+      (await this.deliver(alert) ? delivered : failed).push(alert);
+      budget.spentMs += Math.max(0, Date.now() - started);
     }
-    if (JSON.stringify(evaluation.state) !== JSON.stringify(previous)) await this.store.setState(evaluation.state);
-    return this.dispatch(evaluation.alerts, definition, evaluation.state, now);
+    await this.recordDelivery(name, delivered, failed, dropped, now);
+    return delivered;
   }
-  async dispatch(drafts, definition, state, now) {
-    if (drafts.length === 0) return [];
-    const sent = [];
-    for (const draft of drafts) {
-      const alert = composeAlert(draft, definition, now);
-      if (this.triage && alert.type !== "recovered") {
-        try {
-          const recentRuns = await this.store.listRuns(alert.job, 5);
-          const diagnosis = await withTimeout(this.triage({ alert, recentRuns }), TRIAGE_TIMEOUT_MS);
-          if (diagnosis) alert.triage = diagnosis;
-        } catch (e) {
-          this.onError(e, `triage for ${alert.job}`);
-        }
+  /**
+   * Mark delivered alerts done, drop stale ones, and keep failed ones for the
+   * next check. A failed alert replaces its stored copy, so a triage made on
+   * this attempt is kept. lastAlertAt moves only on a delivery.
+   */
+  async recordDelivery(name, delivered, failed, dropped, now) {
+    try {
+      const { result: trimmed } = await this.updateState(name, (previous) => {
+        const state = normalizeState(previous, name);
+        const done = new Set([...delivered, ...dropped].map(alertKey));
+        const retried = new Map(failed.map((a) => [alertKey(a), a]));
+        const kept = state.undelivered.filter((a) => !done.has(alertKey(a))).map((a) => retried.get(alertKey(a)) ?? a);
+        const known = new Set(kept.map(alertKey));
+        kept.push(...failed.filter((a) => !known.has(alertKey(a))));
+        state.undelivered = kept.slice(-MAX_UNDELIVERED);
+        if (delivered.length > 0) state.lastAlertAt = now;
+        return { state, result: Math.max(0, kept.length - MAX_UNDELIVERED) };
+      });
+      if (trimmed > 0) {
+        this.onError(
+          new Error(`${trimmed} undelivered alert${trimmed === 1 ? "" : "s"} for ${name} dropped: only the newest ${MAX_UNDELIVERED} are kept for retry`),
+          `alert queue for ${name}`
+        );
       }
-      for (const channel of this.alerts) {
+    } catch (e) {
+      this.onError(e, `recording alert delivery for ${name}`);
+    }
+  }
+  /** Send to every channel at once. True when at least one accepted it, or there are none. */
+  async deliver(alert) {
+    if (this.alerts.length === 0) return true;
+    const results = await Promise.all(
+      this.alerts.map(async (channel) => {
         try {
-          await channel.send(alert);
+          await withTimeout(channel.send(alert), CHANNEL_TIMEOUT_MS);
+          return true;
         } catch (e) {
           this.onError(e, `alert channel ${channel.name}`);
+          return false;
         }
-      }
-      sent.push(alert);
+      })
+    );
+    return results.includes(true);
+  }
+  /** Sets alert.triage to the diagnosis, or to null when there is none, so it is tried once per alert. */
+  async addTriage(alert, timeout) {
+    const controller = new AbortController();
+    try {
+      const recentRuns = await this.store.listRuns(alert.job, 5);
+      const diagnosis = await withTimeout(this.triage({ alert, recentRuns, signal: controller.signal }), timeout);
+      alert.triage = typeof diagnosis === "string" && diagnosis !== "" ? diagnosis : null;
+    } catch (e) {
+      controller.abort();
+      alert.triage = null;
+      this.onError(e, `triage for ${alert.job}`);
     }
-    state.lastAlertAt = now;
-    await this.store.setState(state);
-    return sent;
   }
 };
+function isDevelopment() {
+  return process.env.NODE_ENV === "development" || process.env.NODE_ENV === "test";
+}
+function clampLimit(limit, fallback, min) {
+  const n = typeof limit === "number" && Number.isFinite(limit) ? Math.trunc(limit) : fallback;
+  return Math.min(500, Math.max(min, n));
+}
 function withTimeout(promise, ms) {
   return new Promise((resolve, reject) => {
     const t = setTimeout(() => reject(new Error(`timed out after ${ms}ms`)), ms);
+    if (typeof t === "object" && "unref" in t) t.unref();
     promise.then((v) => {
       clearTimeout(t);
       resolve(v);
@@ -1297,6 +1842,6 @@ function cronwatch(options = {}) {
   return new CronWatch(options);
 }
 
-export { CronWatch, composeAlert, consoleChannel, createRoutes, cronwatch, custom, formatDuration, formatRelative, json, memory, nextFire, parseDuration, parseSchedule, previousFire };
+export { CronWatch, composeAlert, consoleChannel, createRoutes, cronwatch, custom, formatDuration, memory, nextFire, parseDuration, parseSchedule };
 //# sourceMappingURL=index.js.map
 //# sourceMappingURL=index.js.map