@cronwatch/sdk 0.13.1 → 0.14.1

package/dist/index.js

@@ -172,6 +172,13 @@ function countFrom(from) {
   return from >= FIRST_DATE_MS ? from : FIRST_DATE_MS - 1;
 }
 function fireAfter(parsed, from) {
+  const found = croneFireAfter(parsed, from);
+  const start = countFrom(from);
+  if (start === null) return found;
+  const moved = gapFireAfter(parsed, start);
+  return moved !== null && (found === null || moved < found) ? moved : found;
+}
+function croneFireAfter(parsed, from) {
   const cron = crons.get(parsed);
   if (!cron) throw new Error(`schedule "${parsed.source}" was not made by parseSchedule`);
   if (never.has(parsed)) return null;
@@ -187,6 +194,43 @@ function fireAfter(parsed, from) {
   }
   return null;
 }
+function springGap(timezone, at) {
+  const jump = springJump(timezone, at);
+  return jump && at < jump.jumpAt + jump.gapMs ? jump : null;
+}
+var JUMP_LOOKBACK_MS = 3 * 36e5;
+function springJump(timezone, at) {
+  if (at - JUMP_LOOKBACK_MS < FIRST_DATE_MS || at >= LAST_DATE_MS) return null;
+  const after2 = utcOffset(at, timezone);
+  const before = utcOffset(at - JUMP_LOOKBACK_MS, timezone);
+  if (after2 <= before) return null;
+  let lo = at - JUMP_LOOKBACK_MS;
+  let hi = at;
+  while (hi - lo > 6e4) {
+    const mid = lo + Math.floor((hi - lo) / 2);
+    if (utcOffset(mid, timezone) === after2) hi = mid;
+    else lo = mid;
+  }
+  return { jumpAt: Math.floor(hi / 6e4) * 6e4, gapMs: after2 - before, beforeMs: before };
+}
+var wallCrons = /* @__PURE__ */ new WeakMap();
+function wallCron(parsed) {
+  let wall = wallCrons.get(parsed);
+  if (!wall) {
+    wall = new Cron(parsed.source, { timezone: "UTC" });
+    wallCrons.set(parsed, wall);
+  }
+  return wall;
+}
+function gapFireAfter(parsed, start) {
+  if (never.has(parsed)) return null;
+  const gap = springGap(parsed.timezone, start);
+  if (!gap) return null;
+  const wall = wallCron(parsed);
+  const [slot] = runsAfter(wall, 1, start + gap.beforeMs);
+  if (slot === void 0 || slot >= gap.jumpAt + gap.beforeMs + gap.gapMs) return null;
+  return slot - gap.beforeMs;
+}
 function firesBetween(parsed, from, to, limit) {
   const cron = crons.get(parsed);
   if (!cron) throw new Error(`schedule "${parsed.source}" was not made by parseSchedule`);
@@ -197,15 +241,34 @@ function firesBetween(parsed, from, to, limit) {
   let probe = start;
   let last = start;
   for (let guard = 0; guard < 1e3; guard++) {
+    if (springGap(parsed.timezone, last)) {
+      const t = fireAfter(parsed, last);
+      if (t === null || t > to) return out;
+      out.push(t);
+      last = probe = t;
+      if (out.length > limit) return null;
+      continue;
+    }
     const batch = runsAfter(cron, Math.min(limit + 1 - out.length, 24), probe);
     if (batch.length === 0) return out;
+    let stepped = false;
     for (const t of batch) {
       if (t <= last) continue;
+      if (springGap(parsed.timezone, t)) {
+        const next = fireAfter(parsed, last);
+        if (next === null || next > to) return out;
+        out.push(next);
+        last = probe = next;
+        if (out.length > limit) return null;
+        stepped = true;
+        break;
+      }
       if (t > to) return out;
       out.push(t);
       last = t;
       if (out.length > limit) return null;
     }
+    if (stepped) continue;
     const end = batch[batch.length - 1];
     probe = end > probe ? end : probe + 36e5;
   }
@@ -218,10 +281,14 @@ function nextFire(parsed, from, lastRunAt) {
   }
   return fireAfter(parsed, from);
 }
-function expectation(parsed, lastRunAt, registeredAt, graceMs2) {
+function intervalDue(everyMs, startedAt, finishedAt) {
+  const due = startedAt + everyMs;
+  return finishedAt !== null && finishedAt > due ? finishedAt + everyMs : due;
+}
+function expectation(parsed, lastRunAt, registeredAt, graceMs2, lastFinishedAt = null) {
   let dueAt;
   if (parsed.kind === "interval") {
-    dueAt = (lastRunAt ?? registeredAt) + parsed.everyMs;
+    dueAt = lastRunAt === null ? registeredAt + parsed.everyMs : intervalDue(parsed.everyMs, lastRunAt, lastFinishedAt);
   } else if (lastRunAt === null) {
     dueAt = fireAfter(parsed, registeredAt - 1);
   } else {
@@ -230,33 +297,25 @@ function expectation(parsed, lastRunAt, registeredAt, graceMs2) {
   return dueAt === null ? null : { dueAt, deadline: dueAt + graceMs2 };
 }
 function dueAfterRun(parsed, startedAt) {
-  const next = fireAfter(parsed, startedAt);
-  if (next === null) return null;
-  const following = fireAfter(parsed, next);
-  const covers = runCovers(startedAt, next, following) || inSpringForwardGap(parsed, startedAt, next);
-  return covers ? following : next;
+  let next = fireAfter(parsed, startedAt);
+  for (let guard = 0; next !== null && guard < 1e5; guard++) {
+    const following = fireAfter(parsed, next);
+    const covers = runCovers(startedAt, next, following) || inSpringForwardGap(parsed, startedAt, next);
+    if (!covers) return next;
+    next = following;
+  }
+  return next;
 }
 function runCovers(startedAt, dueAt, followingAt = null) {
   const slack = followingAt === null ? EARLY_SLACK_MS : Math.min(EARLY_SLACK_MS, Math.floor((followingAt - dueAt) / 2));
   return startedAt >= dueAt - slack;
 }
 function inSpringForwardGap(parsed, startedAt, fireAt) {
-  const LOOKBACK = 3 * 36e5;
-  if (fireAt - LOOKBACK < FIRST_DATE_MS) return false;
-  const after2 = utcOffset(fireAt, parsed.timezone);
-  const before = utcOffset(fireAt - LOOKBACK, parsed.timezone);
-  const gap = after2 - before;
-  if (gap <= 0) return false;
-  let lo = fireAt - LOOKBACK;
-  let hi = fireAt;
-  while (hi - lo > 6e4) {
-    const mid = lo + Math.floor((hi - lo) / 2);
-    if (utcOffset(mid, parsed.timezone) === after2) hi = mid;
-    else lo = mid;
-  }
-  const jumpAt = Math.floor(hi / 6e4) * 6e4;
-  if (fireAt - jumpAt >= gap || startedAt < jumpAt - EARLY_SLACK_MS || startedAt >= fireAt) return false;
-  return fireAfter(parsed, jumpAt - 1) === fireAt;
+  const gap = springGap(parsed.timezone, fireAt);
+  if (!gap || startedAt < gap.jumpAt - EARLY_SLACK_MS || startedAt >= fireAt) return false;
+  const wall = wallCron(parsed);
+  const fires = (at) => runsAfter(wall, 1, at - 1)[0] === at;
+  return fires(fireAt + gap.beforeMs) && !fires(fireAt + gap.beforeMs + gap.gapMs);
 }
 var formatters = /* @__PURE__ */ new Map();
 function utcOffset(at, timezone) {
@@ -353,15 +412,75 @@ var SEND_LEASE_MS = 5 * 6e4;
 function alertKey(alert) {
   return `${alert.type}|${alert.at}|${alert.run?.id ?? ""}`;
 }
+var HELD_TEXT_CAP = 2048;
+var HELD_TEXT_SLACK = 64;
+function heldRun(run) {
+  const error = typeof run.error === "string" && run.error.length > HELD_TEXT_CAP + HELD_TEXT_SLACK ? keepHead(run.error) : run.error;
+  const output = typeof run.output === "string" && run.output.length > HELD_TEXT_CAP + HELD_TEXT_SLACK ? keepTail(run.output) : run.output;
+  return error === run.error && output === run.output ? run : { ...run, error, output };
+}
+var HELD_MESSAGE_CAP = 4096;
+function keepHead(text, cap = HELD_TEXT_CAP) {
+  let end = cap;
+  if (isHighSurrogate(text.charCodeAt(end - 1))) end -= 1;
+  return `${text.slice(0, end)}
+[${text.length - end} more characters trimmed]`;
+}
+function keepTail(text) {
+  let start = text.length - HELD_TEXT_CAP;
+  if (isLowSurrogate(text.charCodeAt(start)) && isHighSurrogate(text.charCodeAt(start - 1))) start += 1;
+  return `[${start} earlier characters trimmed]
+${text.slice(start)}`;
+}
+function isHighSurrogate(code) {
+  return code >= 55296 && code <= 56319;
+}
+function isLowSurrogate(code) {
+  return code >= 56320 && code <= 57343;
+}
+function heldAlert(alert) {
+  const run = alert.run;
+  const trimmedRun = isObject(run) ? heldRun(run) : run;
+  const message = alert.message;
+  const trimmedMessage = typeof message === "string" && message.length > HELD_MESSAGE_CAP + HELD_TEXT_SLACK ? keepHead(message, HELD_MESSAGE_CAP) : message;
+  if (trimmedRun === run && trimmedMessage === message) return alert;
+  return { ...alert, run: trimmedRun, message: trimmedMessage };
+}
+function channelKeys(names) {
+  const seen = /* @__PURE__ */ new Map();
+  const taken = /* @__PURE__ */ new Set();
+  return names.map((name) => {
+    let n = (seen.get(name) ?? 0) + 1;
+    seen.set(name, n);
+    let key = n === 1 ? name : `${name}#${n}`;
+    while (taken.has(key)) key = `${name}#${++n}`;
+    taken.add(key);
+    return key;
+  });
+}
+function owedChannels(alert, configured) {
+  const owed = alert.owed;
+  if (!Array.isArray(owed)) return [...configured];
+  return configured.filter((key) => owed.includes(key));
+}
+function afterSend(alert, tried, failed) {
+  const owed = tried.filter((key) => failed.includes(key));
+  return owed.length === 0 ? null : { ...alert, owed };
+}
+function withoutOwed(alert) {
+  if (!("owed" in alert)) return alert;
+  const { owed: _owed, ...rest } = alert;
+  return rest;
+}
 function newest(list, max) {
   return { kept: list.slice(-max), dropped: Math.max(0, list.length - max) };
 }
 function queueUndelivered(state, alerts) {
   const next = cloneState(state);
-  const byKey = new Map(alerts.map((alert) => [alertKey(alert), alert]));
+  const byKey = new Map(alerts.map((alert) => [alertKey(alert), heldAlert(alert)]));
   const queue = next.undelivered.map((alert) => byKey.get(alertKey(alert)) ?? alert);
   const known = new Set(queue.map(alertKey));
-  queue.push(...alerts.filter((alert) => !known.has(alertKey(alert))));
+  queue.push(...[...byKey.values()].filter((alert) => !known.has(alertKey(alert))));
   const { kept, dropped } = newest(queue, MAX_UNDELIVERED);
   next.undelivered = kept;
   return { state: next, dropped };
@@ -370,7 +489,7 @@ function holdAlerts(state, alerts, until, deferred) {
   if (alerts.length === 0) return { state, dropped: 0 };
   if (deferred) return queueUndelivered(state, alerts);
   const next = cloneState(state);
-  const { kept, dropped } = newest([...next.sending ?? [], ...alerts.map((alert) => ({ until, alert }))], MAX_UNDELIVERED);
+  const { kept, dropped } = newest([...next.sending ?? [], ...alerts.map((alert) => ({ until, alert: heldAlert(alert) }))], MAX_UNDELIVERED);
   next.sending = kept;
   return { state: next, dropped };
 }
@@ -384,7 +503,7 @@ function releaseSending(state, now) {
     lapsed.filter((entry) => typeof entry?.alert === "object" && entry.alert !== null).map((entry) => entry.alert)
   );
 }
-function recordSent(state, delivered, failed, stale, now) {
+function recordSent(state, delivered, failed, stale, now, reached = delivered.length > 0) {
   const next = cloneState(state);
   const done = new Set([...delivered, ...stale].map(alertKey));
   next.undelivered = next.undelivered.filter((alert) => !done.has(alertKey(alert)));
@@ -392,7 +511,7 @@ function recordSent(state, delivered, failed, stale, now) {
   const held = (next.sending ?? []).filter((entry) => !(typeof entry?.alert === "object" && entry.alert !== null && sent.has(alertKey(entry.alert))));
   if (held.length > 0) next.sending = held;
   else delete next.sending;
-  if (delivered.length > 0) next.lastAlertAt = now;
+  if (reached) next.lastAlertAt = now;
   return queueUndelivered(next, failed);
 }
 function claimUndelivered(state, now, until) {
@@ -500,9 +619,21 @@ function formatNumber(n) {
   if (Number.isInteger(n)) return n.toLocaleString("en-US");
   return n.toLocaleString("en-US", { maximumFractionDigits: 4 });
 }
-function onRunStart(state) {
+function finishedAtOf(run) {
+  return run.status !== "running" && typeof run.finishedAt === "number" ? run.finishedAt : null;
+}
+function catchesUp(def, run, now) {
+  if (!def.schedule) return true;
+  try {
+    const exp = expectation(parseSchedule(def.schedule, def.timezone), run.startedAt, run.startedAt, graceMs(def), finishedAtOf(run));
+    return exp === null || now <= exp.deadline;
+  } catch {
+    return true;
+  }
+}
+function onRunStart(def, run, state, now) {
   const next = cloneState(state);
-  closeCondition(next, "missed");
+  if (catchesUp(def, run, now)) closeCondition(next, "missed");
   closeCondition(next, "stuck");
   return next;
 }
@@ -511,7 +642,7 @@ function onRunFinish(def, run, state, history, now) {
   const alerts = [];
   if (run.status === "ok") {
     next.consecutiveFailures = 0;
-    closeCondition(next, "missed");
+    if (catchesUp(def, run, now)) closeCondition(next, "missed");
     closeCondition(next, "stuck");
     closeCondition(next, "failed");
     const slow = slowThreshold(def, history);
@@ -548,7 +679,7 @@ function onRunFinish(def, run, state, history, now) {
     return { state: next, alerts };
   }
   next.consecutiveFailures = Math.min(next.consecutiveFailures + 1, MAX_DURATION_MS);
-  closeCondition(next, "missed");
+  if (catchesUp(def, run, now)) closeCondition(next, "missed");
   const threshold = Math.max(1, def.failuresBeforeAlert ?? 1);
   const condition = run.status === "timeout" ? "stuck" : "failed";
   if (next.consecutiveFailures >= threshold) {
@@ -573,8 +704,8 @@ function onCheck(def, stored, lastRun, state, now) {
   const parsed = parseSchedule(def.schedule, def.timezone);
   const grace = graceMs(def);
   const lastRunAt = lastRun?.startedAt ?? null;
-  const exp = expectation(parsed, lastRunAt, stored.createdAt, grace);
-  const nextExpectedAt = parsed.kind === "interval" ? nextFire(parsed, stored.createdAt, lastRunAt) : nextFire(parsed, now, null);
+  const exp = expectation(parsed, lastRunAt, stored.createdAt, grace, lastRun ? finishedAtOf(lastRun) : null);
+  const nextExpectedAt = parsed.kind === "interval" ? exp.dueAt : nextFire(parsed, now, null);
   if (!exp) return { state: next, alerts, nextExpectedAt, dueAt: null };
   if (parsed.kind === "interval" && lastRun?.status === "running") {
     return { state: next, alerts, nextExpectedAt, dueAt: exp.dueAt };
@@ -860,11 +991,15 @@ function stringify(part) {
   }
 }
 function createRecorder(run) {
+  const window = OUTPUT_CAP + REDACT_EDGE + 2;
+  const copy = (text) => JSON.parse(JSON.stringify(text));
   const lines = [];
   let size = 0;
   const head = [];
   let headSize = 0;
-  let dropped = false;
+  let whole = "";
+  let tail2 = "";
+  let logged = false;
   const metrics = {};
   const controller = new AbortController();
   const context = {
@@ -873,16 +1008,23 @@ function createRecorder(run) {
     startedAt: run.startedAt,
     signal: controller.signal,
     log(...parts) {
-      const line = parts.map(stringify).join(" ");
+      let line = parts.map(stringify).join(" ");
+      const separator = logged ? "\n" : "";
+      logged = true;
+      if (whole !== null) whole = whole.length + separator.length + line.length <= 2 * OUTPUT_CAP ? whole + separator + line : null;
+      tail2 = copy((tail2 + separator + copy(line.slice(-OUTPUT_CAP))).slice(-OUTPUT_CAP));
       if (headSize < OUTPUT_CAP) {
-        head.push(line);
+        head.push(copy(line.slice(0, OUTPUT_CAP - headSize)));
         headSize += line.length + 1;
       }
+      line = stripNul(line);
+      if (line.length >= window) {
+        line = copy(line.slice(-68609));
+      }
       lines.push(line);
       size += line.length + 1;
-      while (size > 64 * 1024 && lines.length > 1) {
+      while (size > window && lines.length > 1) {
         size -= lines.shift().length + 1;
-        dropped = true;
       }
     },
     metric(name, value) {
@@ -899,10 +1041,8 @@ function createRecorder(run) {
     context,
     output: () => lines.length === 0 ? null : lines.join("\n"),
     expectText: () => {
-      if (lines.length === 0) return null;
-      const all = lines.join("\n");
-      if (!dropped && all.length <= 2 * OUTPUT_CAP) return all;
-      return head.join("\n").slice(0, OUTPUT_CAP) + "\n" + all.slice(-OUTPUT_CAP);
+      if (!logged) return null;
+      return whole ?? head.join("\n").slice(0, OUTPUT_CAP) + "\n" + tail2;
     },
     metrics: () => ({ ...metrics }),
     abort: () => controller.abort()
@@ -1076,7 +1216,7 @@ function staticAsset(path, base) {
 }
 
 // src/version.ts
-var VERSION = "0.13.1";
+var VERSION = "0.14.1";
 var API_VERSION = 1;
 
 // src/routes/timeline.ts
@@ -1129,13 +1269,13 @@ function dueTimes(job, parsed, runs, from, to) {
   if (parsed.kind === "interval") {
     const every = parsed.everyMs;
     if ((to - from) / every > MAX_TICKS) return { times: [], dense: true };
-    const starts = runs.map((r) => r.startedAt).sort((a, b) => a - b);
+    const ordered = [...runs].sort((a, b) => a.startedAt - b.startedAt);
+    const dues = ordered.map((r) => intervalDue(every, r.startedAt, finishedAtOf(r)));
     const times2 = /* @__PURE__ */ new Set();
-    for (const start of starts) {
-      const t2 = start + every;
+    for (const t2 of dues) {
       if (t2 >= from && t2 <= to) times2.add(t2);
     }
-    let t = starts.length ? starts[starts.length - 1] + every : job.nextExpectedAt;
+    let t = dues.length ? dues[dues.length - 1] : job.nextExpectedAt;
     if (t !== null) {
       if (t < from) t += Math.ceil((from - t) / every) * every;
       for (; t <= to; t += every) times2.add(t);
@@ -1148,8 +1288,8 @@ function dueTimes(job, parsed, runs, from, to) {
 function missedAt(job, parsed, times, now) {
   if (!parsed || !job.open.includes("missed")) return null;
   const grace = safely(() => graceMs(job.definition), 0);
-  const last = job.lastRun?.startedAt ?? null;
-  if (last !== null) return safely(() => expectation(parsed, last, last, grace)?.dueAt ?? null, null);
+  const lastRun = job.lastRun;
+  if (lastRun) return safely(() => expectation(parsed, lastRun.startedAt, lastRun.startedAt, grace, finishedAtOf(lastRun))?.dueAt ?? null, null);
   const past = times.filter((t) => t + grace < now);
   return past.length ? past[past.length - 1] : null;
 }
@@ -1858,13 +1998,15 @@ function signInReturn(referer, publicOrigin, base) {
   return url.searchParams.has("token") ? `${base}/` : referer;
 }
 function stripBase(pathname, base) {
-  let path = pathname.startsWith(base) ? pathname.slice(base.length) : pathname;
+  if (base !== "" && pathname !== base && !pathname.startsWith(`${base}/`)) return null;
+  let path = pathname.slice(base.length);
   if (path === "") path = "/";
   if (path.length > 1 && path.endsWith("/")) path = path.slice(0, -1);
   return path;
 }
 var MAX_REQUEST_BODY = 1 << 20;
 var TOO_LARGE = /* @__PURE__ */ Symbol("too large");
+var BAD_BODY = /* @__PURE__ */ Symbol("malformed body");
 async function readLimited(request) {
   const length = Number(request.headers.get("content-length"));
   if (Number.isFinite(length) && length > MAX_REQUEST_BODY) return TOO_LARGE;
@@ -1911,14 +2053,14 @@ async function readBody(request) {
   try {
     if (json2) {
       const data = await copy.json();
-      return typeof data === "object" && data !== null ? Object.fromEntries(Object.entries(data).map(([k, v]) => [k, String(v)])) : {};
+      return typeof data === "object" && data !== null && !Array.isArray(data) ? Object.fromEntries(Object.entries(data).map(([k, v]) => [k, String(v)])) : BAD_BODY;
     }
     const fields = await copy.formData();
     const out = {};
     for (const [k, v] of fields.entries()) out[k] = String(v);
     return out;
   } catch {
-    return {};
+    return BAD_BODY;
   }
 }
 function crossSite(request, publicOrigin) {
@@ -1984,6 +2126,7 @@ function buildRoutes(cw, options = {}) {
     const signInPage = () => generated ? html(messagePage("Sign in", "CRONWATCH_TOKEN is not set, so this development server made a token. The sign-in link is in the server log: open it once, or enter the token from it below, and this browser stays signed in.", base, true), 401) : html(messagePage("Sign in", "Enter your CRONWATCH_TOKEN and this browser stays signed in.", base, true), 401);
     if (token && method === "POST" && path === "/signin") {
       const body = await readBody(request);
+      if (body === BAD_BODY) return html(messagePage("Bad request", "Malformed request body", base), 400);
       const sent = body === TOO_LARGE ? void 0 : body.token;
       if (sent === void 0 || !constantTimeEqual(sent, token)) return signInPage();
       return redirect(signInReturn(request.headers.get("referer"), publicOrigin, base), { "set-cookie": await signInCookie() });
@@ -2041,6 +2184,7 @@ function buildRoutes(cw, options = {}) {
       if (parts[2] === "silence") {
         const body = await readBody(request);
         if (body === TOO_LARGE) return html(messagePage("Not silenced", "The request was too large.", base), 413);
+        if (body === BAD_BODY) return html(messagePage("Not silenced", "Malformed request body", base), 400);
         let duration;
         try {
           duration = silenceDuration(body.for);
@@ -2078,6 +2222,7 @@ function buildRoutes(cw, options = {}) {
         if (rest[2] === "silence") {
           const body = await readBody(request);
           if (body === TOO_LARGE) return api({ ok: false, error: "Request body too large" }, 413);
+          if (body === BAD_BODY) return api({ ok: false, error: "Malformed request body" }, 400);
           let duration;
           try {
             duration = silenceDuration(body.for ?? url.searchParams.get("for"));
@@ -2114,6 +2259,7 @@ function buildRoutes(cw, options = {}) {
     try {
       const url = new URL(request.url);
       const path = stripBase(url.pathname, base);
+      if (path === null) return html(messagePage("Not found", url.pathname, base), 404);
       wantsHtml = !path.startsWith("/api");
       return await serve(request, url, path, wantsHtml);
     } catch (e) {
@@ -2468,7 +2614,7 @@ var Cronwatch = class {
   warnNoSecret() {
     if (this.warnedNoSecret) return;
     this.warnedNoSecret = true;
-    this.onError(new Error("handler() refused a request because no CRON_SECRET is set; pass secret: null to allow unauthenticated requests"), "handler");
+    this.report(new Error("handler() refused a request because no CRON_SECRET is set; pass secret: null to allow unauthenticated requests"), "handler");
   }
   /** onError, for places that must carry on even when onError itself throws. */
   report(error, where) {
@@ -2481,6 +2627,13 @@ var Cronwatch = class {
     if (!this.ready) {
       this.ready = (async () => {
         if (this.store.init) await this.store.init();
+        const missing = ["compareAndSetState", "updateRunIf"].filter((method) => typeof this.store[method] !== "function");
+        if (missing.length > 0) {
+          this.report(
+            new Error(`the store has no ${missing.join(" or ")}, so two processes sharing it can overwrite each other's state or both evaluate the same run; it is safe only while one process at a time uses it`),
+            "store"
+          );
+        }
         if (this.usingDefaultStore && isProduction()) {
           console.warn("[cronwatch] using the in-memory store: runs and state are lost on restart. Pass a store from @cronwatch/sdk/sqlite or @cronwatch/sdk/postgres.");
         }
@@ -2628,10 +2781,11 @@ var Cronwatch = class {
       await this.store.insertRun({ ...run });
       recorded = true;
     } catch (e) {
-      this.onError(e, `recording ${name}`);
+      this.report(e, `recording ${name}`);
     }
-    const started = recorded ? this.updateState(name, (before) => ({ state: holdRun(onRunStart(before), run.id), result: void 0 })).then(() => {
-    }, (e) => this.onError(e, `starting ${name}`)) : Promise.resolve();
+    const begun = { ...run };
+    const started = recorded ? this.updateState(name, (before) => ({ state: holdRun(onRunStart(definition, begun, before, this.now()), run.id), result: void 0 })).then(() => {
+    }, (e) => this.report(e, `starting ${name}`)) : Promise.resolve();
     const recorder = createRecorder(run);
     const clearTimer = after(timeoutMs(definition), () => recorder.abort());
     let result;
@@ -2656,7 +2810,7 @@ var Cronwatch = class {
       const ignored = await this.recordFinish(definition, run, recorded, finishedAt);
       if (ignored) this.report(new Error(`run ${run.id} of ${name} ${ignored}; ignored`), `finishing ${name}`);
     } catch (e) {
-      this.onError(e, `recording ${name}`);
+      this.report(e, `recording ${name}`);
     }
     return { run, result, error, threw };
   }
@@ -2704,9 +2858,10 @@ var Cronwatch = class {
         if (stored.job !== run.job) return `belongs to job "${stored.job}"`;
       }
     }
+    const tracked = (await this.readState(run.job)).pendingRuns?.includes(run.id) ?? false;
     const claim = await this.claimFinish(run);
     if ("ignored" in claim) return claim.ignored;
-    if (!claim.lateAfterTimeout || run.status === "ok") await this.finishRun(toStored(definition), run, finishedAt);
+    if (!claim.lateAfterTimeout || run.status === "ok") await this.finishRun(toStored(definition), run, finishedAt, tracked && !claim.lateAfterTimeout);
     return null;
   }
   /** A conditional write (Store.updateRunIf), or for a store without one, a read then a plain write. */
@@ -2759,7 +2914,7 @@ ${id}`;
         await this.ensureReady();
         stored = await this.store.getRun(id);
       } catch (e) {
-        this.onError(e, `recording ${name}`);
+        this.report(e, `recording ${name}`);
       }
       if (stored) return this.existingHandle(definition, stored);
     }
@@ -2783,10 +2938,10 @@ ${id}`;
     } catch (e) {
       const stored = id === void 0 ? null : await this.store.getRun(id).catch(() => null);
       if (stored) return this.existingHandle(definition, stored);
-      this.onError(e, `recording ${name}`);
+      this.report(e, `recording ${name}`);
     }
     if (recorded) {
-      await this.updateState(name, (before) => ({ state: holdRun(onRunStart(before), run.id), result: void 0 })).catch((e) => this.onError(e, `starting ${name}`));
+      await this.updateState(name, (before) => ({ state: holdRun(onRunStart(definition, run, before, this.now()), run.id), result: void 0 })).catch((e) => this.report(e, `starting ${name}`));
     }
     return this.runHandle(definition, run.id, run, recorded, null);
   }
@@ -2798,7 +2953,7 @@ ${id}`;
       await this.ensureReady();
       stored = await this.store.getRun(runId);
     } catch (e) {
-      this.onError(e, `resuming ${definition.name}`);
+      this.report(e, `resuming ${definition.name}`);
       return this.runHandle(definition, runId, null, true, null);
     }
     if (!stored) return this.runHandle(definition, runId, null, true, "was not found");
@@ -2999,9 +3154,9 @@ ${id}`;
       throw e;
     }
     if (!evaluate) return [];
-    await this.updateState(run.job, (before) => ({ state: holdRun(onRunStart(before), run.id), result: void 0 }));
+    await this.updateState(run.job, (before) => ({ state: holdRun(onRunStart(definition, run, before, this.now()), run.id), result: void 0 }));
     if (run.status === "running") return [];
-    return this.finishRun(definition, run, this.now());
+    return this.finishRun(definition, run, this.now(), true);
   }
   /** recordRun() for a run already stored. */
   async recordOver(definition, stored, run, evaluate) {
@@ -3010,6 +3165,7 @@ ${id}`;
       return [];
     }
     if (stored.status !== "running" && stored.status !== "timeout" || run.status === "running") return [];
+    const tracked = (await this.readState(run.job)).pendingRuns?.includes(run.id) ?? false;
     const claim = await this.claimFinish(run);
     if ("ignored" in claim) {
       this.report(new Error(`run ${run.id} of ${run.job} ${claim.ignored}; ignored`), `recording ${run.job}`);
@@ -3020,24 +3176,25 @@ ${id}`;
       return [];
     }
     if (claim.lateAfterTimeout && run.status !== "ok") return [];
-    return this.finishRun(definition, run, this.now());
+    return this.finishRun(definition, run, this.now(), tracked && !claim.lateAfterTimeout);
   }
   /**
    * Evaluate a finished run (ok, failed, or timed out by a check), already
    * written, against the job's state and send what that produces. The
    * alerts are written with that state (see outbox()). Never throws.
    */
-  async finishRun(definition, run, now) {
+  async finishRun(definition, run, now, pendingOnly = false) {
     let held;
     try {
       let history = null;
       ({ result: held } = await this.updateState(run.job, async (previous) => {
+        if (pendingOnly && !(previous.pendingRuns ?? []).includes(run.id)) return { state: previous, result: { alerts: [], dropped: 0 } };
         history ??= await this.history(run);
         const before = releaseRun(previous, run.id);
         return this.outbox(applySilence(before, onRunFinish(definition, run, before, history, now), now), definition, now);
       }));
     } catch (e) {
-      this.onError(e, `evaluating ${run.job}`);
+      this.report(e, `evaluating ${run.job}`);
       return [];
     }
     this.reportDropped(run.job, held.dropped);
@@ -3076,7 +3233,7 @@ ${id}`;
   }
   /**
    * Look for missed and stuck runs across every job, send alerts, retry
-   * alerts no channel accepted, and prune old runs. Call it from an interval
+   * alerts to the channels that refused them, and prune old runs. Call it from an interval
    * (start()), a cron hitting the mounted routes, or by hand. Concurrent
    * calls share one check.
    */
@@ -3176,7 +3333,7 @@ ${id}`;
           continue;
         }
         if (run.status === "running" || typeof run.finishedAt !== "number" || run.finishedAt > now - SEND_LEASE_MS) continue;
-        alerts.push(...await this.finishRun(stored.definition, run, now));
+        alerts.push(...await this.finishRun(stored.definition, run, now, true));
       } catch (e) {
         this.onError(e, `evaluating run ${id} of ${stored.name}`);
       }
@@ -3296,7 +3453,7 @@ ${id}`;
       this.warnedDeferredStart = true;
       console.warn('[cronwatch] startChecking() was called with deliver: "check", so these checks send no alerts. Another process must run checks with deliver: "now" (the default) to send them.');
     }
-    const tick = () => this.check().catch((e) => this.onError(e, "check"));
+    const tick = () => this.check().catch((e) => this.report(e, "check"));
     this.timer = setInterval(tick, ms);
     if (typeof this.timer === "object" && "unref" in this.timer) this.timer.unref();
     this.firstTick = setTimeout(() => {
@@ -3345,21 +3502,28 @@ ${id}`;
     if (alerts.length === 0 || this.deferDelivery) return alerts;
     const delivered = [];
     const failed = [];
+    let reached = false;
     for (const alert of alerts) {
       if (this.triage && alert.type !== "recovered") await this.addTriage(alert, TRIAGE_TIMEOUT_MS);
-      (await this.deliver(alert) ? delivered : failed).push(alert);
+      const sent = await this.deliver(alert);
+      reached ||= sent.reached;
+      if (sent.left) failed.push(sent.left);
+      else delivered.push(alert);
     }
-    await this.recordDelivery(name, delivered, failed, [], now);
+    await this.recordDelivery(name, delivered, failed, [], now, reached);
     return alerts;
   }
   /**
-   * Send the alerts that no channel accepted last time, once each, oldest
-   * first. They are claimed first (claimUndelivered): moved into `sending`
-   * with a lease, in one conditional write, so when several processes check
-   * at once only the one whose write lands sends them. An alert that no
-   * longer describes the job (staleAlert) is dropped instead. Retries across
-   * a check share RETRY_BUDGET_MS of wall-clock time; claimed alerts it has
-   * no time left for go back to the queue for the next check.
+   * Send the alerts some channel refused last time, once each, oldest
+   * first, to the channels that still owe them (HeldAlert.owed) and no
+   * others, from the copy held in state, its run trimmed (heldRun). They
+   * are claimed first (claimUndelivered): moved into `sending` with a lease,
+   * in one conditional write, so when several processes check at once only
+   * the one whose write lands sends them. An alert that no longer describes
+   * the job (staleAlert) is dropped instead. Retries across a check share
+   * RETRY_BUDGET_MS of wall-clock time; claimed alerts it has no time left
+   * for go back to the queue for the next check. Returns the alerts some
+   * channel took on this attempt.
    */
   async retryUndelivered(name, state, now, budget) {
     if ((state.undelivered ?? []).length === 0 || this.deferDelivery) return [];
@@ -3370,12 +3534,13 @@ ${id}`;
         return { state: claim.state, result: claim.claimed };
       }));
     } catch (e) {
-      this.onError(e, `claiming undelivered alerts for ${name}`);
+      this.report(e, `claiming undelivered alerts for ${name}`);
       return [];
     }
     if (claimed.length === 0) return [];
     const delivered = [];
     const failed = [];
+    const sent = [];
     for (const alert of claimed) {
       const left = RETRY_BUDGET_MS - budget.spentMs;
       if (left <= 0) {
@@ -3384,45 +3549,61 @@ ${id}`;
       }
       const started = Date.now();
       if (this.triage && alert.type !== "recovered" && alert.triage === void 0) await this.addTriage(alert, Math.min(TRIAGE_TIMEOUT_MS, left));
-      (await this.deliver(alert) ? delivered : failed).push(alert);
+      const outcome = await this.deliver(alert);
+      if (outcome.reached) sent.push(withoutOwed(alert));
+      if (outcome.left) failed.push(outcome.left);
+      else delivered.push(alert);
       budget.spentMs += Math.max(0, Date.now() - started);
     }
-    await this.recordDelivery(name, delivered, failed, [], now);
-    return delivered;
+    await this.recordDelivery(name, delivered, failed, [], now, sent.length > 0);
+    return sent;
   }
   /**
    * Mark delivered alerts done, drop stale ones, and keep failed ones for the
    * next check, taking them all out of `sending` (recordSent). A failed alert
-   * replaces its stored copy, so a triage made on this attempt is kept.
-   * lastAlertAt moves only on a delivery. When this write fails, alerts
-   * still in `sending` are retried once their lease runs out.
+   * replaces its stored copy, so a triage made on this attempt and the
+   * channels that still owe it are kept. lastAlertAt moves when `reached`,
+   * some channel accepted an alert on this attempt. When this write fails,
+   * alerts still in `sending` are retried once their lease runs out.
    */
-  async recordDelivery(name, delivered, failed, dropped, now) {
+  async recordDelivery(name, delivered, failed, dropped, now, reached) {
     try {
       const { result: trimmed } = await this.updateState(name, (previous) => {
-        const { state, dropped: letGo } = recordSent(normalizeState(previous, name), delivered, failed, dropped, now);
+        const { state, dropped: letGo } = recordSent(normalizeState(previous, name), delivered, failed, dropped, now, reached);
         return { state, result: letGo };
       });
       this.reportDropped(name, trimmed);
     } catch (e) {
-      this.onError(e, `recording alert delivery for ${name}`);
+      this.report(e, `recording alert delivery for ${name}`);
     }
   }
-  /** Send to every channel at once. True when at least one accepted it, or there are none. */
+  /**
+   * Send to every channel that still owes the alert (owedChannels: all of
+   * them for an alert not sent before), at once, each failure reported under
+   * its channel's name. `left` is the alert with `owed` cut to the channels
+   * that refused it, or null when none did (or none is left to owe it);
+   * `reached` is whether any channel accepted it. With no channels at all it
+   * counts as delivered, as it always has.
+   */
   async deliver(alert) {
-    if (this.alerts.length === 0) return true;
-    const results = await Promise.all(
-      this.alerts.map(async (channel) => {
+    if (this.alerts.length === 0) return { left: null, reached: true };
+    const keys = channelKeys(this.alerts.map((channel) => channel.name));
+    const owed = owedChannels(alert, keys);
+    if (owed.length === 0) return { left: null, reached: false };
+    const plain = withoutOwed(alert);
+    const failed = [];
+    await Promise.all(
+      this.alerts.map(async (channel, i) => {
+        if (!owed.includes(keys[i])) return;
         try {
-          await withTimeout(channel.send(alert, { onError: (e) => this.report(e, `alert channel ${channel.name}`) }), CHANNEL_TIMEOUT_MS);
-          return true;
+          await withTimeout(channel.send(plain, { onError: (e) => this.report(e, `alert channel ${channel.name}`) }), CHANNEL_TIMEOUT_MS);
         } catch (e) {
-          this.onError(e, `alert channel ${channel.name}`);
-          return false;
+          failed.push(keys[i]);
+          this.report(e, `alert channel ${channel.name}`);
         }
       })
     );
-    return results.includes(true);
+    return { left: afterSend(alert, owed, failed), reached: failed.length < owed.length };
   }
   /** Sets alert.triage to the diagnosis, or to null when there is none, so it is tried once per alert. */
   async addTriage(alert, timeout) {
@@ -3434,7 +3615,7 @@ ${id}`;
     } catch (e) {
       controller.abort();
       alert.triage = null;
-      this.onError(e, `triage for ${alert.job}`);
+      this.report(e, `triage for ${alert.job}`);
     }
   }
 };