@cronwatch/sdk 0.3.0 → 0.4.0

package/dist/index.js

@@ -1,5 +1,4 @@
 import { Cron } from 'croner';
-import { createHash } from 'crypto';
 
 // src/duration.ts
 var UNIT_MS = {
@@ -53,6 +52,16 @@ function formatRelative(at, now) {
   if (abs < 5e3) return text;
   return diff < 0 ? `${text} ago` : `in ${text}`;
 }
+
+// src/env.ts
+function readEnv(name) {
+  return typeof process === "undefined" || !process.env ? void 0 : process.env[name];
+}
+function isDevelopment() {
+  if (typeof process === "undefined" || !process.env) return false;
+  const mode = process.env.NODE_ENV;
+  return mode === "development" || mode === "test";
+}
 var cache = /* @__PURE__ */ new Map();
 var crons = /* @__PURE__ */ new WeakMap();
 var EARLY_SLACK_MS = 6e4;
@@ -305,7 +314,15 @@ function onRunFinish(def, run, state, history, now) {
 function onCheck(def, stored, lastRun, state, now) {
   const next = cloneState(state);
   const alerts = [];
-  if (!def.schedule) return { state: next, alerts, nextExpectedAt: null, dueAt: null };
+  if (!def.schedule) {
+    const since = next.open.missed;
+    if (since !== void 0) {
+      delete next.open.missed;
+      next.pendingRecovery = (next.pendingRecovery ?? []).filter((c) => c !== "missed");
+      alerts.push({ type: "recovered", run: lastRun, details: { after: ["missed"], reason: "unscheduled", since } });
+    }
+    return { state: next, alerts, nextExpectedAt: null, dueAt: null };
+  }
   const parsed = parseSchedule(def.schedule, def.timezone);
   const grace = graceMs(def);
   const lastRunAt = lastRun?.startedAt ?? null;
@@ -337,6 +354,14 @@ function muteOpens(previous, next) {
 function isSilenced(state, now) {
   return state.silencedUntil !== null && state.silencedUntil > now;
 }
+function applySilence(previous, evaluation, now) {
+  if (!isSilenced(previous, now)) return evaluation;
+  return { state: muteOpens(previous, evaluation.state), alerts: [] };
+}
+function staleAlert(alert, state) {
+  if (alert.type === "recovered") return alert.details.after.some((condition) => state.open[condition] !== void 0);
+  return state.open[alert.type] !== alert.at;
+}
 function jobHealth(def, lastRun, state, now) {
   const open = openConditions(state);
   if (isSilenced(state, now)) return "silenced";
@@ -347,6 +372,12 @@ function jobHealth(def, lastRun, state, now) {
   return "healthy";
 }
 function summarize(stored, recent, state, nextExpectedAt, now) {
+  return summary(stored, recent, state, nextExpectedAt, (lastRun) => jobHealth(stored.definition, lastRun, state, now));
+}
+function unevaluableSummary(stored, recent, state, now) {
+  return summary(stored, recent, state, null, () => isSilenced(state, now) ? "silenced" : "failing");
+}
+function summary(stored, recent, state, nextExpectedAt, health) {
   const window = recent.slice(0, BASELINE_WINDOW);
   const lastRun = window[0] ?? null;
   const finished = window.filter((r) => r.status !== "running");
@@ -354,7 +385,7 @@ function summarize(stored, recent, state, nextExpectedAt, now) {
   return {
     name: stored.name,
     definition: stored.definition,
-    health: jobHealth(stored.definition, lastRun, state, now),
+    health: health(lastRun),
     open: openConditions(state),
     lastRun,
     nextExpectedAt,
@@ -441,6 +472,12 @@ ${out}`);
       break;
     }
     case "recovered": {
+      if (draft.details.reason === "unscheduled") {
+        title = `${name} is no longer scheduled`;
+        const since = draft.details.since;
+        lines.push(`${since === void 0 ? "" : `Missed since ${when(since, now)}. `}It has no schedule now, so nothing is due; the missed alert is closed.`);
+        break;
+      }
       title = `${name} recovered`;
       const after = draft.details.after.map((c) => c.replace("_", " ")).join(", ");
       lines.push(`A run ${run ? when(run.startedAt, now) : "just now"} succeeded${after ? ` after: ${after}` : ""}.`);
@@ -468,8 +505,12 @@ function constantTimeEqual(a, b) {
 // src/output.ts
 var OUTPUT_CAP = 16 * 1024;
 function capOutput(text) {
-  if (text.length <= OUTPUT_CAP) return text;
-  return "[earlier output trimmed]\n" + text.slice(text.length - OUTPUT_CAP);
+  const clean = stripNul(text);
+  if (clean.length <= OUTPUT_CAP) return clean;
+  return "[earlier output trimmed]\n" + clean.slice(clean.length - OUTPUT_CAP);
+}
+function stripNul(text) {
+  return text.includes("\0") ? text.replace(/\u0000/g, "") : text;
 }
 function errorMessage(error) {
   return capOutput(describeError(error));
@@ -489,25 +530,44 @@ ${frames.join("\n")}` : ""}`;
 }
 var REDACTED = "[redacted]";
 var SECRET_PATTERNS = [
-  // password=..., API_KEY: ..., "client_secret": "...", TOKEN='...', token=... (but not max_tokens: 800).
-  // A quoted value is blanked to its closing quote, spaces and all, and keeps its quotes.
+  // A PEM private key, header to footer. Without a footer (the output was
+  // trimmed) it runs to the end of the base64 body. A "-" that starts five
+  // dashes ends the body, so the footer is never swallowed into it.
   [
-    /\b([A-Za-z0-9_-]{0,40}(?:secret|token|passw(?:or)?d|pwd|api[_-]?key|access[_-]?key|private[_-]?key|credential)[A-Za-z0-9_-]{0,40}(?<![Tt][Oo][Kk][Ee][Nn][Ss])"?\s{0,3}[=:]\s{0,3})(?:(")[^"\n]{1,4096}"|(')[^'\n]{1,4096}'|["']?[^\s"',;&]{1,4096})/gi,
+    /-----BEGIN (?:[A-Z0-9]{1,20} ){0,3}PRIVATE KEY-----(?:[A-Za-z0-9+/=\s,:]|-(?!----)){0,16384}(?:-----END (?:[A-Z0-9]{1,20} ){0,3}PRIVATE KEY-----)?/g,
+    REDACTED
+  ],
+  // password=..., API_KEY: ..., "client_secret": "...", TOKEN='...', token=...,
+  // :password=>"..." (but not max_tokens: 800). A quoted value is blanked to
+  // its closing quote, spaces and all, and keeps its quotes.
+  [
+    /\b([A-Za-z0-9_-]{0,40}(?:secret|token|passw(?:or)?d|pwd|api[_-]?key|access[_-]?key|private[_-]?key|credential)[A-Za-z0-9_-]{0,40}(?<![Tt][Oo][Kk][Ee][Nn][Ss])"?\s{0,3}(?:=>|[=:])\s{0,3})(?:(")[^"\n]{1,4096}"|(')[^'\n]{1,4096}'|["']?[^\s"',;&]{1,4096})/gi,
     (_match, name, double, single) => {
       const quote = double ?? single ?? "";
       return `${name}${quote}${REDACTED}${quote}`;
     }
   ],
-  // Credentials inside a URL: postgres://user:password@host
-  [/(\b[a-z][a-z0-9+.-]{0,30}:\/\/[^\s/:@]{0,256}:)[^\s/@]{1,256}@/gi, `$1${REDACTED}@`],
+  // Authorization: Basic <base64> and Authorization: Token <token>, also as a JSON or hash entry.
+  [/\b((?:proxy-)?authorization["']?\s{0,3}(?:=>|[=:])\s{0,3}["']?\s{0,3}(?:basic|token)\s{1,3})[A-Za-z0-9._~+/=:-]{1,4096}/gi, `$1${REDACTED}`],
+  // Credentials inside a URL: postgres://user:password@host. The password
+  // runs to the last "@" before a "/" or a space, so one that contains "@"
+  // is blanked whole.
+  [/(\b[a-z][a-z0-9+.-]{0,30}:\/\/[^\s/:@]{0,256}:)[^\s/]{1,256}@/gi, `$1${REDACTED}@`],
   // Authorization: Bearer <token>
   [/\b(Bearer\s{1,3})[A-Za-z0-9._~+/=-]{8,4096}/g, `$1${REDACTED}`],
-  // Well-known token shapes: AWS, GitHub, Slack, Stripe, Anthropic and OpenAI style keys.
+  // A bare JWT: three base64url segments, the first starting eyJ.
+  [/\beyJ[A-Za-z0-9_-]{4,4096}\.[A-Za-z0-9_-]{4,4096}\.[A-Za-z0-9_-]{0,4096}/g, REDACTED],
+  // Incoming webhook URLs carry their secret in the path.
+  [/(\bhooks\.slack\.com\/(?:services|workflows|triggers)\/)[A-Za-z0-9/_-]{1,255}/gi, `$1${REDACTED}`],
+  [/(\bdiscord(?:app)?\.com\/api\/(?:v\d{1,2}\/)?webhooks\/)[A-Za-z0-9/_-]{1,255}/gi, `$1${REDACTED}`],
+  // Well-known token shapes: AWS, GitHub, Slack, Stripe, Anthropic, OpenAI and Google style keys.
   [/\b(?:AKIA|ASIA)[0-9A-Z]{16}\b/g, REDACTED],
   [/\b(?:gh[pousr]_[A-Za-z0-9]{30,255}|github_pat_[A-Za-z0-9_]{20,255})\b/g, REDACTED],
   [/\bxox[abposr]-[A-Za-z0-9-]{10,255}/g, REDACTED],
   [/\b[rsp]k_(?:live|test)_[A-Za-z0-9]{10,255}\b/g, REDACTED],
-  [/\bsk-[A-Za-z0-9_-]{20,255}/g, REDACTED]
+  [/\bwhsec_[A-Za-z0-9+/=]{16,255}/g, REDACTED],
+  [/\bsk-[A-Za-z0-9_-]{20,255}/g, REDACTED],
+  [/\bAIza[0-9A-Za-z_-]{35}(?![0-9A-Za-z_-])/g, REDACTED]
 ];
 function redactSecrets(text) {
   let out = text;
@@ -779,14 +839,21 @@ function messagePage(title, message, base) {
 
 // src/routes/index.ts
 var COOKIE = "cronwatch_token";
-function cookieValue(token) {
-  return createHash("sha256").update(`cronwatch-cookie:${token}`).digest("hex");
+async function cookieValue(token) {
+  const digest = await crypto.subtle.digest("SHA-256", new TextEncoder().encode(`cronwatch-cookie:${token}`));
+  return Array.from(new Uint8Array(digest), (b) => b.toString(16).padStart(2, "0")).join("");
 }
-var LOOPBACK = /* @__PURE__ */ new Set(["localhost", "127.0.0.1", "[::1]"]);
 var DEFAULT_RUNS = 20;
 var MAX_RUNS = 500;
 var CSP = "default-src 'none'; style-src 'unsafe-inline'; img-src 'self' data:; form-action 'self'; frame-ancestors 'none'; base-uri 'none'";
 var SECURITY_HEADERS = { "x-content-type-options": "nosniff", "referrer-policy": "same-origin", "x-robots-tag": "noindex" };
+function developmentToken() {
+  const bytes = crypto.getRandomValues(new Uint8Array(32));
+  return btoa(String.fromCharCode(...bytes)).replace(/\+/g, "-").replace(/\//g, "_").replace(/=+$/, "");
+}
+function developmentSignInLine(origin, base, token) {
+  return `[cronwatch] CRONWATCH_TOKEN is not set, so this development server made a token for the dashboard. Sign in: ${origin}${base}/?token=${token}`;
+}
 function safeDecode(value) {
   try {
     return decodeURIComponent(value);
@@ -803,6 +870,34 @@ function readCookie(request, name) {
   }
   return null;
 }
+function firstValue(value) {
+  const first = value?.split(",")[0]?.trim();
+  return first ? first : null;
+}
+function configuredOrigin(value) {
+  if (value === void 0 || value === "") return null;
+  let url;
+  try {
+    url = new URL(value);
+  } catch {
+    throw new Error(`routes: origin must be an absolute URL such as "https://app.example.com", got ${JSON.stringify(value)}`);
+  }
+  if (url.protocol !== "http:" && url.protocol !== "https:") throw new Error(`routes: origin must be http or https, got ${JSON.stringify(value)}`);
+  return url.origin;
+}
+function forwardedOrigin(request, url) {
+  const proto = firstValue(request.headers.get("x-forwarded-proto"))?.toLowerCase() ?? null;
+  const host = firstValue(request.headers.get("x-forwarded-host"));
+  if (proto === null && host === null) return url.origin;
+  if (proto !== null && proto !== "http" && proto !== "https") return url.origin;
+  try {
+    const built = new URL(`${proto ?? url.protocol.slice(0, -1)}://${host ?? url.host}`);
+    if (built.pathname !== "/" || built.username || built.password || built.search || built.hash) return url.origin;
+    return built.origin;
+  } catch {
+    return url.origin;
+  }
+}
 function stripBase(pathname, base) {
   let path = pathname.startsWith(base) ? pathname.slice(base.length) : pathname;
   if (path === "") path = "/";
@@ -827,9 +922,9 @@ async function readBody(request) {
   }
   return {};
 }
-function crossSite(request, url) {
+function crossSite(request, publicOrigin) {
   const origin = request.headers.get("origin");
-  if (origin !== null && origin !== url.origin) return true;
+  if (origin !== null && origin !== publicOrigin) return true;
   const site = request.headers.get("sec-fetch-site");
   return site !== null && site !== "same-origin" && site !== "none";
 }
@@ -846,38 +941,54 @@ function runsLimit(value) {
 }
 function createRoutes(cw, options = {}) {
   const optedOut = options.token === null;
-  const token = optedOut ? null : options.token || process.env.CRONWATCH_TOKEN || null;
+  const configured = optedOut ? null : options.token || readEnv("CRONWATCH_TOKEN") || null;
   const base = (options.basePath ?? "/cronwatch").replace(/\/+$/, "");
-  const developing = process.env.NODE_ENV === "development" || process.env.NODE_ENV === "test";
+  const developing = isDevelopment();
+  const fixedOrigin = configuredOrigin(options.origin);
+  const trustProxy = options.trustProxy === true;
+  const originOf = (request, url) => fixedOrigin ?? (trustProxy ? forwardedOrigin(request, url) : url.origin);
+  const generated = !configured && !optedOut && developing;
+  const token = generated ? developmentToken() : configured;
+  let announced = false;
+  let cookie = null;
+  const expectedCookie = (value) => cookie ??= cookieValue(value);
   const serve = async (request, url, path, wantsHtml) => {
     const method = request.method.toUpperCase();
-    if (!token && !optedOut && !(developing && LOOPBACK.has(url.hostname))) {
-      return wantsHtml ? html(messagePage("CronWatch routes are locked", "Set CRONWATCH_TOKEN (or pass token to cw.routes()). Without one the routes only answer localhost in development.", base), 503) : api({ ok: false, error: "CRONWATCH_TOKEN is not set" }, 503);
+    const publicOrigin = originOf(request, url);
+    if (generated && !announced) {
+      announced = true;
+      console.info(developmentSignInLine(publicOrigin, base, token));
+    }
+    if (!token && !optedOut) {
+      return wantsHtml ? html(messagePage("CronWatch routes are locked", "Set CRONWATCH_TOKEN (or pass token to cw.routes()), or pass token: null to serve them open behind your own auth.", base), 503) : api({ ok: false, error: "CRONWATCH_TOKEN is not set" }, 503);
     }
-    if (method !== "GET" && method !== "HEAD" && crossSite(request, url)) {
+    if (method !== "GET" && method !== "HEAD" && crossSite(request, publicOrigin)) {
       return wantsHtml ? html(messagePage("Cross-site request refused", "Changes can only be made from the dashboard itself.", base), 403) : api({ ok: false, error: "Cross-site request refused" }, 403);
     }
     const bearer = request.headers.get("authorization")?.replace(/^Bearer\s+/i, "") ?? null;
     if (token) {
       const query = wantsHtml && method === "GET" ? url.searchParams.get("token") : null;
-      const cookie = readCookie(request, COOKIE);
+      const sent = readCookie(request, COOKIE);
       const isCheck = path === "/api/check";
       const cronSecretOk = isCheck && bearer !== null && cw.cronSecret !== null && constantTimeEqual(bearer, cw.cronSecret);
-      const tokenOk = bearer !== null ? constantTimeEqual(bearer, token) : query !== null ? constantTimeEqual(query, token) : cookie !== null && constantTimeEqual(cookie, cookieValue(token));
+      const tokenOk = bearer !== null ? constantTimeEqual(bearer, token) : query !== null ? constantTimeEqual(query, token) : sent !== null && constantTimeEqual(sent, await expectedCookie(token));
       if (!cronSecretOk && !tokenOk) {
+        if (generated) {
+          return wantsHtml ? html(messagePage("Sign in", "CRONWATCH_TOKEN is not set, so this development server made a token. The sign-in link is in the server log: open it once and this browser stays signed in.", base), 401) : api({ ok: false, error: "Unauthorized: CRONWATCH_TOKEN is not set, so this development server made a token; it is in the server log" }, 401);
+        }
         return wantsHtml ? html(messagePage("Sign in", `Open this page with ?token=<your CRONWATCH_TOKEN> once and it will stay signed in.`, base), 401) : api({ ok: false, error: "Unauthorized" }, 401);
       }
       if (query !== null) {
         url.searchParams.delete("token");
-        const secure = url.protocol === "https:" ? "; Secure" : "";
+        const secure = publicOrigin.startsWith("https:") ? "; Secure" : "";
         return redirect(url.pathname + (url.search || ""), {
-          "set-cookie": `${COOKIE}=${cookieValue(token)}; Path=${base || "/"}; HttpOnly; SameSite=Lax; Max-Age=${60 * 60 * 24 * 30}${secure}`
+          "set-cookie": `${COOKIE}=${await expectedCookie(token)}; Path=${base || "/"}; HttpOnly; SameSite=Lax; Max-Age=${60 * 60 * 24 * 30}${secure}`
         });
       }
     }
     const redirectBack = () => {
       const referer = request.headers.get("referer") ?? "";
-      return redirect(referer.startsWith(url.origin + "/") ? referer : `${base}/`);
+      return redirect(referer.startsWith(publicOrigin + "/") ? referer : `${base}/`);
     };
     const decoded = path.split("/").filter(Boolean).map(safeDecode);
     if (decoded.some((part) => part === null)) {
@@ -1038,6 +1149,7 @@ function memory() {
       }
     },
     async insertRun(run) {
+      if (runs.has(run.id)) throw new Error(`run ${run.id} already exists`);
       runs.set(run.id, clone(run));
       order.set(run.id, ++seq);
     },
@@ -1047,6 +1159,13 @@ function memory() {
       const { status, finishedAt, durationMs, error, output, metrics } = clone(run);
       runs.set(run.id, { ...existing, status, finishedAt, durationMs, error, output, metrics });
     },
+    async updateRunIf(run, fromStatuses) {
+      const existing = runs.get(run.id);
+      if (!existing || !fromStatuses.includes(existing.status)) return false;
+      const { status, finishedAt, durationMs, error, output, metrics } = clone(run);
+      runs.set(run.id, { ...existing, status, finishedAt, durationMs, error, output, metrics });
+      return true;
+    },
     async getRun(id) {
       const r = runs.get(id);
       return r ? clone(r) : null;
@@ -1068,6 +1187,11 @@ function memory() {
     async setState(state) {
       states.set(state.job, clone(state));
     },
+    async compareAndSetState(state, expectedVersion) {
+      if ((states.get(state.job)?.version ?? 0) !== expectedVersion) return false;
+      states.set(state.job, clone(state));
+      return true;
+    },
     async prune(before) {
       const newest = /* @__PURE__ */ new Map();
       for (const run of runs.values()) newest.set(run.job, Math.max(newest.get(run.job) ?? -Infinity, run.startedAt));
@@ -1090,6 +1214,8 @@ var TRIAGE_TIMEOUT_MS = 25e3;
 var CHANNEL_TIMEOUT_MS = 15e3;
 var PRUNE_INTERVAL_MS = 60 * 6e4;
 var MAX_UNDELIVERED = 20;
+var RETRY_BUDGET_MS = 2e4;
+var STATE_ATTEMPTS = 10;
 var HISTORY_PAGE = BASELINE_WINDOW + 5;
 var HISTORY_MAX = 200;
 function validateDefinition(def) {
@@ -1135,6 +1261,7 @@ var CronWatch = class {
   store;
   alerts;
   triage;
+  sources;
   /** The secret handler() requests must carry, or null when none is set. */
   cronSecret;
   retentionMs;
@@ -1150,6 +1277,8 @@ var CronWatch = class {
   synced = /* @__PURE__ */ new Set();
   /** The tail of each job's queue of state updates. See serial(). */
   queues = /* @__PURE__ */ new Map();
+  /** start() calls with an id still in flight, so two at once in this process record one run. */
+  starting = /* @__PURE__ */ new Map();
   ready = null;
   checking = null;
   lastPruneAt = 0;
@@ -1157,17 +1286,29 @@ var CronWatch = class {
   firstTick = null;
   usingDefaultStore = false;
   warnedNoSecret = false;
+  warnedDeferredStart = false;
   constructor(options = {}) {
     this.store = options.store ?? (this.usingDefaultStore = true, memory());
     this.alerts = options.alerts ?? [consoleChannel()];
     this.triage = options.triage;
-    const secret = options.cronSecret === void 0 ? process.env.CRON_SECRET : options.cronSecret;
+    this.sources = options.sources ?? [];
+    const secret = options.cronSecret === void 0 ? readEnv("CRON_SECRET") : options.cronSecret;
     this.cronSecret = secret ? secret : null;
     this.secretOptOut = options.cronSecret === null;
     this.retentionMs = parseDuration(options.retention ?? "30d", "retention");
     this.defaults = options.defaults ?? {};
     this.now = options.now ?? (() => Date.now());
-    this.redact = options.redact === false ? (text) => text : options.redact ?? redactSecrets;
+    const redact = options.redact;
+    this.redact = redact === false ? (text) => text : typeof redact === "function" ? (text) => {
+      try {
+        const out = redact(text);
+        if (typeof out !== "string") throw new TypeError(`redact must return a string, not ${out === null ? "null" : typeof out}`);
+        return out;
+      } catch (e) {
+        this.report(e, "redact");
+        return redactSecrets(text);
+      }
+    } : redactSecrets;
     if (options.deliver !== void 0 && options.deliver !== "now" && options.deliver !== "check") {
       throw new Error(`deliver must be "now" or "check", not ${JSON.stringify(options.deliver)}`);
     }
@@ -1195,6 +1336,12 @@ var CronWatch = class {
   definedJobs() {
     return [...this.definitions.values()];
   }
+  /** A handle on a run started elsewhere, as job(name).resume(runId). The job must be declared in this process. */
+  resumeRun(name, runId) {
+    const definition = this.definitions.get(name);
+    if (!definition) return Promise.reject(new Error(`resumeRun: job "${name}" is not declared; call job() first`));
+    return this.resumeHandle(definition, runId);
+  }
   handle(definition) {
     const self = this;
     return {
@@ -1233,6 +1380,12 @@ var CronWatch = class {
           };
           return json(body, outcome.run.status === "ok" ? 200 : 500);
         };
+      },
+      start(options) {
+        return self.startRun(definition, options ?? {});
+      },
+      resume(runId) {
+        return self.resumeHandle(definition, runId);
       }
     };
   }
@@ -1241,11 +1394,18 @@ var CronWatch = class {
     this.warnedNoSecret = true;
     this.onError(new Error("handler() refused a request because no CRON_SECRET is set; pass secret: null to allow unauthenticated requests"), "handler");
   }
+  /** onError, for places that must carry on even when onError itself throws. */
+  report(error, where) {
+    try {
+      this.onError(error, where);
+    } catch {
+    }
+  }
   async ensureReady() {
     if (!this.ready) {
       this.ready = (async () => {
         if (this.store.init) await this.store.init();
-        if (this.usingDefaultStore && process.env.NODE_ENV === "production") {
+        if (this.usingDefaultStore && readEnv("NODE_ENV") === "production") {
           console.warn("[cronwatch] using the in-memory store: runs and state are lost on restart. Pass a store from @cronwatch/sdk/sqlite or @cronwatch/sdk/postgres.");
         }
       })().catch((error) => {
@@ -1264,7 +1424,8 @@ var CronWatch = class {
   /**
    * Runs `fn` after every earlier state update for the same job has settled,
    * so two runs (or a run and a check) in this process never read and write
-   * the job's state over each other. Other processes are not coordinated.
+   * the job's state over each other. Other processes are coordinated by
+   * updateState() instead.
    */
   serial(job, fn) {
     const result = (this.queues.get(job) ?? Promise.resolve()).then(fn);
@@ -1280,6 +1441,38 @@ var CronWatch = class {
   async readState(job) {
     return normalizeState(await this.store.getState(job), job);
   }
+  /**
+   * Every read-modify-write of a job's state goes through here. In turn with
+   * this process's other updates to the job (serial()), it reads the state,
+   * asks `change` for the next one, and writes it with the version one
+   * higher, only if the stored version is still the one read. When another
+   * process wrote in between, the write is refused and it starts again from
+   * a fresh read, up to STATE_ATTEMPTS times. So `change` may run more than
+   * once and must only compute: whatever it returns from the attempt that
+   * was written is the result. Nothing is written when the state is
+   * unchanged. Returns the state as stored.
+   */
+  updateState(job, change) {
+    return this.serial(job, async () => {
+      for (let attempt = 1; ; attempt++) {
+        const current = await this.readState(job);
+        const { state, result } = await change(current);
+        if (sameState(state, current)) return { state: current, result };
+        const version = current.version ?? 0;
+        const next = { ...state, version: version + 1 };
+        if (await this.writeState(next, version)) return { state: next, result };
+        if (attempt >= STATE_ATTEMPTS) {
+          throw new Error(`the state of ${job} changed under ${STATE_ATTEMPTS} attempts in a row to update it; gave up`);
+        }
+      }
+    });
+  }
+  /** A conditional write, or for a store without compareAndSetState, a plain one that always succeeds. */
+  async writeState(state, expectedVersion) {
+    if (typeof this.store.compareAndSetState === "function") return this.store.compareAndSetState(state, expectedVersion);
+    await this.store.setState(state);
+    return true;
+  }
   /**
    * Runs a function as a recorded run. The function always runs, whatever
    * the store is doing: store errors go to onError, and the result is the
@@ -1308,11 +1501,8 @@ var CronWatch = class {
     } catch (e) {
       this.onError(e, `recording ${name}`);
     }
-    const started = recorded ? this.serial(name, async () => {
-      const before = await this.readState(name);
-      const after = onRunStart(before);
-      if (!sameState(before, after)) await this.store.setState(after);
-    }).catch((e) => this.onError(e, `starting ${name}`)) : Promise.resolve();
+    const started = recorded ? this.updateState(name, (before) => ({ state: onRunStart(before), result: void 0 })).then(() => {
+    }, (e) => this.onError(e, `starting ${name}`)) : Promise.resolve();
     const recorder = createRecorder(run);
     const timer = setTimeout(() => recorder.abort(), timeoutMs(definition));
     if (typeof timer === "object" && "unref" in timer) timer.unref();
@@ -1332,6 +1522,21 @@ var CronWatch = class {
     run.durationMs = Math.max(0, finishedAt - startedAt);
     run.metrics = recorder.metrics();
     run.output = recorder.output() ?? (typeof result === "string" ? capOutput(result) : null);
+    this.conclude(definition, run, result, error, threw, recorder.expectText() ?? (typeof result === "string" ? result : null));
+    await started;
+    try {
+      const ignored = await this.recordFinish(definition, run, recorded, finishedAt);
+      if (ignored) this.report(new Error(`run ${run.id} of ${name} ${ignored}; ignored`), `finishing ${name}`);
+    } catch (e) {
+      this.onError(e, `recording ${name}`);
+    }
+    return { run, result, error, threw };
+  }
+  /**
+   * Sets a finished run's status and error from how it ended, then redacts
+   * its output and error. Shared by execute() and RunHandle.finish().
+   */
+  conclude(definition, run, result, error, threw, expectText) {
     if (threw) {
       run.status = "failed";
       run.error = errorMessage(error);
@@ -1339,7 +1544,6 @@ var CronWatch = class {
       run.status = "failed";
       run.error = `HTTP ${result.status}${result.statusText ? ` ${result.statusText}` : ""}`;
     } else {
-      const expectText = recorder.expectText() ?? (typeof result === "string" ? result : null);
       const unmet = checkExpectation(definition.expect, expectText);
       if (unmet) {
         run.status = "failed";
@@ -1348,51 +1552,344 @@ var CronWatch = class {
         run.status = "ok";
       }
     }
-    if (run.output !== null) run.output = this.redact(run.output);
-    if (run.error !== null) run.error = this.redact(run.error);
-    await started;
+    if (run.output !== null) run.output = stripNul(this.redact(run.output));
+    if (run.error !== null) run.error = stripNul(this.redact(run.error));
+  }
+  /**
+   * Writes a finished run and evaluates it. `recorded` says whether its
+   * start was written; if not, it is inserted now. Returns why nothing was
+   * recorded (another process finished the run first, say), or null. Throws
+   * when the store does, so a handle can be finished again. Shared by
+   * execute() and RunHandle.finish().
+   */
+  async recordFinish(definition, run, recorded, finishedAt) {
     if (!recorded) {
+      await this.sync(definition);
       try {
-        await this.sync(definition);
         await this.store.insertRun(run);
-        recorded = true;
+        await this.finishRun(toStored(definition), run, finishedAt);
+        return null;
       } catch (e) {
-        this.onError(e, `recording ${name}`);
+        const stored = await this.store.getRun(run.id).catch(() => null);
+        if (!stored) throw e;
+        if (stored.job !== run.job) return `belongs to job "${stored.job}"`;
       }
-      if (recorded) await this.finishRun(toStored(definition), run, finishedAt, false);
-    } else if (await this.markedTimedOut(run)) {
+    }
+    const claim = await this.claimFinish(run);
+    if ("ignored" in claim) return claim.ignored;
+    if (!claim.lateAfterTimeout || run.status === "ok") await this.finishRun(toStored(definition), run, finishedAt);
+    return null;
+  }
+  /** A conditional write (Store.updateRunIf), or for a store without one, a read then a plain write. */
+  async writeRunIf(run, fromStatuses) {
+    if (typeof this.store.updateRunIf === "function") return this.store.updateRunIf(run, fromStatuses);
+    const stored = await this.store.getRun(run.id);
+    if (!stored || !fromStatuses.includes(stored.status)) return false;
+    await this.store.updateRun(run);
+    return true;
+  }
+  /**
+   * Writes a finished run over its stored row, only while that row is still
+   * running, or else still marked timeout by a check. Only the process whose
+   * write lands goes on to evaluate the run; for the others it returns why
+   * nothing was written. `lateAfterTimeout` means a check already counted
+   * the run as a stuck failure: a late failure must not count twice, while
+   * a late success still closes stuck and recovers. Throws when the store does.
+   */
+  async claimFinish(run) {
+    if (await this.writeRunIf(run, ["running"])) return { lateAfterTimeout: false };
+    if (await this.writeRunIf(run, ["timeout"])) return { lateAfterTimeout: true };
+    const stored = await this.store.getRun(run.id);
+    return { ignored: stored ? `was already finished as ${stored.status}` : "was not found" };
+  }
+  /** job.start(): records a running run and returns a handle to finish it. See JobHandle.start. */
+  async startRun(definition, options) {
+    const id = options.id;
+    if (id === void 0) return this.recordStart(definition, options.trigger);
+    checkRunId(definition.name, id, "start");
+    const key = `${definition.name}
+${id}`;
+    const inFlight = this.starting.get(key);
+    if (inFlight) return inFlight;
+    const started = this.recordStart(definition, options.trigger, id).finally(() => {
+      if (this.starting.get(key) === started) this.starting.delete(key);
+    });
+    this.starting.set(key, started);
+    return started;
+  }
+  /**
+   * The start of execute() without the function: the run is inserted and
+   * missed and stuck close (onRunStart). A store that fails is reported and
+   * the handle inserts the finished run instead, as execute() does.
+   */
+  async recordStart(definition, trigger = "start", id) {
+    const name = definition.name;
+    if (id !== void 0) {
+      let stored = null;
       try {
-        await this.store.updateRun(run);
+        await this.ensureReady();
+        stored = await this.store.getRun(id);
       } catch (e) {
         this.onError(e, `recording ${name}`);
       }
-    } else {
-      await this.finishRun(toStored(definition), run, finishedAt, true);
+      if (stored) return this.existingHandle(definition, stored);
     }
-    return { run, result, error, threw };
+    const run = {
+      id: id ?? crypto.randomUUID(),
+      job: name,
+      status: "running",
+      startedAt: this.now(),
+      finishedAt: null,
+      durationMs: null,
+      error: null,
+      output: null,
+      metrics: {},
+      trigger
+    };
+    let recorded = false;
+    try {
+      await this.sync(definition);
+      await this.store.insertRun({ ...run });
+      recorded = true;
+    } catch (e) {
+      const stored = id === void 0 ? null : await this.store.getRun(id).catch(() => null);
+      if (stored) return this.existingHandle(definition, stored);
+      this.onError(e, `recording ${name}`);
+    }
+    if (recorded) {
+      await this.updateState(name, (before) => ({ state: onRunStart(before), result: void 0 })).catch((e) => this.onError(e, `starting ${name}`));
+    }
+    return this.runHandle(definition, run.id, run, recorded, null);
   }
-  /** Whether a check already marked this run as timed out, for a failure that finished late. */
-  async markedTimedOut(run) {
-    if (run.status === "ok") return false;
+  /** job.resume() and cw.resumeRun(). A store that cannot be read is reported, and finish() reads it again. */
+  async resumeHandle(definition, runId) {
+    checkRunId(definition.name, runId, "resume");
+    let stored;
     try {
-      return (await this.store.getRun(run.id))?.status === "timeout";
-    } catch {
-      return false;
+      await this.ensureReady();
+      stored = await this.store.getRun(runId);
+    } catch (e) {
+      this.onError(e, `resuming ${definition.name}`);
+      return this.runHandle(definition, runId, null, true, null);
+    }
+    if (!stored) return this.runHandle(definition, runId, null, true, "was not found");
+    return this.existingHandle(definition, stored);
+  }
+  /** A handle on a stored run. One still running, or marked timeout by a check, can be finished. */
+  existingHandle(definition, stored) {
+    if (stored.job !== definition.name) {
+      throw new Error(`run "${stored.id}" belongs to job "${stored.job}", not "${definition.name}"`);
     }
+    const finished = stored.status === "ok" || stored.status === "failed";
+    return this.runHandle(definition, stored.id, stored, true, finished ? `already finished as ${stored.status}` : null);
+  }
+  /**
+   * The handle itself. `base` is the run as last known here, `recorded`
+   * whether its start is in the store, and `inactive` why finish() has
+   * nothing to do, or null. Lines and metrics wait in the handle until
+   * flush() or finish() merges them onto a fresh read of the stored run.
+   */
+  runHandle(definition, id, base, recorded, inactive) {
+    const self = this;
+    const name = definition.name;
+    const fresh = () => createRecorder({ id, job: name, startedAt: base?.startedAt ?? 0 });
+    let recorder = fresh();
+    let finished = inactive !== null;
+    let finishCalled = false;
+    let head = null;
+    let queue = Promise.resolve();
+    const inTurn = (fn) => {
+      const result = queue.then(fn);
+      queue = result.catch(() => {
+      });
+      return result;
+    };
+    const ignored = (why) => self.report(new Error(`run ${id} of ${name} ${why}; ignored`), `finishing ${name}`);
+    const finish = (outcome) => {
+      if (finishCalled) {
+        ignored("was already finished by this handle");
+        return Promise.resolve(null);
+      }
+      finishCalled = true;
+      const wasInactive = finished;
+      finished = true;
+      const retryable = (error) => {
+        finishCalled = false;
+        finished = false;
+        self.report(error, `finishing ${name}`);
+        return null;
+      };
+      return inTurn(async () => {
+        if (wasInactive) {
+          ignored(inactive);
+          return null;
+        }
+        let from = base;
+        if (recorded) {
+          try {
+            from = await self.store.getRun(id) ?? base;
+          } catch (e) {
+            return retryable(e);
+          }
+        }
+        if (!from) {
+          ignored("was not found");
+          return null;
+        }
+        if (from.job !== name) {
+          ignored(`belongs to job "${from.job}"`);
+          return null;
+        }
+        if (from.status === "ok" || from.status === "failed") {
+          ignored(`was already finished as ${from.status}`);
+          return null;
+        }
+        const failed = typeof outcome === "object" && outcome !== null && "error" in outcome;
+        const result = typeof outcome === "string" ? outcome : failed ? void 0 : outcome?.result;
+        const error = failed ? outcome.error : void 0;
+        const finishedAt = self.now();
+        const added = recorder.output() ?? (typeof result === "string" ? capOutput(result) : null);
+        const run = {
+          ...from,
+          status: "running",
+          finishedAt,
+          durationMs: Math.max(0, finishedAt - from.startedAt),
+          error: null,
+          output: joinOutput(from.output, added),
+          metrics: { ...from.metrics, ...recorder.metrics() }
+        };
+        const expectText = joinLines(head, joinLines(from.output, recorder.expectText() ?? (typeof result === "string" ? result : null)));
+        self.conclude(definition, run, result, error, failed, expectText);
+        let why;
+        try {
+          why = await self.recordFinish(definition, run, recorded, finishedAt);
+        } catch (e) {
+          return retryable(e);
+        }
+        if (why) {
+          ignored(why);
+          return null;
+        }
+        return run;
+      });
+    };
+    return {
+      id,
+      job: name,
+      startedAt: base?.startedAt ?? null,
+      get active() {
+        return !finished;
+      },
+      log: (...parts) => recorder.context.log(...parts),
+      metric: (metric, value) => recorder.context.metric(metric, value),
+      metrics: (values) => recorder.context.metrics(values),
+      flush: () => inTurn(async () => {
+        if (finished || !recorded) return;
+        const lines = recorder.output();
+        const metrics = recorder.metrics();
+        if (lines === null && Object.keys(metrics).length === 0) return;
+        const taken = recorder;
+        recorder = fresh();
+        const putBack = () => {
+          const later = recorder;
+          recorder = fresh();
+          for (const text of [taken.expectText(), later.expectText()]) if (text !== null) recorder.context.log(text);
+          recorder.context.metrics({ ...taken.metrics(), ...later.metrics() });
+        };
+        try {
+          const stored = await self.store.getRun(id);
+          if (!stored || stored.status !== "running") return putBack();
+          if (stored.job !== name) {
+            putBack();
+            self.report(new Error(`run ${id} of ${name} belongs to job "${stored.job}"; ignored`), `flushing ${name}`);
+            return;
+          }
+          const output = lines === null ? stored.output : joinOutput(stored.output, stripNul(self.redact(lines)));
+          if (!await self.writeRunIf({ ...stored, output, metrics: { ...stored.metrics, ...metrics } }, ["running"])) return putBack();
+          const text = taken.expectText();
+          if (text !== null && (head === null || head.length < OUTPUT_CAP)) head = (joinLines(head, text) ?? "").slice(0, OUTPUT_CAP);
+        } catch (e) {
+          putBack();
+          self.report(e, `flushing ${name}`);
+        }
+      }),
+      finish,
+      fail: (error) => finish({ error })
+    };
   }
   /**
-   * Record a finished run (ok, failed, or timed out by a check), evaluate it
-   * against the job's state and send what that produces. Never throws.
+   * Record a run that happened outside this process, for a Source. Its job
+   * must be declared with job() first. Runs are keyed by id: a new one is
+   * inserted, a stored one still running (or marked timeout by a check) is
+   * finished when this one is not running, and anything else is left alone,
+   * so recording the same run twice changes nothing. Finishing is
+   * conditional (see Store.updateRunIf): when two processes record the same
+   * finish, only the one whose write lands evaluates it, and the other
+   * reports it as already finished. A stored run of another job is left
+   * alone and reported. A finished run is judged as if it had been wrapped
+   * here (expect, failures, duration, budgets) and its output and error are
+   * redacted the same way; one finishing after a check marked it timeout is
+   * judged only when it succeeded, as RunHandle.finish() does. Returns the
+   * alerts it sent.
    */
-  async finishRun(definition, run, now, write) {
+  async recordRun(input, options = {}) {
+    const declared = this.definitions.get(input.job);
+    if (!declared) throw new Error(`recordRun: job "${input.job}" is not declared; call job() first`);
+    await this.sync(declared);
+    const run = { ...input, metrics: { ...input.metrics } };
+    if (run.status === "ok") {
+      const unmet = checkExpectation(declared.expect, run.output);
+      if (unmet) {
+        run.status = "failed";
+        run.error = unmet;
+      }
+    }
+    if (run.output !== null) run.output = stripNul(this.redact(capOutput(run.output)));
+    if (run.error !== null) run.error = stripNul(this.redact(capOutput(run.error)));
+    const evaluate = options.evaluate !== false;
+    const definition = toStored(declared);
+    const stored = await this.store.getRun(run.id);
+    if (stored) return this.recordOver(definition, stored, run, evaluate);
+    try {
+      await this.store.insertRun(run);
+    } catch (e) {
+      const again = await this.store.getRun(run.id).catch(() => null);
+      if (again) return this.recordOver(definition, again, run, evaluate);
+      throw e;
+    }
+    if (!evaluate) return [];
+    await this.updateState(run.job, (before) => ({ state: onRunStart(before), result: void 0 }));
+    if (run.status === "running") return [];
+    return this.finishRun(definition, run, this.now());
+  }
+  /** recordRun() for a run already stored. */
+  async recordOver(definition, stored, run, evaluate) {
+    if (stored.job !== run.job) {
+      this.report(new Error(`run ${run.id} of ${run.job} belongs to job "${stored.job}"; ignored`), `recording ${run.job}`);
+      return [];
+    }
+    if (stored.status !== "running" && stored.status !== "timeout" || run.status === "running") return [];
+    const claim = await this.claimFinish(run);
+    if ("ignored" in claim) {
+      this.report(new Error(`run ${run.id} of ${run.job} ${claim.ignored}; ignored`), `recording ${run.job}`);
+      return [];
+    }
+    if (!evaluate || claim.lateAfterTimeout && run.status !== "ok") return [];
+    return this.finishRun(definition, run, this.now());
+  }
+  /**
+   * Evaluate a finished run (ok, failed, or timed out by a check), already
+   * written, against the job's state and send what that produces. Never throws.
+   */
+  async finishRun(definition, run, now) {
     let drafts;
     try {
-      if (write) await this.store.updateRun(run);
-      drafts = await this.serial(run.job, async () => {
-        const history = await this.history(run);
-        const previous = await this.readState(run.job);
-        return (await this.settle(previous, onRunFinish(definition, run, previous, history, now), now)).alerts;
-      });
+      let history = null;
+      ({ result: drafts } = await this.updateState(run.job, async (previous) => {
+        history ??= await this.history(run);
+        const settled = applySilence(previous, onRunFinish(definition, run, previous, history, now), now);
+        return { state: settled.state, result: settled.alerts };
+      }));
     } catch (e) {
       this.onError(e, `evaluating ${run.job}`);
       return [];
@@ -1427,30 +1924,51 @@ var CronWatch = class {
   }
   async runCheck() {
     await this.ensureReady();
+    const alerts = [];
+    for (const source of this.sources) {
+      try {
+        alerts.push(...await source.sync(this) ?? []);
+      } catch (e) {
+        this.onError(e, `source ${source.name}`);
+      }
+    }
     for (const definition of this.definitions.values()) await this.sync(definition);
     const now = this.now();
-    const alerts = [];
     for (const run of await this.store.runningRuns()) {
-      const declared = this.definitions.get(run.job);
-      const definition = declared ? toStored(declared) : (await this.store.getJob(run.job))?.definition;
-      if (!definition || !isStuck(definition, run, now)) continue;
-      run.status = "timeout";
-      run.finishedAt = now;
-      run.durationMs = now - run.startedAt;
-      run.error = `Still running after ${formatDuration(timeoutMs(definition))}; marked as timed out`;
-      alerts.push(...await this.finishRun(definition, run, now, true));
+      try {
+        const declared = this.definitions.get(run.job);
+        const definition = declared ? toStored(declared) : (await this.store.getJob(run.job))?.definition;
+        if (!definition || !isStuck(definition, run, now)) continue;
+        const timeout = timeoutMs(definition);
+        run.status = "timeout";
+        run.finishedAt = now;
+        run.durationMs = now - run.startedAt;
+        run.error = `Still running after ${formatDuration(timeout)}; marked as timed out`;
+        if (!await this.writeRunIf(run, ["running"])) continue;
+        alerts.push(...await this.finishRun(definition, run, now));
+      } catch (e) {
+        this.onError(e, `checking ${run.job}`);
+      }
     }
     const jobs = [];
+    const retries = { spentMs: 0 };
     for (const stored of await this.store.listJobs()) {
-      const recent = await this.store.listRuns(stored.name, BASELINE_WINDOW);
-      const { previous, evaluation, settled } = await this.serial(stored.name, async () => {
-        const previous2 = await this.readState(stored.name);
-        const evaluation2 = onCheck(stored.definition, stored, recent[0] ?? null, previous2, now);
-        return { previous: previous2, evaluation: evaluation2, settled: await this.settle(previous2, evaluation2, now) };
-      });
-      alerts.push(...await this.retryUndelivered(stored.name, previous, now));
-      alerts.push(...await this.dispatch(settled.alerts, stored.definition, now));
-      jobs.push(summarize(stored, recent, settled.state, evaluation.nextExpectedAt, now));
+      try {
+        const recent = await this.store.listRuns(stored.name, BASELINE_WINDOW);
+        let nextExpectedAt = null;
+        const { state, result: drafts } = await this.updateState(stored.name, (previous) => {
+          const evaluation = onCheck(stored.definition, stored, recent[0] ?? null, previous, now);
+          nextExpectedAt = evaluation.nextExpectedAt;
+          const settled = applySilence(previous, evaluation, now);
+          return { state: settled.state, result: settled.alerts };
+        });
+        alerts.push(...await this.retryUndelivered(stored.name, state, now, retries));
+        alerts.push(...await this.dispatch(drafts, stored.definition, now));
+        jobs.push(summarize(stored, recent, state, nextExpectedAt, now));
+      } catch (e) {
+        this.onError(e, `checking ${stored.name}`);
+        jobs.push(await this.unevaluable(stored, now));
+      }
     }
     let pruned = 0;
     if (now - this.lastPruneAt > PRUNE_INTERVAL_MS) {
@@ -1463,12 +1981,24 @@ var CronWatch = class {
     }
     return { checkedAt: now, jobs, alerts, pruned };
   }
-  /** A job's summary and its newest runs, without alerting. */
+  /** A job's summary and its newest runs, without alerting. A job that cannot be evaluated is reported and shown as failing. */
   async snapshot(stored, now, runs) {
-    const recent = await this.store.listRuns(stored.name, Math.max(runs, BASELINE_WINDOW));
-    const state = await this.readState(stored.name);
-    const { nextExpectedAt } = onCheck(stored.definition, stored, recent[0] ?? null, state, now);
-    return { job: summarize(stored, recent, state, nextExpectedAt, now), runs: recent.slice(0, runs) };
+    let recent = [];
+    try {
+      recent = await this.store.listRuns(stored.name, Math.max(runs, BASELINE_WINDOW));
+      const state = await this.readState(stored.name);
+      const { nextExpectedAt } = onCheck(stored.definition, stored, recent[0] ?? null, state, now);
+      return { job: summarize(stored, recent, state, nextExpectedAt, now), runs: recent.slice(0, runs) };
+    } catch (e) {
+      this.onError(e, `reading ${stored.name}`);
+      return { job: await this.unevaluable(stored, now), runs: recent.slice(0, runs) };
+    }
+  }
+  /** The summary of a job whose evaluation failed, from whatever can still be read. */
+  async unevaluable(stored, now) {
+    const recent = await this.store.listRuns(stored.name, BASELINE_WINDOW).catch(() => []);
+    const state = await this.readState(stored.name).catch(() => emptyState(stored.name));
+    return unevaluableSummary(stored, recent, state, now);
   }
   /** Every job the store knows about, with its health. Does not send alerts. */
   async jobs() {
@@ -1515,12 +2045,12 @@ var CronWatch = class {
   /** Read, change and write one job's state, in turn with every other update to it. */
   async patchState(name, change) {
     await this.ensureReady();
-    return this.serial(name, async () => {
-      const state = await this.readState(name);
-      change(state);
-      await this.store.setState(state);
-      return state;
+    const { state } = await this.updateState(name, (current) => {
+      const next = normalizeState(current, name);
+      change(next);
+      return { state: next, result: void 0 };
     });
+    return state;
   }
   /** Remove a job and its runs from the store. A job still declared in code comes back on its next run. */
   async forget(name) {
@@ -1533,10 +2063,18 @@ var CronWatch = class {
   routes(options) {
     return createRoutes(this, options);
   }
-  /** Check on an interval, for long-running servers. Default every minute. */
+  /**
+   * Check on an interval, for long-running servers. Default every minute.
+   * Not for serverless functions or Cloudflare Workers, where nothing runs
+   * between requests: call check() from a cron there instead.
+   */
   start(every = "1m") {
     if (this.timer) return;
     const ms = Math.max(5e3, parseDuration(every, "check interval"));
+    if (this.deferDelivery && !this.warnedDeferredStart) {
+      this.warnedDeferredStart = true;
+      console.warn('[cronwatch] start() was called with deliver: "check", so these checks send no alerts. Another process must run checks with deliver: "now" (the default) to send them.');
+    }
     const tick = () => this.check().catch((e) => this.onError(e, "check"));
     this.timer = setInterval(tick, ms);
     if (typeof this.timer === "object" && "unref" in this.timer) this.timer.unref();
@@ -1556,20 +2094,10 @@ var CronWatch = class {
     this.stop();
     if (this.store.close) await this.store.close();
   }
-  /** Save an evaluation's state, honouring silence, and return what should be sent. Call inside serial(). */
-  async settle(previous, evaluation, now) {
-    let { state, alerts } = evaluation;
-    if (isSilenced(previous, now)) {
-      state = muteOpens(previous, state);
-      alerts = [];
-    }
-    if (!sameState(state, previous)) await this.store.setState(state);
-    return { state, alerts };
-  }
   /**
    * Compose, triage and send each draft. The state was saved before this
-   * (settle), so a slow channel holds up nothing else; afterwards only the
-   * delivery fields are written back, onto a fresh read of the state.
+   * (updateState), so a slow channel holds up nothing else; afterwards only
+   * the delivery fields are written back, onto a fresh read of the state.
    */
   async dispatch(drafts, definition, now) {
     if (drafts.length === 0) return [];
@@ -1581,41 +2109,63 @@ var CronWatch = class {
       if (this.deferDelivery) {
         failed.push(alert);
       } else {
-        if (this.triage && alert.type !== "recovered") await this.addTriage(alert);
+        if (this.triage && alert.type !== "recovered") await this.addTriage(alert, TRIAGE_TIMEOUT_MS);
         (await this.deliver(alert) ? delivered : failed).push(alert);
       }
       composed.push(alert);
     }
-    await this.recordDelivery(definition.name, delivered, failed, now);
+    await this.recordDelivery(definition.name, delivered, failed, [], now);
     return composed;
   }
-  /** Send the alerts that no channel accepted last time, once each. */
-  async retryUndelivered(name, state, now) {
+  /**
+   * Send the alerts that no channel accepted last time, once each, oldest
+   * first. `state` is the job's state as this check left it: an alert that
+   * no longer describes it (staleAlert) is dropped instead. Retries across a
+   * check share RETRY_BUDGET_MS of wall-clock time; once it is spent the rest
+   * stay queued for the next check.
+   */
+  async retryUndelivered(name, state, now, budget) {
     const pending = state.undelivered ?? [];
     if (pending.length === 0 || isSilenced(state, now) || this.deferDelivery) return [];
     const delivered = [];
     const failed = [];
+    const dropped = pending.filter((alert) => staleAlert(alert, state));
     for (const alert of pending) {
-      if (this.triage && alert.type !== "recovered" && alert.triage === void 0) await this.addTriage(alert);
+      if (dropped.includes(alert)) continue;
+      const left = RETRY_BUDGET_MS - budget.spentMs;
+      if (left <= 0) break;
+      const started = Date.now();
+      if (this.triage && alert.type !== "recovered" && alert.triage === void 0) await this.addTriage(alert, Math.min(TRIAGE_TIMEOUT_MS, left));
       (await this.deliver(alert) ? delivered : failed).push(alert);
+      budget.spentMs += Math.max(0, Date.now() - started);
     }
-    await this.recordDelivery(name, delivered, failed, now);
+    await this.recordDelivery(name, delivered, failed, dropped, now);
     return delivered;
   }
-  /** Mark delivered alerts done and keep failed ones for the next check. lastAlertAt moves only on a delivery. */
-  async recordDelivery(name, delivered, failed, now) {
+  /**
+   * Mark delivered alerts done, drop stale ones, and keep failed ones for the
+   * next check. A failed alert replaces its stored copy, so a triage made on
+   * this attempt is kept. lastAlertAt moves only on a delivery.
+   */
+  async recordDelivery(name, delivered, failed, dropped, now) {
     try {
-      await this.serial(name, async () => {
-        const previous = await this.readState(name);
+      const { result: trimmed } = await this.updateState(name, (previous) => {
         const state = normalizeState(previous, name);
-        const done = new Set(delivered.map(alertKey));
-        const kept = state.undelivered.filter((a) => !done.has(alertKey(a)));
+        const done = new Set([...delivered, ...dropped].map(alertKey));
+        const retried = new Map(failed.map((a) => [alertKey(a), a]));
+        const kept = state.undelivered.filter((a) => !done.has(alertKey(a))).map((a) => retried.get(alertKey(a)) ?? a);
         const known = new Set(kept.map(alertKey));
         kept.push(...failed.filter((a) => !known.has(alertKey(a))));
         state.undelivered = kept.slice(-MAX_UNDELIVERED);
         if (delivered.length > 0) state.lastAlertAt = now;
-        if (!sameState(state, previous)) await this.store.setState(state);
+        return { state, result: Math.max(0, kept.length - MAX_UNDELIVERED) };
       });
+      if (trimmed > 0) {
+        this.onError(
+          new Error(`${trimmed} undelivered alert${trimmed === 1 ? "" : "s"} for ${name} dropped: only the newest ${MAX_UNDELIVERED} are kept for retry`),
+          `alert queue for ${name}`
+        );
+      }
     } catch (e) {
       this.onError(e, `recording alert delivery for ${name}`);
     }
@@ -1626,7 +2176,7 @@ var CronWatch = class {
     const results = await Promise.all(
       this.alerts.map(async (channel) => {
         try {
-          await withTimeout(channel.send(alert), CHANNEL_TIMEOUT_MS);
+          await withTimeout(channel.send(alert, { onError: (e) => this.report(e, `alert channel ${channel.name}`) }), CHANNEL_TIMEOUT_MS);
           return true;
         } catch (e) {
           this.onError(e, `alert channel ${channel.name}`);
@@ -1636,20 +2186,38 @@ var CronWatch = class {
     );
     return results.includes(true);
   }
-  async addTriage(alert) {
+  /** Sets alert.triage to the diagnosis, or to null when there is none, so it is tried once per alert. */
+  async addTriage(alert, timeout) {
     const controller = new AbortController();
     try {
       const recentRuns = await this.store.listRuns(alert.job, 5);
-      const diagnosis = await withTimeout(this.triage({ alert, recentRuns, signal: controller.signal }), TRIAGE_TIMEOUT_MS);
-      if (diagnosis) alert.triage = diagnosis;
+      const diagnosis = await withTimeout(this.triage({ alert, recentRuns, signal: controller.signal }), timeout);
+      alert.triage = typeof diagnosis === "string" && diagnosis !== "" ? diagnosis : null;
     } catch (e) {
       controller.abort();
+      alert.triage = null;
       this.onError(e, `triage for ${alert.job}`);
     }
   }
 };
-function isDevelopment() {
-  return process.env.NODE_ENV === "development" || process.env.NODE_ENV === "test";
+var RESERVED_RUN_ID_PREFIX = "pgcron:";
+function checkRunId(job, id, method) {
+  if (typeof id !== "string" || id.length === 0 || id.length > 200) {
+    throw new Error(`job "${job}": ${method}() needs a run id of 1 to 200 characters (got ${typeof id === "string" ? `${id.length} characters` : typeof id})`);
+  }
+  if (id.startsWith(RESERVED_RUN_ID_PREFIX)) {
+    throw new Error(`job "${job}": ${method}() cannot take a run id starting with "${RESERVED_RUN_ID_PREFIX}", which the pg_cron source uses for its runs`);
+  }
+}
+function joinLines(before, after) {
+  if (before === null || before === "") return after;
+  if (after === null) return before;
+  return `${before}
+${after}`;
+}
+function joinOutput(before, after) {
+  const joined = joinLines(before, after);
+  return joined === null ? null : capOutput(joined);
 }
 function clampLimit(limit, fallback, min) {
   const n = typeof limit === "number" && Number.isFinite(limit) ? Math.trunc(limit) : fallback;