@cronwatch/sdk 0.3.0 → 0.3.1

package/dist/index.js

@@ -1,5 +1,5 @@
 import { Cron } from 'croner';
-import { createHash } from 'crypto';
+import { randomBytes, createHash } from 'crypto';
 
 // src/duration.ts
 var UNIT_MS = {
@@ -337,6 +337,14 @@ function muteOpens(previous, next) {
 function isSilenced(state, now) {
   return state.silencedUntil !== null && state.silencedUntil > now;
 }
+function applySilence(previous, evaluation, now) {
+  if (!isSilenced(previous, now)) return evaluation;
+  return { state: muteOpens(previous, evaluation.state), alerts: [] };
+}
+function staleAlert(alert, state) {
+  if (alert.type === "recovered") return alert.details.after.some((condition) => state.open[condition] !== void 0);
+  return state.open[alert.type] !== alert.at;
+}
 function jobHealth(def, lastRun, state, now) {
   const open = openConditions(state);
   if (isSilenced(state, now)) return "silenced";
@@ -347,6 +355,12 @@ function jobHealth(def, lastRun, state, now) {
   return "healthy";
 }
 function summarize(stored, recent, state, nextExpectedAt, now) {
+  return summary(stored, recent, state, nextExpectedAt, (lastRun) => jobHealth(stored.definition, lastRun, state, now));
+}
+function unevaluableSummary(stored, recent, state, now) {
+  return summary(stored, recent, state, null, () => isSilenced(state, now) ? "silenced" : "failing");
+}
+function summary(stored, recent, state, nextExpectedAt, health) {
   const window = recent.slice(0, BASELINE_WINDOW);
   const lastRun = window[0] ?? null;
   const finished = window.filter((r) => r.status !== "running");
@@ -354,7 +368,7 @@ function summarize(stored, recent, state, nextExpectedAt, now) {
   return {
     name: stored.name,
     definition: stored.definition,
-    health: jobHealth(stored.definition, lastRun, state, now),
+    health: health(lastRun),
     open: openConditions(state),
     lastRun,
     nextExpectedAt,
@@ -468,8 +482,12 @@ function constantTimeEqual(a, b) {
 // src/output.ts
 var OUTPUT_CAP = 16 * 1024;
 function capOutput(text) {
-  if (text.length <= OUTPUT_CAP) return text;
-  return "[earlier output trimmed]\n" + text.slice(text.length - OUTPUT_CAP);
+  const clean = stripNul(text);
+  if (clean.length <= OUTPUT_CAP) return clean;
+  return "[earlier output trimmed]\n" + clean.slice(clean.length - OUTPUT_CAP);
+}
+function stripNul(text) {
+  return text.includes("\0") ? text.replace(/\u0000/g, "") : text;
 }
 function errorMessage(error) {
   return capOutput(describeError(error));
@@ -489,25 +507,44 @@ ${frames.join("\n")}` : ""}`;
 }
 var REDACTED = "[redacted]";
 var SECRET_PATTERNS = [
-  // password=..., API_KEY: ..., "client_secret": "...", TOKEN='...', token=... (but not max_tokens: 800).
-  // A quoted value is blanked to its closing quote, spaces and all, and keeps its quotes.
+  // A PEM private key, header to footer. Without a footer (the output was
+  // trimmed) it runs to the end of the base64 body. A "-" that starts five
+  // dashes ends the body, so the footer is never swallowed into it.
+  [
+    /-----BEGIN (?:[A-Z0-9]{1,20} ){0,3}PRIVATE KEY-----(?:[A-Za-z0-9+/=\s,:]|-(?!----)){0,16384}(?:-----END (?:[A-Z0-9]{1,20} ){0,3}PRIVATE KEY-----)?/g,
+    REDACTED
+  ],
+  // password=..., API_KEY: ..., "client_secret": "...", TOKEN='...', token=...,
+  // :password=>"..." (but not max_tokens: 800). A quoted value is blanked to
+  // its closing quote, spaces and all, and keeps its quotes.
   [
-    /\b([A-Za-z0-9_-]{0,40}(?:secret|token|passw(?:or)?d|pwd|api[_-]?key|access[_-]?key|private[_-]?key|credential)[A-Za-z0-9_-]{0,40}(?<![Tt][Oo][Kk][Ee][Nn][Ss])"?\s{0,3}[=:]\s{0,3})(?:(")[^"\n]{1,4096}"|(')[^'\n]{1,4096}'|["']?[^\s"',;&]{1,4096})/gi,
+    /\b([A-Za-z0-9_-]{0,40}(?:secret|token|passw(?:or)?d|pwd|api[_-]?key|access[_-]?key|private[_-]?key|credential)[A-Za-z0-9_-]{0,40}(?<![Tt][Oo][Kk][Ee][Nn][Ss])"?\s{0,3}(?:=>|[=:])\s{0,3})(?:(")[^"\n]{1,4096}"|(')[^'\n]{1,4096}'|["']?[^\s"',;&]{1,4096})/gi,
     (_match, name, double, single) => {
       const quote = double ?? single ?? "";
       return `${name}${quote}${REDACTED}${quote}`;
     }
   ],
-  // Credentials inside a URL: postgres://user:password@host
-  [/(\b[a-z][a-z0-9+.-]{0,30}:\/\/[^\s/:@]{0,256}:)[^\s/@]{1,256}@/gi, `$1${REDACTED}@`],
+  // Authorization: Basic <base64> and Authorization: Token <token>, also as a JSON or hash entry.
+  [/\b((?:proxy-)?authorization["']?\s{0,3}(?:=>|[=:])\s{0,3}["']?\s{0,3}(?:basic|token)\s{1,3})[A-Za-z0-9._~+/=:-]{1,4096}/gi, `$1${REDACTED}`],
+  // Credentials inside a URL: postgres://user:password@host. The password
+  // runs to the last "@" before a "/" or a space, so one that contains "@"
+  // is blanked whole.
+  [/(\b[a-z][a-z0-9+.-]{0,30}:\/\/[^\s/:@]{0,256}:)[^\s/]{1,256}@/gi, `$1${REDACTED}@`],
   // Authorization: Bearer <token>
   [/\b(Bearer\s{1,3})[A-Za-z0-9._~+/=-]{8,4096}/g, `$1${REDACTED}`],
-  // Well-known token shapes: AWS, GitHub, Slack, Stripe, Anthropic and OpenAI style keys.
+  // A bare JWT: three base64url segments, the first starting eyJ.
+  [/\beyJ[A-Za-z0-9_-]{4,4096}\.[A-Za-z0-9_-]{4,4096}\.[A-Za-z0-9_-]{0,4096}/g, REDACTED],
+  // Incoming webhook URLs carry their secret in the path.
+  [/(\bhooks\.slack\.com\/(?:services|workflows|triggers)\/)[A-Za-z0-9/_-]{1,255}/gi, `$1${REDACTED}`],
+  [/(\bdiscord(?:app)?\.com\/api\/(?:v\d{1,2}\/)?webhooks\/)[A-Za-z0-9/_-]{1,255}/gi, `$1${REDACTED}`],
+  // Well-known token shapes: AWS, GitHub, Slack, Stripe, Anthropic, OpenAI and Google style keys.
   [/\b(?:AKIA|ASIA)[0-9A-Z]{16}\b/g, REDACTED],
   [/\b(?:gh[pousr]_[A-Za-z0-9]{30,255}|github_pat_[A-Za-z0-9_]{20,255})\b/g, REDACTED],
   [/\bxox[abposr]-[A-Za-z0-9-]{10,255}/g, REDACTED],
   [/\b[rsp]k_(?:live|test)_[A-Za-z0-9]{10,255}\b/g, REDACTED],
-  [/\bsk-[A-Za-z0-9_-]{20,255}/g, REDACTED]
+  [/\bwhsec_[A-Za-z0-9+/=]{16,255}/g, REDACTED],
+  [/\bsk-[A-Za-z0-9_-]{20,255}/g, REDACTED],
+  [/\bAIza[0-9A-Za-z_-]{35}(?![0-9A-Za-z_-])/g, REDACTED]
 ];
 function redactSecrets(text) {
   let out = text;
@@ -782,11 +819,16 @@ var COOKIE = "cronwatch_token";
 function cookieValue(token) {
   return createHash("sha256").update(`cronwatch-cookie:${token}`).digest("hex");
 }
-var LOOPBACK = /* @__PURE__ */ new Set(["localhost", "127.0.0.1", "[::1]"]);
 var DEFAULT_RUNS = 20;
 var MAX_RUNS = 500;
 var CSP = "default-src 'none'; style-src 'unsafe-inline'; img-src 'self' data:; form-action 'self'; frame-ancestors 'none'; base-uri 'none'";
 var SECURITY_HEADERS = { "x-content-type-options": "nosniff", "referrer-policy": "same-origin", "x-robots-tag": "noindex" };
+function developmentToken() {
+  return randomBytes(32).toString("base64url");
+}
+function developmentSignInLine(origin, base, token) {
+  return `[cronwatch] CRONWATCH_TOKEN is not set, so this development server made a token for the dashboard. Sign in: ${origin}${base}/?token=${token}`;
+}
 function safeDecode(value) {
   try {
     return decodeURIComponent(value);
@@ -846,13 +888,20 @@ function runsLimit(value) {
 }
 function createRoutes(cw, options = {}) {
   const optedOut = options.token === null;
-  const token = optedOut ? null : options.token || process.env.CRONWATCH_TOKEN || null;
+  const configured = optedOut ? null : options.token || process.env.CRONWATCH_TOKEN || null;
   const base = (options.basePath ?? "/cronwatch").replace(/\/+$/, "");
   const developing = process.env.NODE_ENV === "development" || process.env.NODE_ENV === "test";
+  const generated = !configured && !optedOut && developing;
+  const token = generated ? developmentToken() : configured;
+  let announced = false;
   const serve = async (request, url, path, wantsHtml) => {
     const method = request.method.toUpperCase();
-    if (!token && !optedOut && !(developing && LOOPBACK.has(url.hostname))) {
-      return wantsHtml ? html(messagePage("CronWatch routes are locked", "Set CRONWATCH_TOKEN (or pass token to cw.routes()). Without one the routes only answer localhost in development.", base), 503) : api({ ok: false, error: "CRONWATCH_TOKEN is not set" }, 503);
+    if (generated && !announced) {
+      announced = true;
+      console.info(developmentSignInLine(url.origin, base, token));
+    }
+    if (!token && !optedOut) {
+      return wantsHtml ? html(messagePage("CronWatch routes are locked", "Set CRONWATCH_TOKEN (or pass token to cw.routes()), or pass token: null to serve them open behind your own auth.", base), 503) : api({ ok: false, error: "CRONWATCH_TOKEN is not set" }, 503);
     }
     if (method !== "GET" && method !== "HEAD" && crossSite(request, url)) {
       return wantsHtml ? html(messagePage("Cross-site request refused", "Changes can only be made from the dashboard itself.", base), 403) : api({ ok: false, error: "Cross-site request refused" }, 403);
@@ -865,6 +914,9 @@ function createRoutes(cw, options = {}) {
       const cronSecretOk = isCheck && bearer !== null && cw.cronSecret !== null && constantTimeEqual(bearer, cw.cronSecret);
       const tokenOk = bearer !== null ? constantTimeEqual(bearer, token) : query !== null ? constantTimeEqual(query, token) : cookie !== null && constantTimeEqual(cookie, cookieValue(token));
       if (!cronSecretOk && !tokenOk) {
+        if (generated) {
+          return wantsHtml ? html(messagePage("Sign in", "CRONWATCH_TOKEN is not set, so this development server made a token. The sign-in link is in the server log: open it once and this browser stays signed in.", base), 401) : api({ ok: false, error: "Unauthorized: CRONWATCH_TOKEN is not set, so this development server made a token; it is in the server log" }, 401);
+        }
         return wantsHtml ? html(messagePage("Sign in", `Open this page with ?token=<your CRONWATCH_TOKEN> once and it will stay signed in.`, base), 401) : api({ ok: false, error: "Unauthorized" }, 401);
       }
       if (query !== null) {
@@ -1068,6 +1120,11 @@ function memory() {
     async setState(state) {
       states.set(state.job, clone(state));
     },
+    async compareAndSetState(state, expectedVersion) {
+      if ((states.get(state.job)?.version ?? 0) !== expectedVersion) return false;
+      states.set(state.job, clone(state));
+      return true;
+    },
     async prune(before) {
       const newest = /* @__PURE__ */ new Map();
       for (const run of runs.values()) newest.set(run.job, Math.max(newest.get(run.job) ?? -Infinity, run.startedAt));
@@ -1090,6 +1147,8 @@ var TRIAGE_TIMEOUT_MS = 25e3;
 var CHANNEL_TIMEOUT_MS = 15e3;
 var PRUNE_INTERVAL_MS = 60 * 6e4;
 var MAX_UNDELIVERED = 20;
+var RETRY_BUDGET_MS = 2e4;
+var STATE_ATTEMPTS = 10;
 var HISTORY_PAGE = BASELINE_WINDOW + 5;
 var HISTORY_MAX = 200;
 function validateDefinition(def) {
@@ -1157,6 +1216,7 @@ var CronWatch = class {
   firstTick = null;
   usingDefaultStore = false;
   warnedNoSecret = false;
+  warnedDeferredStart = false;
   constructor(options = {}) {
     this.store = options.store ?? (this.usingDefaultStore = true, memory());
     this.alerts = options.alerts ?? [consoleChannel()];
@@ -1167,7 +1227,17 @@ var CronWatch = class {
     this.retentionMs = parseDuration(options.retention ?? "30d", "retention");
     this.defaults = options.defaults ?? {};
     this.now = options.now ?? (() => Date.now());
-    this.redact = options.redact === false ? (text) => text : options.redact ?? redactSecrets;
+    const redact = options.redact;
+    this.redact = redact === false ? (text) => text : typeof redact === "function" ? (text) => {
+      try {
+        const out = redact(text);
+        if (typeof out !== "string") throw new TypeError(`redact must return a string, not ${out === null ? "null" : typeof out}`);
+        return out;
+      } catch (e) {
+        this.report(e, "redact");
+        return redactSecrets(text);
+      }
+    } : redactSecrets;
     if (options.deliver !== void 0 && options.deliver !== "now" && options.deliver !== "check") {
       throw new Error(`deliver must be "now" or "check", not ${JSON.stringify(options.deliver)}`);
     }
@@ -1241,6 +1311,13 @@ var CronWatch = class {
     this.warnedNoSecret = true;
     this.onError(new Error("handler() refused a request because no CRON_SECRET is set; pass secret: null to allow unauthenticated requests"), "handler");
   }
+  /** onError, for places that must carry on even when onError itself throws. */
+  report(error, where) {
+    try {
+      this.onError(error, where);
+    } catch {
+    }
+  }
   async ensureReady() {
     if (!this.ready) {
       this.ready = (async () => {
@@ -1264,7 +1341,8 @@ var CronWatch = class {
   /**
    * Runs `fn` after every earlier state update for the same job has settled,
    * so two runs (or a run and a check) in this process never read and write
-   * the job's state over each other. Other processes are not coordinated.
+   * the job's state over each other. Other processes are coordinated by
+   * updateState() instead.
    */
   serial(job, fn) {
     const result = (this.queues.get(job) ?? Promise.resolve()).then(fn);
@@ -1280,6 +1358,38 @@ var CronWatch = class {
   async readState(job) {
     return normalizeState(await this.store.getState(job), job);
   }
+  /**
+   * Every read-modify-write of a job's state goes through here. In turn with
+   * this process's other updates to the job (serial()), it reads the state,
+   * asks `change` for the next one, and writes it with the version one
+   * higher, only if the stored version is still the one read. When another
+   * process wrote in between, the write is refused and it starts again from
+   * a fresh read, up to STATE_ATTEMPTS times. So `change` may run more than
+   * once and must only compute: whatever it returns from the attempt that
+   * was written is the result. Nothing is written when the state is
+   * unchanged. Returns the state as stored.
+   */
+  updateState(job, change) {
+    return this.serial(job, async () => {
+      for (let attempt = 1; ; attempt++) {
+        const current = await this.readState(job);
+        const { state, result } = await change(current);
+        if (sameState(state, current)) return { state: current, result };
+        const version = current.version ?? 0;
+        const next = { ...state, version: version + 1 };
+        if (await this.writeState(next, version)) return { state: next, result };
+        if (attempt >= STATE_ATTEMPTS) {
+          throw new Error(`the state of ${job} changed under ${STATE_ATTEMPTS} attempts in a row to update it; gave up`);
+        }
+      }
+    });
+  }
+  /** A conditional write, or for a store without compareAndSetState, a plain one that always succeeds. */
+  async writeState(state, expectedVersion) {
+    if (typeof this.store.compareAndSetState === "function") return this.store.compareAndSetState(state, expectedVersion);
+    await this.store.setState(state);
+    return true;
+  }
   /**
    * Runs a function as a recorded run. The function always runs, whatever
    * the store is doing: store errors go to onError, and the result is the
@@ -1308,11 +1418,8 @@ var CronWatch = class {
     } catch (e) {
       this.onError(e, `recording ${name}`);
     }
-    const started = recorded ? this.serial(name, async () => {
-      const before = await this.readState(name);
-      const after = onRunStart(before);
-      if (!sameState(before, after)) await this.store.setState(after);
-    }).catch((e) => this.onError(e, `starting ${name}`)) : Promise.resolve();
+    const started = recorded ? this.updateState(name, (before) => ({ state: onRunStart(before), result: void 0 })).then(() => {
+    }, (e) => this.onError(e, `starting ${name}`)) : Promise.resolve();
     const recorder = createRecorder(run);
     const timer = setTimeout(() => recorder.abort(), timeoutMs(definition));
     if (typeof timer === "object" && "unref" in timer) timer.unref();
@@ -1348,8 +1455,8 @@ var CronWatch = class {
         run.status = "ok";
       }
     }
-    if (run.output !== null) run.output = this.redact(run.output);
-    if (run.error !== null) run.error = this.redact(run.error);
+    if (run.output !== null) run.output = stripNul(this.redact(run.output));
+    if (run.error !== null) run.error = stripNul(this.redact(run.error));
     await started;
     if (!recorded) {
       try {
@@ -1388,11 +1495,12 @@ var CronWatch = class {
     let drafts;
     try {
       if (write) await this.store.updateRun(run);
-      drafts = await this.serial(run.job, async () => {
-        const history = await this.history(run);
-        const previous = await this.readState(run.job);
-        return (await this.settle(previous, onRunFinish(definition, run, previous, history, now), now)).alerts;
-      });
+      let history = null;
+      ({ result: drafts } = await this.updateState(run.job, async (previous) => {
+        history ??= await this.history(run);
+        const settled = applySilence(previous, onRunFinish(definition, run, previous, history, now), now);
+        return { state: settled.state, result: settled.alerts };
+      }));
     } catch (e) {
       this.onError(e, `evaluating ${run.job}`);
       return [];
@@ -1431,26 +1539,39 @@ var CronWatch = class {
     const now = this.now();
     const alerts = [];
     for (const run of await this.store.runningRuns()) {
-      const declared = this.definitions.get(run.job);
-      const definition = declared ? toStored(declared) : (await this.store.getJob(run.job))?.definition;
-      if (!definition || !isStuck(definition, run, now)) continue;
-      run.status = "timeout";
-      run.finishedAt = now;
-      run.durationMs = now - run.startedAt;
-      run.error = `Still running after ${formatDuration(timeoutMs(definition))}; marked as timed out`;
-      alerts.push(...await this.finishRun(definition, run, now, true));
+      try {
+        const declared = this.definitions.get(run.job);
+        const definition = declared ? toStored(declared) : (await this.store.getJob(run.job))?.definition;
+        if (!definition || !isStuck(definition, run, now)) continue;
+        const timeout = timeoutMs(definition);
+        run.status = "timeout";
+        run.finishedAt = now;
+        run.durationMs = now - run.startedAt;
+        run.error = `Still running after ${formatDuration(timeout)}; marked as timed out`;
+        alerts.push(...await this.finishRun(definition, run, now, true));
+      } catch (e) {
+        this.onError(e, `checking ${run.job}`);
+      }
     }
     const jobs = [];
+    const retries = { spentMs: 0 };
     for (const stored of await this.store.listJobs()) {
-      const recent = await this.store.listRuns(stored.name, BASELINE_WINDOW);
-      const { previous, evaluation, settled } = await this.serial(stored.name, async () => {
-        const previous2 = await this.readState(stored.name);
-        const evaluation2 = onCheck(stored.definition, stored, recent[0] ?? null, previous2, now);
-        return { previous: previous2, evaluation: evaluation2, settled: await this.settle(previous2, evaluation2, now) };
-      });
-      alerts.push(...await this.retryUndelivered(stored.name, previous, now));
-      alerts.push(...await this.dispatch(settled.alerts, stored.definition, now));
-      jobs.push(summarize(stored, recent, settled.state, evaluation.nextExpectedAt, now));
+      try {
+        const recent = await this.store.listRuns(stored.name, BASELINE_WINDOW);
+        let nextExpectedAt = null;
+        const { state, result: drafts } = await this.updateState(stored.name, (previous) => {
+          const evaluation = onCheck(stored.definition, stored, recent[0] ?? null, previous, now);
+          nextExpectedAt = evaluation.nextExpectedAt;
+          const settled = applySilence(previous, evaluation, now);
+          return { state: settled.state, result: settled.alerts };
+        });
+        alerts.push(...await this.retryUndelivered(stored.name, state, now, retries));
+        alerts.push(...await this.dispatch(drafts, stored.definition, now));
+        jobs.push(summarize(stored, recent, state, nextExpectedAt, now));
+      } catch (e) {
+        this.onError(e, `checking ${stored.name}`);
+        jobs.push(await this.unevaluable(stored, now));
+      }
     }
     let pruned = 0;
     if (now - this.lastPruneAt > PRUNE_INTERVAL_MS) {
@@ -1463,12 +1584,24 @@ var CronWatch = class {
     }
     return { checkedAt: now, jobs, alerts, pruned };
   }
-  /** A job's summary and its newest runs, without alerting. */
+  /** A job's summary and its newest runs, without alerting. A job that cannot be evaluated is reported and shown as failing. */
   async snapshot(stored, now, runs) {
-    const recent = await this.store.listRuns(stored.name, Math.max(runs, BASELINE_WINDOW));
-    const state = await this.readState(stored.name);
-    const { nextExpectedAt } = onCheck(stored.definition, stored, recent[0] ?? null, state, now);
-    return { job: summarize(stored, recent, state, nextExpectedAt, now), runs: recent.slice(0, runs) };
+    let recent = [];
+    try {
+      recent = await this.store.listRuns(stored.name, Math.max(runs, BASELINE_WINDOW));
+      const state = await this.readState(stored.name);
+      const { nextExpectedAt } = onCheck(stored.definition, stored, recent[0] ?? null, state, now);
+      return { job: summarize(stored, recent, state, nextExpectedAt, now), runs: recent.slice(0, runs) };
+    } catch (e) {
+      this.onError(e, `reading ${stored.name}`);
+      return { job: await this.unevaluable(stored, now), runs: recent.slice(0, runs) };
+    }
+  }
+  /** The summary of a job whose evaluation failed, from whatever can still be read. */
+  async unevaluable(stored, now) {
+    const recent = await this.store.listRuns(stored.name, BASELINE_WINDOW).catch(() => []);
+    const state = await this.readState(stored.name).catch(() => emptyState(stored.name));
+    return unevaluableSummary(stored, recent, state, now);
   }
   /** Every job the store knows about, with its health. Does not send alerts. */
   async jobs() {
@@ -1515,12 +1648,12 @@ var CronWatch = class {
   /** Read, change and write one job's state, in turn with every other update to it. */
   async patchState(name, change) {
     await this.ensureReady();
-    return this.serial(name, async () => {
-      const state = await this.readState(name);
-      change(state);
-      await this.store.setState(state);
-      return state;
+    const { state } = await this.updateState(name, (current) => {
+      const next = normalizeState(current, name);
+      change(next);
+      return { state: next, result: void 0 };
     });
+    return state;
   }
   /** Remove a job and its runs from the store. A job still declared in code comes back on its next run. */
   async forget(name) {
@@ -1537,6 +1670,10 @@ var CronWatch = class {
   start(every = "1m") {
     if (this.timer) return;
     const ms = Math.max(5e3, parseDuration(every, "check interval"));
+    if (this.deferDelivery && !this.warnedDeferredStart) {
+      this.warnedDeferredStart = true;
+      console.warn('[cronwatch] start() was called with deliver: "check", so these checks send no alerts. Another process must run checks with deliver: "now" (the default) to send them.');
+    }
     const tick = () => this.check().catch((e) => this.onError(e, "check"));
     this.timer = setInterval(tick, ms);
     if (typeof this.timer === "object" && "unref" in this.timer) this.timer.unref();
@@ -1556,20 +1693,10 @@ var CronWatch = class {
     this.stop();
     if (this.store.close) await this.store.close();
   }
-  /** Save an evaluation's state, honouring silence, and return what should be sent. Call inside serial(). */
-  async settle(previous, evaluation, now) {
-    let { state, alerts } = evaluation;
-    if (isSilenced(previous, now)) {
-      state = muteOpens(previous, state);
-      alerts = [];
-    }
-    if (!sameState(state, previous)) await this.store.setState(state);
-    return { state, alerts };
-  }
   /**
    * Compose, triage and send each draft. The state was saved before this
-   * (settle), so a slow channel holds up nothing else; afterwards only the
-   * delivery fields are written back, onto a fresh read of the state.
+   * (updateState), so a slow channel holds up nothing else; afterwards only
+   * the delivery fields are written back, onto a fresh read of the state.
    */
   async dispatch(drafts, definition, now) {
     if (drafts.length === 0) return [];
@@ -1581,41 +1708,63 @@ var CronWatch = class {
       if (this.deferDelivery) {
         failed.push(alert);
       } else {
-        if (this.triage && alert.type !== "recovered") await this.addTriage(alert);
+        if (this.triage && alert.type !== "recovered") await this.addTriage(alert, TRIAGE_TIMEOUT_MS);
         (await this.deliver(alert) ? delivered : failed).push(alert);
       }
       composed.push(alert);
     }
-    await this.recordDelivery(definition.name, delivered, failed, now);
+    await this.recordDelivery(definition.name, delivered, failed, [], now);
     return composed;
   }
-  /** Send the alerts that no channel accepted last time, once each. */
-  async retryUndelivered(name, state, now) {
+  /**
+   * Send the alerts that no channel accepted last time, once each, oldest
+   * first. `state` is the job's state as this check left it: an alert that
+   * no longer describes it (staleAlert) is dropped instead. Retries across a
+   * check share RETRY_BUDGET_MS of wall-clock time; once it is spent the rest
+   * stay queued for the next check.
+   */
+  async retryUndelivered(name, state, now, budget) {
     const pending = state.undelivered ?? [];
     if (pending.length === 0 || isSilenced(state, now) || this.deferDelivery) return [];
     const delivered = [];
     const failed = [];
+    const dropped = pending.filter((alert) => staleAlert(alert, state));
     for (const alert of pending) {
-      if (this.triage && alert.type !== "recovered" && alert.triage === void 0) await this.addTriage(alert);
+      if (dropped.includes(alert)) continue;
+      const left = RETRY_BUDGET_MS - budget.spentMs;
+      if (left <= 0) break;
+      const started = Date.now();
+      if (this.triage && alert.type !== "recovered" && alert.triage === void 0) await this.addTriage(alert, Math.min(TRIAGE_TIMEOUT_MS, left));
       (await this.deliver(alert) ? delivered : failed).push(alert);
+      budget.spentMs += Math.max(0, Date.now() - started);
     }
-    await this.recordDelivery(name, delivered, failed, now);
+    await this.recordDelivery(name, delivered, failed, dropped, now);
     return delivered;
   }
-  /** Mark delivered alerts done and keep failed ones for the next check. lastAlertAt moves only on a delivery. */
-  async recordDelivery(name, delivered, failed, now) {
+  /**
+   * Mark delivered alerts done, drop stale ones, and keep failed ones for the
+   * next check. A failed alert replaces its stored copy, so a triage made on
+   * this attempt is kept. lastAlertAt moves only on a delivery.
+   */
+  async recordDelivery(name, delivered, failed, dropped, now) {
     try {
-      await this.serial(name, async () => {
-        const previous = await this.readState(name);
+      const { result: trimmed } = await this.updateState(name, (previous) => {
         const state = normalizeState(previous, name);
-        const done = new Set(delivered.map(alertKey));
-        const kept = state.undelivered.filter((a) => !done.has(alertKey(a)));
+        const done = new Set([...delivered, ...dropped].map(alertKey));
+        const retried = new Map(failed.map((a) => [alertKey(a), a]));
+        const kept = state.undelivered.filter((a) => !done.has(alertKey(a))).map((a) => retried.get(alertKey(a)) ?? a);
         const known = new Set(kept.map(alertKey));
         kept.push(...failed.filter((a) => !known.has(alertKey(a))));
         state.undelivered = kept.slice(-MAX_UNDELIVERED);
         if (delivered.length > 0) state.lastAlertAt = now;
-        if (!sameState(state, previous)) await this.store.setState(state);
+        return { state, result: Math.max(0, kept.length - MAX_UNDELIVERED) };
       });
+      if (trimmed > 0) {
+        this.onError(
+          new Error(`${trimmed} undelivered alert${trimmed === 1 ? "" : "s"} for ${name} dropped: only the newest ${MAX_UNDELIVERED} are kept for retry`),
+          `alert queue for ${name}`
+        );
+      }
     } catch (e) {
       this.onError(e, `recording alert delivery for ${name}`);
     }
@@ -1636,14 +1785,16 @@ var CronWatch = class {
     );
     return results.includes(true);
   }
-  async addTriage(alert) {
+  /** Sets alert.triage to the diagnosis, or to null when there is none, so it is tried once per alert. */
+  async addTriage(alert, timeout) {
     const controller = new AbortController();
     try {
       const recentRuns = await this.store.listRuns(alert.job, 5);
-      const diagnosis = await withTimeout(this.triage({ alert, recentRuns, signal: controller.signal }), TRIAGE_TIMEOUT_MS);
-      if (diagnosis) alert.triage = diagnosis;
+      const diagnosis = await withTimeout(this.triage({ alert, recentRuns, signal: controller.signal }), timeout);
+      alert.triage = typeof diagnosis === "string" && diagnosis !== "" ? diagnosis : null;
     } catch (e) {
       controller.abort();
+      alert.triage = null;
       this.onError(e, `triage for ${alert.job}`);
     }
   }