@cronwatch/sdk 0.12.4 → 0.13.0

package/dist/index.cjs

@@ -30,7 +30,7 @@ function parseDuration(value, label = "duration") {
     consumed += match[0];
   }
   if (consumed.replace(/\s+/g, "") !== text.replace(/\s+/g, "")) {
-    throw new Error(`${label} "${value}" is not a duration like "15m", "1h30m" or "90s"`);
+    throw new Error(`${label} "${value}" is not a duration like "15m", "1h30m", or "90s"`);
   }
   return Math.round(total);
 }
@@ -81,27 +81,31 @@ function readEnv(name) {
   return typeof process === "undefined" || !process.env ? void 0 : process.env[name];
 }
 function readSecretEnv(name) {
-  const value = readEnv(name);
-  return value === void 0 || value.trim() === "" ? void 0 : value;
+  const value = readEnv(name)?.trim();
+  return value === void 0 || value === "" ? void 0 : value;
 }
 function secretOption(value, what) {
   if (value === void 0 || value === null) return value;
   if (typeof value !== "string") {
     throw new TypeError(`${what} must be a string, or null to opt out, not ${Array.isArray(value) ? "an array" : typeof value}`);
   }
-  return value.trim() === "" ? void 0 : value;
+  const trimmed = value.trim();
+  return trimmed === "" ? void 0 : trimmed;
 }
-function environment() {
-  if (typeof process === "undefined" || !process.env) return "";
-  for (const value of [process.env.CRONWATCH_ENV, process.env.APP_ENV, process.env.NODE_ENV]) {
+function environmentOf(values) {
+  for (const value of values) {
     const name = typeof value === "string" ? value.trim().toLowerCase() : "";
     if (name === "") continue;
     if (name === "prod") return "production";
-    if (name === "dev" || name === "local" || name === "test" || name === "testing") return "development";
+    if (name === "dev" || name === "local") return "development";
     return name;
   }
   return "";
 }
+function environment() {
+  if (typeof process === "undefined" || !process.env) return "";
+  return environmentOf([process.env.CRONWATCH_ENV, process.env.APP_ENV, process.env.NODE_ENV]);
+}
 function isDevelopment() {
   return environment() === "development";
 }
@@ -328,7 +332,7 @@ function isObject(value) {
 }
 function normalizeState(state, job) {
   if (!isObject(state)) return emptyState(job);
-  const { sending, underFloor, ...rest } = state;
+  const { sending, underFloor, pendingRuns, ...rest } = state;
   const open = isObject(state.open) ? Object.fromEntries(Object.entries(state.open).filter(([, at]) => typeof at === "number")) : {};
   const number = (value) => typeof value === "number" ? value : null;
   const list = (value) => Array.isArray(value) ? value : [];
@@ -342,7 +346,8 @@ function normalizeState(state, job) {
     pendingRecovery: list(state.pendingRecovery).filter((c) => typeof c === "string"),
     undelivered: list(state.undelivered).filter((alert) => isObject(alert)),
     ...Array.isArray(sending) && sending.length > 0 ? { sending: [...sending] } : {},
-    ...Array.isArray(underFloor) && underFloor.some((m) => typeof m === "string") ? { underFloor: underFloor.filter((m) => typeof m === "string") } : {}
+    ...Array.isArray(underFloor) && underFloor.some((m) => typeof m === "string") ? { underFloor: underFloor.filter((m) => typeof m === "string") } : {},
+    ...Array.isArray(pendingRuns) && pendingRuns.some((id) => typeof id === "string") ? { pendingRuns: pendingRuns.filter((id) => typeof id === "string") } : {}
   };
 }
 var MAX_UNDELIVERED = 20;
@@ -392,6 +397,34 @@ function recordSent(state, delivered, failed, stale, now) {
   if (delivered.length > 0) next.lastAlertAt = now;
   return queueUndelivered(next, failed);
 }
+function claimUndelivered(state, now, until) {
+  const next = cloneState(state);
+  const queue = next.undelivered;
+  const stale = queue.filter((alert) => staleAlert(alert, next));
+  const silenced = isSilenced(next, now);
+  const room = Math.max(0, MAX_UNDELIVERED - (next.sending ?? []).length);
+  const claimed = queue.filter((alert) => !stale.includes(alert) && (!silenced || alert.type === "recovered")).slice(0, room);
+  if (stale.length === 0 && claimed.length === 0) return { state, claimed, stale };
+  next.undelivered = queue.filter((alert) => !stale.includes(alert) && !claimed.includes(alert));
+  if (claimed.length > 0) next.sending = [...next.sending ?? [], ...claimed.map((alert) => ({ until, alert }))];
+  return { state: next, claimed, stale };
+}
+function holdRun(state, runId) {
+  const next = cloneState(state);
+  const held = (next.pendingRuns ?? []).filter((id) => id !== runId);
+  held.push(runId);
+  next.pendingRuns = held.slice(-100);
+  return next;
+}
+function releaseRun(state, runId) {
+  const held = state.pendingRuns ?? [];
+  if (!held.includes(runId)) return state;
+  const next = cloneState(state);
+  const left = held.filter((id) => id !== runId);
+  if (left.length > 0) next.pendingRuns = left;
+  else delete next.pendingRuns;
+  return next;
+}
 function cloneState(state) {
   return normalizeState(state, state.job);
 }
@@ -572,7 +605,7 @@ function isSilenced(state, now) {
 }
 function applySilence(previous, evaluation, now) {
   if (!isSilenced(previous, now)) return evaluation;
-  return { state: muteOpens(previous, evaluation.state), alerts: [] };
+  return { state: muteOpens(previous, evaluation.state), alerts: evaluation.alerts.filter((alert) => alert.type === "recovered") };
 }
 function staleAlert(alert, state) {
   if (alert.type === "recovered") {
@@ -636,6 +669,10 @@ function tail(text, n) {
   const lines = text.trimEnd().split("\n");
   return lines.slice(Math.max(0, lines.length - n)).join("\n");
 }
+function andList(words2) {
+  if (words2.length <= 2) return words2.join(" and ");
+  return `${words2.slice(0, -1).join(", ")}, and ${words2[words2.length - 1]}`;
+}
 function errorLine(error) {
   const text = firstLines(error, 4);
   return /^[A-Za-z_$][\w$]*: /.test(text) ? text : `Error: ${text}`;
@@ -709,7 +746,7 @@ ${out}`);
         break;
       }
       title = `${name} recovered`;
-      const after2 = draft.details.after.map((c) => c.replace("_", " ")).join(", ");
+      const after2 = andList(draft.details.after.map((c) => c.replace("_", " ")));
       lines.push(`A run ${run ? when(run.startedAt, now) : "just now"} succeeded${after2 ? ` after: ${after2}` : ""}.`);
       if (run?.durationMs !== null && run?.durationMs !== void 0) lines.push(`Ran ${formatDuration(run.durationMs)}.`);
       break;
@@ -725,10 +762,10 @@ function json(body, status = 200, headers = {}) {
     headers: { "content-type": "application/json; charset=utf-8", "cache-control": "no-store", ...headers }
   });
 }
-function constantTimeEqual(a, b) {
-  if (a.length !== b.length) return false;
-  let diff = 0;
-  for (let i = 0; i < a.length; i++) diff |= a.charCodeAt(i) ^ b.charCodeAt(i);
+function constantTimeEqual(sent, secret) {
+  if (secret.length === 0) return sent.length === 0;
+  let diff = sent.length ^ secret.length;
+  for (let i = 0; i < sent.length; i++) diff |= sent.charCodeAt(i) ^ secret.charCodeAt(i % secret.length);
   return diff === 0;
 }
 
@@ -799,7 +836,7 @@ var SECRET_PATTERNS = [
   // Incoming webhook URLs carry their secret in the path.
   [/(\bhooks\.slack\.com\/(?:services|workflows|triggers)\/)[A-Za-z0-9/_-]{1,255}/gi, `$1${REDACTED}`],
   [/(\bdiscord(?:app)?\.com\/api\/(?:v\d{1,2}\/)?webhooks\/)[A-Za-z0-9/_-]{1,255}/gi, `$1${REDACTED}`],
-  // Well-known token shapes: AWS, GitHub, Slack, Stripe, Anthropic, OpenAI and Google style keys.
+  // Well-known token shapes: AWS, GitHub, Slack, Stripe, Anthropic, OpenAI, and Google style keys.
   [/\b(?:AKIA|ASIA)[0-9A-Z]{16}\b/g, REDACTED],
   [/\b(?:gh[pousr]_[A-Za-z0-9]{30,255}|github_pat_[A-Za-z0-9_]{20,255})\b/g, REDACTED],
   [/\bxox[abposr]-[A-Za-z0-9-]{10,255}/g, REDACTED],
@@ -937,7 +974,7 @@ function manifest(base) {
     id: `${base}/`,
     name: "CronWatch",
     short_name: "CronWatch",
-    description: "The scheduled jobs of this app: their health, their last day and their runs.",
+    description: "The scheduled jobs of this app: their health, their last day, and their runs.",
     start_url: `${base}/`,
     scope: `${base}/`,
     display: "standalone",
@@ -1041,7 +1078,7 @@ function staticAsset(path, base) {
 }
 
 // src/version.ts
-var VERSION = "0.12.4";
+var VERSION = "0.13.0";
 var API_VERSION = 1;
 
 // src/routes/timeline.ts
@@ -1157,11 +1194,11 @@ function laneNote(job, missed, now) {
   if (job.open.includes("stuck")) return "stuck";
   if (job.open.includes("over_budget") && last) {
     const over = overCeilings(job);
-    return `went over budget${over.length ? ` on ${over.join(" and ")}` : ""} at ${when2(last.startedAt, now)}`;
+    return `went over budget${over.length ? ` on ${andList(over)}` : ""} at ${when2(last.startedAt, now)}`;
   }
   if (job.open.includes("under_floor") && last) {
     const under = underFloors(job);
-    return `fell short${under.length ? ` on ${under.join(" and ")}` : ""} at ${when2(last.startedAt, now)}`;
+    return `fell short${under.length ? ` on ${andList(under)}` : ""} at ${when2(last.startedAt, now)}`;
   }
   if (job.open.includes("slow") && last?.durationMs != null) return `slow: took ${formatDuration(last.durationMs)}`;
   if (job.open.includes("failed")) return "failing";
@@ -1279,7 +1316,7 @@ function legend() {
     [box("run ok"), "ran"],
     [box("run bad"), "failed"],
     [box("run timeout"), "timed out"],
-    [box("run warn"), "over budget, under floor or slow"],
+    [box("run warn"), "over budget, under floor, or slow"],
     [box("run running"), "running"],
     [box("missed"), "missed"]
   ];
@@ -1828,23 +1865,63 @@ function stripBase(pathname, base) {
   if (path.length > 1 && path.endsWith("/")) path = path.slice(0, -1);
   return path;
 }
+var MAX_REQUEST_BODY = 1 << 20;
+var TOO_LARGE = /* @__PURE__ */ Symbol("too large");
+async function readLimited(request) {
+  const length = Number(request.headers.get("content-length"));
+  if (Number.isFinite(length) && length > MAX_REQUEST_BODY) return TOO_LARGE;
+  if (!request.body) return new Uint8Array(new ArrayBuffer(0));
+  const reader = request.body.getReader();
+  const chunks = [];
+  let size = 0;
+  try {
+    for (; ; ) {
+      const { done, value } = await reader.read();
+      if (done) break;
+      size += value.byteLength;
+      if (size > MAX_REQUEST_BODY) {
+        await reader.cancel().catch(() => {
+        });
+        return TOO_LARGE;
+      }
+      chunks.push(value);
+    }
+  } finally {
+    reader.releaseLock();
+  }
+  const out = new Uint8Array(new ArrayBuffer(size));
+  let at = 0;
+  for (const chunk of chunks) {
+    out.set(chunk, at);
+    at += chunk.byteLength;
+  }
+  return out;
+}
 async function readBody(request) {
   const type = request.headers.get("content-type") ?? "";
+  const json2 = type.includes("application/json");
+  const form = type.includes("application/x-www-form-urlencoded") || type.includes("multipart/form-data");
+  if (!json2 && !form) return {};
+  let bytes;
+  try {
+    bytes = await readLimited(request);
+  } catch {
+    return {};
+  }
+  if (bytes === TOO_LARGE) return TOO_LARGE;
+  const copy = new Request(request.url, { method: "POST", headers: { "content-type": type }, body: bytes });
   try {
-    if (type.includes("application/json")) {
-      const data = await request.json();
+    if (json2) {
+      const data = await copy.json();
       return typeof data === "object" && data !== null ? Object.fromEntries(Object.entries(data).map(([k, v]) => [k, String(v)])) : {};
     }
-    if (type.includes("application/x-www-form-urlencoded") || type.includes("multipart/form-data")) {
-      const form = await request.formData();
-      const out = {};
-      for (const [k, v] of form.entries()) out[k] = String(v);
-      return out;
-    }
+    const fields = await copy.formData();
+    const out = {};
+    for (const [k, v] of fields.entries()) out[k] = String(v);
+    return out;
   } catch {
     return {};
   }
-  return {};
 }
 function crossSite(request, publicOrigin) {
   const origin = request.headers.get("origin");
@@ -1908,7 +1985,8 @@ function buildRoutes(cw, options = {}) {
     };
     const signInPage = () => generated ? html(messagePage("Sign in", "CRONWATCH_TOKEN is not set, so this development server made a token. The sign-in link is in the server log: open it once, or enter the token from it below, and this browser stays signed in.", base, true), 401) : html(messagePage("Sign in", "Enter your CRONWATCH_TOKEN and this browser stays signed in.", base, true), 401);
     if (token && method === "POST" && path === "/signin") {
-      const sent = (await readBody(request)).token;
+      const body = await readBody(request);
+      const sent = body === TOO_LARGE ? void 0 : body.token;
       if (sent === void 0 || !constantTimeEqual(sent, token)) return signInPage();
       return redirect(signInReturn(request.headers.get("referer"), publicOrigin, base), { "set-cookie": await signInCookie() });
     }
@@ -1924,7 +2002,7 @@ function buildRoutes(cw, options = {}) {
       }
       if (query !== null) {
         url.searchParams.delete("token");
-        return redirect(url.pathname + (url.search || ""), { "set-cookie": await signInCookie() });
+        return redirect(url.pathname.replace(/^\/+/, "/") + (url.search || ""), { "set-cookie": await signInCookie() });
       }
     }
     const redirectBack = () => {
@@ -1963,9 +2041,11 @@ function buildRoutes(cw, options = {}) {
       if (parts[2] !== "silence" && parts[2] !== "unsilence") return html(messagePage("Not found", path, base), 404);
       if (!await cw.jobSummary(name)) return html(messagePage("No such job", `${name} is not in the store.`, base), 404);
       if (parts[2] === "silence") {
+        const body = await readBody(request);
+        if (body === TOO_LARGE) return html(messagePage("Not silenced", "The request was too large.", base), 413);
         let duration;
         try {
-          duration = silenceDuration((await readBody(request)).for);
+          duration = silenceDuration(body.for);
         } catch (e) {
           return html(messagePage("Not silenced", e.message, base), 400);
         }
@@ -1999,6 +2079,7 @@ function buildRoutes(cw, options = {}) {
         if (!await cw.jobSummary(name)) return api({ ok: false, error: "No such job" }, 404);
         if (rest[2] === "silence") {
           const body = await readBody(request);
+          if (body === TOO_LARGE) return api({ ok: false, error: "Request body too large" }, 413);
           let duration;
           try {
             duration = silenceDuration(body.for ?? url.searchParams.get("for"));
@@ -2225,7 +2306,7 @@ function validateDefinition(def) {
     }
   }
   if (def.expect !== void 0 && typeof def.expect !== "string" && !(def.expect instanceof RegExp) && typeof def.expect !== "function") {
-    throw new Error(`job "${name}": expect must be a string, a RegExp or a function`);
+    throw new Error(`job "${name}": expect must be a string, a RegExp, or a function`);
   }
 }
 function sameState(a, b) {
@@ -2248,6 +2329,11 @@ var Cronwatch = class {
   deferDelivery;
   defaults;
   definitions = /* @__PURE__ */ new Map();
+  /**
+   * Declarations forgotten here (forget()), kept so that run() or resumeRun()
+   * by name declares the job again as it was written, not bare.
+   */
+  forgotten = /* @__PURE__ */ new Map();
   synced = /* @__PURE__ */ new Set();
   /** The tail of each job's queue of state updates. See serial(). */
   queues = /* @__PURE__ */ new Map();
@@ -2297,18 +2383,31 @@ var Cronwatch = class {
   /** Declare a job. Call it once, at module level, and keep the handle. */
   job(name, options = {}) {
     if (!NAME_RE.test(name)) {
-      throw new Error(`job name "${name}" must be 1 to 120 characters of letters, digits, ".", "_", ":" or "-"`);
+      throw new Error(`job name "${name}" must be 1 to 120 characters of letters, digits, ".", "_", ":", or "-"`);
     }
     const definition = { ...this.defaults, ...options, name };
     validateDefinition(definition);
     this.definitions.set(name, definition);
+    this.forgotten.delete(name);
     this.synced.delete(name);
     return this.handle(definition);
   }
+  /** The declaration of `name`: the standing one, else one forgotten here, declared again as it was. */
+  declared(name) {
+    const standing = this.definitions.get(name);
+    if (standing) return standing;
+    const forgotten = this.forgotten.get(name);
+    if (!forgotten) return void 0;
+    this.forgotten.delete(name);
+    this.definitions.set(name, forgotten);
+    this.synced.delete(name);
+    return forgotten;
+  }
   run(name, optionsOrFn, maybeFn) {
     const fn = typeof optionsOrFn === "function" ? optionsOrFn : maybeFn;
     const options = typeof optionsOrFn === "function" ? void 0 : optionsOrFn;
-    const handle = options || !this.definitions.has(name) ? this.job(name, options) : this.handle(this.definitions.get(name));
+    const declared = options ? void 0 : this.declared(name);
+    const handle = declared ? this.handle(declared) : this.job(name, options);
     return handle.run(fn);
   }
   /** The definitions declared in this process. */
@@ -2317,7 +2416,7 @@ var Cronwatch = class {
   }
   /** A handle on a run started elsewhere, as job(name).resume(runId). The job must be declared in this process. */
   resumeRun(name, runId) {
-    const definition = this.definitions.get(name);
+    const definition = this.declared(name);
     if (!definition) return Promise.reject(new Error(`resumeRun: job "${name}" is not declared; call job() first`));
     return this.resumeHandle(definition, runId);
   }
@@ -2533,7 +2632,7 @@ var Cronwatch = class {
     } catch (e) {
       this.onError(e, `recording ${name}`);
     }
-    const started = recorded ? this.updateState(name, (before) => ({ state: onRunStart(before), result: void 0 })).then(() => {
+    const started = recorded ? this.updateState(name, (before) => ({ state: holdRun(onRunStart(before), run.id), result: void 0 })).then(() => {
     }, (e) => this.onError(e, `starting ${name}`)) : Promise.resolve();
     const recorder = createRecorder(run);
     const clearTimer = after(timeoutMs(definition), () => recorder.abort());
@@ -2689,7 +2788,7 @@ ${id}`;
       this.onError(e, `recording ${name}`);
     }
     if (recorded) {
-      await this.updateState(name, (before) => ({ state: onRunStart(before), result: void 0 })).catch((e) => this.onError(e, `starting ${name}`));
+      await this.updateState(name, (before) => ({ state: holdRun(onRunStart(before), run.id), result: void 0 })).catch((e) => this.onError(e, `starting ${name}`));
     }
     return this.runHandle(definition, run.id, run, recorded, null);
   }
@@ -2902,7 +3001,7 @@ ${id}`;
       throw e;
     }
     if (!evaluate) return [];
-    await this.updateState(run.job, (before) => ({ state: onRunStart(before), result: void 0 }));
+    await this.updateState(run.job, (before) => ({ state: holdRun(onRunStart(before), run.id), result: void 0 }));
     if (run.status === "running") return [];
     return this.finishRun(definition, run, this.now());
   }
@@ -2918,7 +3017,11 @@ ${id}`;
       this.report(new Error(`run ${run.id} of ${run.job} ${claim.ignored}; ignored`), `recording ${run.job}`);
       return [];
     }
-    if (!evaluate || claim.lateAfterTimeout && run.status !== "ok") return [];
+    if (!evaluate) {
+      await this.updateState(run.job, (current) => ({ state: releaseRun(current, run.id), result: void 0 })).catch((e) => this.onError(e, `recording ${run.job}`));
+      return [];
+    }
+    if (claim.lateAfterTimeout && run.status !== "ok") return [];
     return this.finishRun(definition, run, this.now());
   }
   /**
@@ -2932,7 +3035,8 @@ ${id}`;
       let history = null;
       ({ result: held } = await this.updateState(run.job, async (previous) => {
         history ??= await this.history(run);
-        return this.outbox(applySilence(previous, onRunFinish(definition, run, previous, history, now), now), definition, now);
+        const before = releaseRun(previous, run.id);
+        return this.outbox(applySilence(before, onRunFinish(definition, run, before, history, now), now), definition, now);
       }));
     } catch (e) {
       this.onError(e, `evaluating ${run.job}`);
@@ -3037,6 +3141,7 @@ ${id}`;
         this.reportDropped(stored.name, held.dropped);
         alerts.push(...await this.retryUndelivered(stored.name, state, now, retries));
         alerts.push(...await this.dispatch(stored.name, held.alerts, now));
+        alerts.push(...await this.evaluatePending(stored, recent, state, now));
         jobs.push(summarize(stored, recent, state, nextExpectedAt, now));
       } catch (e) {
         this.onError(e, `checking ${stored.name}`);
@@ -3054,6 +3159,32 @@ ${id}`;
     }
     return { checkedAt: now, jobs, alerts, pruned };
   }
+  /**
+   * Runs the job's state still holds as pending (holdRun) whose finish was
+   * written but never evaluated: the process that finished them stopped, or
+   * its state write failed, between the two writes. Each one finished more
+   * than SEND_LEASE_MS ago is evaluated now, as its finish would have been;
+   * one still running is left to the stuck check, and one the store no
+   * longer has (given back, or pruned) is let go. The wait leaves a finish
+   * under way in another process to write its own evaluation.
+   */
+  async evaluatePending(stored, recent, state, now) {
+    const alerts = [];
+    for (const id of state.pendingRuns ?? []) {
+      try {
+        const run = recent.find((r) => r.id === id) ?? await this.store.getRun(id);
+        if (!run || run.job !== stored.name) {
+          await this.updateState(stored.name, (current) => ({ state: releaseRun(current, id), result: void 0 }));
+          continue;
+        }
+        if (run.status === "running" || typeof run.finishedAt !== "number" || run.finishedAt > now - SEND_LEASE_MS) continue;
+        alerts.push(...await this.finishRun(stored.definition, run, now));
+      } catch (e) {
+        this.onError(e, `evaluating run ${id} of ${stored.name}`);
+      }
+    }
+    return alerts;
+  }
   /** A job's summary and its newest runs, without alerting. A job that cannot be evaluated is reported and shown as failing. */
   async snapshot(stored, now, runs) {
     let recent = [];
@@ -3120,7 +3251,7 @@ ${id}`;
       state.silencedUntil = null;
     });
   }
-  /** Read, change and write one job's state, in turn with every other update to it. */
+  /** Read, change, and write one job's state, in turn with every other update to it. */
   async patchState(name, change) {
     await this.ensureReady();
     const { state } = await this.updateState(name, (current) => {
@@ -3132,11 +3263,15 @@ ${id}`;
   }
   /**
    * Remove a job and its runs from the store. A job still declared in code
-   * comes back: here on its next run, and in any other process that declares
-   * it on its next run there, or at that process's next check or dashboard read.
+   * comes back, with no history: here on its next run, declared as it was
+   * written (cw.run() and cw.resumeRun() by name included), and in any other
+   * process that declares it on its next run there, or at that process's
+   * next check or dashboard read.
    */
   async forget(name) {
     await this.ensureReady();
+    const definition = this.definitions.get(name);
+    if (definition) this.forgotten.set(name, definition);
     this.definitions.delete(name);
     this.synced.delete(name);
     await this.store.deleteJob(name);
@@ -3189,7 +3324,7 @@ ${id}`;
   }
   /**
    * Stop the interval, wait for a check already under way (bounded by its
-   * own channel, triage and retry timeouts; what it throws was reported to
+   * own channel, triage, and retry timeouts; what it throws was reported to
    * whoever started it), then close the store, so that check neither writes
    * after the store is closed nor loses the alerts it would queue.
    */
@@ -3221,27 +3356,40 @@ ${id}`;
   }
   /**
    * Send the alerts that no channel accepted last time, once each, oldest
-   * first. `state` is the job's state as this check left it: an alert that
-   * no longer describes it (staleAlert) is dropped instead. Retries across a
-   * check share RETRY_BUDGET_MS of wall-clock time; once it is spent the rest
-   * stay queued for the next check.
+   * first. They are claimed first (claimUndelivered): moved into `sending`
+   * with a lease, in one conditional write, so when several processes check
+   * at once only the one whose write lands sends them. An alert that no
+   * longer describes the job (staleAlert) is dropped instead. Retries across
+   * a check share RETRY_BUDGET_MS of wall-clock time; claimed alerts it has
+   * no time left for go back to the queue for the next check.
    */
   async retryUndelivered(name, state, now, budget) {
-    const pending = state.undelivered ?? [];
-    if (pending.length === 0 || isSilenced(state, now) || this.deferDelivery) return [];
+    if ((state.undelivered ?? []).length === 0 || this.deferDelivery) return [];
+    let claimed;
+    try {
+      ({ result: claimed } = await this.updateState(name, (previous) => {
+        const claim = claimUndelivered(previous, now, this.now() + SEND_LEASE_MS);
+        return { state: claim.state, result: claim.claimed };
+      }));
+    } catch (e) {
+      this.onError(e, `claiming undelivered alerts for ${name}`);
+      return [];
+    }
+    if (claimed.length === 0) return [];
     const delivered = [];
     const failed = [];
-    const dropped = pending.filter((alert) => staleAlert(alert, state));
-    for (const alert of pending) {
-      if (dropped.includes(alert)) continue;
+    for (const alert of claimed) {
       const left = RETRY_BUDGET_MS - budget.spentMs;
-      if (left <= 0) break;
+      if (left <= 0) {
+        failed.push(alert);
+        continue;
+      }
       const started = Date.now();
       if (this.triage && alert.type !== "recovered" && alert.triage === void 0) await this.addTriage(alert, Math.min(TRIAGE_TIMEOUT_MS, left));
       (await this.deliver(alert) ? delivered : failed).push(alert);
       budget.spentMs += Math.max(0, Date.now() - started);
     }
-    await this.recordDelivery(name, delivered, failed, dropped, now);
+    await this.recordDelivery(name, delivered, failed, [], now);
     return delivered;
   }
   /**