@cronwatch/sdk 0.8.0 → 0.10.0

package/dist/index.js

@@ -266,23 +266,81 @@ function runDuration(startedAt, finishedAt) {
   const ms = finishedAt - startedAt;
   return ms > 0 ? Math.min(ms, MAX_DURATION_MS) : 0;
 }
+function silenceEnd(now, ms) {
+  return Math.min(now + Math.floor(Math.min(ms, MAX_DURATION_MS)), MAX_DURATION_MS);
+}
 function stateVersion(state) {
   const version = state?.version;
   return typeof version === "number" && Number.isSafeInteger(version) && version >= 0 ? version + 0 : 0;
 }
+function failureCount(state) {
+  const count = state?.consecutiveFailures;
+  if (typeof count !== "number" || !Number.isInteger(count) || count <= 0) return 0;
+  return Math.min(count, MAX_DURATION_MS);
+}
 function emptyState(job) {
   return { job, open: {}, consecutiveFailures: 0, silencedUntil: null, lastAlertAt: null, pendingRecovery: [], undelivered: [] };
 }
 function normalizeState(state, job) {
   if (!state) return emptyState(job);
+  const { sending, ...rest } = state;
   return {
     ...emptyState(job),
-    ...state,
+    ...rest,
     open: { ...state.open },
+    consecutiveFailures: failureCount(state),
     pendingRecovery: [...state.pendingRecovery ?? []],
-    undelivered: [...state.undelivered ?? []]
+    undelivered: [...state.undelivered ?? []],
+    ...Array.isArray(sending) && sending.length > 0 ? { sending: [...sending] } : {}
   };
 }
+var MAX_UNDELIVERED = 20;
+var SEND_LEASE_MS = 5 * 6e4;
+function alertKey(alert) {
+  return `${alert.type}|${alert.at}|${alert.run?.id ?? ""}`;
+}
+function newest(list, max) {
+  return { kept: list.slice(-max), dropped: Math.max(0, list.length - max) };
+}
+function queueUndelivered(state, alerts) {
+  const next = cloneState(state);
+  const byKey = new Map(alerts.map((alert) => [alertKey(alert), alert]));
+  const queue = next.undelivered.map((alert) => byKey.get(alertKey(alert)) ?? alert);
+  const known = new Set(queue.map(alertKey));
+  queue.push(...alerts.filter((alert) => !known.has(alertKey(alert))));
+  const { kept, dropped } = newest(queue, MAX_UNDELIVERED);
+  next.undelivered = kept;
+  return { state: next, dropped };
+}
+function holdAlerts(state, alerts, until, deferred) {
+  if (alerts.length === 0) return { state, dropped: 0 };
+  if (deferred) return queueUndelivered(state, alerts);
+  const next = cloneState(state);
+  const { kept, dropped } = newest([...next.sending ?? [], ...alerts.map((alert) => ({ until, alert }))], MAX_UNDELIVERED);
+  next.sending = kept;
+  return { state: next, dropped };
+}
+function releaseSending(state, now) {
+  const sending = state.sending ?? [];
+  const lapsed = sending.filter((entry) => !(typeof entry?.until === "number" && entry.until > now));
+  if (lapsed.length === 0) return { state, dropped: 0 };
+  const held = sending.filter((entry) => !lapsed.includes(entry));
+  return queueUndelivered(
+    { ...state, sending: held },
+    lapsed.filter((entry) => typeof entry?.alert === "object" && entry.alert !== null).map((entry) => entry.alert)
+  );
+}
+function recordSent(state, delivered, failed, stale, now) {
+  const next = cloneState(state);
+  const done = new Set([...delivered, ...stale].map(alertKey));
+  next.undelivered = next.undelivered.filter((alert) => !done.has(alertKey(alert)));
+  const sent = new Set([...delivered, ...failed, ...stale].map(alertKey));
+  const held = (next.sending ?? []).filter((entry) => !(typeof entry?.alert === "object" && entry.alert !== null && sent.has(alertKey(entry.alert))));
+  if (held.length > 0) next.sending = held;
+  else delete next.sending;
+  if (delivered.length > 0) next.lastAlertAt = now;
+  return queueUndelivered(next, failed);
+}
 function cloneState(state) {
   return normalizeState(state, state.job);
 }
@@ -377,7 +435,7 @@ function onRunFinish(def, run, state, history, now) {
     }
     return { state: next, alerts };
   }
-  next.consecutiveFailures += 1;
+  next.consecutiveFailures = Math.min(next.consecutiveFailures + 1, MAX_DURATION_MS);
   closeCondition(next, "missed");
   const threshold = Math.max(1, def.failuresBeforeAlert ?? 1);
   const condition = run.status === "timeout" ? "stuck" : "failed";
@@ -586,13 +644,22 @@ var OUTPUT_CAP = 16 * 1024;
 function capOutput(text) {
   const clean = stripNul(text);
   if (clean.length <= OUTPUT_CAP) return clean;
-  return "[earlier output trimmed]\n" + clean.slice(clean.length - OUTPUT_CAP);
+  return TRIMMED + clean.slice(clean.length - OUTPUT_CAP);
+}
+var TRIMMED = "[earlier output trimmed]\n";
+var REDACT_EDGE = 3 * (OUTPUT_CAP + 1024);
+function redactAndCap(text, redact) {
+  const clean = stripNul(text);
+  const from = clean.length - (OUTPUT_CAP + REDACT_EDGE);
+  if (from <= 0) return capOutput(redact(clean));
+  const redacted = stripNul(redact(clean.slice(from)));
+  return TRIMMED + redacted.slice(Math.max(redacted.length - OUTPUT_CAP, REDACT_EDGE));
 }
 function stripNul(text) {
   return text.includes("\0") ? text.replace(/\u0000/g, "") : text;
 }
-function errorMessage(error) {
-  return capOutput(describeError(error));
+function stripJsonNul(json2) {
+  return json2.includes("\\u0000") ? json2.replace(/\\(u0000|[\s\S])/g, (escape, next) => next === "u0000" ? "" : escape) : json2;
 }
 function describeError(error) {
   if (error instanceof Error) {
@@ -702,7 +769,7 @@ function createRecorder(run) {
   };
   return {
     context,
-    output: () => lines.length === 0 ? null : capOutput(lines.join("\n")),
+    output: () => lines.length === 0 ? null : lines.join("\n"),
     expectText: () => {
       if (lines.length === 0) return null;
       const all = lines.join("\n");
@@ -1443,7 +1510,7 @@ function jobPage(job, runs, now, base, complete = true) {
       run.error ? `<details class="out error" open><summary>error</summary><pre>${h(run.error)}</pre></details>` : "",
       run.output ? `<details class="out"${run.status === "ok" ? "" : " open"}><summary>output</summary><pre>${h(run.output)}</pre></details>` : ""
     ].join("");
-    const metrics = Object.entries(run.metrics).map(([k, v]) => `<span><span class="k">${h(k)}</span> ${h(Number.isInteger(v) ? v : v.toFixed(4))}</span>`).join("");
+    const metrics = Object.entries(run.metrics ?? {}).filter(([, v]) => typeof v === "number" && Number.isFinite(v)).map(([k, v]) => `<span><span class="k">${h(k)}</span> ${h(Number.isInteger(v) ? v : v.toFixed(4))}</span>`).join("");
     return `<tr${detail ? ` class="has-detail"` : ""}>
 <td class="nowrap">${runState(run)}</td>
 <td class="nowrap">${h(when2(run.startedAt, now))} <span class="muted">UTC</span><span class="sub">${stamp(run.startedAt, now)}</span></td>
@@ -1843,13 +1910,21 @@ function memory() {
   const states = /* @__PURE__ */ new Map();
   let seq = 0;
   const clone = (v) => JSON.parse(JSON.stringify(v));
+  const kept = (v) => JSON.parse(stripJsonNul(JSON.stringify(v)));
+  const keptRun = (run) => ({
+    ...clone(run),
+    trigger: stripNul(run.trigger),
+    output: run.output === null ? null : stripNul(run.output),
+    error: run.error === null ? null : stripNul(run.error),
+    metrics: kept(run.metrics)
+  });
   const byName = (a, b) => a.name < b.name ? -1 : a.name > b.name ? 1 : 0;
   return {
     async upsertJob(definition, now) {
       const existing = jobs.get(definition.name);
       jobs.set(definition.name, {
         name: definition.name,
-        definition: clone(definition),
+        definition: kept(definition),
         createdAt: existing?.createdAt ?? now,
         updatedAt: now
       });
@@ -1871,19 +1946,19 @@ function memory() {
     },
     async insertRun(run) {
       if (runs.has(run.id)) throw new Error(`run ${run.id} already exists`);
-      runs.set(run.id, clone(run));
+      runs.set(run.id, keptRun(run));
       order.set(run.id, ++seq);
     },
     async updateRun(run) {
       const existing = runs.get(run.id);
       if (!existing) return;
-      const { status, finishedAt, durationMs, error, output, metrics } = clone(run);
+      const { status, finishedAt, durationMs, error, output, metrics } = keptRun(run);
       runs.set(run.id, { ...existing, status, finishedAt, durationMs, error, output, metrics });
     },
     async updateRunIf(run, fromStatuses) {
       const existing = runs.get(run.id);
       if (!existing || !fromStatuses.includes(existing.status)) return false;
-      const { status, finishedAt, durationMs, error, output, metrics } = clone(run);
+      const { status, finishedAt, durationMs, error, output, metrics } = keptRun(run);
       runs.set(run.id, { ...existing, status, finishedAt, durationMs, error, output, metrics });
       return true;
     },
@@ -1906,19 +1981,19 @@ function memory() {
       return s ? clone(s) : null;
     },
     async setState(state) {
-      states.set(state.job, clone(state));
+      states.set(state.job, kept(state));
     },
     async compareAndSetState(state, expectedVersion) {
       if (stateVersion(states.get(state.job)) !== expectedVersion) return false;
-      states.set(state.job, clone(state));
+      states.set(state.job, kept(state));
       return true;
     },
     async prune(before) {
-      const newest = /* @__PURE__ */ new Map();
-      for (const run of runs.values()) newest.set(run.job, Math.max(newest.get(run.job) ?? -Infinity, run.startedAt));
+      const newest2 = /* @__PURE__ */ new Map();
+      for (const run of runs.values()) newest2.set(run.job, Math.max(newest2.get(run.job) ?? -Infinity, run.startedAt));
       let n = 0;
       for (const [id, run] of runs) {
-        if (run.status !== "running" && run.startedAt < before && run.startedAt < newest.get(run.job)) {
+        if (run.status !== "running" && run.startedAt < before && run.startedAt < newest2.get(run.job)) {
           runs.delete(id);
           order.delete(id);
           n++;
@@ -1934,7 +2009,6 @@ var NAME_RE = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,119}$/;
 var TRIAGE_TIMEOUT_MS = 25e3;
 var CHANNEL_TIMEOUT_MS = 15e3;
 var PRUNE_INTERVAL_MS = 60 * 6e4;
-var MAX_UNDELIVERED = 20;
 var RETRY_BUDGET_MS = 2e4;
 var STATE_ATTEMPTS = 10;
 var HISTORY_PAGE = BASELINE_WINDOW + 5;
@@ -1975,9 +2049,6 @@ function validateDefinition(def) {
 function sameState(a, b) {
   return JSON.stringify(a) === JSON.stringify(b);
 }
-function alertKey(alert) {
-  return `${alert.type}|${alert.at}|${alert.run?.id ?? ""}`;
-}
 var CronWatch = class {
   store;
   alerts;
@@ -1998,6 +2069,8 @@ var CronWatch = class {
   synced = /* @__PURE__ */ new Set();
   /** The tail of each job's queue of state updates. See serial(). */
   queues = /* @__PURE__ */ new Map();
+  /** The tail of each job's queue of declaration writes. See sync(). */
+  syncing = /* @__PURE__ */ new Map();
   /** start() calls with an id still in flight, so two at once in this process record one run. */
   starting = /* @__PURE__ */ new Map();
   ready = null;
@@ -2136,26 +2209,69 @@ var CronWatch = class {
     }
     await this.ready;
   }
-  async sync(definition) {
+  /**
+   * Writes the declaration of `definition`'s name as it stands, unless the
+   * store has it. A handle kept from an earlier declaration writes the one
+   * that replaced it, never its own over it, and one forgotten since writes
+   * its own. The writes of one name take turns, in the order they were asked
+   * for, so one still under way cannot land after a later one; and a name
+   * declared again while its write was under way is still to be written. A
+   * name is marked as written only while that same declaration stands, so a
+   * forget that lands during the write (deleting the row after it) leaves
+   * the name to be written again, as does one forgotten before it.
+   *
+   * With `confirm`, as a run starts, a name already written is read back:
+   * another process may have forgotten the job since, and a job still
+   * declared here comes back on its next run.
+   */
+  async sync(definition, confirm = false) {
     await this.ensureReady();
-    if (this.synced.has(definition.name)) return;
-    await this.store.upsertJob(toStored(definition), this.now());
-    this.synced.add(definition.name);
+    const name = definition.name;
+    if (this.synced.has(name)) {
+      if (!confirm || await this.store.getJob(name)) return;
+      this.synced.delete(name);
+    }
+    await this.serial(name, async () => {
+      if (this.synced.has(name)) return;
+      const standing = this.definitions.get(name) ?? definition;
+      await this.store.upsertJob(toStored(standing), this.now());
+      if (this.definitions.get(name) === standing) this.synced.add(name);
+    }, this.syncing);
+  }
+  /**
+   * Every stored job, once each declaration has been written. A job declared
+   * here that the store no longer has was forgotten by another process after
+   * this one wrote it: it is written again, as its next run would, so it is
+   * checked and shown while any process still declares it.
+   */
+  async storedJobs() {
+    for (const definition of this.definitions.values()) await this.sync(definition);
+    const jobs = await this.store.listJobs();
+    const listed = new Set(jobs.map((job) => job.name));
+    const missing = [...this.definitions.values()].filter((definition) => !listed.has(definition.name));
+    if (missing.length === 0) return jobs;
+    for (const definition of missing) {
+      if (this.definitions.get(definition.name) !== definition) continue;
+      this.synced.delete(definition.name);
+      await this.sync(definition);
+    }
+    return this.store.listJobs();
   }
   /**
    * Runs `fn` after every earlier state update for the same job has settled,
    * so two runs (or a run and a check) in this process never read and write
    * the job's state over each other. Other processes are coordinated by
-   * updateState() instead.
+   * updateState() instead. sync() queues a job's declaration writes the same
+   * way, in `syncing`.
    */
-  serial(job, fn) {
-    const result = (this.queues.get(job) ?? Promise.resolve()).then(fn);
+  serial(job, fn, queues = this.queues) {
+    const result = (queues.get(job) ?? Promise.resolve()).then(fn);
     const tail2 = result.then(() => {
     }, () => {
     });
-    this.queues.set(job, tail2);
+    queues.set(job, tail2);
     void tail2.then(() => {
-      if (this.queues.get(job) === tail2) this.queues.delete(job);
+      if (queues.get(job) === tail2) queues.delete(job);
     });
     return result;
   }
@@ -2216,7 +2332,7 @@ var CronWatch = class {
     };
     let recorded = false;
     try {
-      await this.sync(definition);
+      await this.sync(definition, true);
       await this.store.insertRun({ ...run });
       recorded = true;
     } catch (e) {
@@ -2241,7 +2357,7 @@ var CronWatch = class {
     run.finishedAt = finishedAt;
     run.durationMs = runDuration(startedAt, finishedAt);
     run.metrics = recorder.metrics();
-    run.output = recorder.output() ?? (typeof result === "string" ? capOutput(result) : null);
+    run.output = recorder.output() ?? (typeof result === "string" ? result : null);
     this.conclude(definition, run, result, error, threw, recorder.expectText() ?? (typeof result === "string" ? result : null));
     await started;
     try {
@@ -2254,12 +2370,13 @@ var CronWatch = class {
   }
   /**
    * Sets a finished run's status and error from how it ended, then redacts
-   * its output and error. Shared by execute() and RunHandle.finish().
+   * its output and error and caps them, in that order. Shared by execute()
+   * and RunHandle.finish().
    */
   conclude(definition, run, result, error, threw, expectText) {
     if (threw) {
       run.status = "failed";
-      run.error = errorMessage(error);
+      run.error = describeError(error);
     } else if (result instanceof Response && result.status >= 400) {
       run.status = "failed";
       run.error = `HTTP ${result.status}${result.statusText ? ` ${result.statusText}` : ""}`;
@@ -2272,8 +2389,8 @@ var CronWatch = class {
         run.status = "ok";
       }
     }
-    if (run.output !== null) run.output = stripNul(this.redact(run.output));
-    if (run.error !== null) run.error = stripNul(this.redact(run.error));
+    if (run.output !== null) run.output = redactAndCap(run.output, this.redact);
+    if (run.error !== null) run.error = redactAndCap(run.error, this.redact);
   }
   /**
    * Writes a finished run and evaluates it. `recorded` says whether its
@@ -2368,7 +2485,7 @@ ${id}`;
     };
     let recorded = false;
     try {
-      await this.sync(definition);
+      await this.sync(definition, true);
       await this.store.insertRun({ ...run });
       recorded = true;
     } catch (e) {
@@ -2468,14 +2585,15 @@ ${id}`;
         const result = typeof outcome === "string" ? outcome : failed ? void 0 : outcome?.result;
         const error = failed ? outcome.error : void 0;
         const finishedAt = self.now();
-        const added = recorder.output() ?? (typeof result === "string" ? capOutput(result) : null);
+        const added = recorder.output() ?? (typeof result === "string" ? result : null);
         const run = {
           ...from,
           status: "running",
           finishedAt,
           durationMs: runDuration(from.startedAt, finishedAt),
           error: null,
-          output: joinOutput(from.output, added),
+          // Capped by conclude(), after it is redacted.
+          output: joinLines(from.output, added),
           metrics: { ...from.metrics, ...recorder.metrics() }
         };
         const expectText = joinLines(head, joinLines(from.output, recorder.expectText() ?? (typeof result === "string" ? result : null)));
@@ -2524,7 +2642,7 @@ ${id}`;
             self.report(new Error(`run ${id} of ${name} belongs to job "${stored.job}"; ignored`), `flushing ${name}`);
             return;
           }
-          const output = lines === null ? stored.output : joinOutput(stored.output, stripNul(self.redact(lines)));
+          const output = lines === null ? stored.output : joinOutput(stored.output, redactAndCap(lines, self.redact));
           if (!await self.writeRunIf({ ...stored, output, metrics: { ...stored.metrics, ...metrics } }, ["running"])) return putBack();
           const text = taken.expectText();
           if (text !== null && (head === null || head.length < OUTPUT_CAP)) head = (joinLines(head, text) ?? "").slice(0, OUTPUT_CAP);
@@ -2549,12 +2667,19 @@ ${id}`;
    * alone and reported. A finished run is judged as if it had been wrapped
    * here (expect, failures, duration, budgets) and its output and error are
    * redacted the same way; one finishing after a check marked it timeout is
-   * judged only when it succeeded, as RunHandle.finish() does. Returns the
-   * alerts it sent.
+   * judged only when it succeeded, as RunHandle.finish() does. A metric that
+   * is not a finite number throws before anything is written, as
+   * job.metric() does. Returns the alerts it sent.
    */
   async recordRun(input, options = {}) {
     const declared = this.definitions.get(input.job);
     if (!declared) throw new Error(`recordRun: job "${input.job}" is not declared; call job() first`);
+    if (input.id.includes("\0")) throw new Error(`recordRun: run ids cannot contain a NUL character (job "${input.job}")`);
+    for (const [metric, value] of Object.entries(input.metrics ?? {})) {
+      if (typeof value !== "number" || !Number.isFinite(value)) {
+        throw new Error(`recordRun: metric "${metric}" must be a finite number (job "${input.job}", run "${input.id}")`);
+      }
+    }
     await this.sync(declared);
     const run = { ...input, metrics: { ...input.metrics } };
     if (run.status === "ok") {
@@ -2564,8 +2689,8 @@ ${id}`;
         run.error = unmet;
       }
     }
-    if (run.output !== null) run.output = stripNul(this.redact(capOutput(run.output)));
-    if (run.error !== null) run.error = stripNul(this.redact(capOutput(run.error)));
+    if (run.output !== null) run.output = redactAndCap(run.output, this.redact);
+    if (run.error !== null) run.error = redactAndCap(run.error, this.redact);
     const evaluate = options.evaluate !== false;
     const definition = toStored(declared);
     const stored = await this.store.getRun(run.id);
@@ -2599,22 +2724,42 @@ ${id}`;
   }
   /**
    * Evaluate a finished run (ok, failed, or timed out by a check), already
-   * written, against the job's state and send what that produces. Never throws.
+   * written, against the job's state and send what that produces. The
+   * alerts are written with that state (see outbox()). Never throws.
    */
   async finishRun(definition, run, now) {
-    let drafts;
+    let held;
     try {
       let history = null;
-      ({ result: drafts } = await this.updateState(run.job, async (previous) => {
+      ({ result: held } = await this.updateState(run.job, async (previous) => {
         history ??= await this.history(run);
-        const settled = applySilence(previous, onRunFinish(definition, run, previous, history, now), now);
-        return { state: settled.state, result: settled.alerts };
+        return this.outbox(applySilence(previous, onRunFinish(definition, run, previous, history, now), now), definition, now);
       }));
     } catch (e) {
       this.onError(e, `evaluating ${run.job}`);
       return [];
     }
-    return this.dispatch(drafts, definition, now);
+    this.reportDropped(run.job, held.dropped);
+    return this.dispatch(run.job, held.alerts, now);
+  }
+  /**
+   * An evaluation as it is written: its drafts composed into alerts and held
+   * in the same state (holdAlerts), so the write that opens a condition also
+   * keeps its alerts, and a process that stops before sending them does not
+   * lose them. Called inside updateState(), so it only computes.
+   */
+  outbox(settled, definition, now) {
+    const alerts = settled.alerts.map((draft) => composeAlert(draft, definition, now));
+    const { state, dropped } = holdAlerts(settled.state, alerts, this.now() + SEND_LEASE_MS, this.deferDelivery);
+    return { state, result: { alerts, dropped } };
+  }
+  /** Reports alerts let go because a job's queue was full. */
+  reportDropped(name, dropped) {
+    if (dropped <= 0) return;
+    this.report(
+      new Error(`${dropped} undelivered alert${dropped === 1 ? "" : "s"} for ${name} dropped: only the newest ${MAX_UNDELIVERED} are kept for retry`),
+      `alert queue for ${name}`
+    );
   }
   /**
    * The runs before `run`, newest first, with up to BASELINE_WINDOW
@@ -2654,11 +2799,13 @@ ${id}`;
     }
     for (const definition of this.definitions.values()) await this.sync(definition);
     const now = this.now();
-    for (const run of await this.store.runningRuns()) {
+    for (const listed of await this.store.runningRuns()) {
       try {
-        const declared = this.definitions.get(run.job);
-        const definition = declared ? toStored(declared) : (await this.store.getJob(run.job))?.definition;
-        if (!definition || !isStuck(definition, run, now)) continue;
+        const declared = this.definitions.get(listed.job);
+        const definition = declared ? toStored(declared) : (await this.store.getJob(listed.job))?.definition;
+        if (!definition || !isStuck(definition, listed, now)) continue;
+        const run = await this.store.getRun(listed.id);
+        if (!run || run.status !== "running" || run.job !== listed.job) continue;
         const timeout = timeoutMs(definition);
         run.status = "timeout";
         run.finishedAt = now;
@@ -2667,23 +2814,26 @@ ${id}`;
         if (!await this.writeRunIf(run, ["running"])) continue;
         alerts.push(...await this.finishRun(definition, run, now));
       } catch (e) {
-        this.onError(e, `checking ${run.job}`);
+        this.onError(e, `checking ${listed.job}`);
       }
     }
     const jobs = [];
     const retries = { spentMs: 0 };
-    for (const stored of await this.store.listJobs()) {
+    for (const stored of await this.storedJobs()) {
       try {
         const recent = await this.store.listRuns(stored.name, BASELINE_WINDOW);
         let nextExpectedAt = null;
-        const { state, result: drafts } = await this.updateState(stored.name, (previous) => {
+        const { state, result: held } = await this.updateState(stored.name, (previous) => {
           const evaluation = onCheck(stored.definition, stored, recent[0] ?? null, previous, now);
           nextExpectedAt = evaluation.nextExpectedAt;
           const settled = applySilence(previous, evaluation, now);
-          return { state: settled.state, result: settled.alerts };
+          const released = releaseSending(settled.state, this.now());
+          const out = this.outbox({ state: released.state, alerts: settled.alerts }, stored.definition, now);
+          return { state: out.state, result: { alerts: out.result.alerts, dropped: released.dropped + out.result.dropped } };
         });
+        this.reportDropped(stored.name, held.dropped);
         alerts.push(...await this.retryUndelivered(stored.name, state, now, retries));
-        alerts.push(...await this.dispatch(drafts, stored.definition, now));
+        alerts.push(...await this.dispatch(stored.name, held.alerts, now));
         jobs.push(summarize(stored, recent, state, nextExpectedAt, now));
       } catch (e) {
         this.onError(e, `checking ${stored.name}`);
@@ -2727,16 +2877,17 @@ ${id}`;
   /** Every job's summary with its newest `limit` runs, read together. What the dashboard shows. */
   async jobsWithRuns(limit = 20) {
     await this.ensureReady();
-    for (const definition of this.definitions.values()) await this.sync(definition);
+    const jobs = await this.storedJobs();
     const now = this.now();
     const out = [];
-    for (const stored of await this.store.listJobs()) out.push(await this.snapshot(stored, now, clampLimit(limit, 20, 0)));
+    for (const stored of jobs) out.push(await this.snapshot(stored, now, clampLimit(limit, 20, 0)));
     return out;
   }
+  /** A job's summary, or null for one the store does not have. One declared here and forgotten elsewhere is written again, as storedJobs() does. */
   async jobSummary(name) {
     await this.ensureReady();
     const definition = this.definitions.get(name);
-    if (definition) await this.sync(definition);
+    if (definition) await this.sync(definition, true);
     const stored = await this.store.getJob(name);
     if (!stored) return null;
     return (await this.snapshot(stored, this.now(), 0)).job;
@@ -2750,11 +2901,14 @@ ${id}`;
     await this.ensureReady();
     return this.store.getRun(id);
   }
-  /** Stop alerts for a job for a while. State keeps updating underneath. */
+  /**
+   * Stop alerts for a job for a while. State keeps updating underneath. The
+   * end is a whole millisecond, held at 2^53 - 1 (see silenceEnd).
+   */
   async silence(name, duration) {
     const ms = parseDuration(duration, "silence duration");
     return this.patchState(name, (state) => {
-      state.silencedUntil = this.now() + ms;
+      state.silencedUntil = silenceEnd(this.now(), ms);
     });
   }
   async unsilence(name) {
@@ -2772,7 +2926,11 @@ ${id}`;
     });
     return state;
   }
-  /** Remove a job and its runs from the store. A job still declared in code comes back on its next run. */
+  /**
+   * Remove a job and its runs from the store. A job still declared in code
+   * comes back: here on its next run, and in any other process that declares
+   * it on its next run there, or at that process's next check or dashboard read.
+   */
   async forget(name) {
     await this.ensureReady();
     this.definitions.delete(name);
@@ -2810,32 +2968,37 @@ ${id}`;
     this.timer = null;
     this.firstTick = null;
   }
+  /**
+   * Stop the interval, wait for a check already under way (bounded by its
+   * own channel, triage and retry timeouts; what it throws was reported to
+   * whoever started it), then close the store, so that check neither writes
+   * after the store is closed nor loses the alerts it would queue.
+   */
   async close() {
     this.stop();
+    if (this.checking) await this.checking.catch(() => {
+    });
     if (this.store.close) await this.store.close();
   }
   /**
-   * Compose, triage and send each draft. The state was saved before this
-   * (updateState), so a slow channel holds up nothing else; afterwards only
-   * the delivery fields are written back, onto a fresh read of the state.
+   * Triage and send each alert the outbox holds (see outbox()). The state,
+   * with the alerts in it, was saved before this, so a slow channel holds up
+   * nothing else; afterwards only the delivery fields are written back, onto
+   * a fresh read of the state, and the alerts leave `sending`. Triage is made
+   * here, never stored with the held alert: the write that opens a condition
+   * cannot wait for it, and a retry triages an alert that has none. With
+   * deliver: "check" the alerts were queued for a check elsewhere instead.
    */
-  async dispatch(drafts, definition, now) {
-    if (drafts.length === 0) return [];
-    const composed = [];
+  async dispatch(name, alerts, now) {
+    if (alerts.length === 0 || this.deferDelivery) return alerts;
     const delivered = [];
     const failed = [];
-    for (const draft of drafts) {
-      const alert = composeAlert(draft, definition, now);
-      if (this.deferDelivery) {
-        failed.push(alert);
-      } else {
-        if (this.triage && alert.type !== "recovered") await this.addTriage(alert, TRIAGE_TIMEOUT_MS);
-        (await this.deliver(alert) ? delivered : failed).push(alert);
-      }
-      composed.push(alert);
+    for (const alert of alerts) {
+      if (this.triage && alert.type !== "recovered") await this.addTriage(alert, TRIAGE_TIMEOUT_MS);
+      (await this.deliver(alert) ? delivered : failed).push(alert);
     }
-    await this.recordDelivery(definition.name, delivered, failed, [], now);
-    return composed;
+    await this.recordDelivery(name, delivered, failed, [], now);
+    return alerts;
   }
   /**
    * Send the alerts that no channel accepted last time, once each, oldest
@@ -2864,28 +3027,18 @@ ${id}`;
   }
   /**
    * Mark delivered alerts done, drop stale ones, and keep failed ones for the
-   * next check. A failed alert replaces its stored copy, so a triage made on
-   * this attempt is kept. lastAlertAt moves only on a delivery.
+   * next check, taking them all out of `sending` (recordSent). A failed alert
+   * replaces its stored copy, so a triage made on this attempt is kept.
+   * lastAlertAt moves only on a delivery. When this write fails, alerts
+   * still in `sending` are retried once their lease runs out.
    */
   async recordDelivery(name, delivered, failed, dropped, now) {
     try {
       const { result: trimmed } = await this.updateState(name, (previous) => {
-        const state = normalizeState(previous, name);
-        const done = new Set([...delivered, ...dropped].map(alertKey));
-        const retried = new Map(failed.map((a) => [alertKey(a), a]));
-        const kept = state.undelivered.filter((a) => !done.has(alertKey(a))).map((a) => retried.get(alertKey(a)) ?? a);
-        const known = new Set(kept.map(alertKey));
-        kept.push(...failed.filter((a) => !known.has(alertKey(a))));
-        state.undelivered = kept.slice(-MAX_UNDELIVERED);
-        if (delivered.length > 0) state.lastAlertAt = now;
-        return { state, result: Math.max(0, kept.length - MAX_UNDELIVERED) };
+        const { state, dropped: letGo } = recordSent(normalizeState(previous, name), delivered, failed, dropped, now);
+        return { state, result: letGo };
       });
-      if (trimmed > 0) {
-        this.onError(
-          new Error(`${trimmed} undelivered alert${trimmed === 1 ? "" : "s"} for ${name} dropped: only the newest ${MAX_UNDELIVERED} are kept for retry`),
-          `alert queue for ${name}`
-        );
-      }
+      this.reportDropped(name, trimmed);
     } catch (e) {
       this.onError(e, `recording alert delivery for ${name}`);
     }
@@ -2925,6 +3078,7 @@ function checkRunId(job, id, method) {
   if (typeof id !== "string" || id.length === 0 || id.length > 200) {
     throw new Error(`job "${job}": ${method}() needs a run id of 1 to 200 characters (got ${typeof id === "string" ? `${id.length} characters` : typeof id})`);
   }
+  if (id.includes("\0")) throw new Error(`job "${job}": ${method}() cannot take a run id containing a NUL character`);
   if (id.startsWith(RESERVED_RUN_ID_PREFIX)) {
     throw new Error(`job "${job}": ${method}() cannot take a run id starting with "${RESERVED_RUN_ID_PREFIX}", which the pg_cron source uses for its runs`);
   }