@cronwatch/sdk 0.3.1 → 0.4.0

package/dist/index.js

@@ -1,5 +1,4 @@
 import { Cron } from 'croner';
-import { randomBytes, createHash } from 'crypto';
 
 // src/duration.ts
 var UNIT_MS = {
@@ -53,6 +52,16 @@ function formatRelative(at, now) {
   if (abs < 5e3) return text;
   return diff < 0 ? `${text} ago` : `in ${text}`;
 }
+
+// src/env.ts
+function readEnv(name) {
+  return typeof process === "undefined" || !process.env ? void 0 : process.env[name];
+}
+function isDevelopment() {
+  if (typeof process === "undefined" || !process.env) return false;
+  const mode = process.env.NODE_ENV;
+  return mode === "development" || mode === "test";
+}
 var cache = /* @__PURE__ */ new Map();
 var crons = /* @__PURE__ */ new WeakMap();
 var EARLY_SLACK_MS = 6e4;
@@ -305,7 +314,15 @@ function onRunFinish(def, run, state, history, now) {
 function onCheck(def, stored, lastRun, state, now) {
   const next = cloneState(state);
   const alerts = [];
-  if (!def.schedule) return { state: next, alerts, nextExpectedAt: null, dueAt: null };
+  if (!def.schedule) {
+    const since = next.open.missed;
+    if (since !== void 0) {
+      delete next.open.missed;
+      next.pendingRecovery = (next.pendingRecovery ?? []).filter((c) => c !== "missed");
+      alerts.push({ type: "recovered", run: lastRun, details: { after: ["missed"], reason: "unscheduled", since } });
+    }
+    return { state: next, alerts, nextExpectedAt: null, dueAt: null };
+  }
   const parsed = parseSchedule(def.schedule, def.timezone);
   const grace = graceMs(def);
   const lastRunAt = lastRun?.startedAt ?? null;
@@ -455,6 +472,12 @@ ${out}`);
       break;
     }
     case "recovered": {
+      if (draft.details.reason === "unscheduled") {
+        title = `${name} is no longer scheduled`;
+        const since = draft.details.since;
+        lines.push(`${since === void 0 ? "" : `Missed since ${when(since, now)}. `}It has no schedule now, so nothing is due; the missed alert is closed.`);
+        break;
+      }
       title = `${name} recovered`;
       const after = draft.details.after.map((c) => c.replace("_", " ")).join(", ");
       lines.push(`A run ${run ? when(run.startedAt, now) : "just now"} succeeded${after ? ` after: ${after}` : ""}.`);
@@ -816,15 +839,17 @@ function messagePage(title, message, base) {
 
 // src/routes/index.ts
 var COOKIE = "cronwatch_token";
-function cookieValue(token) {
-  return createHash("sha256").update(`cronwatch-cookie:${token}`).digest("hex");
+async function cookieValue(token) {
+  const digest = await crypto.subtle.digest("SHA-256", new TextEncoder().encode(`cronwatch-cookie:${token}`));
+  return Array.from(new Uint8Array(digest), (b) => b.toString(16).padStart(2, "0")).join("");
 }
 var DEFAULT_RUNS = 20;
 var MAX_RUNS = 500;
 var CSP = "default-src 'none'; style-src 'unsafe-inline'; img-src 'self' data:; form-action 'self'; frame-ancestors 'none'; base-uri 'none'";
 var SECURITY_HEADERS = { "x-content-type-options": "nosniff", "referrer-policy": "same-origin", "x-robots-tag": "noindex" };
 function developmentToken() {
-  return randomBytes(32).toString("base64url");
+  const bytes = crypto.getRandomValues(new Uint8Array(32));
+  return btoa(String.fromCharCode(...bytes)).replace(/\+/g, "-").replace(/\//g, "_").replace(/=+$/, "");
 }
 function developmentSignInLine(origin, base, token) {
   return `[cronwatch] CRONWATCH_TOKEN is not set, so this development server made a token for the dashboard. Sign in: ${origin}${base}/?token=${token}`;
@@ -845,6 +870,34 @@ function readCookie(request, name) {
   }
   return null;
 }
+function firstValue(value) {
+  const first = value?.split(",")[0]?.trim();
+  return first ? first : null;
+}
+function configuredOrigin(value) {
+  if (value === void 0 || value === "") return null;
+  let url;
+  try {
+    url = new URL(value);
+  } catch {
+    throw new Error(`routes: origin must be an absolute URL such as "https://app.example.com", got ${JSON.stringify(value)}`);
+  }
+  if (url.protocol !== "http:" && url.protocol !== "https:") throw new Error(`routes: origin must be http or https, got ${JSON.stringify(value)}`);
+  return url.origin;
+}
+function forwardedOrigin(request, url) {
+  const proto = firstValue(request.headers.get("x-forwarded-proto"))?.toLowerCase() ?? null;
+  const host = firstValue(request.headers.get("x-forwarded-host"));
+  if (proto === null && host === null) return url.origin;
+  if (proto !== null && proto !== "http" && proto !== "https") return url.origin;
+  try {
+    const built = new URL(`${proto ?? url.protocol.slice(0, -1)}://${host ?? url.host}`);
+    if (built.pathname !== "/" || built.username || built.password || built.search || built.hash) return url.origin;
+    return built.origin;
+  } catch {
+    return url.origin;
+  }
+}
 function stripBase(pathname, base) {
   let path = pathname.startsWith(base) ? pathname.slice(base.length) : pathname;
   if (path === "") path = "/";
@@ -869,9 +922,9 @@ async function readBody(request) {
   }
   return {};
 }
-function crossSite(request, url) {
+function crossSite(request, publicOrigin) {
   const origin = request.headers.get("origin");
-  if (origin !== null && origin !== url.origin) return true;
+  if (origin !== null && origin !== publicOrigin) return true;
   const site = request.headers.get("sec-fetch-site");
   return site !== null && site !== "same-origin" && site !== "none";
 }
@@ -888,31 +941,37 @@ function runsLimit(value) {
 }
 function createRoutes(cw, options = {}) {
   const optedOut = options.token === null;
-  const configured = optedOut ? null : options.token || process.env.CRONWATCH_TOKEN || null;
+  const configured = optedOut ? null : options.token || readEnv("CRONWATCH_TOKEN") || null;
   const base = (options.basePath ?? "/cronwatch").replace(/\/+$/, "");
-  const developing = process.env.NODE_ENV === "development" || process.env.NODE_ENV === "test";
+  const developing = isDevelopment();
+  const fixedOrigin = configuredOrigin(options.origin);
+  const trustProxy = options.trustProxy === true;
+  const originOf = (request, url) => fixedOrigin ?? (trustProxy ? forwardedOrigin(request, url) : url.origin);
   const generated = !configured && !optedOut && developing;
   const token = generated ? developmentToken() : configured;
   let announced = false;
+  let cookie = null;
+  const expectedCookie = (value) => cookie ??= cookieValue(value);
   const serve = async (request, url, path, wantsHtml) => {
     const method = request.method.toUpperCase();
+    const publicOrigin = originOf(request, url);
     if (generated && !announced) {
       announced = true;
-      console.info(developmentSignInLine(url.origin, base, token));
+      console.info(developmentSignInLine(publicOrigin, base, token));
     }
     if (!token && !optedOut) {
       return wantsHtml ? html(messagePage("CronWatch routes are locked", "Set CRONWATCH_TOKEN (or pass token to cw.routes()), or pass token: null to serve them open behind your own auth.", base), 503) : api({ ok: false, error: "CRONWATCH_TOKEN is not set" }, 503);
     }
-    if (method !== "GET" && method !== "HEAD" && crossSite(request, url)) {
+    if (method !== "GET" && method !== "HEAD" && crossSite(request, publicOrigin)) {
       return wantsHtml ? html(messagePage("Cross-site request refused", "Changes can only be made from the dashboard itself.", base), 403) : api({ ok: false, error: "Cross-site request refused" }, 403);
     }
     const bearer = request.headers.get("authorization")?.replace(/^Bearer\s+/i, "") ?? null;
     if (token) {
       const query = wantsHtml && method === "GET" ? url.searchParams.get("token") : null;
-      const cookie = readCookie(request, COOKIE);
+      const sent = readCookie(request, COOKIE);
       const isCheck = path === "/api/check";
       const cronSecretOk = isCheck && bearer !== null && cw.cronSecret !== null && constantTimeEqual(bearer, cw.cronSecret);
-      const tokenOk = bearer !== null ? constantTimeEqual(bearer, token) : query !== null ? constantTimeEqual(query, token) : cookie !== null && constantTimeEqual(cookie, cookieValue(token));
+      const tokenOk = bearer !== null ? constantTimeEqual(bearer, token) : query !== null ? constantTimeEqual(query, token) : sent !== null && constantTimeEqual(sent, await expectedCookie(token));
       if (!cronSecretOk && !tokenOk) {
         if (generated) {
           return wantsHtml ? html(messagePage("Sign in", "CRONWATCH_TOKEN is not set, so this development server made a token. The sign-in link is in the server log: open it once and this browser stays signed in.", base), 401) : api({ ok: false, error: "Unauthorized: CRONWATCH_TOKEN is not set, so this development server made a token; it is in the server log" }, 401);
@@ -921,15 +980,15 @@ function createRoutes(cw, options = {}) {
       }
       if (query !== null) {
         url.searchParams.delete("token");
-        const secure = url.protocol === "https:" ? "; Secure" : "";
+        const secure = publicOrigin.startsWith("https:") ? "; Secure" : "";
         return redirect(url.pathname + (url.search || ""), {
-          "set-cookie": `${COOKIE}=${cookieValue(token)}; Path=${base || "/"}; HttpOnly; SameSite=Lax; Max-Age=${60 * 60 * 24 * 30}${secure}`
+          "set-cookie": `${COOKIE}=${await expectedCookie(token)}; Path=${base || "/"}; HttpOnly; SameSite=Lax; Max-Age=${60 * 60 * 24 * 30}${secure}`
         });
       }
     }
     const redirectBack = () => {
       const referer = request.headers.get("referer") ?? "";
-      return redirect(referer.startsWith(url.origin + "/") ? referer : `${base}/`);
+      return redirect(referer.startsWith(publicOrigin + "/") ? referer : `${base}/`);
     };
     const decoded = path.split("/").filter(Boolean).map(safeDecode);
     if (decoded.some((part) => part === null)) {
@@ -1090,6 +1149,7 @@ function memory() {
       }
     },
     async insertRun(run) {
+      if (runs.has(run.id)) throw new Error(`run ${run.id} already exists`);
       runs.set(run.id, clone(run));
       order.set(run.id, ++seq);
     },
@@ -1099,6 +1159,13 @@ function memory() {
       const { status, finishedAt, durationMs, error, output, metrics } = clone(run);
       runs.set(run.id, { ...existing, status, finishedAt, durationMs, error, output, metrics });
     },
+    async updateRunIf(run, fromStatuses) {
+      const existing = runs.get(run.id);
+      if (!existing || !fromStatuses.includes(existing.status)) return false;
+      const { status, finishedAt, durationMs, error, output, metrics } = clone(run);
+      runs.set(run.id, { ...existing, status, finishedAt, durationMs, error, output, metrics });
+      return true;
+    },
     async getRun(id) {
       const r = runs.get(id);
       return r ? clone(r) : null;
@@ -1194,6 +1261,7 @@ var CronWatch = class {
   store;
   alerts;
   triage;
+  sources;
   /** The secret handler() requests must carry, or null when none is set. */
   cronSecret;
   retentionMs;
@@ -1209,6 +1277,8 @@ var CronWatch = class {
   synced = /* @__PURE__ */ new Set();
   /** The tail of each job's queue of state updates. See serial(). */
   queues = /* @__PURE__ */ new Map();
+  /** start() calls with an id still in flight, so two at once in this process record one run. */
+  starting = /* @__PURE__ */ new Map();
   ready = null;
   checking = null;
   lastPruneAt = 0;
@@ -1221,7 +1291,8 @@ var CronWatch = class {
     this.store = options.store ?? (this.usingDefaultStore = true, memory());
     this.alerts = options.alerts ?? [consoleChannel()];
     this.triage = options.triage;
-    const secret = options.cronSecret === void 0 ? process.env.CRON_SECRET : options.cronSecret;
+    this.sources = options.sources ?? [];
+    const secret = options.cronSecret === void 0 ? readEnv("CRON_SECRET") : options.cronSecret;
     this.cronSecret = secret ? secret : null;
     this.secretOptOut = options.cronSecret === null;
     this.retentionMs = parseDuration(options.retention ?? "30d", "retention");
@@ -1265,6 +1336,12 @@ var CronWatch = class {
   definedJobs() {
     return [...this.definitions.values()];
   }
+  /** A handle on a run started elsewhere, as job(name).resume(runId). The job must be declared in this process. */
+  resumeRun(name, runId) {
+    const definition = this.definitions.get(name);
+    if (!definition) return Promise.reject(new Error(`resumeRun: job "${name}" is not declared; call job() first`));
+    return this.resumeHandle(definition, runId);
+  }
   handle(definition) {
     const self = this;
     return {
@@ -1303,6 +1380,12 @@ var CronWatch = class {
           };
           return json(body, outcome.run.status === "ok" ? 200 : 500);
         };
+      },
+      start(options) {
+        return self.startRun(definition, options ?? {});
+      },
+      resume(runId) {
+        return self.resumeHandle(definition, runId);
       }
     };
   }
@@ -1322,7 +1405,7 @@ var CronWatch = class {
     if (!this.ready) {
       this.ready = (async () => {
         if (this.store.init) await this.store.init();
-        if (this.usingDefaultStore && process.env.NODE_ENV === "production") {
+        if (this.usingDefaultStore && readEnv("NODE_ENV") === "production") {
           console.warn("[cronwatch] using the in-memory store: runs and state are lost on restart. Pass a store from @cronwatch/sdk/sqlite or @cronwatch/sdk/postgres.");
         }
       })().catch((error) => {
@@ -1439,6 +1522,21 @@ var CronWatch = class {
     run.durationMs = Math.max(0, finishedAt - startedAt);
     run.metrics = recorder.metrics();
     run.output = recorder.output() ?? (typeof result === "string" ? capOutput(result) : null);
+    this.conclude(definition, run, result, error, threw, recorder.expectText() ?? (typeof result === "string" ? result : null));
+    await started;
+    try {
+      const ignored = await this.recordFinish(definition, run, recorded, finishedAt);
+      if (ignored) this.report(new Error(`run ${run.id} of ${name} ${ignored}; ignored`), `finishing ${name}`);
+    } catch (e) {
+      this.onError(e, `recording ${name}`);
+    }
+    return { run, result, error, threw };
+  }
+  /**
+   * Sets a finished run's status and error from how it ended, then redacts
+   * its output and error. Shared by execute() and RunHandle.finish().
+   */
+  conclude(definition, run, result, error, threw, expectText) {
     if (threw) {
       run.status = "failed";
       run.error = errorMessage(error);
@@ -1446,7 +1544,6 @@ var CronWatch = class {
       run.status = "failed";
       run.error = `HTTP ${result.status}${result.statusText ? ` ${result.statusText}` : ""}`;
     } else {
-      const expectText = recorder.expectText() ?? (typeof result === "string" ? result : null);
       const unmet = checkExpectation(definition.expect, expectText);
       if (unmet) {
         run.status = "failed";
@@ -1457,44 +1554,336 @@ var CronWatch = class {
     }
     if (run.output !== null) run.output = stripNul(this.redact(run.output));
     if (run.error !== null) run.error = stripNul(this.redact(run.error));
-    await started;
+  }
+  /**
+   * Writes a finished run and evaluates it. `recorded` says whether its
+   * start was written; if not, it is inserted now. Returns why nothing was
+   * recorded (another process finished the run first, say), or null. Throws
+   * when the store does, so a handle can be finished again. Shared by
+   * execute() and RunHandle.finish().
+   */
+  async recordFinish(definition, run, recorded, finishedAt) {
     if (!recorded) {
+      await this.sync(definition);
       try {
-        await this.sync(definition);
         await this.store.insertRun(run);
-        recorded = true;
+        await this.finishRun(toStored(definition), run, finishedAt);
+        return null;
       } catch (e) {
-        this.onError(e, `recording ${name}`);
+        const stored = await this.store.getRun(run.id).catch(() => null);
+        if (!stored) throw e;
+        if (stored.job !== run.job) return `belongs to job "${stored.job}"`;
       }
-      if (recorded) await this.finishRun(toStored(definition), run, finishedAt, false);
-    } else if (await this.markedTimedOut(run)) {
+    }
+    const claim = await this.claimFinish(run);
+    if ("ignored" in claim) return claim.ignored;
+    if (!claim.lateAfterTimeout || run.status === "ok") await this.finishRun(toStored(definition), run, finishedAt);
+    return null;
+  }
+  /** A conditional write (Store.updateRunIf), or for a store without one, a read then a plain write. */
+  async writeRunIf(run, fromStatuses) {
+    if (typeof this.store.updateRunIf === "function") return this.store.updateRunIf(run, fromStatuses);
+    const stored = await this.store.getRun(run.id);
+    if (!stored || !fromStatuses.includes(stored.status)) return false;
+    await this.store.updateRun(run);
+    return true;
+  }
+  /**
+   * Writes a finished run over its stored row, only while that row is still
+   * running, or else still marked timeout by a check. Only the process whose
+   * write lands goes on to evaluate the run; for the others it returns why
+   * nothing was written. `lateAfterTimeout` means a check already counted
+   * the run as a stuck failure: a late failure must not count twice, while
+   * a late success still closes stuck and recovers. Throws when the store does.
+   */
+  async claimFinish(run) {
+    if (await this.writeRunIf(run, ["running"])) return { lateAfterTimeout: false };
+    if (await this.writeRunIf(run, ["timeout"])) return { lateAfterTimeout: true };
+    const stored = await this.store.getRun(run.id);
+    return { ignored: stored ? `was already finished as ${stored.status}` : "was not found" };
+  }
+  /** job.start(): records a running run and returns a handle to finish it. See JobHandle.start. */
+  async startRun(definition, options) {
+    const id = options.id;
+    if (id === void 0) return this.recordStart(definition, options.trigger);
+    checkRunId(definition.name, id, "start");
+    const key = `${definition.name}
+${id}`;
+    const inFlight = this.starting.get(key);
+    if (inFlight) return inFlight;
+    const started = this.recordStart(definition, options.trigger, id).finally(() => {
+      if (this.starting.get(key) === started) this.starting.delete(key);
+    });
+    this.starting.set(key, started);
+    return started;
+  }
+  /**
+   * The start of execute() without the function: the run is inserted and
+   * missed and stuck close (onRunStart). A store that fails is reported and
+   * the handle inserts the finished run instead, as execute() does.
+   */
+  async recordStart(definition, trigger = "start", id) {
+    const name = definition.name;
+    if (id !== void 0) {
+      let stored = null;
       try {
-        await this.store.updateRun(run);
+        await this.ensureReady();
+        stored = await this.store.getRun(id);
       } catch (e) {
         this.onError(e, `recording ${name}`);
       }
-    } else {
-      await this.finishRun(toStored(definition), run, finishedAt, true);
+      if (stored) return this.existingHandle(definition, stored);
     }
-    return { run, result, error, threw };
+    const run = {
+      id: id ?? crypto.randomUUID(),
+      job: name,
+      status: "running",
+      startedAt: this.now(),
+      finishedAt: null,
+      durationMs: null,
+      error: null,
+      output: null,
+      metrics: {},
+      trigger
+    };
+    let recorded = false;
+    try {
+      await this.sync(definition);
+      await this.store.insertRun({ ...run });
+      recorded = true;
+    } catch (e) {
+      const stored = id === void 0 ? null : await this.store.getRun(id).catch(() => null);
+      if (stored) return this.existingHandle(definition, stored);
+      this.onError(e, `recording ${name}`);
+    }
+    if (recorded) {
+      await this.updateState(name, (before) => ({ state: onRunStart(before), result: void 0 })).catch((e) => this.onError(e, `starting ${name}`));
+    }
+    return this.runHandle(definition, run.id, run, recorded, null);
   }
-  /** Whether a check already marked this run as timed out, for a failure that finished late. */
-  async markedTimedOut(run) {
-    if (run.status === "ok") return false;
+  /** job.resume() and cw.resumeRun(). A store that cannot be read is reported, and finish() reads it again. */
+  async resumeHandle(definition, runId) {
+    checkRunId(definition.name, runId, "resume");
+    let stored;
     try {
-      return (await this.store.getRun(run.id))?.status === "timeout";
-    } catch {
-      return false;
+      await this.ensureReady();
+      stored = await this.store.getRun(runId);
+    } catch (e) {
+      this.onError(e, `resuming ${definition.name}`);
+      return this.runHandle(definition, runId, null, true, null);
+    }
+    if (!stored) return this.runHandle(definition, runId, null, true, "was not found");
+    return this.existingHandle(definition, stored);
+  }
+  /** A handle on a stored run. One still running, or marked timeout by a check, can be finished. */
+  existingHandle(definition, stored) {
+    if (stored.job !== definition.name) {
+      throw new Error(`run "${stored.id}" belongs to job "${stored.job}", not "${definition.name}"`);
+    }
+    const finished = stored.status === "ok" || stored.status === "failed";
+    return this.runHandle(definition, stored.id, stored, true, finished ? `already finished as ${stored.status}` : null);
+  }
+  /**
+   * The handle itself. `base` is the run as last known here, `recorded`
+   * whether its start is in the store, and `inactive` why finish() has
+   * nothing to do, or null. Lines and metrics wait in the handle until
+   * flush() or finish() merges them onto a fresh read of the stored run.
+   */
+  runHandle(definition, id, base, recorded, inactive) {
+    const self = this;
+    const name = definition.name;
+    const fresh = () => createRecorder({ id, job: name, startedAt: base?.startedAt ?? 0 });
+    let recorder = fresh();
+    let finished = inactive !== null;
+    let finishCalled = false;
+    let head = null;
+    let queue = Promise.resolve();
+    const inTurn = (fn) => {
+      const result = queue.then(fn);
+      queue = result.catch(() => {
+      });
+      return result;
+    };
+    const ignored = (why) => self.report(new Error(`run ${id} of ${name} ${why}; ignored`), `finishing ${name}`);
+    const finish = (outcome) => {
+      if (finishCalled) {
+        ignored("was already finished by this handle");
+        return Promise.resolve(null);
+      }
+      finishCalled = true;
+      const wasInactive = finished;
+      finished = true;
+      const retryable = (error) => {
+        finishCalled = false;
+        finished = false;
+        self.report(error, `finishing ${name}`);
+        return null;
+      };
+      return inTurn(async () => {
+        if (wasInactive) {
+          ignored(inactive);
+          return null;
+        }
+        let from = base;
+        if (recorded) {
+          try {
+            from = await self.store.getRun(id) ?? base;
+          } catch (e) {
+            return retryable(e);
+          }
+        }
+        if (!from) {
+          ignored("was not found");
+          return null;
+        }
+        if (from.job !== name) {
+          ignored(`belongs to job "${from.job}"`);
+          return null;
+        }
+        if (from.status === "ok" || from.status === "failed") {
+          ignored(`was already finished as ${from.status}`);
+          return null;
+        }
+        const failed = typeof outcome === "object" && outcome !== null && "error" in outcome;
+        const result = typeof outcome === "string" ? outcome : failed ? void 0 : outcome?.result;
+        const error = failed ? outcome.error : void 0;
+        const finishedAt = self.now();
+        const added = recorder.output() ?? (typeof result === "string" ? capOutput(result) : null);
+        const run = {
+          ...from,
+          status: "running",
+          finishedAt,
+          durationMs: Math.max(0, finishedAt - from.startedAt),
+          error: null,
+          output: joinOutput(from.output, added),
+          metrics: { ...from.metrics, ...recorder.metrics() }
+        };
+        const expectText = joinLines(head, joinLines(from.output, recorder.expectText() ?? (typeof result === "string" ? result : null)));
+        self.conclude(definition, run, result, error, failed, expectText);
+        let why;
+        try {
+          why = await self.recordFinish(definition, run, recorded, finishedAt);
+        } catch (e) {
+          return retryable(e);
+        }
+        if (why) {
+          ignored(why);
+          return null;
+        }
+        return run;
+      });
+    };
+    return {
+      id,
+      job: name,
+      startedAt: base?.startedAt ?? null,
+      get active() {
+        return !finished;
+      },
+      log: (...parts) => recorder.context.log(...parts),
+      metric: (metric, value) => recorder.context.metric(metric, value),
+      metrics: (values) => recorder.context.metrics(values),
+      flush: () => inTurn(async () => {
+        if (finished || !recorded) return;
+        const lines = recorder.output();
+        const metrics = recorder.metrics();
+        if (lines === null && Object.keys(metrics).length === 0) return;
+        const taken = recorder;
+        recorder = fresh();
+        const putBack = () => {
+          const later = recorder;
+          recorder = fresh();
+          for (const text of [taken.expectText(), later.expectText()]) if (text !== null) recorder.context.log(text);
+          recorder.context.metrics({ ...taken.metrics(), ...later.metrics() });
+        };
+        try {
+          const stored = await self.store.getRun(id);
+          if (!stored || stored.status !== "running") return putBack();
+          if (stored.job !== name) {
+            putBack();
+            self.report(new Error(`run ${id} of ${name} belongs to job "${stored.job}"; ignored`), `flushing ${name}`);
+            return;
+          }
+          const output = lines === null ? stored.output : joinOutput(stored.output, stripNul(self.redact(lines)));
+          if (!await self.writeRunIf({ ...stored, output, metrics: { ...stored.metrics, ...metrics } }, ["running"])) return putBack();
+          const text = taken.expectText();
+          if (text !== null && (head === null || head.length < OUTPUT_CAP)) head = (joinLines(head, text) ?? "").slice(0, OUTPUT_CAP);
+        } catch (e) {
+          putBack();
+          self.report(e, `flushing ${name}`);
+        }
+      }),
+      finish,
+      fail: (error) => finish({ error })
+    };
+  }
+  /**
+   * Record a run that happened outside this process, for a Source. Its job
+   * must be declared with job() first. Runs are keyed by id: a new one is
+   * inserted, a stored one still running (or marked timeout by a check) is
+   * finished when this one is not running, and anything else is left alone,
+   * so recording the same run twice changes nothing. Finishing is
+   * conditional (see Store.updateRunIf): when two processes record the same
+   * finish, only the one whose write lands evaluates it, and the other
+   * reports it as already finished. A stored run of another job is left
+   * alone and reported. A finished run is judged as if it had been wrapped
+   * here (expect, failures, duration, budgets) and its output and error are
+   * redacted the same way; one finishing after a check marked it timeout is
+   * judged only when it succeeded, as RunHandle.finish() does. Returns the
+   * alerts it sent.
+   */
+  async recordRun(input, options = {}) {
+    const declared = this.definitions.get(input.job);
+    if (!declared) throw new Error(`recordRun: job "${input.job}" is not declared; call job() first`);
+    await this.sync(declared);
+    const run = { ...input, metrics: { ...input.metrics } };
+    if (run.status === "ok") {
+      const unmet = checkExpectation(declared.expect, run.output);
+      if (unmet) {
+        run.status = "failed";
+        run.error = unmet;
+      }
+    }
+    if (run.output !== null) run.output = stripNul(this.redact(capOutput(run.output)));
+    if (run.error !== null) run.error = stripNul(this.redact(capOutput(run.error)));
+    const evaluate = options.evaluate !== false;
+    const definition = toStored(declared);
+    const stored = await this.store.getRun(run.id);
+    if (stored) return this.recordOver(definition, stored, run, evaluate);
+    try {
+      await this.store.insertRun(run);
+    } catch (e) {
+      const again = await this.store.getRun(run.id).catch(() => null);
+      if (again) return this.recordOver(definition, again, run, evaluate);
+      throw e;
     }
+    if (!evaluate) return [];
+    await this.updateState(run.job, (before) => ({ state: onRunStart(before), result: void 0 }));
+    if (run.status === "running") return [];
+    return this.finishRun(definition, run, this.now());
+  }
+  /** recordRun() for a run already stored. */
+  async recordOver(definition, stored, run, evaluate) {
+    if (stored.job !== run.job) {
+      this.report(new Error(`run ${run.id} of ${run.job} belongs to job "${stored.job}"; ignored`), `recording ${run.job}`);
+      return [];
+    }
+    if (stored.status !== "running" && stored.status !== "timeout" || run.status === "running") return [];
+    const claim = await this.claimFinish(run);
+    if ("ignored" in claim) {
+      this.report(new Error(`run ${run.id} of ${run.job} ${claim.ignored}; ignored`), `recording ${run.job}`);
+      return [];
+    }
+    if (!evaluate || claim.lateAfterTimeout && run.status !== "ok") return [];
+    return this.finishRun(definition, run, this.now());
   }
   /**
-   * Record a finished run (ok, failed, or timed out by a check), evaluate it
-   * against the job's state and send what that produces. Never throws.
+   * Evaluate a finished run (ok, failed, or timed out by a check), already
+   * written, against the job's state and send what that produces. Never throws.
    */
-  async finishRun(definition, run, now, write) {
+  async finishRun(definition, run, now) {
     let drafts;
     try {
-      if (write) await this.store.updateRun(run);
       let history = null;
       ({ result: drafts } = await this.updateState(run.job, async (previous) => {
         history ??= await this.history(run);
@@ -1535,9 +1924,16 @@ var CronWatch = class {
   }
   async runCheck() {
     await this.ensureReady();
+    const alerts = [];
+    for (const source of this.sources) {
+      try {
+        alerts.push(...await source.sync(this) ?? []);
+      } catch (e) {
+        this.onError(e, `source ${source.name}`);
+      }
+    }
     for (const definition of this.definitions.values()) await this.sync(definition);
     const now = this.now();
-    const alerts = [];
     for (const run of await this.store.runningRuns()) {
       try {
         const declared = this.definitions.get(run.job);
@@ -1548,7 +1944,8 @@ var CronWatch = class {
         run.finishedAt = now;
         run.durationMs = now - run.startedAt;
         run.error = `Still running after ${formatDuration(timeout)}; marked as timed out`;
-        alerts.push(...await this.finishRun(definition, run, now, true));
+        if (!await this.writeRunIf(run, ["running"])) continue;
+        alerts.push(...await this.finishRun(definition, run, now));
       } catch (e) {
         this.onError(e, `checking ${run.job}`);
       }
@@ -1666,7 +2063,11 @@ var CronWatch = class {
   routes(options) {
     return createRoutes(this, options);
   }
-  /** Check on an interval, for long-running servers. Default every minute. */
+  /**
+   * Check on an interval, for long-running servers. Default every minute.
+   * Not for serverless functions or Cloudflare Workers, where nothing runs
+   * between requests: call check() from a cron there instead.
+   */
   start(every = "1m") {
     if (this.timer) return;
     const ms = Math.max(5e3, parseDuration(every, "check interval"));
@@ -1775,7 +2176,7 @@ var CronWatch = class {
     const results = await Promise.all(
       this.alerts.map(async (channel) => {
         try {
-          await withTimeout(channel.send(alert), CHANNEL_TIMEOUT_MS);
+          await withTimeout(channel.send(alert, { onError: (e) => this.report(e, `alert channel ${channel.name}`) }), CHANNEL_TIMEOUT_MS);
           return true;
         } catch (e) {
           this.onError(e, `alert channel ${channel.name}`);
@@ -1799,8 +2200,24 @@ var CronWatch = class {
     }
   }
 };
-function isDevelopment() {
-  return process.env.NODE_ENV === "development" || process.env.NODE_ENV === "test";
+var RESERVED_RUN_ID_PREFIX = "pgcron:";
+function checkRunId(job, id, method) {
+  if (typeof id !== "string" || id.length === 0 || id.length > 200) {
+    throw new Error(`job "${job}": ${method}() needs a run id of 1 to 200 characters (got ${typeof id === "string" ? `${id.length} characters` : typeof id})`);
+  }
+  if (id.startsWith(RESERVED_RUN_ID_PREFIX)) {
+    throw new Error(`job "${job}": ${method}() cannot take a run id starting with "${RESERVED_RUN_ID_PREFIX}", which the pg_cron source uses for its runs`);
+  }
+}
+function joinLines(before, after) {
+  if (before === null || before === "") return after;
+  if (after === null) return before;
+  return `${before}
+${after}`;
+}
+function joinOutput(before, after) {
+  const joined = joinLines(before, after);
+  return joined === null ? null : capOutput(joined);
 }
 function clampLimit(limit, fallback, min) {
   const n = typeof limit === "number" && Number.isFinite(limit) ? Math.trunc(limit) : fallback;