@cronwatch/sdk 0.14.2 → 0.15.0

package/dist/index.cjs

@@ -636,7 +636,6 @@ function catchesUp(def, run, now) {
 function onRunStart(def, run, state, now) {
   const next = cloneState(state);
   if (catchesUp(def, run, now)) closeCondition(next, "missed");
-  closeCondition(next, "stuck");
   return next;
 }
 function onRunFinish(def, run, state, history, now) {
@@ -724,6 +723,9 @@ function onCheck(def, stored, lastRun, state, now) {
 function isStuck(def, run, now) {
   return run.status === "running" && run.finishedAt === null && now - run.startedAt > timeoutMs(def);
 }
+function isStranded(run, now) {
+  return run.status === "running" && run.finishedAt !== null && now - run.finishedAt > SEND_LEASE_MS;
+}
 function muteOpens(previous, next) {
   const muted = cloneState(next);
   for (const condition of openConditions(muted)) {
@@ -899,6 +901,10 @@ function constantTimeEqual(sent, secret) {
   for (let i = 0; i < sent.length; i++) diff |= sent.charCodeAt(i) ^ secret.charCodeAt(i % secret.length);
   return diff === 0;
 }
+function bearerToken(header) {
+  const match = header === null ? null : /^Bearer\s+([\s\S]*)$/i.exec(header);
+  return match ? match[1] : null;
+}
 
 // src/output.ts
 var OUTPUT_CAP = 16 * 1024;
@@ -992,15 +998,45 @@ function stringify(part) {
     return String(part);
   }
 }
+var LineWindow = class {
+  items = [];
+  start = 0;
+  size = 0;
+  get count() {
+    return this.items.length - this.start;
+  }
+  push(line) {
+    this.items.push(line);
+    this.size += line.length + 1;
+  }
+  /** Drops the oldest lines while over `max`, always keeping the newest. */
+  keepAtMost(max) {
+    while (this.size > max && this.count > 1) this.drop();
+  }
+  /** Drops the oldest lines only while what is left still holds `min`. */
+  keepAtLeast(min) {
+    while (this.count > 1 && this.size - (this.items[this.start].length + 1) > min) this.drop();
+  }
+  join() {
+    return this.items.slice(this.start).join("\n");
+  }
+  drop() {
+    this.size -= this.items[this.start].length + 1;
+    this.start += 1;
+    if (this.start >= 1024 && this.start * 2 >= this.items.length) {
+      this.items = this.items.slice(this.start);
+      this.start = 0;
+    }
+  }
+};
 function createRecorder(run) {
   const window = OUTPUT_CAP + REDACT_EDGE + 2;
   const copy = (text) => JSON.parse(JSON.stringify(text));
-  const lines = [];
-  let size = 0;
+  const lines = new LineWindow();
   const head = [];
   let headSize = 0;
   let whole = "";
-  let tail2 = "";
+  const raw = new LineWindow();
   let logged = false;
   const metrics = {};
   const controller = new AbortController();
@@ -1014,7 +1050,8 @@ function createRecorder(run) {
       const separator = logged ? "\n" : "";
       logged = true;
       if (whole !== null) whole = whole.length + separator.length + line.length <= 2 * OUTPUT_CAP ? whole + separator + line : null;
-      tail2 = copy((tail2 + separator + copy(line.slice(-OUTPUT_CAP))).slice(-OUTPUT_CAP));
+      raw.push(copy(line.slice(-OUTPUT_CAP)));
+      raw.keepAtLeast(OUTPUT_CAP);
       if (headSize < OUTPUT_CAP) {
         head.push(copy(line.slice(0, OUTPUT_CAP - headSize)));
         headSize += line.length + 1;
@@ -1024,10 +1061,7 @@ function createRecorder(run) {
         line = copy(line.slice(-68609));
       }
       lines.push(line);
-      size += line.length + 1;
-      while (size > window && lines.length > 1) {
-        size -= lines.shift().length + 1;
-      }
+      lines.keepAtMost(window);
     },
     metric(name, value) {
       if (typeof value !== "number" || !Number.isFinite(value)) {
@@ -1041,10 +1075,10 @@ function createRecorder(run) {
   };
   return {
     context,
-    output: () => lines.length === 0 ? null : lines.join("\n"),
+    output: () => lines.count === 0 ? null : lines.join(),
     expectText: () => {
       if (!logged) return null;
-      return whole ?? head.join("\n").slice(0, OUTPUT_CAP) + "\n" + tail2;
+      return whole ?? head.join("\n").slice(0, OUTPUT_CAP) + "\n" + raw.join().slice(-OUTPUT_CAP);
     },
     metrics: () => ({ ...metrics }),
     abort: () => controller.abort()
@@ -1218,7 +1252,7 @@ function staticAsset(path, base) {
 }
 
 // src/version.ts
-var VERSION = "0.14.2";
+var VERSION = "0.15.0";
 var API_VERSION = 1;
 
 // src/routes/timeline.ts
@@ -1985,10 +2019,6 @@ function forwardedOrigin(request, url) {
     return url.origin;
   }
 }
-function bearerToken(header) {
-  const match = header === null ? null : /^Bearer\s+([\s\S]*)$/i.exec(header);
-  return match ? match[1] : null;
-}
 function signInReturn(referer, publicOrigin, base) {
   if (referer === null || !referer.startsWith(publicOrigin + "/")) return `${base}/`;
   let url;
@@ -2157,13 +2187,14 @@ function buildRoutes(cw, options = {}) {
       return wantsHtml ? html(messagePage("Bad request", "The path is not valid.", base), 400) : api({ ok: false, error: "Bad path" }, 400);
     }
     const parts = decoded;
-    if (method === "GET" && path === "/") {
+    const reads = method === "GET" || method === "HEAD";
+    if (reads && path === "/") {
       const entries = await cw.jobsWithRuns(BOARD_PAGE_RUNS);
       const now = cw.now();
       const runsByJob = new Map(entries.map((entry) => [entry.job.name, entry.runs]));
       return html(dashboardPage(entries.map((entry) => entry.job), runsByJob, now, base, null, await boardLanes(cw, entries, now)));
     }
-    if (method === "GET" && parts[0] === "jobs" && parts.length === 2) {
+    if (reads && parts[0] === "jobs" && parts.length === 2) {
       const job = await cw.jobSummary(parts[1]);
       if (!job) return html(messagePage("No such job", `${parts[1]} is not in the store.`, base), 404);
       const now = cw.now();
@@ -2199,15 +2230,15 @@ function buildRoutes(cw, options = {}) {
     }
     if (parts[0] === "api") {
       const rest = parts.slice(1);
-      if (method === "GET" && rest.length === 0) {
+      if (reads && rest.length === 0) {
         return api({ ok: true, library: LIBRARY, language: LANGUAGE, version: VERSION, api: API_VERSION });
       }
-      if (method === "GET" && rest[0] === "jobs" && rest.length === 1) {
+      if (reads && rest[0] === "jobs" && rest.length === 1) {
         return api({ ok: true, jobs: await cw.jobs() });
       }
       if (rest[0] === "jobs" && rest.length === 2) {
         const name = rest[1];
-        if (method === "GET") {
+        if (reads) {
           const job = await cw.jobSummary(name);
           if (!job) return api({ ok: false, error: "No such job" }, 404);
           return api({ ok: true, job, runs: await cw.runs(name, runsLimit(url.searchParams.get("runs"))) });
@@ -2248,7 +2279,7 @@ function buildRoutes(cw, options = {}) {
           return api({ ok: true, ...result });
         }
       }
-      if (method === "GET" && rest[0] === "runs" && rest.length === 2) {
+      if (reads && rest[0] === "runs" && rest.length === 2) {
         const run = await cw.getRun(rest[1]);
         return run ? api({ ok: true, run }) : api({ ok: false, error: "No such run" }, 404);
       }
@@ -2263,7 +2294,8 @@ function buildRoutes(cw, options = {}) {
       const path = stripBase(url.pathname, base);
       if (path === null) return html(messagePage("Not found", url.pathname, base), 404);
       wantsHtml = !path.startsWith("/api");
-      return await serve(request, url, path, wantsHtml);
+      const response = await serve(request, url, path, wantsHtml);
+      return request.method.toUpperCase() === "HEAD" ? new Response(null, { status: response.status, headers: response.headers }) : response;
     } catch (e) {
       try {
         cw.onError(e, "routes");
@@ -2407,9 +2439,11 @@ var TRIAGE_TIMEOUT_MS = 25e3;
 var CHANNEL_TIMEOUT_MS = 15e3;
 var PRUNE_INTERVAL_MS = 60 * 6e4;
 var RETRY_BUDGET_MS = 2e4;
+var START_DEADLINE_MS = 5e3;
 var STATE_ATTEMPTS = 10;
 var HISTORY_PAGE = BASELINE_WINDOW + 5;
 var HISTORY_MAX = 200;
+var STRANDED_ERROR = "Imported, but its finish was never written; marked as timed out";
 function validateDefinition(def) {
   const name = def.name;
   if (def.schedule !== void 0) {
@@ -2586,8 +2620,8 @@ var Cronwatch = class {
             return json({ ok: false, error: "CRON_SECRET is not set, so this job will not run for an unauthenticated request. Set it, or pass secret: null to handler() to allow anyone." }, 503);
           }
           if (secret) {
-            const header = request.headers.get("authorization") ?? "";
-            if (!constantTimeEqual(header, `Bearer ${secret}`)) {
+            const sent = bearerToken(request.headers.get("authorization"));
+            if (sent === null || !constantTimeEqual(sent, secret)) {
               return json({ ok: false, error: "Unauthorized" }, 401);
             }
           }
@@ -2649,8 +2683,8 @@ var Cronwatch = class {
   /**
    * Writes the declaration of `definition`'s name as it stands, unless the
    * store has it. A handle kept from an earlier declaration writes the one
-   * that replaced it, never its own over it, and one forgotten since writes
-   * its own. The writes of one name take turns, in the order they were asked
+   * that replaced it, never its own over it, and one forgotten since
+   * declares the job again as it was last declared here. The writes of one name take turns, in the order they were asked
    * for, so one still under way cannot land after a later one; and a name
    * declared again while its write was under way is still to be written. A
    * name is marked as written only while that same declaration stands, so a
@@ -2659,9 +2693,12 @@ var Cronwatch = class {
    *
    * With `confirm`, as a run starts, a name already written is read back:
    * another process may have forgotten the job since, and a job still
-   * declared here comes back on its next run.
+   * declared here comes back on its next run. `running` says the write is
+   * for a run (by default, whenever `confirm` is): a name forgotten here is
+   * then declared again, where a check or a dashboard read, whose loop may
+   * have picked the name up just before the forget, leaves it forgotten.
    */
-  async sync(definition, confirm = false) {
+  async sync(definition, confirm = false, running = confirm) {
     await this.ensureReady();
     const name = definition.name;
     if (this.synced.has(name)) {
@@ -2670,6 +2707,10 @@ var Cronwatch = class {
     }
     await this.serial(name, async () => {
       if (this.synced.has(name)) return;
+      if (!this.definitions.has(name)) {
+        if (!running) return;
+        this.declared(name);
+      }
       const standing = this.definitions.get(name) ?? definition;
       await this.store.upsertJob(toStored(standing), this.now());
       if (this.definitions.get(name) === standing) this.synced.add(name);
@@ -2777,15 +2818,32 @@ var Cronwatch = class {
       metrics: {},
       trigger
     };
+    const begun = { ...run };
     let recorded = false;
-    try {
+    let landedLate = false;
+    let concluded = false;
+    const writing = (async () => {
       await this.sync(definition, true);
-      await this.store.insertRun({ ...run });
-      recorded = true;
+      await this.store.insertRun({ ...begun });
+    })();
+    try {
+      if (await within(writing, START_DEADLINE_MS) === TIMED_OUT) {
+        this.report(new Error(`the store took longer than ${formatDuration(START_DEADLINE_MS)} to record the start of ${name}; running it unrecorded`), `recording ${name}`);
+        void writing.then(
+          () => {
+            landedLate = true;
+            if (concluded) return this.recordFinish(definition, run, true, run.finishedAt).then(() => {
+            }, (e) => this.report(e, `recording ${name}`));
+          },
+          () => {
+          }
+        );
+      } else {
+        recorded = true;
+      }
     } catch (e) {
       this.report(e, `recording ${name}`);
     }
-    const begun = { ...run };
     const started = recorded ? this.updateState(name, (before) => ({ state: holdRun(onRunStart(definition, begun, before, this.now()), run.id), result: void 0 })).then(() => {
     }, (e) => this.report(e, `starting ${name}`)) : Promise.resolve();
     const recorder = createRecorder(run);
@@ -2808,11 +2866,23 @@ var Cronwatch = class {
     run.output = recorder.output() ?? (typeof result === "string" ? result : null);
     this.conclude(definition, run, result, error, threw, recorder.expectText() ?? (typeof result === "string" ? result : null));
     await started;
+    const ignore = (why) => {
+      if (why) this.report(new Error(`run ${run.id} of ${name} ${why}; ignored`), `finishing ${name}`);
+    };
+    const over = recorded || landedLate;
     try {
-      const ignored = await this.recordFinish(definition, run, recorded, finishedAt);
-      if (ignored) this.report(new Error(`run ${run.id} of ${name} ${ignored}; ignored`), `finishing ${name}`);
+      ignore(await this.recordFinish(definition, run, over, finishedAt));
     } catch (e) {
       this.report(e, `recording ${name}`);
+      if (!over && landedLate) {
+        try {
+          ignore(await this.recordFinish(definition, run, true, finishedAt));
+        } catch (again) {
+          this.report(again, `recording ${name}`);
+        }
+      }
+    } finally {
+      concluded = true;
     }
     return { run, result, error, threw };
   }
@@ -2849,7 +2919,7 @@ var Cronwatch = class {
    */
   async recordFinish(definition, run, recorded, finishedAt) {
     if (!recorded) {
-      await this.sync(definition);
+      await this.sync(definition, false, true);
       try {
         await this.store.insertRun(run);
         await this.finishRun(toStored(definition), run, finishedAt);
@@ -2905,7 +2975,7 @@ ${id}`;
   }
   /**
    * The start of execute() without the function: the run is inserted and
-   * missed and stuck close (onRunStart). A store that fails is reported and
+   * missed closes (onRunStart). A store that fails is reported and
    * the handle inserts the finished run instead, as execute() does.
    */
   async recordStart(definition, trigger = "start", id) {
@@ -2983,6 +3053,21 @@ ${id}`;
     let finished = inactive !== null;
     let finishCalled = false;
     let head = null;
+    let flushed = null;
+    const appendRedacted = (stored, lines) => {
+      const added = stripNul(lines);
+      const joins = flushed !== null && stored !== null && stored.endsWith(flushed.redacted);
+      let raw = joins ? `${flushed.raw}
+${added}` : added;
+      const redacted = redactAndCap(raw, self.redact);
+      const output = joins ? stored.slice(0, stored.length - flushed.redacted.length) + redacted : joinLines(stored, redacted);
+      if (raw.length <= OUTPUT_CAP) return { output, tail: { raw, redacted } };
+      const cut = raw.indexOf("\n", raw.length - OUTPUT_CAP);
+      if (cut === -1) return { output, tail: null };
+      raw = raw.slice(cut + 1);
+      const kept = redactAndCap(raw, self.redact);
+      return { output, tail: redacted.endsWith(kept) ? { raw, redacted: kept } : null };
+    };
     let queue = Promise.resolve();
     const inTurn = (fn) => {
       const result = queue.then(fn);
@@ -3042,7 +3127,7 @@ ${id}`;
           durationMs: runDuration(from.startedAt, finishedAt),
           error: null,
           // Capped by conclude(), after it is redacted.
-          output: joinLines(from.output, added),
+          output: added === null ? from.output : appendRedacted(from.output, added).output,
           metrics: { ...from.metrics, ...recorder.metrics() }
         };
         const expectText = joinLines(head, joinLines(from.output, recorder.expectText() ?? (typeof result === "string" ? result : null)));
@@ -3091,8 +3176,10 @@ ${id}`;
             self.report(new Error(`run ${id} of ${name} belongs to job "${stored.job}"; ignored`), `flushing ${name}`);
             return;
           }
-          const output = lines === null ? stored.output : joinOutput(stored.output, redactAndCap(lines, self.redact));
+          const next = lines === null ? null : appendRedacted(stored.output, lines);
+          const output = next === null ? stored.output : next.output === null ? null : capOutput(next.output);
           if (!await self.writeRunIf({ ...stored, output, metrics: { ...stored.metrics, ...metrics } }, ["running"])) return putBack();
+          if (next !== null) flushed = next.tail !== null && output !== null && output.endsWith(next.tail.redacted) ? next.tail : null;
           const text = taken.expectText();
           if (text !== null && (head === null || head.length < OUTPUT_CAP)) head = (joinLines(head, text) ?? "").slice(0, OUTPUT_CAP);
         } catch (e) {
@@ -3128,6 +3215,17 @@ ${id}`;
       throw new Error(`recordRun: run ids must be 1 to ${MAX_RUN_ID} characters (got ${got}; job "${input.job}")`);
     }
     if (input.id.includes("\0")) throw new Error(`recordRun: run ids cannot contain a NUL character (job "${input.job}")`);
+    const where = `job "${input.job}", run "${input.id}"`;
+    if (!["running", "ok", "failed", "timeout"].includes(input.status)) {
+      throw new Error(`recordRun: status must be "running", "ok", "failed", or "timeout" (${where})`);
+    }
+    if (!Number.isFinite(input.startedAt)) throw new Error(`recordRun: startedAt must be a finite number (${where})`);
+    if (input.finishedAt !== null && !Number.isFinite(input.finishedAt)) throw new Error(`recordRun: finishedAt must be a finite number or null (${where})`);
+    if (input.durationMs !== null && !Number.isFinite(input.durationMs)) throw new Error(`recordRun: durationMs must be a finite number or null (${where})`);
+    for (const field of ["error", "output"]) {
+      if (input[field] !== null && typeof input[field] !== "string") throw new Error(`recordRun: ${field} must be a string or null (${where})`);
+    }
+    if (typeof input.trigger !== "string") throw new Error(`recordRun: trigger must be a string (${where})`);
     for (const [metric, value] of Object.entries(input.metrics ?? {})) {
       if (typeof value !== "number" || !Number.isFinite(value)) {
         throw new Error(`recordRun: metric "${metric}" must be a finite number (job "${input.job}", run "${input.id}")`);
@@ -3148,7 +3246,7 @@ ${id}`;
     const definition = toStored(declared);
     const stored = await this.store.getRun(run.id);
     if (stored) return this.recordOver(definition, stored, run, evaluate);
-    const inserted = evaluate ? { ...run, status: "running" } : run;
+    const inserted = evaluate ? { ...run, status: "running", finishedAt: run.status === "running" ? null : this.now() } : run;
     try {
       await this.store.insertRun(inserted);
     } catch (e) {
@@ -3181,7 +3279,7 @@ ${id}`;
       return [];
     }
     if (!evaluate) {
-      await this.updateState(run.job, (current) => ({ state: releaseRun(current, run.id), result: void 0 })).catch((e) => this.onError(e, `recording ${run.job}`));
+      await this.updateState(run.job, (current) => ({ state: releaseRun(current, run.id), result: void 0 })).catch((e) => this.report(e, `recording ${run.job}`));
       return [];
     }
     if (claim.lateAfterTimeout && run.status !== "ok") return [];
@@ -3261,7 +3359,7 @@ ${id}`;
       try {
         alerts.push(...await source.sync(this) ?? []);
       } catch (e) {
-        this.onError(e, `source ${source.name}`);
+        this.report(e, `source ${source.name}`);
       }
     }
     for (const definition of this.definitions.values()) await this.sync(definition);
@@ -3273,18 +3371,27 @@ ${id}`;
         const stored = found ? this.readJob(found) : null;
         if (stored) this.evaluable(stored);
         const definition = declared ? toStored(declared) : stored?.definition;
-        if (!definition || !isStuck(definition, listed, now)) continue;
+        const stranded = isStranded(listed, now);
+        if (!definition || !(stranded || isStuck(definition, listed, now))) continue;
         const run = await this.store.getRun(listed.id);
         if (!run || run.status !== "running" || run.job !== listed.job) continue;
-        const timeout = timeoutMs(definition);
         run.status = "timeout";
-        run.finishedAt = now;
-        run.durationMs = runDuration(run.startedAt, now);
-        run.error = `Still running after ${formatDuration(timeout)}; marked as timed out`;
+        if (stranded && run.error !== null) {
+          run.status = "failed";
+          if (run.durationMs !== null) run.finishedAt = run.startedAt + run.durationMs;
+          else run.durationMs = runDuration(run.startedAt, run.finishedAt);
+        } else if (stranded) {
+          run.durationMs ??= runDuration(run.startedAt, run.finishedAt);
+          run.error = STRANDED_ERROR;
+        } else {
+          run.finishedAt = now;
+          run.durationMs = runDuration(run.startedAt, now);
+          run.error = `Still running after ${formatDuration(timeoutMs(definition))}; marked as timed out`;
+        }
         if (!await this.writeRunIf(run, ["running"])) continue;
         alerts.push(...await this.finishRun(definition, run, now));
       } catch (e) {
-        this.onError(e, `checking ${listed.job}`);
+        this.report(e, `checking ${listed.job}`);
       }
     }
     const jobs = [];
@@ -3308,7 +3415,7 @@ ${id}`;
         alerts.push(...await this.evaluatePending(stored, recent, state, now));
         jobs.push(summarize(stored, recent, state, nextExpectedAt, now));
       } catch (e) {
-        this.onError(e, `checking ${stored.name}`);
+        this.report(e, `checking ${stored.name}`);
         jobs.push(await this.unevaluable(stored, now));
       }
     }
@@ -3318,7 +3425,7 @@ ${id}`;
       try {
         pruned = await this.store.prune(now - this.retentionMs);
       } catch (e) {
-        this.onError(e, "pruning");
+        this.report(e, "pruning");
       }
     }
     return { checkedAt: now, jobs, alerts, pruned };
@@ -3344,7 +3451,7 @@ ${id}`;
         if (run.status === "running" || typeof run.finishedAt !== "number" || run.finishedAt > now - SEND_LEASE_MS) continue;
         alerts.push(...await this.finishRun(stored.definition, run, now, true));
       } catch (e) {
-        this.onError(e, `evaluating run ${id} of ${stored.name}`);
+        this.report(e, `evaluating run ${id} of ${stored.name}`);
       }
     }
     return alerts;
@@ -3359,7 +3466,7 @@ ${id}`;
       const { nextExpectedAt } = onCheck(stored.definition, stored, recent[0] ?? null, state, now);
       return { job: summarize(stored, recent, state, nextExpectedAt, now), runs: recent.slice(0, runs) };
     } catch (e) {
-      this.onError(e, `reading ${stored.name}`);
+      this.report(e, `reading ${stored.name}`);
       return { job: await this.unevaluable(stored, now), runs: recent.slice(0, runs) };
     }
   }
@@ -3386,7 +3493,7 @@ ${id}`;
   async jobSummary(name) {
     await this.ensureReady();
     const definition = this.definitions.get(name);
-    if (definition) await this.sync(definition, true);
+    if (definition) await this.sync(definition, true, false);
     const stored = await this.store.getJob(name);
     if (!stored) return null;
     return (await this.snapshot(this.readJob(stored), this.now(), 0)).job;
@@ -3645,10 +3752,6 @@ function joinLines(before, after2) {
   return `${before}
 ${after2}`;
 }
-function joinOutput(before, after2) {
-  const joined = joinLines(before, after2);
-  return joined === null ? null : capOutput(joined);
-}
 function clampLimit(limit, fallback, min) {
   const n = typeof limit === "number" && Number.isFinite(limit) ? Math.trunc(limit) : fallback;
   return Math.min(500, Math.max(min, n));
@@ -3663,6 +3766,20 @@ function after(ms, fn) {
   arm(ms);
   return () => clearTimeout(timer);
 }
+var TIMED_OUT = /* @__PURE__ */ Symbol("timed out");
+function within(promise, ms) {
+  return new Promise((resolve, reject) => {
+    const t = setTimeout(() => resolve(TIMED_OUT), ms);
+    if (typeof t === "object" && "unref" in t) t.unref();
+    promise.then((v) => {
+      clearTimeout(t);
+      resolve(v);
+    }, (e) => {
+      clearTimeout(t);
+      reject(e);
+    });
+  });
+}
 function withTimeout(promise, ms) {
   return new Promise((resolve, reject) => {
     const t = setTimeout(() => reject(new Error(`timed out after ${ms}ms`)), ms);