@cronwatch/sdk 0.14.3 → 0.15.0

package/dist/index.d.cts

@@ -1,5 +1,5 @@
-import { C as CronwatchOptions, a as Cronwatch } from './client-CHjrNHH-.cjs';
-export { b as CronWatch, c as CronWatchOptions, F as FetchHandler, H as HandlerFn, d as HandlerOptions, J as JobContext, e as JobFn, f as JobHandle, R as RecordRunOptions, g as Routes, h as RoutesOptions, i as RunHandle, j as RunOutcome, S as Source, k as SourceHost, l as StartOptions, m as consoleChannel, n as createRoutes, o as custom } from './client-CHjrNHH-.cjs';
+import { C as CronwatchOptions, a as Cronwatch } from './client-IuAn2-v9.cjs';
+export { b as CronWatch, c as CronWatchOptions, F as FetchHandler, H as HandlerFn, d as HandlerOptions, J as JobContext, e as JobFn, f as JobHandle, R as RecordRunOptions, g as Routes, h as RoutesOptions, i as RunHandle, j as RunOutcome, S as Source, k as SourceHost, l as StartOptions, m as consoleChannel, n as createRoutes, o as custom } from './client-IuAn2-v9.cjs';
 import { S as Store, D as Duration, b as AlertDraft, c as StoredJobDefinition, A as Alert } from './types-B8IFVK5k.cjs';
 export { a as AlertChannel, d as AlertDetails, e as AlertType, B as BudgetBreach, C as ChannelContext, f as CheckResult, g as Condition, E as ExpectRule, H as HeldAlert, h as JobDefinition, i as JobHealth, J as JobOptions, j as JobState, k as JobSummary, R as Run, l as RunStatus, m as SendingAlert, n as StoredJob, T as TriageContext, o as TriageFn } from './types-B8IFVK5k.cjs';
 
@@ -11,7 +11,7 @@ export { a as AlertChannel, d as AlertDetails, e as AlertType, B as BudgetBreach
 declare function memory(): Store & Required<Pick<Store, "lastRun">>;
 
 /** This release of @cronwatch/sdk. scripts/release.mjs keeps it in step with package.json. */
-declare const VERSION = "0.14.3";
+declare const VERSION = "0.15.0";
 
 /**
  * "15m" -> 900000. Accepts a plain number of milliseconds, and compound

package/dist/index.d.ts

@@ -1,5 +1,5 @@
-import { C as CronwatchOptions, a as Cronwatch } from './client-C8j7B2Bo.js';
-export { b as CronWatch, c as CronWatchOptions, F as FetchHandler, H as HandlerFn, d as HandlerOptions, J as JobContext, e as JobFn, f as JobHandle, R as RecordRunOptions, g as Routes, h as RoutesOptions, i as RunHandle, j as RunOutcome, S as Source, k as SourceHost, l as StartOptions, m as consoleChannel, n as createRoutes, o as custom } from './client-C8j7B2Bo.js';
+import { C as CronwatchOptions, a as Cronwatch } from './client-MLN3rDFE.js';
+export { b as CronWatch, c as CronWatchOptions, F as FetchHandler, H as HandlerFn, d as HandlerOptions, J as JobContext, e as JobFn, f as JobHandle, R as RecordRunOptions, g as Routes, h as RoutesOptions, i as RunHandle, j as RunOutcome, S as Source, k as SourceHost, l as StartOptions, m as consoleChannel, n as createRoutes, o as custom } from './client-MLN3rDFE.js';
 import { S as Store, D as Duration, b as AlertDraft, c as StoredJobDefinition, A as Alert } from './types-B8IFVK5k.js';
 export { a as AlertChannel, d as AlertDetails, e as AlertType, B as BudgetBreach, C as ChannelContext, f as CheckResult, g as Condition, E as ExpectRule, H as HeldAlert, h as JobDefinition, i as JobHealth, J as JobOptions, j as JobState, k as JobSummary, R as Run, l as RunStatus, m as SendingAlert, n as StoredJob, T as TriageContext, o as TriageFn } from './types-B8IFVK5k.js';
 
@@ -11,7 +11,7 @@ export { a as AlertChannel, d as AlertDetails, e as AlertType, B as BudgetBreach
 declare function memory(): Store & Required<Pick<Store, "lastRun">>;
 
 /** This release of @cronwatch/sdk. scripts/release.mjs keeps it in step with package.json. */
-declare const VERSION = "0.14.3";
+declare const VERSION = "0.15.0";
 
 /**
  * "15m" -> 900000. Accepts a plain number of milliseconds, and compound

package/dist/index.js

@@ -634,7 +634,6 @@ function catchesUp(def, run, now) {
 function onRunStart(def, run, state, now) {
   const next = cloneState(state);
   if (catchesUp(def, run, now)) closeCondition(next, "missed");
-  closeCondition(next, "stuck");
   return next;
 }
 function onRunFinish(def, run, state, history, now) {
@@ -1251,7 +1250,7 @@ function staticAsset(path, base) {
 }
 
 // src/version.ts
-var VERSION = "0.14.3";
+var VERSION = "0.15.0";
 var API_VERSION = 1;
 
 // src/routes/timeline.ts
@@ -2438,6 +2437,7 @@ var TRIAGE_TIMEOUT_MS = 25e3;
 var CHANNEL_TIMEOUT_MS = 15e3;
 var PRUNE_INTERVAL_MS = 60 * 6e4;
 var RETRY_BUDGET_MS = 2e4;
+var START_DEADLINE_MS = 5e3;
 var STATE_ATTEMPTS = 10;
 var HISTORY_PAGE = BASELINE_WINDOW + 5;
 var HISTORY_MAX = 200;
@@ -2681,8 +2681,8 @@ var Cronwatch = class {
   /**
    * Writes the declaration of `definition`'s name as it stands, unless the
    * store has it. A handle kept from an earlier declaration writes the one
-   * that replaced it, never its own over it, and one forgotten since writes
-   * its own. The writes of one name take turns, in the order they were asked
+   * that replaced it, never its own over it, and one forgotten since
+   * declares the job again as it was last declared here. The writes of one name take turns, in the order they were asked
    * for, so one still under way cannot land after a later one; and a name
    * declared again while its write was under way is still to be written. A
    * name is marked as written only while that same declaration stands, so a
@@ -2691,9 +2691,12 @@ var Cronwatch = class {
    *
    * With `confirm`, as a run starts, a name already written is read back:
    * another process may have forgotten the job since, and a job still
-   * declared here comes back on its next run.
+   * declared here comes back on its next run. `running` says the write is
+   * for a run (by default, whenever `confirm` is): a name forgotten here is
+   * then declared again, where a check or a dashboard read, whose loop may
+   * have picked the name up just before the forget, leaves it forgotten.
    */
-  async sync(definition, confirm = false) {
+  async sync(definition, confirm = false, running = confirm) {
     await this.ensureReady();
     const name = definition.name;
     if (this.synced.has(name)) {
@@ -2702,6 +2705,10 @@ var Cronwatch = class {
     }
     await this.serial(name, async () => {
       if (this.synced.has(name)) return;
+      if (!this.definitions.has(name)) {
+        if (!running) return;
+        this.declared(name);
+      }
       const standing = this.definitions.get(name) ?? definition;
       await this.store.upsertJob(toStored(standing), this.now());
       if (this.definitions.get(name) === standing) this.synced.add(name);
@@ -2809,15 +2816,32 @@ var Cronwatch = class {
       metrics: {},
       trigger
     };
+    const begun = { ...run };
     let recorded = false;
-    try {
+    let landedLate = false;
+    let concluded = false;
+    const writing = (async () => {
       await this.sync(definition, true);
-      await this.store.insertRun({ ...run });
-      recorded = true;
+      await this.store.insertRun({ ...begun });
+    })();
+    try {
+      if (await within(writing, START_DEADLINE_MS) === TIMED_OUT) {
+        this.report(new Error(`the store took longer than ${formatDuration(START_DEADLINE_MS)} to record the start of ${name}; running it unrecorded`), `recording ${name}`);
+        void writing.then(
+          () => {
+            landedLate = true;
+            if (concluded) return this.recordFinish(definition, run, true, run.finishedAt).then(() => {
+            }, (e) => this.report(e, `recording ${name}`));
+          },
+          () => {
+          }
+        );
+      } else {
+        recorded = true;
+      }
     } catch (e) {
       this.report(e, `recording ${name}`);
     }
-    const begun = { ...run };
     const started = recorded ? this.updateState(name, (before) => ({ state: holdRun(onRunStart(definition, begun, before, this.now()), run.id), result: void 0 })).then(() => {
     }, (e) => this.report(e, `starting ${name}`)) : Promise.resolve();
     const recorder = createRecorder(run);
@@ -2840,11 +2864,23 @@ var Cronwatch = class {
     run.output = recorder.output() ?? (typeof result === "string" ? result : null);
     this.conclude(definition, run, result, error, threw, recorder.expectText() ?? (typeof result === "string" ? result : null));
     await started;
+    const ignore = (why) => {
+      if (why) this.report(new Error(`run ${run.id} of ${name} ${why}; ignored`), `finishing ${name}`);
+    };
+    const over = recorded || landedLate;
     try {
-      const ignored = await this.recordFinish(definition, run, recorded, finishedAt);
-      if (ignored) this.report(new Error(`run ${run.id} of ${name} ${ignored}; ignored`), `finishing ${name}`);
+      ignore(await this.recordFinish(definition, run, over, finishedAt));
     } catch (e) {
       this.report(e, `recording ${name}`);
+      if (!over && landedLate) {
+        try {
+          ignore(await this.recordFinish(definition, run, true, finishedAt));
+        } catch (again) {
+          this.report(again, `recording ${name}`);
+        }
+      }
+    } finally {
+      concluded = true;
     }
     return { run, result, error, threw };
   }
@@ -2881,7 +2917,7 @@ var Cronwatch = class {
    */
   async recordFinish(definition, run, recorded, finishedAt) {
     if (!recorded) {
-      await this.sync(definition);
+      await this.sync(definition, false, true);
       try {
         await this.store.insertRun(run);
         await this.finishRun(toStored(definition), run, finishedAt);
@@ -2937,7 +2973,7 @@ ${id}`;
   }
   /**
    * The start of execute() without the function: the run is inserted and
-   * missed and stuck close (onRunStart). A store that fails is reported and
+   * missed closes (onRunStart). A store that fails is reported and
    * the handle inserts the finished run instead, as execute() does.
    */
   async recordStart(definition, trigger = "start", id) {
@@ -3015,6 +3051,21 @@ ${id}`;
     let finished = inactive !== null;
     let finishCalled = false;
     let head = null;
+    let flushed = null;
+    const appendRedacted = (stored, lines) => {
+      const added = stripNul(lines);
+      const joins = flushed !== null && stored !== null && stored.endsWith(flushed.redacted);
+      let raw = joins ? `${flushed.raw}
+${added}` : added;
+      const redacted = redactAndCap(raw, self.redact);
+      const output = joins ? stored.slice(0, stored.length - flushed.redacted.length) + redacted : joinLines(stored, redacted);
+      if (raw.length <= OUTPUT_CAP) return { output, tail: { raw, redacted } };
+      const cut = raw.indexOf("\n", raw.length - OUTPUT_CAP);
+      if (cut === -1) return { output, tail: null };
+      raw = raw.slice(cut + 1);
+      const kept = redactAndCap(raw, self.redact);
+      return { output, tail: redacted.endsWith(kept) ? { raw, redacted: kept } : null };
+    };
     let queue = Promise.resolve();
     const inTurn = (fn) => {
       const result = queue.then(fn);
@@ -3074,7 +3125,7 @@ ${id}`;
           durationMs: runDuration(from.startedAt, finishedAt),
           error: null,
           // Capped by conclude(), after it is redacted.
-          output: joinLines(from.output, added),
+          output: added === null ? from.output : appendRedacted(from.output, added).output,
           metrics: { ...from.metrics, ...recorder.metrics() }
         };
         const expectText = joinLines(head, joinLines(from.output, recorder.expectText() ?? (typeof result === "string" ? result : null)));
@@ -3123,8 +3174,10 @@ ${id}`;
             self.report(new Error(`run ${id} of ${name} belongs to job "${stored.job}"; ignored`), `flushing ${name}`);
             return;
           }
-          const output = lines === null ? stored.output : joinOutput(stored.output, redactAndCap(lines, self.redact));
+          const next = lines === null ? null : appendRedacted(stored.output, lines);
+          const output = next === null ? stored.output : next.output === null ? null : capOutput(next.output);
           if (!await self.writeRunIf({ ...stored, output, metrics: { ...stored.metrics, ...metrics } }, ["running"])) return putBack();
+          if (next !== null) flushed = next.tail !== null && output !== null && output.endsWith(next.tail.redacted) ? next.tail : null;
           const text = taken.expectText();
           if (text !== null && (head === null || head.length < OUTPUT_CAP)) head = (joinLines(head, text) ?? "").slice(0, OUTPUT_CAP);
         } catch (e) {
@@ -3224,7 +3277,7 @@ ${id}`;
       return [];
     }
     if (!evaluate) {
-      await this.updateState(run.job, (current) => ({ state: releaseRun(current, run.id), result: void 0 })).catch((e) => this.onError(e, `recording ${run.job}`));
+      await this.updateState(run.job, (current) => ({ state: releaseRun(current, run.id), result: void 0 })).catch((e) => this.report(e, `recording ${run.job}`));
       return [];
     }
     if (claim.lateAfterTimeout && run.status !== "ok") return [];
@@ -3304,7 +3357,7 @@ ${id}`;
       try {
         alerts.push(...await source.sync(this) ?? []);
       } catch (e) {
-        this.onError(e, `source ${source.name}`);
+        this.report(e, `source ${source.name}`);
       }
     }
     for (const definition of this.definitions.values()) await this.sync(definition);
@@ -3321,10 +3374,13 @@ ${id}`;
         const run = await this.store.getRun(listed.id);
         if (!run || run.status !== "running" || run.job !== listed.job) continue;
         run.status = "timeout";
-        if (stranded) {
+        if (stranded && run.error !== null) {
+          run.status = "failed";
+          if (run.durationMs !== null) run.finishedAt = run.startedAt + run.durationMs;
+          else run.durationMs = runDuration(run.startedAt, run.finishedAt);
+        } else if (stranded) {
           run.durationMs ??= runDuration(run.startedAt, run.finishedAt);
-          run.error = STRANDED_ERROR + (run.error === null ? "" : `
-${run.error}`);
+          run.error = STRANDED_ERROR;
         } else {
           run.finishedAt = now;
           run.durationMs = runDuration(run.startedAt, now);
@@ -3333,7 +3389,7 @@ ${run.error}`);
         if (!await this.writeRunIf(run, ["running"])) continue;
         alerts.push(...await this.finishRun(definition, run, now));
       } catch (e) {
-        this.onError(e, `checking ${listed.job}`);
+        this.report(e, `checking ${listed.job}`);
       }
     }
     const jobs = [];
@@ -3357,7 +3413,7 @@ ${run.error}`);
         alerts.push(...await this.evaluatePending(stored, recent, state, now));
         jobs.push(summarize(stored, recent, state, nextExpectedAt, now));
       } catch (e) {
-        this.onError(e, `checking ${stored.name}`);
+        this.report(e, `checking ${stored.name}`);
         jobs.push(await this.unevaluable(stored, now));
       }
     }
@@ -3367,7 +3423,7 @@ ${run.error}`);
       try {
         pruned = await this.store.prune(now - this.retentionMs);
       } catch (e) {
-        this.onError(e, "pruning");
+        this.report(e, "pruning");
       }
     }
     return { checkedAt: now, jobs, alerts, pruned };
@@ -3393,7 +3449,7 @@ ${run.error}`);
         if (run.status === "running" || typeof run.finishedAt !== "number" || run.finishedAt > now - SEND_LEASE_MS) continue;
         alerts.push(...await this.finishRun(stored.definition, run, now, true));
       } catch (e) {
-        this.onError(e, `evaluating run ${id} of ${stored.name}`);
+        this.report(e, `evaluating run ${id} of ${stored.name}`);
       }
     }
     return alerts;
@@ -3408,7 +3464,7 @@ ${run.error}`);
       const { nextExpectedAt } = onCheck(stored.definition, stored, recent[0] ?? null, state, now);
       return { job: summarize(stored, recent, state, nextExpectedAt, now), runs: recent.slice(0, runs) };
     } catch (e) {
-      this.onError(e, `reading ${stored.name}`);
+      this.report(e, `reading ${stored.name}`);
       return { job: await this.unevaluable(stored, now), runs: recent.slice(0, runs) };
     }
   }
@@ -3435,7 +3491,7 @@ ${run.error}`);
   async jobSummary(name) {
     await this.ensureReady();
     const definition = this.definitions.get(name);
-    if (definition) await this.sync(definition, true);
+    if (definition) await this.sync(definition, true, false);
     const stored = await this.store.getJob(name);
     if (!stored) return null;
     return (await this.snapshot(this.readJob(stored), this.now(), 0)).job;
@@ -3694,10 +3750,6 @@ function joinLines(before, after2) {
   return `${before}
 ${after2}`;
 }
-function joinOutput(before, after2) {
-  const joined = joinLines(before, after2);
-  return joined === null ? null : capOutput(joined);
-}
 function clampLimit(limit, fallback, min) {
   const n = typeof limit === "number" && Number.isFinite(limit) ? Math.trunc(limit) : fallback;
   return Math.min(500, Math.max(min, n));
@@ -3712,6 +3764,20 @@ function after(ms, fn) {
   arm(ms);
   return () => clearTimeout(timer);
 }
+var TIMED_OUT = /* @__PURE__ */ Symbol("timed out");
+function within(promise, ms) {
+  return new Promise((resolve, reject) => {
+    const t = setTimeout(() => resolve(TIMED_OUT), ms);
+    if (typeof t === "object" && "unref" in t) t.unref();
+    promise.then((v) => {
+      clearTimeout(t);
+      resolve(v);
+    }, (e) => {
+      clearTimeout(t);
+      reject(e);
+    });
+  });
+}
 function withTimeout(promise, ms) {
   return new Promise((resolve, reject) => {
     const t = setTimeout(() => reject(new Error(`timed out after ${ms}ms`)), ms);