@cronwatch/sdk 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +69 -0
- package/dist/anthropic.cjs +84 -0
- package/dist/anthropic.cjs.map +1 -0
- package/dist/anthropic.d.cts +35 -0
- package/dist/anthropic.d.ts +35 -0
- package/dist/anthropic.js +78 -0
- package/dist/anthropic.js.map +1 -0
- package/dist/discord.cjs +42 -0
- package/dist/discord.cjs.map +1 -0
- package/dist/discord.d.cts +11 -0
- package/dist/discord.d.ts +11 -0
- package/dist/discord.js +40 -0
- package/dist/discord.js.map +1 -0
- package/dist/index.cjs +1317 -0
- package/dist/index.cjs.map +1 -0
- package/dist/index.d.cts +219 -0
- package/dist/index.d.ts +219 -0
- package/dist/index.js +1302 -0
- package/dist/index.js.map +1 -0
- package/dist/postgres.cjs +142 -0
- package/dist/postgres.cjs.map +1 -0
- package/dist/postgres.d.cts +19 -0
- package/dist/postgres.d.ts +19 -0
- package/dist/postgres.js +136 -0
- package/dist/postgres.js.map +1 -0
- package/dist/slack.cjs +44 -0
- package/dist/slack.cjs.map +1 -0
- package/dist/slack.d.cts +12 -0
- package/dist/slack.d.ts +12 -0
- package/dist/slack.js +42 -0
- package/dist/slack.js.map +1 -0
- package/dist/sqlite.cjs +151 -0
- package/dist/sqlite.cjs.map +1 -0
- package/dist/sqlite.d.cts +17 -0
- package/dist/sqlite.d.ts +17 -0
- package/dist/sqlite.js +144 -0
- package/dist/sqlite.js.map +1 -0
- package/dist/types-BQ1P8z55.d.cts +159 -0
- package/dist/types-BQ1P8z55.d.ts +159 -0
- package/dist/webhook.cjs +24 -0
- package/dist/webhook.cjs.map +1 -0
- package/dist/webhook.d.cts +20 -0
- package/dist/webhook.d.ts +20 -0
- package/dist/webhook.js +22 -0
- package/dist/webhook.js.map +1 -0
- package/package.json +114 -0
package/dist/index.cjs
ADDED
|
@@ -0,0 +1,1317 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
var croner = require('croner');
|
|
4
|
+
|
|
5
|
+
// src/duration.ts
|
|
6
|
+
var UNIT_MS = {
|
|
7
|
+
ms: 1,
|
|
8
|
+
s: 1e3,
|
|
9
|
+
m: 6e4,
|
|
10
|
+
h: 36e5,
|
|
11
|
+
d: 864e5,
|
|
12
|
+
w: 6048e5
|
|
13
|
+
};
|
|
14
|
+
function parseDuration(value, label = "duration") {
|
|
15
|
+
if (typeof value === "number") {
|
|
16
|
+
if (!Number.isFinite(value) || value < 0) throw new Error(`${label} must be a non-negative number of milliseconds`);
|
|
17
|
+
return value;
|
|
18
|
+
}
|
|
19
|
+
const text = value.trim().toLowerCase();
|
|
20
|
+
if (text === "") throw new Error(`${label} is empty`);
|
|
21
|
+
const re = /(\d+(?:\.\d+)?)\s*(ms|s|m|h|d|w)/g;
|
|
22
|
+
let total = 0;
|
|
23
|
+
let consumed = "";
|
|
24
|
+
let match;
|
|
25
|
+
while ((match = re.exec(text)) !== null) {
|
|
26
|
+
total += parseFloat(match[1]) * UNIT_MS[match[2]];
|
|
27
|
+
consumed += match[0];
|
|
28
|
+
}
|
|
29
|
+
if (consumed.replace(/\s+/g, "") !== text.replace(/\s+/g, "")) {
|
|
30
|
+
throw new Error(`${label} "${value}" is not a duration like "15m", "1h30m" or "90s"`);
|
|
31
|
+
}
|
|
32
|
+
return Math.round(total);
|
|
33
|
+
}
|
|
34
|
+
function formatDuration(ms) {
|
|
35
|
+
if (!Number.isFinite(ms)) return "?";
|
|
36
|
+
if (ms < 1e3) return `${Math.round(ms)}ms`;
|
|
37
|
+
const parts = [];
|
|
38
|
+
let rest = Math.round(ms / 1e3);
|
|
39
|
+
const units = [["d", 86400], ["h", 3600], ["m", 60], ["s", 1]];
|
|
40
|
+
for (const [unit, size] of units) {
|
|
41
|
+
if (rest >= size) {
|
|
42
|
+
const n = Math.floor(rest / size);
|
|
43
|
+
rest -= n * size;
|
|
44
|
+
parts.push(`${n}${unit}`);
|
|
45
|
+
}
|
|
46
|
+
if (parts.length === 2) break;
|
|
47
|
+
}
|
|
48
|
+
return parts.join(" ") || "0s";
|
|
49
|
+
}
|
|
50
|
+
function formatRelative(at, now) {
|
|
51
|
+
const diff = at - now;
|
|
52
|
+
const abs = Math.abs(diff);
|
|
53
|
+
const text = abs < 5e3 ? "now" : formatDuration(abs);
|
|
54
|
+
if (abs < 5e3) return text;
|
|
55
|
+
return diff < 0 ? `${text} ago` : `in ${text}`;
|
|
56
|
+
}
|
|
57
|
+
var cache = /* @__PURE__ */ new Map();
|
|
58
|
+
function parseSchedule(schedule, timezone) {
|
|
59
|
+
const key = `${timezone ?? ""}|${schedule}`;
|
|
60
|
+
const hit = cache.get(key);
|
|
61
|
+
if (hit) return hit;
|
|
62
|
+
const text = schedule.trim();
|
|
63
|
+
const every = /^every\s+(.+)$/i.exec(text);
|
|
64
|
+
let parsed;
|
|
65
|
+
if (every) {
|
|
66
|
+
const everyMs = parseDuration(every[1], "schedule interval");
|
|
67
|
+
if (everyMs < 1e3) throw new Error(`schedule "${schedule}" is shorter than one second`);
|
|
68
|
+
parsed = { kind: "interval", source: text, everyMs };
|
|
69
|
+
} else {
|
|
70
|
+
let cron;
|
|
71
|
+
try {
|
|
72
|
+
cron = new croner.Cron(text, timezone ? { timezone } : {});
|
|
73
|
+
} catch (error) {
|
|
74
|
+
throw new Error(`schedule "${schedule}" is not a cron expression or "every <duration>": ${error.message}`);
|
|
75
|
+
}
|
|
76
|
+
parsed = { kind: "cron", source: text, cron };
|
|
77
|
+
}
|
|
78
|
+
cache.set(key, parsed);
|
|
79
|
+
return parsed;
|
|
80
|
+
}
|
|
81
|
+
function nextFire(parsed, from, lastRunAt) {
|
|
82
|
+
if (parsed.kind === "interval") {
|
|
83
|
+
const base = lastRunAt ?? from;
|
|
84
|
+
return base + parsed.everyMs;
|
|
85
|
+
}
|
|
86
|
+
const next = parsed.cron.nextRun(new Date(from));
|
|
87
|
+
return next ? next.getTime() : null;
|
|
88
|
+
}
|
|
89
|
+
function previousFire(parsed, now) {
|
|
90
|
+
if (parsed.kind !== "cron") throw new Error("previousFire is for cron schedules");
|
|
91
|
+
const cron = parsed.cron;
|
|
92
|
+
const fireAtOrBefore = (from) => {
|
|
93
|
+
const next = cron.nextRun(new Date(from - 1));
|
|
94
|
+
if (!next) return null;
|
|
95
|
+
const t = next.getTime();
|
|
96
|
+
return t <= now ? t : null;
|
|
97
|
+
};
|
|
98
|
+
const YEAR = 366 * 864e5;
|
|
99
|
+
let step = 6e4;
|
|
100
|
+
let lo = null;
|
|
101
|
+
while (step <= YEAR) {
|
|
102
|
+
if (fireAtOrBefore(now - step) !== null) {
|
|
103
|
+
lo = now - step;
|
|
104
|
+
break;
|
|
105
|
+
}
|
|
106
|
+
step *= 2;
|
|
107
|
+
}
|
|
108
|
+
if (lo === null) return null;
|
|
109
|
+
let hi = now;
|
|
110
|
+
while (hi - lo > 1) {
|
|
111
|
+
const mid = Math.floor((lo + hi) / 2);
|
|
112
|
+
if (fireAtOrBefore(mid) !== null) lo = mid;
|
|
113
|
+
else hi = mid;
|
|
114
|
+
}
|
|
115
|
+
return fireAtOrBefore(lo);
|
|
116
|
+
}
|
|
117
|
+
function expectation(parsed, now, lastRunAt, registeredAt, graceMs2) {
|
|
118
|
+
if (parsed.kind === "interval") {
|
|
119
|
+
const dueAt = (lastRunAt ?? registeredAt) + parsed.everyMs;
|
|
120
|
+
return { dueAt, deadline: dueAt + graceMs2 };
|
|
121
|
+
}
|
|
122
|
+
const prev = previousFire(parsed, now);
|
|
123
|
+
if (prev === null || prev < registeredAt) return null;
|
|
124
|
+
return { dueAt: prev, deadline: prev + graceMs2 };
|
|
125
|
+
}
|
|
126
|
+
function runCovers(startedAt, dueAt) {
|
|
127
|
+
return startedAt >= dueAt - 6e4;
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
// src/stats.ts
|
|
131
|
+
function percentile(values, p) {
|
|
132
|
+
if (values.length === 0) return null;
|
|
133
|
+
const sorted = [...values].sort((a, b) => a - b);
|
|
134
|
+
const index = Math.min(sorted.length - 1, Math.max(0, Math.ceil(p / 100 * sorted.length) - 1));
|
|
135
|
+
return sorted[index];
|
|
136
|
+
}
|
|
137
|
+
function median(values) {
|
|
138
|
+
if (values.length === 0) return null;
|
|
139
|
+
const sorted = [...values].sort((a, b) => a - b);
|
|
140
|
+
const mid = Math.floor(sorted.length / 2);
|
|
141
|
+
return sorted.length % 2 === 0 ? (sorted[mid - 1] + sorted[mid]) / 2 : sorted[mid];
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
// src/evaluate.ts
|
|
145
|
+
var DEFAULT_GRACE_MS = 10 * 6e4;
|
|
146
|
+
var DEFAULT_TIMEOUT_MS = 60 * 6e4;
|
|
147
|
+
var SLOW_FLOOR_MS = 1e4;
|
|
148
|
+
var BASELINE_MIN_RUNS = 5;
|
|
149
|
+
var BASELINE_WINDOW = 20;
|
|
150
|
+
function emptyState(job) {
|
|
151
|
+
return { job, open: {}, consecutiveFailures: 0, silencedUntil: null, lastAlertAt: null };
|
|
152
|
+
}
|
|
153
|
+
function cloneState(state) {
|
|
154
|
+
return { ...state, open: { ...state.open } };
|
|
155
|
+
}
|
|
156
|
+
function openCondition(state, condition, now) {
|
|
157
|
+
if (state.open[condition] !== void 0) return false;
|
|
158
|
+
state.open[condition] = now;
|
|
159
|
+
return true;
|
|
160
|
+
}
|
|
161
|
+
function closeCondition(state, condition) {
|
|
162
|
+
if (state.open[condition] === void 0) return false;
|
|
163
|
+
delete state.open[condition];
|
|
164
|
+
return true;
|
|
165
|
+
}
|
|
166
|
+
function openConditions(state) {
|
|
167
|
+
return Object.keys(state.open);
|
|
168
|
+
}
|
|
169
|
+
function graceMs(def) {
|
|
170
|
+
return def.grace === void 0 ? DEFAULT_GRACE_MS : parseDuration(def.grace, "grace");
|
|
171
|
+
}
|
|
172
|
+
function timeoutMs(def) {
|
|
173
|
+
return def.timeout === void 0 ? DEFAULT_TIMEOUT_MS : parseDuration(def.timeout, "timeout");
|
|
174
|
+
}
|
|
175
|
+
function slowThreshold(def, history) {
|
|
176
|
+
if (def.maxDuration !== void 0) {
|
|
177
|
+
return { thresholdMs: parseDuration(def.maxDuration, "maxDuration"), basis: "maxDuration" };
|
|
178
|
+
}
|
|
179
|
+
const durations = history.filter((r) => r.status === "ok" && r.durationMs !== null).slice(0, BASELINE_WINDOW).map((r) => r.durationMs);
|
|
180
|
+
if (durations.length < BASELINE_MIN_RUNS) return null;
|
|
181
|
+
const p95 = percentile(durations, 95);
|
|
182
|
+
return { thresholdMs: Math.max(2 * p95, SLOW_FLOOR_MS), basis: `twice the p95 of the last ${durations.length} runs (${formatDuration(p95)})` };
|
|
183
|
+
}
|
|
184
|
+
function budgetBreaches(def, run, history) {
|
|
185
|
+
const breaches = [];
|
|
186
|
+
for (const [metric, value] of Object.entries(run.metrics)) {
|
|
187
|
+
const ceiling = def.budget?.[metric];
|
|
188
|
+
if (ceiling !== void 0) {
|
|
189
|
+
if (value > ceiling) breaches.push({ metric, value, limit: ceiling, basis: "budget" });
|
|
190
|
+
continue;
|
|
191
|
+
}
|
|
192
|
+
const past = history.filter((r) => r.status === "ok" && typeof r.metrics[metric] === "number").slice(0, BASELINE_WINDOW).map((r) => r.metrics[metric]);
|
|
193
|
+
if (past.length < BASELINE_MIN_RUNS) continue;
|
|
194
|
+
const usual = median(past);
|
|
195
|
+
if (usual > 0 && value > 3 * usual) {
|
|
196
|
+
breaches.push({ metric, value, limit: 3 * usual, basis: `three times the usual ${formatNumber(usual)}` });
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
return breaches;
|
|
200
|
+
}
|
|
201
|
+
function formatNumber(n) {
|
|
202
|
+
if (Number.isInteger(n)) return n.toLocaleString("en-US");
|
|
203
|
+
return n.toLocaleString("en-US", { maximumFractionDigits: 4 });
|
|
204
|
+
}
|
|
205
|
+
function onRunStart(state) {
|
|
206
|
+
const next = cloneState(state);
|
|
207
|
+
const openBefore = openConditions(next);
|
|
208
|
+
closeCondition(next, "missed");
|
|
209
|
+
closeCondition(next, "stuck");
|
|
210
|
+
return { state: next, openBefore };
|
|
211
|
+
}
|
|
212
|
+
function onRunFinish(def, run, state, history, openBefore, now) {
|
|
213
|
+
const next = cloneState(state);
|
|
214
|
+
const alerts = [];
|
|
215
|
+
if (run.status === "ok") {
|
|
216
|
+
next.consecutiveFailures = 0;
|
|
217
|
+
closeCondition(next, "failed");
|
|
218
|
+
closeCondition(next, "missed");
|
|
219
|
+
closeCondition(next, "stuck");
|
|
220
|
+
const slow = slowThreshold(def, history);
|
|
221
|
+
if (slow && run.durationMs !== null && run.durationMs > slow.thresholdMs) {
|
|
222
|
+
if (openCondition(next, "slow", now)) {
|
|
223
|
+
alerts.push({ type: "slow", run, details: { durationMs: run.durationMs, thresholdMs: slow.thresholdMs, basis: slow.basis } });
|
|
224
|
+
}
|
|
225
|
+
} else {
|
|
226
|
+
closeCondition(next, "slow");
|
|
227
|
+
}
|
|
228
|
+
const breaches = budgetBreaches(def, run, history);
|
|
229
|
+
if (breaches.length > 0) {
|
|
230
|
+
if (openCondition(next, "over_budget", now)) {
|
|
231
|
+
alerts.push({ type: "over_budget", run, details: { breaches } });
|
|
232
|
+
}
|
|
233
|
+
} else {
|
|
234
|
+
closeCondition(next, "over_budget");
|
|
235
|
+
}
|
|
236
|
+
const wasOpen = /* @__PURE__ */ new Set([...openBefore, ...openConditions(state)]);
|
|
237
|
+
if (wasOpen.size > 0 && openConditions(next).length === 0) {
|
|
238
|
+
alerts.push({ type: "recovered", run, details: { after: [...wasOpen] } });
|
|
239
|
+
}
|
|
240
|
+
return { state: next, alerts };
|
|
241
|
+
}
|
|
242
|
+
next.consecutiveFailures += 1;
|
|
243
|
+
closeCondition(next, "missed");
|
|
244
|
+
const threshold = Math.max(1, def.failuresBeforeAlert ?? 1);
|
|
245
|
+
const condition = run.status === "timeout" ? "stuck" : "failed";
|
|
246
|
+
if (next.consecutiveFailures >= threshold) {
|
|
247
|
+
if (openCondition(next, condition, now)) {
|
|
248
|
+
alerts.push({ type: condition, run, details: { consecutiveFailures: next.consecutiveFailures, threshold } });
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
return { state: next, alerts };
|
|
252
|
+
}
|
|
253
|
+
function onCheck(def, stored, lastRun, state, now) {
|
|
254
|
+
const next = cloneState(state);
|
|
255
|
+
const alerts = [];
|
|
256
|
+
if (!def.schedule) return { state: next, alerts, nextExpectedAt: null, dueAt: null };
|
|
257
|
+
const parsed = parseSchedule(def.schedule, def.timezone);
|
|
258
|
+
const grace = graceMs(def);
|
|
259
|
+
const exp = expectation(parsed, now, lastRun?.startedAt ?? null, stored.createdAt, grace);
|
|
260
|
+
let nextExpectedAt = null;
|
|
261
|
+
if (parsed.kind === "interval") {
|
|
262
|
+
nextExpectedAt = (lastRun?.startedAt ?? stored.createdAt) + parsed.everyMs;
|
|
263
|
+
} else {
|
|
264
|
+
const upcoming = parsed.cron.nextRun(new Date(now));
|
|
265
|
+
nextExpectedAt = upcoming ? upcoming.getTime() : null;
|
|
266
|
+
}
|
|
267
|
+
if (!exp) return { state: next, alerts, nextExpectedAt, dueAt: null };
|
|
268
|
+
const covered = parsed.kind === "cron" && lastRun !== null && runCovers(lastRun.startedAt, exp.dueAt);
|
|
269
|
+
if (covered) {
|
|
270
|
+
closeCondition(next, "missed");
|
|
271
|
+
} else if (now > exp.deadline) {
|
|
272
|
+
if (openCondition(next, "missed", now)) {
|
|
273
|
+
alerts.push({
|
|
274
|
+
type: "missed",
|
|
275
|
+
run: lastRun,
|
|
276
|
+
details: { dueAt: exp.dueAt, deadline: exp.deadline, graceMs: grace, lastRunAt: lastRun?.startedAt ?? null }
|
|
277
|
+
});
|
|
278
|
+
}
|
|
279
|
+
}
|
|
280
|
+
return { state: next, alerts, nextExpectedAt, dueAt: exp.dueAt };
|
|
281
|
+
}
|
|
282
|
+
function isStuck(def, run, now) {
|
|
283
|
+
return run.status === "running" && now - run.startedAt > timeoutMs(def);
|
|
284
|
+
}
|
|
285
|
+
function muteOpens(previous, next) {
|
|
286
|
+
const muted = cloneState(next);
|
|
287
|
+
for (const condition of openConditions(muted)) {
|
|
288
|
+
if (previous.open[condition] === void 0) delete muted.open[condition];
|
|
289
|
+
}
|
|
290
|
+
return muted;
|
|
291
|
+
}
|
|
292
|
+
function isSilenced(state, now) {
|
|
293
|
+
return state.silencedUntil !== null && state.silencedUntil > now;
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
// src/format.ts
|
|
297
|
+
function when(at, now) {
|
|
298
|
+
if (at === null || at === void 0) return "never";
|
|
299
|
+
return `${new Date(at).toISOString().replace("T", " ").slice(0, 19)} UTC (${formatRelative(at, now)})`;
|
|
300
|
+
}
|
|
301
|
+
function firstLines(text, n) {
|
|
302
|
+
if (!text) return "";
|
|
303
|
+
return text.split("\n").slice(0, n).join("\n");
|
|
304
|
+
}
|
|
305
|
+
function tail(text, n) {
|
|
306
|
+
if (!text) return "";
|
|
307
|
+
const lines = text.trimEnd().split("\n");
|
|
308
|
+
return lines.slice(Math.max(0, lines.length - n)).join("\n");
|
|
309
|
+
}
|
|
310
|
+
function composeAlert(draft, def, now) {
|
|
311
|
+
const name = def.name;
|
|
312
|
+
const run = draft.run;
|
|
313
|
+
const d = draft.details;
|
|
314
|
+
let title;
|
|
315
|
+
const lines = [];
|
|
316
|
+
switch (draft.type) {
|
|
317
|
+
case "missed": {
|
|
318
|
+
title = `${name} missed its scheduled run`;
|
|
319
|
+
lines.push(`Due ${when(d.dueAt, now)}, and no run had started by ${when(d.deadline, now)} (grace ${formatDuration(d.graceMs)}).`);
|
|
320
|
+
lines.push(`Schedule: ${def.schedule}${def.timezone ? ` (${def.timezone})` : ""}.`);
|
|
321
|
+
lines.push(`Last run: ${run ? `${run.status} ${when(run.startedAt, now)}` : "never"}.`);
|
|
322
|
+
break;
|
|
323
|
+
}
|
|
324
|
+
case "failed": {
|
|
325
|
+
title = `${name} failed`;
|
|
326
|
+
const n = d.consecutiveFailures;
|
|
327
|
+
if (n > 1) lines.push(`${n} consecutive failures.`);
|
|
328
|
+
if (run) {
|
|
329
|
+
lines.push(`Started ${when(run.startedAt, now)}${run.durationMs !== null ? `, ran ${formatDuration(run.durationMs)}` : ""}.`);
|
|
330
|
+
if (run.error) lines.push(`Error: ${firstLines(run.error, 4)}`);
|
|
331
|
+
const out = tail(run.output, 8);
|
|
332
|
+
if (out) lines.push(`Output (tail):
|
|
333
|
+
${out}`);
|
|
334
|
+
}
|
|
335
|
+
break;
|
|
336
|
+
}
|
|
337
|
+
case "stuck": {
|
|
338
|
+
title = `${name} is stuck`;
|
|
339
|
+
if (run) {
|
|
340
|
+
lines.push(`Started ${when(run.startedAt, now)} and never reported finishing. Marked as timed out after ${formatDuration(run.durationMs ?? now - run.startedAt)}.`);
|
|
341
|
+
const out = tail(run.output, 8);
|
|
342
|
+
if (out) lines.push(`Output so far (tail):
|
|
343
|
+
${out}`);
|
|
344
|
+
}
|
|
345
|
+
lines.push(`If the process was killed mid-run (a serverless timeout, a deploy), this is what that looks like.`);
|
|
346
|
+
break;
|
|
347
|
+
}
|
|
348
|
+
case "slow": {
|
|
349
|
+
title = `${name} was slow`;
|
|
350
|
+
lines.push(`Took ${formatDuration(d.durationMs)}; the limit is ${formatDuration(d.thresholdMs)} (${d.basis}).`);
|
|
351
|
+
if (run) lines.push(`Started ${when(run.startedAt, now)}.`);
|
|
352
|
+
break;
|
|
353
|
+
}
|
|
354
|
+
case "over_budget": {
|
|
355
|
+
title = `${name} went over budget`;
|
|
356
|
+
for (const b of d.breaches) {
|
|
357
|
+
lines.push(`${b.metric}: ${formatNumber(b.value)}, limit ${formatNumber(b.limit)} (${b.basis}).`);
|
|
358
|
+
}
|
|
359
|
+
if (run) lines.push(`Started ${when(run.startedAt, now)}.`);
|
|
360
|
+
break;
|
|
361
|
+
}
|
|
362
|
+
case "recovered": {
|
|
363
|
+
title = `${name} recovered`;
|
|
364
|
+
const after = d.after.map((c) => c.replace("_", " ")).join(", ");
|
|
365
|
+
lines.push(`A run ${run ? when(run.startedAt, now) : "just now"} succeeded${after ? ` after: ${after}` : ""}.`);
|
|
366
|
+
if (run?.durationMs !== null && run?.durationMs !== void 0) lines.push(`Ran ${formatDuration(run.durationMs)}.`);
|
|
367
|
+
break;
|
|
368
|
+
}
|
|
369
|
+
}
|
|
370
|
+
return {
|
|
371
|
+
type: draft.type,
|
|
372
|
+
job: name,
|
|
373
|
+
definition: def,
|
|
374
|
+
run,
|
|
375
|
+
title,
|
|
376
|
+
message: lines.join("\n"),
|
|
377
|
+
details: d,
|
|
378
|
+
at: now
|
|
379
|
+
};
|
|
380
|
+
}
|
|
381
|
+
|
|
382
|
+
// src/http.ts
|
|
383
|
+
function json(body, status = 200, headers = {}) {
|
|
384
|
+
return new Response(JSON.stringify(body), {
|
|
385
|
+
status,
|
|
386
|
+
headers: { "content-type": "application/json; charset=utf-8", "cache-control": "no-store", ...headers }
|
|
387
|
+
});
|
|
388
|
+
}
|
|
389
|
+
|
|
390
|
+
// src/output.ts
|
|
391
|
+
var OUTPUT_CAP = 16 * 1024;
|
|
392
|
+
function capOutput(text) {
|
|
393
|
+
if (text.length <= OUTPUT_CAP) return text;
|
|
394
|
+
return "[earlier output trimmed]\n" + text.slice(text.length - OUTPUT_CAP);
|
|
395
|
+
}
|
|
396
|
+
function errorMessage(error) {
|
|
397
|
+
if (error instanceof Error) {
|
|
398
|
+
const stack = error.stack ? `
|
|
399
|
+
${error.stack.split("\n").slice(1, 6).join("\n")}` : "";
|
|
400
|
+
return `${error.name}: ${error.message}${stack}`;
|
|
401
|
+
}
|
|
402
|
+
if (typeof error === "string") return error;
|
|
403
|
+
try {
|
|
404
|
+
return JSON.stringify(error);
|
|
405
|
+
} catch {
|
|
406
|
+
return String(error);
|
|
407
|
+
}
|
|
408
|
+
}
|
|
409
|
+
|
|
410
|
+
// src/job.ts
|
|
411
|
+
function stringify(part) {
|
|
412
|
+
if (typeof part === "string") return part;
|
|
413
|
+
if (part instanceof Error) return `${part.name}: ${part.message}`;
|
|
414
|
+
try {
|
|
415
|
+
return JSON.stringify(part);
|
|
416
|
+
} catch {
|
|
417
|
+
return String(part);
|
|
418
|
+
}
|
|
419
|
+
}
|
|
420
|
+
function createRecorder(run) {
|
|
421
|
+
const lines = [];
|
|
422
|
+
let size = 0;
|
|
423
|
+
const metrics = {};
|
|
424
|
+
const controller = new AbortController();
|
|
425
|
+
const context = {
|
|
426
|
+
name: run.job,
|
|
427
|
+
runId: run.id,
|
|
428
|
+
startedAt: run.startedAt,
|
|
429
|
+
signal: controller.signal,
|
|
430
|
+
log(...parts) {
|
|
431
|
+
const line = parts.map(stringify).join(" ");
|
|
432
|
+
lines.push(line);
|
|
433
|
+
size += line.length + 1;
|
|
434
|
+
while (size > 64 * 1024 && lines.length > 1) {
|
|
435
|
+
size -= lines.shift().length + 1;
|
|
436
|
+
}
|
|
437
|
+
},
|
|
438
|
+
metric(name, value) {
|
|
439
|
+
if (typeof value !== "number" || !Number.isFinite(value)) {
|
|
440
|
+
throw new Error(`metric "${name}" must be a finite number`);
|
|
441
|
+
}
|
|
442
|
+
metrics[name] = value;
|
|
443
|
+
},
|
|
444
|
+
metrics(values) {
|
|
445
|
+
for (const [k, v] of Object.entries(values)) context.metric(k, v);
|
|
446
|
+
}
|
|
447
|
+
};
|
|
448
|
+
return {
|
|
449
|
+
context,
|
|
450
|
+
output: () => lines.length === 0 ? null : capOutput(lines.join("\n")),
|
|
451
|
+
metrics: () => ({ ...metrics }),
|
|
452
|
+
abort: () => controller.abort()
|
|
453
|
+
};
|
|
454
|
+
}
|
|
455
|
+
|
|
456
|
+
// src/serialize.ts
|
|
457
|
+
function toStored(def) {
|
|
458
|
+
const { expect, ...rest } = def;
|
|
459
|
+
const stored = { ...rest };
|
|
460
|
+
if (expect !== void 0) {
|
|
461
|
+
stored.expect = typeof expect === "string" ? `contains ${JSON.stringify(expect)}` : expect instanceof RegExp ? `matches ${expect.toString()}` : "custom function";
|
|
462
|
+
}
|
|
463
|
+
return stored;
|
|
464
|
+
}
|
|
465
|
+
function checkExpectation(expect, output) {
|
|
466
|
+
if (expect === void 0) return null;
|
|
467
|
+
const text = output ?? "";
|
|
468
|
+
if (typeof expect === "string") {
|
|
469
|
+
return text.includes(expect) ? null : `Output did not contain ${JSON.stringify(expect)}`;
|
|
470
|
+
}
|
|
471
|
+
if (expect instanceof RegExp) {
|
|
472
|
+
return expect.test(text) ? null : `Output did not match ${expect.toString()}`;
|
|
473
|
+
}
|
|
474
|
+
let ok = false;
|
|
475
|
+
try {
|
|
476
|
+
ok = expect(text);
|
|
477
|
+
} catch (error) {
|
|
478
|
+
return `Output check threw: ${error.message}`;
|
|
479
|
+
}
|
|
480
|
+
return ok ? null : "Output did not pass the expect() check";
|
|
481
|
+
}
|
|
482
|
+
|
|
483
|
+
// src/routes/html.ts
|
|
484
|
+
function escapeHtml(value) {
|
|
485
|
+
return String(value ?? "").replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">").replace(/"/g, """).replace(/'/g, "'");
|
|
486
|
+
}
|
|
487
|
+
var h = escapeHtml;
|
|
488
|
+
var CSS = `
|
|
489
|
+
:root{--bg:#fbfbf9;--fg:#1b1b18;--muted:#6b6b64;--line:#e6e5df;--card:#fff;--ok:#1f8a4c;--warn:#b7791f;--bad:#c62828;--info:#2b5fb3;--pill:#f1f0ea;--mono:ui-monospace,SFMono-Regular,Menlo,Consolas,monospace;--sans:-apple-system,BlinkMacSystemFont,"Segoe UI",Inter,Roboto,sans-serif}
|
|
490
|
+
@media(prefers-color-scheme:dark){:root{--bg:#121311;--fg:#ecece6;--muted:#9a9a91;--line:#2a2b27;--card:#1a1b18;--pill:#24251f}}
|
|
491
|
+
*{box-sizing:border-box}html{-webkit-text-size-adjust:100%}
|
|
492
|
+
body{margin:0;background:var(--bg);color:var(--fg);font:15px/1.5 var(--sans)}
|
|
493
|
+
a{color:inherit}main{max-width:1080px;margin:0 auto;padding:24px 16px 64px}
|
|
494
|
+
header{display:flex;align-items:baseline;justify-content:space-between;gap:16px;flex-wrap:wrap;margin-bottom:20px}
|
|
495
|
+
header h1{font-size:18px;margin:0;letter-spacing:-.01em}header h1 a{text-decoration:none}
|
|
496
|
+
header .meta{color:var(--muted);font-size:13px}
|
|
497
|
+
.card{background:var(--card);border:1px solid var(--line);border-radius:10px;overflow:hidden}
|
|
498
|
+
table{width:100%;border-collapse:collapse;font-size:14px}
|
|
499
|
+
th{text-align:left;font-weight:600;color:var(--muted);font-size:12px;text-transform:uppercase;letter-spacing:.04em;padding:10px 12px;border-bottom:1px solid var(--line);white-space:nowrap}
|
|
500
|
+
td{padding:10px 12px;border-bottom:1px solid var(--line);vertical-align:top}
|
|
501
|
+
tr:last-child td{border-bottom:0}
|
|
502
|
+
.name{font-weight:600;white-space:nowrap}.name a{text-decoration:none}.name a:hover{text-decoration:underline}
|
|
503
|
+
.mono{font-family:var(--mono);font-size:13px}.muted{color:var(--muted)}.nowrap{white-space:nowrap}
|
|
504
|
+
.pill{display:inline-flex;align-items:center;gap:6px;padding:2px 9px;border-radius:999px;background:var(--pill);font-size:12px;font-weight:600;white-space:nowrap}
|
|
505
|
+
.pill::before{content:"";width:7px;height:7px;border-radius:50%;background:currentColor}
|
|
506
|
+
.ok{color:var(--ok)}.warn{color:var(--warn)}.bad{color:var(--bad)}.info{color:var(--info)}.mutedpill{color:var(--muted)}
|
|
507
|
+
.spark{display:block}
|
|
508
|
+
form.inline{display:inline}
|
|
509
|
+
button,select{font:inherit;font-size:13px;padding:5px 10px;border:1px solid var(--line);border-radius:7px;background:var(--card);color:var(--fg);cursor:pointer}
|
|
510
|
+
button:hover{border-color:var(--muted)}
|
|
511
|
+
.actions{display:flex;gap:8px;align-items:center;flex-wrap:wrap}
|
|
512
|
+
.grid{display:grid;grid-template-columns:repeat(auto-fit,minmax(180px,1fr));gap:12px;margin:0 0 20px}
|
|
513
|
+
.stat{padding:12px 14px}.stat .k{font-size:12px;color:var(--muted);text-transform:uppercase;letter-spacing:.04em}.stat .v{font-size:20px;font-weight:600;margin-top:2px}
|
|
514
|
+
pre{margin:0;padding:10px 12px;background:var(--pill);border-radius:8px;font:12.5px/1.45 var(--mono);white-space:pre-wrap;word-break:break-word;max-height:320px;overflow:auto}
|
|
515
|
+
details summary{cursor:pointer;color:var(--muted);font-size:13px}details{margin-top:6px}
|
|
516
|
+
.empty{padding:40px 16px;text-align:center;color:var(--muted)}
|
|
517
|
+
dl{display:grid;grid-template-columns:max-content 1fr;gap:6px 16px;margin:0;padding:14px 16px;font-size:14px}dt{color:var(--muted)}dd{margin:0}
|
|
518
|
+
footer{margin-top:28px;color:var(--muted);font-size:12px}
|
|
519
|
+
@media(max-width:720px){.hide-sm{display:none}main{padding:16px 16px 48px}}
|
|
520
|
+
`;
|
|
521
|
+
function layout(title, body, options = {}) {
|
|
522
|
+
return `<!doctype html>
|
|
523
|
+
<html lang="en">
|
|
524
|
+
<head>
|
|
525
|
+
<meta charset="utf-8">
|
|
526
|
+
<meta name="viewport" content="width=device-width,initial-scale=1">
|
|
527
|
+
<meta name="robots" content="noindex,nofollow">
|
|
528
|
+
${options.refresh ? `<meta http-equiv="refresh" content="${options.refresh}">` : ""}
|
|
529
|
+
<title>${h(title)}</title>
|
|
530
|
+
<style>${CSS}</style>
|
|
531
|
+
</head>
|
|
532
|
+
<body><main>${body}</main></body>
|
|
533
|
+
</html>`;
|
|
534
|
+
}
|
|
535
|
+
function healthPill(job) {
|
|
536
|
+
const map = {
|
|
537
|
+
healthy: ["ok", "healthy"],
|
|
538
|
+
late: ["warn", "late"],
|
|
539
|
+
failing: ["bad", "failing"],
|
|
540
|
+
stuck: ["bad", "stuck"],
|
|
541
|
+
silenced: ["mutedpill", "silenced"],
|
|
542
|
+
never_ran: ["info", "never ran"]
|
|
543
|
+
};
|
|
544
|
+
const [cls, label] = map[job.health];
|
|
545
|
+
const extras = job.open.filter((c) => !["missed", "failed", "stuck"].includes(c)).map((c) => c.replace("_", " "));
|
|
546
|
+
return `<span class="pill ${cls}">${label}</span>${extras.length ? ` <span class="pill warn">${h(extras.join(", "))}</span>` : ""}`;
|
|
547
|
+
}
|
|
548
|
+
function runPill(run) {
|
|
549
|
+
const cls = run.status === "ok" ? "ok" : run.status === "running" ? "info" : "bad";
|
|
550
|
+
return `<span class="pill ${cls}">${h(run.status)}</span>`;
|
|
551
|
+
}
|
|
552
|
+
function sparkline(runs) {
|
|
553
|
+
const points = [...runs].reverse().filter((r) => r.durationMs !== null).slice(-20);
|
|
554
|
+
if (points.length < 2) return "";
|
|
555
|
+
const w = 96, hgt = 22;
|
|
556
|
+
const max = Math.max(...points.map((r) => r.durationMs), 1);
|
|
557
|
+
const step = w / (points.length - 1);
|
|
558
|
+
const path = points.map((r, i) => `${i === 0 ? "M" : "L"}${(i * step).toFixed(1)},${(hgt - 2 - r.durationMs / max * (hgt - 4)).toFixed(1)}`).join(" ");
|
|
559
|
+
const dots = points.map((r, i) => r.status === "ok" ? "" : `<circle cx="${(i * step).toFixed(1)}" cy="${(hgt - 2 - r.durationMs / max * (hgt - 4)).toFixed(1)}" r="2.2" fill="var(--bad)"/>`).join("");
|
|
560
|
+
return `<svg class="spark" width="${w}" height="${hgt}" viewBox="0 0 ${w} ${hgt}" aria-hidden="true"><path d="${path}" fill="none" stroke="var(--muted)" stroke-width="1.5"/>${dots}</svg>`;
|
|
561
|
+
}
|
|
562
|
+
function stamp(at, now) {
|
|
563
|
+
if (at === null) return `<span class="muted">never</span>`;
|
|
564
|
+
const iso = new Date(at).toISOString().replace("T", " ").slice(0, 19);
|
|
565
|
+
return `<span class="nowrap" title="${iso} UTC">${h(formatRelative(at, now))}</span>`;
|
|
566
|
+
}
|
|
567
|
+
function dashboardPage(jobs, runsByJob, now, base, checkedAt) {
|
|
568
|
+
const rows = jobs.map((job) => {
|
|
569
|
+
const last = job.lastRun;
|
|
570
|
+
return `<tr>
|
|
571
|
+
<td class="name"><a href="${h(base)}/jobs/${encodeURIComponent(job.name)}">${h(job.name)}</a>${job.definition.description ? `<div class="muted" style="font-weight:400;font-size:13px;white-space:normal">${h(job.definition.description)}</div>` : ""}</td>
|
|
572
|
+
<td>${healthPill(job)}</td>
|
|
573
|
+
<td class="mono nowrap">${h(job.definition.schedule ?? "")}<span class="muted">${job.definition.schedule ? "" : "no schedule"}</span></td>
|
|
574
|
+
<td class="nowrap">${last ? `${runPill(last)} ${stamp(last.startedAt, now)}${last.durationMs !== null ? ` <span class="muted">${h(formatDuration(last.durationMs))}</span>` : ""}` : `<span class="muted">never</span>`}</td>
|
|
575
|
+
<td class="nowrap hide-sm">${stamp(job.nextExpectedAt, now)}</td>
|
|
576
|
+
<td class="hide-sm">${sparkline(runsByJob.get(job.name) ?? [])}</td>
|
|
577
|
+
</tr>`;
|
|
578
|
+
}).join("\n");
|
|
579
|
+
const body = `
|
|
580
|
+
<header>
|
|
581
|
+
<h1><a href="${h(base)}/">CronWatch</a></h1>
|
|
582
|
+
<div class="actions">
|
|
583
|
+
<span class="meta">${jobs.length} job${jobs.length === 1 ? "" : "s"}${""}</span>
|
|
584
|
+
<form class="inline" method="post" action="${h(base)}/check"><button type="submit">Run check now</button></form>
|
|
585
|
+
</div>
|
|
586
|
+
</header>
|
|
587
|
+
<div class="card">
|
|
588
|
+
${jobs.length === 0 ? `<div class="empty">No jobs yet. Declare one with <code class="mono">cw.job("name", { schedule: "0 2 * * *" })</code> and run it once.</div>` : `<table>
|
|
589
|
+
<thead><tr><th>Job</th><th>Health</th><th>Schedule</th><th>Last run</th><th class="hide-sm">Next due</th><th class="hide-sm">Durations</th></tr></thead>
|
|
590
|
+
<tbody>${rows}</tbody></table>`}
|
|
591
|
+
</div>
|
|
592
|
+
<footer>Refreshes every minute. <a href="${h(base)}/api/jobs">JSON</a></footer>`;
|
|
593
|
+
return layout("CronWatch", body, { refresh: 60 });
|
|
594
|
+
}
|
|
595
|
+
function jobPage(job, runs, now, base) {
|
|
596
|
+
const d = job.definition;
|
|
597
|
+
const okRate = `${Math.round(job.stats.okRate * 100)}%`;
|
|
598
|
+
const runRows = runs.map((run) => {
|
|
599
|
+
const detail = [
|
|
600
|
+
run.error ? `<details open><summary>error</summary><pre>${h(run.error)}</pre></details>` : "",
|
|
601
|
+
run.output ? `<details${run.status === "ok" ? "" : " open"}><summary>output</summary><pre>${h(run.output)}</pre></details>` : ""
|
|
602
|
+
].join("");
|
|
603
|
+
const metrics = Object.entries(run.metrics).map(([k, v]) => `<span class="pill mutedpill">${h(k)} ${h(Number.isInteger(v) ? v : v.toFixed(4))}</span>`).join(" ");
|
|
604
|
+
return `<tr>
|
|
605
|
+
<td class="nowrap">${runPill(run)}</td>
|
|
606
|
+
<td class="nowrap">${stamp(run.startedAt, now)}</td>
|
|
607
|
+
<td class="nowrap">${run.durationMs !== null ? h(formatDuration(run.durationMs)) : `<span class="muted">running</span>`}</td>
|
|
608
|
+
<td class="hide-sm">${metrics}</td>
|
|
609
|
+
<td class="mono hide-sm muted">${h(run.trigger)}</td>
|
|
610
|
+
</tr>${detail ? `<tr><td colspan="5" style="padding-top:0">${detail}</td></tr>` : ""}`;
|
|
611
|
+
}).join("\n");
|
|
612
|
+
const silenced = job.silencedUntil !== null && job.silencedUntil > now;
|
|
613
|
+
const body = `
|
|
614
|
+
<header>
|
|
615
|
+
<h1><a href="${h(base)}/">CronWatch</a> <span class="muted">/</span> ${h(job.name)}</h1>
|
|
616
|
+
<div class="actions">
|
|
617
|
+
${healthPill(job)}
|
|
618
|
+
${silenced ? `<form class="inline" method="post" action="${h(base)}/jobs/${encodeURIComponent(job.name)}/unsilence"><button type="submit">Unsilence (until ${h(formatRelative(job.silencedUntil, now))})</button></form>` : `<form class="inline" method="post" action="${h(base)}/jobs/${encodeURIComponent(job.name)}/silence"><select name="for"><option value="1h">1 hour</option><option value="4h">4 hours</option><option value="1d">1 day</option><option value="7d">1 week</option></select> <button type="submit">Silence</button></form>`}
|
|
619
|
+
<form class="inline" method="post" action="${h(base)}/jobs/${encodeURIComponent(job.name)}/forget" onsubmit="return confirm('Remove this job and its runs from the store?')"><button type="submit">Forget</button></form>
|
|
620
|
+
</div>
|
|
621
|
+
</header>
|
|
622
|
+
<div class="grid">
|
|
623
|
+
<div class="card stat"><div class="k">Last run</div><div class="v">${job.lastRun ? h(formatRelative(job.lastRun.startedAt, now)) : "never"}</div></div>
|
|
624
|
+
<div class="card stat"><div class="k">Next due</div><div class="v">${job.nextExpectedAt ? h(formatRelative(job.nextExpectedAt, now)) : "no schedule"}</div></div>
|
|
625
|
+
<div class="card stat"><div class="k">Success, last ${job.stats.runs}</div><div class="v">${okRate}</div></div>
|
|
626
|
+
<div class="card stat"><div class="k">p50 / p95</div><div class="v">${job.stats.p50Ms !== null ? h(formatDuration(job.stats.p50Ms)) : "?"} <span class="muted">/</span> ${job.stats.p95Ms !== null ? h(formatDuration(job.stats.p95Ms)) : "?"}</div></div>
|
|
627
|
+
</div>
|
|
628
|
+
<div class="card" style="margin-bottom:20px">
|
|
629
|
+
<dl>
|
|
630
|
+
<dt>Schedule</dt><dd class="mono">${d.schedule ? h(d.schedule) + (d.timezone ? ` <span class="muted">${h(d.timezone)}</span>` : "") : `<span class="muted">none</span>`}</dd>
|
|
631
|
+
<dt>Grace</dt><dd class="mono">${h(d.grace ?? "10m")}</dd>
|
|
632
|
+
<dt>Timeout</dt><dd class="mono">${h(d.timeout ?? "1h")}</dd>
|
|
633
|
+
${d.maxDuration ? `<dt>Max duration</dt><dd class="mono">${h(d.maxDuration)}</dd>` : ""}
|
|
634
|
+
${d.budget ? `<dt>Budget</dt><dd class="mono">${h(Object.entries(d.budget).map(([k, v]) => `${k} \u2264 ${v}`).join(", "))}</dd>` : ""}
|
|
635
|
+
${d.expect ? `<dt>Expect</dt><dd class="mono">${h(d.expect)}</dd>` : ""}
|
|
636
|
+
${d.failuresBeforeAlert && d.failuresBeforeAlert > 1 ? `<dt>Alert after</dt><dd>${d.failuresBeforeAlert} consecutive failures</dd>` : ""}
|
|
637
|
+
${d.description ? `<dt>Description</dt><dd>${h(d.description)}</dd>` : ""}
|
|
638
|
+
${d.tags?.length ? `<dt>Tags</dt><dd>${d.tags.map((t) => `<span class="pill mutedpill">${h(t)}</span>`).join(" ")}</dd>` : ""}
|
|
639
|
+
${job.open.length ? `<dt>Open</dt><dd>${job.open.map((c) => `<span class="pill warn">${h(c.replace("_", " "))}</span>`).join(" ")}</dd>` : ""}
|
|
640
|
+
${job.consecutiveFailures > 0 ? `<dt>Consecutive failures</dt><dd>${job.consecutiveFailures}</dd>` : ""}
|
|
641
|
+
</dl>
|
|
642
|
+
</div>
|
|
643
|
+
<div class="card">
|
|
644
|
+
${runs.length === 0 ? `<div class="empty">No runs yet.</div>` : `<table>
|
|
645
|
+
<thead><tr><th>Status</th><th>Started</th><th>Duration</th><th class="hide-sm">Metrics</th><th class="hide-sm">Trigger</th></tr></thead>
|
|
646
|
+
<tbody>${runRows}</tbody></table>`}
|
|
647
|
+
</div>
|
|
648
|
+
<footer><a href="${h(base)}/api/jobs/${encodeURIComponent(job.name)}">JSON</a></footer>`;
|
|
649
|
+
return layout(`${job.name}: CronWatch`, body, { refresh: 60 });
|
|
650
|
+
}
|
|
651
|
+
function messagePage(title, message, base) {
|
|
652
|
+
return layout(title, `<header><h1><a href="${h(base)}/">CronWatch</a></h1></header><div class="card"><div class="empty"><strong>${h(title)}</strong><br>${h(message)}</div></div>`);
|
|
653
|
+
}
|
|
654
|
+
|
|
655
|
+
// src/routes/index.ts
|
|
656
|
+
var COOKIE = "cronwatch_token";
|
|
657
|
+
function constantTimeEqual(a, b) {
|
|
658
|
+
if (a.length !== b.length) return false;
|
|
659
|
+
let diff = 0;
|
|
660
|
+
for (let i = 0; i < a.length; i++) diff |= a.charCodeAt(i) ^ b.charCodeAt(i);
|
|
661
|
+
return diff === 0;
|
|
662
|
+
}
|
|
663
|
+
function readCookie(request, name) {
|
|
664
|
+
const header = request.headers.get("cookie");
|
|
665
|
+
if (!header) return null;
|
|
666
|
+
for (const part of header.split(";")) {
|
|
667
|
+
const [k, ...rest] = part.trim().split("=");
|
|
668
|
+
if (k === name) return decodeURIComponent(rest.join("="));
|
|
669
|
+
}
|
|
670
|
+
return null;
|
|
671
|
+
}
|
|
672
|
+
function stripBase(pathname, base) {
|
|
673
|
+
let path = pathname.startsWith(base) ? pathname.slice(base.length) : pathname;
|
|
674
|
+
if (path === "") path = "/";
|
|
675
|
+
if (path.length > 1 && path.endsWith("/")) path = path.slice(0, -1);
|
|
676
|
+
return path;
|
|
677
|
+
}
|
|
678
|
+
async function readBody(request) {
|
|
679
|
+
const type = request.headers.get("content-type") ?? "";
|
|
680
|
+
try {
|
|
681
|
+
if (type.includes("application/json")) {
|
|
682
|
+
const data = await request.json();
|
|
683
|
+
return typeof data === "object" && data !== null ? Object.fromEntries(Object.entries(data).map(([k, v]) => [k, String(v)])) : {};
|
|
684
|
+
}
|
|
685
|
+
if (type.includes("application/x-www-form-urlencoded") || type.includes("multipart/form-data")) {
|
|
686
|
+
const form = await request.formData();
|
|
687
|
+
const out = {};
|
|
688
|
+
for (const [k, v] of form.entries()) out[k] = String(v);
|
|
689
|
+
return out;
|
|
690
|
+
}
|
|
691
|
+
} catch {
|
|
692
|
+
return {};
|
|
693
|
+
}
|
|
694
|
+
return {};
|
|
695
|
+
}
|
|
696
|
+
function createRoutes(cw, options = {}) {
|
|
697
|
+
const token = options.token === void 0 ? process.env.CRONWATCH_TOKEN ?? null : options.token;
|
|
698
|
+
const base = (options.basePath ?? "/cronwatch").replace(/\/+$/, "");
|
|
699
|
+
const production = process.env.NODE_ENV === "production";
|
|
700
|
+
const handler = async (request) => {
|
|
701
|
+
const url = new URL(request.url);
|
|
702
|
+
const path = stripBase(url.pathname, base);
|
|
703
|
+
const wantsHtml = !path.startsWith("/api");
|
|
704
|
+
const method = request.method.toUpperCase();
|
|
705
|
+
if (!token && production) {
|
|
706
|
+
return wantsHtml ? html(messagePage("CronWatch routes are locked", "Set CRONWATCH_TOKEN (or pass token to cw.routes()) to use them in production.", base), 503) : json({ ok: false, error: "CRONWATCH_TOKEN is not set" }, 503);
|
|
707
|
+
}
|
|
708
|
+
if (token) {
|
|
709
|
+
const bearer = request.headers.get("authorization")?.replace(/^Bearer\s+/i, "") ?? null;
|
|
710
|
+
const query = url.searchParams.get("token");
|
|
711
|
+
const cookie = readCookie(request, COOKIE);
|
|
712
|
+
const presented = bearer ?? query ?? cookie;
|
|
713
|
+
const isCheck = path === "/api/check";
|
|
714
|
+
const cronSecretOk = isCheck && bearer !== null && cw.cronSecret !== null && constantTimeEqual(bearer, cw.cronSecret);
|
|
715
|
+
if (!cronSecretOk && (presented === null || !constantTimeEqual(presented, token))) {
|
|
716
|
+
return wantsHtml ? html(messagePage("Sign in", `Open this page with ?token=<your CRONWATCH_TOKEN> once and it will stay signed in.`, base), 401) : json({ ok: false, error: "Unauthorized" }, 401);
|
|
717
|
+
}
|
|
718
|
+
if (query !== null && wantsHtml && method === "GET") {
|
|
719
|
+
url.searchParams.delete("token");
|
|
720
|
+
const secure = url.protocol === "https:" ? "; Secure" : "";
|
|
721
|
+
return new Response(null, {
|
|
722
|
+
status: 303,
|
|
723
|
+
headers: {
|
|
724
|
+
location: url.pathname + (url.search || ""),
|
|
725
|
+
"set-cookie": `${COOKIE}=${encodeURIComponent(token)}; Path=${base || "/"}; HttpOnly; SameSite=Lax; Max-Age=${60 * 60 * 24 * 30}${secure}`
|
|
726
|
+
}
|
|
727
|
+
});
|
|
728
|
+
}
|
|
729
|
+
}
|
|
730
|
+
const redirectBack = () => {
|
|
731
|
+
const referer = request.headers.get("referer") ?? "";
|
|
732
|
+
const location = referer.startsWith(url.origin + "/") ? referer : `${base}/`;
|
|
733
|
+
return new Response(null, { status: 303, headers: { location } });
|
|
734
|
+
};
|
|
735
|
+
const jobName = (segment) => segment ? decodeURIComponent(segment) : "";
|
|
736
|
+
const parts = path.split("/").filter(Boolean);
|
|
737
|
+
if (method === "GET" && path === "/") {
|
|
738
|
+
const jobs = await cw.jobs();
|
|
739
|
+
const runsByJob = /* @__PURE__ */ new Map();
|
|
740
|
+
for (const job of jobs) runsByJob.set(job.name, await cw.runs(job.name, 20));
|
|
741
|
+
return html(dashboardPage(jobs, runsByJob, cw.now(), base));
|
|
742
|
+
}
|
|
743
|
+
if (method === "GET" && parts[0] === "jobs" && parts.length === 2) {
|
|
744
|
+
const job = await cw.jobSummary(jobName(parts[1]));
|
|
745
|
+
if (!job) return html(messagePage("No such job", `${jobName(parts[1])} is not in the store.`, base), 404);
|
|
746
|
+
return html(jobPage(job, await cw.runs(job.name, 50), cw.now(), base));
|
|
747
|
+
}
|
|
748
|
+
if (method === "POST" && path === "/check") {
|
|
749
|
+
await cw.check();
|
|
750
|
+
return redirectBack();
|
|
751
|
+
}
|
|
752
|
+
if (method === "POST" && parts[0] === "jobs" && parts.length === 3) {
|
|
753
|
+
const name = jobName(parts[1]);
|
|
754
|
+
const body = await readBody(request);
|
|
755
|
+
if (parts[2] === "silence") await cw.silence(name, safeDuration(body.for, "1h"));
|
|
756
|
+
else if (parts[2] === "unsilence") await cw.unsilence(name);
|
|
757
|
+
else if (parts[2] === "forget") {
|
|
758
|
+
await cw.forget(name);
|
|
759
|
+
return new Response(null, { status: 303, headers: { location: `${base}/` } });
|
|
760
|
+
} else return html(messagePage("Not found", path, base), 404);
|
|
761
|
+
return redirectBack();
|
|
762
|
+
}
|
|
763
|
+
if (parts[0] === "api") {
|
|
764
|
+
const rest = parts.slice(1);
|
|
765
|
+
if (method === "GET" && rest[0] === "jobs" && rest.length === 1) {
|
|
766
|
+
return json({ ok: true, jobs: await cw.jobs() });
|
|
767
|
+
}
|
|
768
|
+
if (rest[0] === "jobs" && rest.length === 2) {
|
|
769
|
+
const name = jobName(rest[1]);
|
|
770
|
+
if (method === "GET") {
|
|
771
|
+
const job = await cw.jobSummary(name);
|
|
772
|
+
if (!job) return json({ ok: false, error: "No such job" }, 404);
|
|
773
|
+
const limit = Number(url.searchParams.get("runs") ?? 20);
|
|
774
|
+
return json({ ok: true, job, runs: await cw.runs(name, Number.isFinite(limit) ? limit : 20) });
|
|
775
|
+
}
|
|
776
|
+
if (method === "DELETE") {
|
|
777
|
+
await cw.forget(name);
|
|
778
|
+
return json({ ok: true });
|
|
779
|
+
}
|
|
780
|
+
}
|
|
781
|
+
if (method === "POST" && rest[0] === "jobs" && rest.length === 3) {
|
|
782
|
+
const name = jobName(rest[1]);
|
|
783
|
+
if (!await cw.jobSummary(name)) return json({ ok: false, error: "No such job" }, 404);
|
|
784
|
+
const body = await readBody(request);
|
|
785
|
+
if (rest[2] === "silence") return json({ ok: true, state: await cw.silence(name, safeDuration(body.for ?? url.searchParams.get("for"), "1h")) });
|
|
786
|
+
if (rest[2] === "unsilence") return json({ ok: true, state: await cw.unsilence(name) });
|
|
787
|
+
}
|
|
788
|
+
if ((method === "GET" || method === "POST") && rest[0] === "check" && rest.length === 1) {
|
|
789
|
+
const result = await cw.check();
|
|
790
|
+
return json({ ok: true, ...result });
|
|
791
|
+
}
|
|
792
|
+
if (method === "GET" && rest[0] === "runs" && rest.length === 2) {
|
|
793
|
+
const run = await cw.getRun(rest[1]);
|
|
794
|
+
return run ? json({ ok: true, run }) : json({ ok: false, error: "No such run" }, 404);
|
|
795
|
+
}
|
|
796
|
+
return json({ ok: false, error: "Not found" }, 404);
|
|
797
|
+
}
|
|
798
|
+
return html(messagePage("Not found", path, base), 404);
|
|
799
|
+
};
|
|
800
|
+
return { handler, GET: handler, POST: handler, DELETE: handler };
|
|
801
|
+
}
|
|
802
|
+
function safeDuration(value, fallback) {
|
|
803
|
+
if (!value) return fallback;
|
|
804
|
+
try {
|
|
805
|
+
parseDuration(value);
|
|
806
|
+
return value;
|
|
807
|
+
} catch {
|
|
808
|
+
return fallback;
|
|
809
|
+
}
|
|
810
|
+
}
|
|
811
|
+
function html(body, status = 200) {
|
|
812
|
+
return new Response(body, {
|
|
813
|
+
status,
|
|
814
|
+
headers: { "content-type": "text/html; charset=utf-8", "cache-control": "no-store", "x-robots-tag": "noindex" }
|
|
815
|
+
});
|
|
816
|
+
}
|
|
817
|
+
|
|
818
|
+
// src/stores/memory.ts
|
|
819
|
+
function memory() {
|
|
820
|
+
const jobs = /* @__PURE__ */ new Map();
|
|
821
|
+
const runs = /* @__PURE__ */ new Map();
|
|
822
|
+
const order = /* @__PURE__ */ new Map();
|
|
823
|
+
const states = /* @__PURE__ */ new Map();
|
|
824
|
+
let seq = 0;
|
|
825
|
+
const clone = (v) => JSON.parse(JSON.stringify(v));
|
|
826
|
+
return {
|
|
827
|
+
async upsertJob(definition, now) {
|
|
828
|
+
const existing = jobs.get(definition.name);
|
|
829
|
+
jobs.set(definition.name, {
|
|
830
|
+
name: definition.name,
|
|
831
|
+
definition: clone(definition),
|
|
832
|
+
createdAt: existing?.createdAt ?? now,
|
|
833
|
+
updatedAt: now
|
|
834
|
+
});
|
|
835
|
+
},
|
|
836
|
+
async getJob(name) {
|
|
837
|
+
const j = jobs.get(name);
|
|
838
|
+
return j ? clone(j) : null;
|
|
839
|
+
},
|
|
840
|
+
async listJobs() {
|
|
841
|
+
return [...jobs.values()].map(clone).sort((a, b) => a.name.localeCompare(b.name));
|
|
842
|
+
},
|
|
843
|
+
async deleteJob(name) {
|
|
844
|
+
jobs.delete(name);
|
|
845
|
+
states.delete(name);
|
|
846
|
+
for (const [id, run] of runs) if (run.job === name) {
|
|
847
|
+
runs.delete(id);
|
|
848
|
+
order.delete(id);
|
|
849
|
+
}
|
|
850
|
+
},
|
|
851
|
+
async insertRun(run) {
|
|
852
|
+
runs.set(run.id, clone(run));
|
|
853
|
+
order.set(run.id, ++seq);
|
|
854
|
+
},
|
|
855
|
+
async updateRun(run) {
|
|
856
|
+
runs.set(run.id, clone(run));
|
|
857
|
+
},
|
|
858
|
+
async getRun(id) {
|
|
859
|
+
const r = runs.get(id);
|
|
860
|
+
return r ? clone(r) : null;
|
|
861
|
+
},
|
|
862
|
+
async listRuns(job, limit) {
|
|
863
|
+
return [...runs.values()].filter((r) => r.job === job).sort((a, b) => b.startedAt - a.startedAt || order.get(b.id) - order.get(a.id)).slice(0, limit).map(clone);
|
|
864
|
+
},
|
|
865
|
+
async lastRun(job) {
|
|
866
|
+
const list = await this.listRuns(job, 1);
|
|
867
|
+
return list[0] ?? null;
|
|
868
|
+
},
|
|
869
|
+
async runningRuns() {
|
|
870
|
+
return [...runs.values()].filter((r) => r.status === "running").map(clone);
|
|
871
|
+
},
|
|
872
|
+
async getState(job) {
|
|
873
|
+
const s = states.get(job);
|
|
874
|
+
return s ? clone(s) : null;
|
|
875
|
+
},
|
|
876
|
+
async setState(state) {
|
|
877
|
+
states.set(state.job, clone(state));
|
|
878
|
+
},
|
|
879
|
+
async prune(before) {
|
|
880
|
+
let n = 0;
|
|
881
|
+
for (const [id, run] of runs) {
|
|
882
|
+
if (run.status !== "running" && run.startedAt < before) {
|
|
883
|
+
runs.delete(id);
|
|
884
|
+
order.delete(id);
|
|
885
|
+
n++;
|
|
886
|
+
}
|
|
887
|
+
}
|
|
888
|
+
return n;
|
|
889
|
+
}
|
|
890
|
+
};
|
|
891
|
+
}
|
|
892
|
+
|
|
893
|
+
// src/client.ts
|
|
894
|
+
var NAME_RE = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,119}$/;
|
|
895
|
+
var TRIAGE_TIMEOUT_MS = 25e3;
|
|
896
|
+
var PRUNE_INTERVAL_MS = 60 * 6e4;
|
|
897
|
+
function constantTimeEqual2(a, b) {
|
|
898
|
+
if (a.length !== b.length) return false;
|
|
899
|
+
let diff = 0;
|
|
900
|
+
for (let i = 0; i < a.length; i++) diff |= a.charCodeAt(i) ^ b.charCodeAt(i);
|
|
901
|
+
return diff === 0;
|
|
902
|
+
}
|
|
903
|
+
var CronWatch = class {
|
|
904
|
+
store;
|
|
905
|
+
alerts;
|
|
906
|
+
triage;
|
|
907
|
+
cronSecret;
|
|
908
|
+
retentionMs;
|
|
909
|
+
now;
|
|
910
|
+
onError;
|
|
911
|
+
defaults;
|
|
912
|
+
definitions = /* @__PURE__ */ new Map();
|
|
913
|
+
synced = /* @__PURE__ */ new Set();
|
|
914
|
+
ready = null;
|
|
915
|
+
checking = null;
|
|
916
|
+
lastPruneAt = 0;
|
|
917
|
+
timer = null;
|
|
918
|
+
usingDefaultStore = false;
|
|
919
|
+
constructor(options = {}) {
|
|
920
|
+
this.store = options.store ?? (this.usingDefaultStore = true, memory());
|
|
921
|
+
this.alerts = options.alerts ?? [consoleChannel()];
|
|
922
|
+
this.triage = options.triage;
|
|
923
|
+
this.cronSecret = options.cronSecret === void 0 ? process.env.CRON_SECRET ?? null : options.cronSecret;
|
|
924
|
+
this.retentionMs = parseDuration(options.retention ?? "30d", "retention");
|
|
925
|
+
this.defaults = options.defaults ?? {};
|
|
926
|
+
this.now = options.now ?? (() => Date.now());
|
|
927
|
+
this.onError = options.onError ?? ((error, where) => console.error(`[cronwatch] ${where}:`, error));
|
|
928
|
+
}
|
|
929
|
+
/** Declare a job. Call it once, at module level, and keep the handle. */
|
|
930
|
+
job(name, options = {}) {
|
|
931
|
+
if (!NAME_RE.test(name)) {
|
|
932
|
+
throw new Error(`job name "${name}" must be 1 to 120 characters of letters, digits, ".", "_", ":" or "-"`);
|
|
933
|
+
}
|
|
934
|
+
const definition = { ...this.defaults, ...options, name };
|
|
935
|
+
if (definition.schedule) parseSchedule(definition.schedule, definition.timezone);
|
|
936
|
+
if (definition.grace !== void 0) parseDuration(definition.grace, "grace");
|
|
937
|
+
if (definition.timeout !== void 0) parseDuration(definition.timeout, "timeout");
|
|
938
|
+
if (definition.maxDuration !== void 0) parseDuration(definition.maxDuration, "maxDuration");
|
|
939
|
+
this.definitions.set(name, definition);
|
|
940
|
+
this.synced.delete(name);
|
|
941
|
+
return this.handle(definition);
|
|
942
|
+
}
|
|
943
|
+
run(name, optionsOrFn, maybeFn) {
|
|
944
|
+
const fn = typeof optionsOrFn === "function" ? optionsOrFn : maybeFn;
|
|
945
|
+
const options = typeof optionsOrFn === "function" ? void 0 : optionsOrFn;
|
|
946
|
+
const handle = options || !this.definitions.has(name) ? this.job(name, options) : this.handle(this.definitions.get(name));
|
|
947
|
+
return handle.run(fn);
|
|
948
|
+
}
|
|
949
|
+
/** The definitions declared in this process. */
|
|
950
|
+
definedJobs() {
|
|
951
|
+
return [...this.definitions.values()];
|
|
952
|
+
}
|
|
953
|
+
handle(definition) {
|
|
954
|
+
const self = this;
|
|
955
|
+
return {
|
|
956
|
+
name: definition.name,
|
|
957
|
+
definition,
|
|
958
|
+
async run(fn, options) {
|
|
959
|
+
const outcome = await self.execute(definition, fn, options?.trigger ?? "run");
|
|
960
|
+
if (outcome.threw) throw outcome.error;
|
|
961
|
+
return outcome.result;
|
|
962
|
+
},
|
|
963
|
+
handler(fn, options) {
|
|
964
|
+
const secret = options?.secret === void 0 ? self.cronSecret : options.secret;
|
|
965
|
+
return async (request) => {
|
|
966
|
+
if (secret) {
|
|
967
|
+
const header = request.headers.get("authorization") ?? "";
|
|
968
|
+
if (!constantTimeEqual2(header, `Bearer ${secret}`)) {
|
|
969
|
+
return json({ ok: false, error: "Unauthorized" }, 401);
|
|
970
|
+
}
|
|
971
|
+
}
|
|
972
|
+
const outcome = await self.execute(definition, (job) => fn(job, request), "handler");
|
|
973
|
+
if (outcome.result instanceof Response) return outcome.result;
|
|
974
|
+
const body = {
|
|
975
|
+
ok: outcome.run.status === "ok",
|
|
976
|
+
job: definition.name,
|
|
977
|
+
run: outcome.run.id,
|
|
978
|
+
status: outcome.run.status,
|
|
979
|
+
durationMs: outcome.run.durationMs,
|
|
980
|
+
...outcome.run.error ? { error: outcome.run.error.split("\n")[0] } : {}
|
|
981
|
+
};
|
|
982
|
+
return json(body, outcome.run.status === "ok" ? 200 : 500);
|
|
983
|
+
};
|
|
984
|
+
}
|
|
985
|
+
};
|
|
986
|
+
}
|
|
987
|
+
async ensureReady() {
|
|
988
|
+
if (!this.ready) {
|
|
989
|
+
this.ready = (async () => {
|
|
990
|
+
if (this.store.init) await this.store.init();
|
|
991
|
+
if (this.usingDefaultStore && process.env.NODE_ENV === "production") {
|
|
992
|
+
console.warn("[cronwatch] using the in-memory store: runs and state are lost on restart. Pass a store from @cronwatch/sdk/sqlite or @cronwatch/sdk/postgres.");
|
|
993
|
+
}
|
|
994
|
+
})();
|
|
995
|
+
}
|
|
996
|
+
await this.ready;
|
|
997
|
+
}
|
|
998
|
+
async sync(definition) {
|
|
999
|
+
await this.ensureReady();
|
|
1000
|
+
if (this.synced.has(definition.name)) return;
|
|
1001
|
+
await this.store.upsertJob(toStored(definition), this.now());
|
|
1002
|
+
this.synced.add(definition.name);
|
|
1003
|
+
}
|
|
1004
|
+
/** Runs a function as a recorded run. Never throws for the job's own error; see `threw`. */
|
|
1005
|
+
async execute(definition, fn, trigger) {
|
|
1006
|
+
await this.sync(definition);
|
|
1007
|
+
const startedAt = this.now();
|
|
1008
|
+
const run = {
|
|
1009
|
+
id: crypto.randomUUID(),
|
|
1010
|
+
job: definition.name,
|
|
1011
|
+
status: "running",
|
|
1012
|
+
startedAt,
|
|
1013
|
+
finishedAt: null,
|
|
1014
|
+
durationMs: null,
|
|
1015
|
+
error: null,
|
|
1016
|
+
output: null,
|
|
1017
|
+
metrics: {},
|
|
1018
|
+
trigger
|
|
1019
|
+
};
|
|
1020
|
+
await this.store.insertRun(run);
|
|
1021
|
+
const before = await this.store.getState(definition.name) ?? emptyState(definition.name);
|
|
1022
|
+
const started = onRunStart(before);
|
|
1023
|
+
if (openConditions(started.state).length !== openConditions(before).length) {
|
|
1024
|
+
await this.store.setState(started.state);
|
|
1025
|
+
}
|
|
1026
|
+
const recorder = createRecorder(run);
|
|
1027
|
+
const timer = setTimeout(() => recorder.abort(), timeoutMs(definition));
|
|
1028
|
+
if (typeof timer === "object" && "unref" in timer) timer.unref();
|
|
1029
|
+
let result;
|
|
1030
|
+
let error;
|
|
1031
|
+
let threw = false;
|
|
1032
|
+
try {
|
|
1033
|
+
result = await fn(recorder.context);
|
|
1034
|
+
} catch (e) {
|
|
1035
|
+
error = e;
|
|
1036
|
+
threw = true;
|
|
1037
|
+
} finally {
|
|
1038
|
+
clearTimeout(timer);
|
|
1039
|
+
}
|
|
1040
|
+
const finishedAt = this.now();
|
|
1041
|
+
run.finishedAt = finishedAt;
|
|
1042
|
+
run.durationMs = Math.max(0, finishedAt - startedAt);
|
|
1043
|
+
run.metrics = recorder.metrics();
|
|
1044
|
+
run.output = recorder.output() ?? (typeof result === "string" ? result : null);
|
|
1045
|
+
if (threw) {
|
|
1046
|
+
run.status = "failed";
|
|
1047
|
+
run.error = errorMessage(error);
|
|
1048
|
+
} else if (result instanceof Response && result.status >= 400) {
|
|
1049
|
+
run.status = "failed";
|
|
1050
|
+
run.error = `HTTP ${result.status}${result.statusText ? ` ${result.statusText}` : ""}`;
|
|
1051
|
+
} else {
|
|
1052
|
+
const unmet = checkExpectation(definition.expect, run.output);
|
|
1053
|
+
if (unmet) {
|
|
1054
|
+
run.status = "failed";
|
|
1055
|
+
run.error = unmet;
|
|
1056
|
+
} else {
|
|
1057
|
+
run.status = "ok";
|
|
1058
|
+
}
|
|
1059
|
+
}
|
|
1060
|
+
await this.store.updateRun(run);
|
|
1061
|
+
try {
|
|
1062
|
+
const history = (await this.store.listRuns(definition.name, BASELINE_WINDOW + 5)).filter((r) => r.id !== run.id);
|
|
1063
|
+
const state = await this.store.getState(definition.name) ?? emptyState(definition.name);
|
|
1064
|
+
const evaluation = onRunFinish(definition, run, state, history, started.openBefore, finishedAt);
|
|
1065
|
+
await this.settle(state, evaluation, toStored(definition), finishedAt);
|
|
1066
|
+
} catch (e) {
|
|
1067
|
+
this.onError(e, `evaluating ${definition.name}`);
|
|
1068
|
+
}
|
|
1069
|
+
return { run, result, error, threw };
|
|
1070
|
+
}
|
|
1071
|
+
/**
|
|
1072
|
+
* Look for missed and stuck runs across every job, send alerts, and prune
|
|
1073
|
+
* old runs. Call it from an interval (start()), a cron hitting the mounted
|
|
1074
|
+
* routes, or by hand. Concurrent calls share one check.
|
|
1075
|
+
*/
|
|
1076
|
+
check() {
|
|
1077
|
+
if (!this.checking) {
|
|
1078
|
+
this.checking = this.runCheck().finally(() => {
|
|
1079
|
+
this.checking = null;
|
|
1080
|
+
});
|
|
1081
|
+
}
|
|
1082
|
+
return this.checking;
|
|
1083
|
+
}
|
|
1084
|
+
async runCheck() {
|
|
1085
|
+
await this.ensureReady();
|
|
1086
|
+
for (const definition of this.definitions.values()) await this.sync(definition);
|
|
1087
|
+
const now = this.now();
|
|
1088
|
+
const alerts = [];
|
|
1089
|
+
for (const run of await this.store.runningRuns()) {
|
|
1090
|
+
const stored = await this.store.getJob(run.job);
|
|
1091
|
+
const declared = this.definitions.get(run.job);
|
|
1092
|
+
const definition = declared ? toStored(declared) : stored?.definition;
|
|
1093
|
+
if (!definition || !isStuck(definition, run, now)) continue;
|
|
1094
|
+
run.status = "timeout";
|
|
1095
|
+
run.finishedAt = now;
|
|
1096
|
+
run.durationMs = now - run.startedAt;
|
|
1097
|
+
run.error = `Still running after ${Math.round(timeoutMs(definition) / 6e4)} minutes; marked as timed out`;
|
|
1098
|
+
await this.store.updateRun(run);
|
|
1099
|
+
const history = (await this.store.listRuns(run.job, BASELINE_WINDOW + 5)).filter((r) => r.id !== run.id);
|
|
1100
|
+
const state = await this.store.getState(run.job) ?? emptyState(run.job);
|
|
1101
|
+
const evaluation = onRunFinish(definition, run, state, history, [], now);
|
|
1102
|
+
alerts.push(...await this.settle(state, evaluation, definition, now));
|
|
1103
|
+
}
|
|
1104
|
+
const jobs = [];
|
|
1105
|
+
for (const stored of await this.store.listJobs()) {
|
|
1106
|
+
const definition = stored.definition;
|
|
1107
|
+
const lastRun = await this.store.lastRun(stored.name);
|
|
1108
|
+
const state = await this.store.getState(stored.name) ?? emptyState(stored.name);
|
|
1109
|
+
const evaluation = onCheck(definition, stored, lastRun, state, now);
|
|
1110
|
+
alerts.push(...await this.settle(state, evaluation, definition, now));
|
|
1111
|
+
jobs.push(await this.summarize(stored, lastRun, evaluation.state, evaluation.nextExpectedAt, now));
|
|
1112
|
+
}
|
|
1113
|
+
let pruned = 0;
|
|
1114
|
+
if (now - this.lastPruneAt > PRUNE_INTERVAL_MS) {
|
|
1115
|
+
this.lastPruneAt = now;
|
|
1116
|
+
try {
|
|
1117
|
+
pruned = await this.store.prune(now - this.retentionMs);
|
|
1118
|
+
} catch (e) {
|
|
1119
|
+
this.onError(e, "pruning");
|
|
1120
|
+
}
|
|
1121
|
+
}
|
|
1122
|
+
return { checkedAt: now, jobs, alerts, pruned };
|
|
1123
|
+
}
|
|
1124
|
+
/** Every job the store knows about, with its health. Does not send alerts. */
|
|
1125
|
+
async jobs() {
|
|
1126
|
+
await this.ensureReady();
|
|
1127
|
+
for (const definition of this.definitions.values()) await this.sync(definition);
|
|
1128
|
+
const now = this.now();
|
|
1129
|
+
const out = [];
|
|
1130
|
+
for (const stored of await this.store.listJobs()) {
|
|
1131
|
+
const lastRun = await this.store.lastRun(stored.name);
|
|
1132
|
+
const state = await this.store.getState(stored.name) ?? emptyState(stored.name);
|
|
1133
|
+
const { nextExpectedAt } = onCheck(stored.definition, stored, lastRun, state, now);
|
|
1134
|
+
out.push(await this.summarize(stored, lastRun, state, nextExpectedAt, now));
|
|
1135
|
+
}
|
|
1136
|
+
return out;
|
|
1137
|
+
}
|
|
1138
|
+
async jobSummary(name) {
|
|
1139
|
+
await this.ensureReady();
|
|
1140
|
+
const definition = this.definitions.get(name);
|
|
1141
|
+
if (definition) await this.sync(definition);
|
|
1142
|
+
const stored = await this.store.getJob(name);
|
|
1143
|
+
if (!stored) return null;
|
|
1144
|
+
const now = this.now();
|
|
1145
|
+
const lastRun = await this.store.lastRun(name);
|
|
1146
|
+
const state = await this.store.getState(name) ?? emptyState(name);
|
|
1147
|
+
const { nextExpectedAt } = onCheck(stored.definition, stored, lastRun, state, now);
|
|
1148
|
+
return this.summarize(stored, lastRun, state, nextExpectedAt, now);
|
|
1149
|
+
}
|
|
1150
|
+
async runs(name, limit = 50) {
|
|
1151
|
+
await this.ensureReady();
|
|
1152
|
+
return this.store.listRuns(name, Math.min(500, Math.max(1, limit)));
|
|
1153
|
+
}
|
|
1154
|
+
async getRun(id) {
|
|
1155
|
+
await this.ensureReady();
|
|
1156
|
+
return this.store.getRun(id);
|
|
1157
|
+
}
|
|
1158
|
+
/** Stop alerts for a job for a while. State keeps updating underneath. */
|
|
1159
|
+
async silence(name, duration) {
|
|
1160
|
+
await this.ensureReady();
|
|
1161
|
+
const state = await this.store.getState(name) ?? emptyState(name);
|
|
1162
|
+
state.silencedUntil = this.now() + parseDuration(duration, "silence duration");
|
|
1163
|
+
await this.store.setState(state);
|
|
1164
|
+
return state;
|
|
1165
|
+
}
|
|
1166
|
+
async unsilence(name) {
|
|
1167
|
+
await this.ensureReady();
|
|
1168
|
+
const state = await this.store.getState(name) ?? emptyState(name);
|
|
1169
|
+
state.silencedUntil = null;
|
|
1170
|
+
await this.store.setState(state);
|
|
1171
|
+
return state;
|
|
1172
|
+
}
|
|
1173
|
+
/** Remove a job and its runs from the store. A job still declared in code comes back on its next run. */
|
|
1174
|
+
async forget(name) {
|
|
1175
|
+
await this.ensureReady();
|
|
1176
|
+
this.definitions.delete(name);
|
|
1177
|
+
this.synced.delete(name);
|
|
1178
|
+
await this.store.deleteJob(name);
|
|
1179
|
+
}
|
|
1180
|
+
/** The dashboard and JSON API as fetch-style handlers. See createRoutes(). */
|
|
1181
|
+
routes(options) {
|
|
1182
|
+
return createRoutes(this, options);
|
|
1183
|
+
}
|
|
1184
|
+
/** Check on an interval, for long-running servers. Default every minute. */
|
|
1185
|
+
start(every = "1m") {
|
|
1186
|
+
if (this.timer) return;
|
|
1187
|
+
const ms = Math.max(5e3, parseDuration(every, "check interval"));
|
|
1188
|
+
const tick = () => this.check().catch((e) => this.onError(e, "check"));
|
|
1189
|
+
this.timer = setInterval(tick, ms);
|
|
1190
|
+
if (typeof this.timer === "object" && "unref" in this.timer) this.timer.unref();
|
|
1191
|
+
setTimeout(tick, 1e3).unref?.();
|
|
1192
|
+
}
|
|
1193
|
+
stop() {
|
|
1194
|
+
if (this.timer) clearInterval(this.timer);
|
|
1195
|
+
this.timer = null;
|
|
1196
|
+
}
|
|
1197
|
+
async close() {
|
|
1198
|
+
this.stop();
|
|
1199
|
+
if (this.store.close) await this.store.close();
|
|
1200
|
+
}
|
|
1201
|
+
async summarize(stored, lastRun, state, nextExpectedAt, now) {
|
|
1202
|
+
const recent = await this.store.listRuns(stored.name, BASELINE_WINDOW);
|
|
1203
|
+
const finished = recent.filter((r) => r.status !== "running");
|
|
1204
|
+
const okDurations = recent.filter((r) => r.status === "ok" && r.durationMs !== null).map((r) => r.durationMs);
|
|
1205
|
+
const open = openConditions(state);
|
|
1206
|
+
let health;
|
|
1207
|
+
if (state.silencedUntil !== null && state.silencedUntil > now) health = "silenced";
|
|
1208
|
+
else if (open.includes("stuck") || lastRun && isStuck(stored.definition, lastRun, now)) health = "stuck";
|
|
1209
|
+
else if (open.includes("failed") || lastRun?.status === "failed" || lastRun?.status === "timeout") health = "failing";
|
|
1210
|
+
else if (open.includes("missed")) health = "late";
|
|
1211
|
+
else if (!lastRun) health = "never_ran";
|
|
1212
|
+
else health = "healthy";
|
|
1213
|
+
return {
|
|
1214
|
+
name: stored.name,
|
|
1215
|
+
definition: stored.definition,
|
|
1216
|
+
health,
|
|
1217
|
+
open,
|
|
1218
|
+
lastRun,
|
|
1219
|
+
nextExpectedAt,
|
|
1220
|
+
consecutiveFailures: state.consecutiveFailures,
|
|
1221
|
+
silencedUntil: state.silencedUntil,
|
|
1222
|
+
stats: {
|
|
1223
|
+
runs: finished.length,
|
|
1224
|
+
okRate: finished.length === 0 ? 1 : finished.filter((r) => r.status === "ok").length / finished.length,
|
|
1225
|
+
p50Ms: percentile(okDurations, 50),
|
|
1226
|
+
p95Ms: percentile(okDurations, 95)
|
|
1227
|
+
}
|
|
1228
|
+
};
|
|
1229
|
+
}
|
|
1230
|
+
/** Save an evaluation's state, honouring silence, and send its alerts. */
|
|
1231
|
+
async settle(previous, evaluation, definition, now) {
|
|
1232
|
+
if (isSilenced(previous, now)) {
|
|
1233
|
+
evaluation.state = muteOpens(previous, evaluation.state);
|
|
1234
|
+
evaluation.alerts = [];
|
|
1235
|
+
}
|
|
1236
|
+
if (JSON.stringify(evaluation.state) !== JSON.stringify(previous)) await this.store.setState(evaluation.state);
|
|
1237
|
+
return this.dispatch(evaluation.alerts, definition, evaluation.state, now);
|
|
1238
|
+
}
|
|
1239
|
+
async dispatch(drafts, definition, state, now) {
|
|
1240
|
+
if (drafts.length === 0) return [];
|
|
1241
|
+
const sent = [];
|
|
1242
|
+
for (const draft of drafts) {
|
|
1243
|
+
const alert = composeAlert(draft, definition, now);
|
|
1244
|
+
if (this.triage && alert.type !== "recovered") {
|
|
1245
|
+
try {
|
|
1246
|
+
const recentRuns = await this.store.listRuns(alert.job, 5);
|
|
1247
|
+
const diagnosis = await withTimeout(this.triage({ alert, recentRuns }), TRIAGE_TIMEOUT_MS);
|
|
1248
|
+
if (diagnosis) alert.triage = diagnosis;
|
|
1249
|
+
} catch (e) {
|
|
1250
|
+
this.onError(e, `triage for ${alert.job}`);
|
|
1251
|
+
}
|
|
1252
|
+
}
|
|
1253
|
+
for (const channel of this.alerts) {
|
|
1254
|
+
try {
|
|
1255
|
+
await channel.send(alert);
|
|
1256
|
+
} catch (e) {
|
|
1257
|
+
this.onError(e, `alert channel ${channel.name}`);
|
|
1258
|
+
}
|
|
1259
|
+
}
|
|
1260
|
+
sent.push(alert);
|
|
1261
|
+
}
|
|
1262
|
+
state.lastAlertAt = now;
|
|
1263
|
+
await this.store.setState(state);
|
|
1264
|
+
return sent;
|
|
1265
|
+
}
|
|
1266
|
+
};
|
|
1267
|
+
function withTimeout(promise, ms) {
|
|
1268
|
+
return new Promise((resolve, reject) => {
|
|
1269
|
+
const t = setTimeout(() => reject(new Error(`timed out after ${ms}ms`)), ms);
|
|
1270
|
+
promise.then((v) => {
|
|
1271
|
+
clearTimeout(t);
|
|
1272
|
+
resolve(v);
|
|
1273
|
+
}, (e) => {
|
|
1274
|
+
clearTimeout(t);
|
|
1275
|
+
reject(e);
|
|
1276
|
+
});
|
|
1277
|
+
});
|
|
1278
|
+
}
|
|
1279
|
+
function consoleChannel() {
|
|
1280
|
+
return {
|
|
1281
|
+
name: "console",
|
|
1282
|
+
async send(alert) {
|
|
1283
|
+
const line = `[cronwatch] ${alert.title}
|
|
1284
|
+
${alert.message}${alert.triage ? `
|
|
1285
|
+
Triage: ${alert.triage}` : ""}`;
|
|
1286
|
+
if (alert.type === "recovered") console.info(line);
|
|
1287
|
+
else console.error(line);
|
|
1288
|
+
}
|
|
1289
|
+
};
|
|
1290
|
+
}
|
|
1291
|
+
function custom(name, send) {
|
|
1292
|
+
return { name, send: async (alert) => {
|
|
1293
|
+
await send(alert);
|
|
1294
|
+
} };
|
|
1295
|
+
}
|
|
1296
|
+
|
|
1297
|
+
// src/index.ts
|
|
1298
|
+
function cronwatch(options = {}) {
|
|
1299
|
+
return new CronWatch(options);
|
|
1300
|
+
}
|
|
1301
|
+
|
|
1302
|
+
exports.CronWatch = CronWatch;
|
|
1303
|
+
exports.composeAlert = composeAlert;
|
|
1304
|
+
exports.consoleChannel = consoleChannel;
|
|
1305
|
+
exports.createRoutes = createRoutes;
|
|
1306
|
+
exports.cronwatch = cronwatch;
|
|
1307
|
+
exports.custom = custom;
|
|
1308
|
+
exports.formatDuration = formatDuration;
|
|
1309
|
+
exports.formatRelative = formatRelative;
|
|
1310
|
+
exports.json = json;
|
|
1311
|
+
exports.memory = memory;
|
|
1312
|
+
exports.nextFire = nextFire;
|
|
1313
|
+
exports.parseDuration = parseDuration;
|
|
1314
|
+
exports.parseSchedule = parseSchedule;
|
|
1315
|
+
exports.previousFire = previousFire;
|
|
1316
|
+
//# sourceMappingURL=index.cjs.map
|
|
1317
|
+
//# sourceMappingURL=index.cjs.map
|